@matware/e2e-runner 1.1.1 → 1.2.1

package/src/learner-neo4j.js

@@ -0,0 +1,255 @@
+/**
+ * Neo4j knowledge graph integration for the learning system.
+ *
+ * Optional — all operations are no-ops if neo4j-driver is not installed.
+ * Enable with learningsNeo4j: true in config.
+ *
+ * Nodes: Project, Test, Page, Selector, ApiEndpoint, ErrorPattern, Run
+ * Relationships: VISITS, USES_SELECTOR, CALLS_API, FAILED_WITH, EXECUTED_IN, SELECTOR_ON
+ */
+
+let neo4j = null;
+let driver = null;
+
+/** Try to load neo4j-driver. Returns false if not available. */
+async function ensureDriver(config) {
+  if (driver) return true;
+  if (neo4j === false) return false; // already tried and failed
+
+  try {
+    neo4j = (await import('neo4j-driver')).default;
+    driver = neo4j.driver(
+      config.neo4jBoltUrl || 'bolt://localhost:7687',
+      neo4j.auth.basic(config.neo4jUser || 'neo4j', config.neo4jPassword || 'e2erunner')
+    );
+    // Verify connectivity
+    await driver.verifyConnectivity();
+    return true;
+  } catch {
+    neo4j = false;
+    driver = null;
+    return false;
+  }
+}
+
+/** Close the Neo4j driver. */
+export async function closeNeo4j() {
+  if (driver) {
+    await driver.close();
+    driver = null;
+  }
+}
+
+/**
+ * Writes learning data to Neo4j graph.
+ * Called from learnFromRun() when learningsNeo4j is enabled.
+ * All operations are no-ops if neo4j-driver is not installed or connection fails.
+ */
+export async function writeToGraph(projectId, runDbId, report, config, suiteName) {
+  if (!config?.learningsNeo4j) return;
+  const connected = await ensureDriver(config);
+  if (!connected) return;
+
+  const session = driver.session();
+  try {
+    const projectName = config.projectName || 'unknown';
+    const cwd = config._cwd || process.cwd();
+
+    // Ensure Project node
+    await session.run(
+      'MERGE (p:Project {cwd: $cwd}) SET p.name = $name, p.updatedAt = datetime()',
+      { cwd, name: projectName }
+    );
+
+    // Ensure Run node
+    await session.run(
+      `MERGE (r:Run {dbId: $runDbId})
+       SET r.total = $total, r.passed = $passed, r.failed = $failed,
+           r.passRate = $passRate, r.duration = $duration, r.suiteName = $suiteName,
+           r.createdAt = datetime()
+       WITH r
+       MATCH (p:Project {cwd: $cwd})
+       MERGE (r)-[:EXECUTED_IN]->(p)`,
+      {
+        runDbId: neo4j.int(runDbId),
+        total: neo4j.int(report.summary.total),
+        passed: neo4j.int(report.summary.passed),
+        failed: neo4j.int(report.summary.failed),
+        passRate: report.summary.passRate,
+        duration: report.summary.duration,
+        suiteName: suiteName || null,
+        cwd,
+      }
+    );
+
+    for (const result of report.results) {
+      // Test node
+      await session.run(
+        `MERGE (t:Test {name: $name, projectCwd: $cwd})
+         SET t.lastSuccess = $success, t.lastDuration = $duration, t.updatedAt = datetime()
+         WITH t
+         MATCH (r:Run {dbId: $runDbId})
+         MERGE (t)-[:EXECUTED_IN]->(r)`,
+        {
+          name: result.name,
+          cwd,
+          success: result.success,
+          duration: result.endTime && result.startTime
+            ? new Date(result.endTime) - new Date(result.startTime)
+            : 0,
+          runDbId: neo4j.int(runDbId),
+        }
+      );
+
+      // Page nodes + VISITS relationships
+      if (result.actions) {
+        for (const action of result.actions) {
+          if ((action.type === 'goto' || action.type === 'navigate') && action.value) {
+            let urlPath = action.value;
+            try { urlPath = new URL(action.value, 'http://placeholder').pathname; } catch { /* */ }
+
+            await session.run(
+              `MERGE (pg:Page {path: $path, projectCwd: $cwd})
+               SET pg.updatedAt = datetime()
+               WITH pg
+               MATCH (t:Test {name: $testName, projectCwd: $cwd})
+               MERGE (t)-[:VISITS]->(pg)`,
+              { path: urlPath, cwd, testName: result.name }
+            );
+          }
+
+          // Selector nodes + USES_SELECTOR relationships
+          if (action.selector) {
+            await session.run(
+              `MERGE (s:Selector {value: $selector, projectCwd: $cwd})
+               SET s.updatedAt = datetime()
+               WITH s
+               MATCH (t:Test {name: $testName, projectCwd: $cwd})
+               MERGE (t)-[:USES_SELECTOR {actionType: $actionType}]->(s)`,
+              { selector: action.selector, cwd, testName: result.name, actionType: action.type }
+            );
+          }
+        }
+      }
+
+      // API endpoint nodes + CALLS_API relationships
+      if (result.networkLogs?.length) {
+        for (const log of result.networkLogs) {
+          if (!log.url || !log.method) continue;
+          let urlPath;
+          try { urlPath = new URL(log.url).pathname; } catch { urlPath = log.url; }
+          urlPath = urlPath
+            .replace(/\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi, '/:uuid')
+            .replace(/\/\d+/g, '/:id');
+          const endpoint = `${log.method} ${urlPath}`;
+
+          await session.run(
+            `MERGE (a:ApiEndpoint {endpoint: $endpoint, projectCwd: $cwd})
+             SET a.updatedAt = datetime()
+             WITH a
+             MATCH (t:Test {name: $testName, projectCwd: $cwd})
+             MERGE (t)-[:CALLS_API]->(a)`,
+            { endpoint, cwd, testName: result.name }
+          );
+        }
+      }
+
+      // Error pattern nodes + FAILED_WITH relationships
+      if (result.error) {
+        const normalized = result.error.replace(/\d+ms/g, 'Nms').replace(/"[^"]+"/g, '"..."').slice(0, 200);
+        await session.run(
+          `MERGE (e:ErrorPattern {pattern: $pattern, projectCwd: $cwd})
+           SET e.count = COALESCE(e.count, 0) + 1, e.lastSeen = datetime()
+           WITH e
+           MATCH (t:Test {name: $testName, projectCwd: $cwd})
+           MERGE (t)-[:FAILED_WITH]->(e)`,
+          { pattern: normalized, cwd, testName: result.name }
+        );
+      }
+    }
+  } finally {
+    await session.close();
+  }
+}
+
+/**
+ * Query the graph for relationships — used by e2e_learnings MCP tool.
+ * Returns enriched insights about test dependencies, shared selectors, etc.
+ */
+export async function queryGraph(config, queryType, params = {}) {
+  if (!config?.learningsNeo4j) return null;
+  const connected = await ensureDriver(config);
+  if (!connected) return null;
+
+  const session = driver.session();
+  const cwd = config._cwd || process.cwd();
+
+  try {
+    switch (queryType) {
+      case 'test-dependencies': {
+        // Tests that share selectors or pages with a given test
+        const result = await session.run(
+          `MATCH (t1:Test {name: $testName, projectCwd: $cwd})-[:USES_SELECTOR]->(s)<-[:USES_SELECTOR]-(t2:Test)
+           WHERE t1 <> t2
+           RETURN DISTINCT t2.name AS related, COLLECT(DISTINCT s.value) AS sharedSelectors
+           LIMIT 20`,
+          { testName: params.testName, cwd }
+        );
+        return result.records.map(r => ({
+          test: r.get('related'),
+          sharedSelectors: r.get('sharedSelectors'),
+        }));
+      }
+
+      case 'page-impact': {
+        // All tests that visit a given page
+        const result = await session.run(
+          `MATCH (t:Test {projectCwd: $cwd})-[:VISITS]->(pg:Page {path: $path, projectCwd: $cwd})
+           RETURN t.name AS test, t.lastSuccess AS lastSuccess`,
+          { path: params.path, cwd }
+        );
+        return result.records.map(r => ({
+          test: r.get('test'),
+          lastSuccess: r.get('lastSuccess'),
+        }));
+      }
+
+      case 'error-impact': {
+        // Tests that failed with a given error pattern
+        const result = await session.run(
+          `MATCH (t:Test {projectCwd: $cwd})-[:FAILED_WITH]->(e:ErrorPattern)
+           WHERE e.pattern CONTAINS $search
+           RETURN t.name AS test, e.pattern AS pattern, e.count AS count
+           ORDER BY e.count DESC
+           LIMIT 20`,
+          { search: params.search || '', cwd }
+        );
+        return result.records.map(r => ({
+          test: r.get('test'),
+          pattern: r.get('pattern'),
+          count: r.get('count')?.toNumber?.() || r.get('count'),
+        }));
+      }
+
+      case 'selector-usage': {
+        // All tests and pages using a given selector
+        const result = await session.run(
+          `MATCH (t:Test {projectCwd: $cwd})-[r:USES_SELECTOR]->(s:Selector {value: $selector, projectCwd: $cwd})
+           OPTIONAL MATCH (t)-[:VISITS]->(pg:Page)
+           RETURN t.name AS test, r.actionType AS action, COLLECT(DISTINCT pg.path) AS pages`,
+          { selector: params.selector, cwd }
+        );
+        return result.records.map(r => ({
+          test: r.get('test'),
+          action: r.get('action'),
+          pages: r.get('pages'),
+        }));
+      }
+
+      default:
+        return null;
+    }
+  } finally {
+    await session.close();
+  }
+}

package/src/learner-sqlite.js

@@ -0,0 +1,354 @@
+/**
+ * Learning system — SQLite read queries.
+ *
+ * All functions return plain objects/arrays ready for JSON serialization.
+ * Used by MCP tools, dashboard REST endpoints, CLI, and markdown generator.
+ */
+
+import { getDb } from './db.js';
+
+/**
+ * Full learning summary for a project — reads from the cached learning_summary table.
+ * Falls back to live queries if cache is empty.
+ */
+export function getLearningsSummary(projectId) {
+  const d = getDb();
+  const empty = {
+    totalRuns: 0, totalTests: 0, overallPassRate: 0, avgDurationMs: 0,
+    flakyTests: [], slowTests: [], unstableSelectors: [],
+    failingPages: [], apiIssues: [], topErrors: [], updatedAt: null,
+  };
+
+  // Cross-project aggregate when projectId is null
+  if (projectId === null || projectId === undefined) {
+    const rows = d.prepare('SELECT * FROM learning_summary').all();
+    if (!rows.length) return empty;
+    let totalRuns = 0, totalTests = 0, passSumW = 0, durSumW = 0;
+    let allFlaky = [], allSlow = [], allSelectors = [], allPages = [], allApis = [], allErrors = [];
+    let latestUpdate = null;
+    for (const row of rows) {
+      totalRuns += row.total_runs;
+      totalTests += row.total_tests;
+      passSumW += row.overall_pass_rate * row.total_tests;
+      durSumW += row.avg_duration_ms * row.total_tests;
+      allFlaky = allFlaky.concat(JSON.parse(row.flaky_tests || '[]'));
+      allSlow = allSlow.concat(JSON.parse(row.slow_tests || '[]'));
+      allSelectors = allSelectors.concat(JSON.parse(row.unstable_selectors || '[]'));
+      allPages = allPages.concat(JSON.parse(row.failing_pages || '[]'));
+      allApis = allApis.concat(JSON.parse(row.api_issues || '[]'));
+      allErrors = allErrors.concat(JSON.parse(row.top_errors || '[]'));
+      if (!latestUpdate || (row.updated_at && row.updated_at > latestUpdate)) latestUpdate = row.updated_at;
+    }
+    return {
+      totalRuns, totalTests,
+      overallPassRate: totalTests > 0 ? Math.round(passSumW / totalTests * 10) / 10 : 0,
+      avgDurationMs: totalTests > 0 ? Math.round(durSumW / totalTests) : 0,
+      flakyTests: allFlaky, slowTests: allSlow, unstableSelectors: allSelectors,
+      failingPages: allPages, apiIssues: allApis, topErrors: allErrors, updatedAt: latestUpdate,
+    };
+  }
+
+  const row = d.prepare('SELECT * FROM learning_summary WHERE project_id = ?').get(projectId);
+  if (!row) return empty;
+
+  return {
+    totalRuns: row.total_runs,
+    totalTests: row.total_tests,
+    overallPassRate: Math.round(row.overall_pass_rate * 10) / 10,
+    avgDurationMs: Math.round(row.avg_duration_ms),
+    flakyTests: JSON.parse(row.flaky_tests || '[]'),
+    slowTests: JSON.parse(row.slow_tests || '[]'),
+    unstableSelectors: JSON.parse(row.unstable_selectors || '[]'),
+    failingPages: JSON.parse(row.failing_pages || '[]'),
+    apiIssues: JSON.parse(row.api_issues || '[]'),
+    topErrors: JSON.parse(row.top_errors || '[]'),
+    updatedAt: row.updated_at,
+  };
+}
+
+/** Flaky test details — tests that pass only after retries. */
+export function getFlakySummary(projectId, days = 30) {
+  const d = getDb();
+  return d.prepare(`
+    SELECT
+      test_name,
+      COUNT(*) AS total_runs,
+      SUM(flaky) AS flaky_count,
+      ROUND(AVG(flaky) * 100, 1) AS flaky_rate,
+      ROUND(AVG(duration_ms)) AS avg_duration_ms,
+      MAX(CASE WHEN flaky = 1 THEN created_at END) AS last_flaky,
+      ROUND(AVG(attempt), 1) AS avg_attempts
+    FROM test_learnings
+    WHERE project_id = ? AND created_at >= datetime('now', '-' || ? || ' days')
+    GROUP BY test_name
+    HAVING flaky_count > 0
+    ORDER BY flaky_rate DESC
+  `).all(projectId, days);
+}
+
+/** Selector stability — selectors with failure rates. */
+export function getSelectorStability(projectId, days = 30) {
+  const d = getDb();
+  return d.prepare(`
+    SELECT
+      selector,
+      action_type,
+      COUNT(*) AS total_uses,
+      SUM(CASE WHEN success = 0 THEN 1 ELSE 0 END) AS fail_count,
+      ROUND(AVG(CASE WHEN success = 0 THEN 100.0 ELSE 0.0 END), 1) AS fail_rate,
+      COUNT(DISTINCT test_name) AS used_by_tests,
+      page_url,
+      MAX(CASE WHEN success = 0 THEN error END) AS last_error
+    FROM selector_learnings
+    WHERE project_id = ? AND created_at >= datetime('now', '-' || ? || ' days')
+    GROUP BY selector, action_type
+    HAVING fail_rate > 0
+    ORDER BY fail_rate DESC
+  `).all(projectId, days);
+}
+
+/** Page health — pages with failure rates, console/network errors. */
+export function getPageHealth(projectId, days = 30) {
+  const d = getDb();
+  return d.prepare(`
+    SELECT
+      url_path,
+      COUNT(*) AS total_visits,
+      SUM(CASE WHEN success = 0 THEN 1 ELSE 0 END) AS fail_count,
+      ROUND(AVG(CASE WHEN success = 0 THEN 100.0 ELSE 0.0 END), 1) AS fail_rate,
+      COUNT(DISTINCT test_name) AS tested_by,
+      SUM(console_errors) AS console_errors,
+      SUM(console_warns) AS console_warns,
+      SUM(network_errors) AS network_errors,
+      ROUND(AVG(load_time_ms)) AS avg_load_ms
+    FROM page_learnings
+    WHERE project_id = ? AND created_at >= datetime('now', '-' || ? || ' days')
+    GROUP BY url_path
+    ORDER BY fail_rate DESC
+  `).all(projectId, days);
+}
+
+/** API health — endpoints with error rates and latency. */
+export function getApiHealth(projectId, days = 30) {
+  const d = getDb();
+  return d.prepare(`
+    SELECT
+      endpoint,
+      COUNT(*) AS total_calls,
+      SUM(is_error) AS error_count,
+      ROUND(AVG(CASE WHEN is_error = 1 THEN 100.0 ELSE 0.0 END), 1) AS error_rate,
+      ROUND(AVG(duration_ms)) AS avg_duration_ms,
+      MAX(duration_ms) AS max_duration_ms,
+      GROUP_CONCAT(DISTINCT status) AS status_codes
+    FROM api_learnings
+    WHERE project_id = ? AND created_at >= datetime('now', '-' || ? || ' days')
+    GROUP BY endpoint
+    ORDER BY error_rate DESC, total_calls DESC
+  `).all(projectId, days);
+}
+
+/** Error patterns — most frequent errors with categories. */
+export function getErrorPatterns(projectId) {
+  const d = getDb();
+  return d.prepare(`
+    SELECT
+      pattern,
+      category,
+      occurrence_count,
+      first_seen,
+      last_seen,
+      example_error,
+      example_test
+    FROM error_patterns
+    WHERE project_id = ?
+    ORDER BY occurrence_count DESC
+  `).all(projectId);
+}
+
+/** Test pass/fail trends over time — aggregated by day, or by hour when all data is from a single day. */
+export function getTestTrends(projectId, days = 7) {
+  const d = getDb();
+  const projectClause = (projectId !== null && projectId !== undefined) ? 'project_id = ? AND' : '';
+  const params = (projectId !== null && projectId !== undefined) ? [projectId, days] : [days];
+
+  const daily = d.prepare(`
+    SELECT
+      DATE(created_at) AS date,
+      COUNT(*) AS total_tests,
+      SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END) AS passed,
+      SUM(CASE WHEN success = 0 THEN 1 ELSE 0 END) AS failed,
+      ROUND(AVG(CASE WHEN success = 1 THEN 100.0 ELSE 0.0 END), 1) AS pass_rate,
+      ROUND(AVG(duration_ms)) AS avg_duration_ms,
+      SUM(flaky) AS flaky_count
+    FROM test_learnings
+    WHERE ${projectClause} created_at >= datetime('now', '-' || ? || ' days')
+    GROUP BY DATE(created_at)
+    ORDER BY date ASC
+  `).all(...params);
+
+  // If all data is from a single day, provide hourly breakdown instead
+  if (daily.length <= 1 && daily[0]?.total_tests > 1) {
+    const hourly = d.prepare(`
+      SELECT
+        strftime('%Y-%m-%d %H:00', created_at) AS date,
+        COUNT(*) AS total_tests,
+        SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END) AS passed,
+        SUM(CASE WHEN success = 0 THEN 1 ELSE 0 END) AS failed,
+        ROUND(AVG(CASE WHEN success = 1 THEN 100.0 ELSE 0.0 END), 1) AS pass_rate,
+        ROUND(AVG(duration_ms)) AS avg_duration_ms,
+        SUM(flaky) AS flaky_count
+      FROM test_learnings
+      WHERE ${projectClause} created_at >= datetime('now', '-' || ? || ' days')
+      GROUP BY strftime('%Y-%m-%d %H', created_at)
+      ORDER BY date ASC
+    `).all(...params);
+    if (hourly.length > 1) {
+      return { granularity: 'hourly', data: hourly };
+    }
+  }
+
+  return { granularity: 'daily', data: daily };
+}
+
+/**
+ * Contextual insights for the current run — identifies known flaky tests,
+ * new failures, recovered tests, and unstable selectors used.
+ */
+export function getRunInsights(projectId, report) {
+  const d = getDb();
+  const insights = [];
+
+  if (!report?.results) return insights;
+
+  for (const result of report.results) {
+    const history = d.prepare(`
+      SELECT
+        COUNT(*) AS total_runs,
+        SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END) AS pass_count,
+        SUM(flaky) AS flaky_count,
+        ROUND(AVG(CASE WHEN success = 1 THEN 100.0 ELSE 0.0 END), 1) AS pass_rate
+      FROM test_learnings
+      WHERE project_id = ? AND test_name = ?
+    `).get(projectId, result.name);
+
+    if (!history || history.total_runs === 0) continue;
+
+    // Known flaky test
+    if (history.flaky_count > 0 && result.success) {
+      insights.push({
+        type: 'flaky',
+        test: result.name,
+        message: `Known flaky test (${history.flaky_count} flaky runs out of ${history.total_runs}). Passed this time.`,
+      });
+    }
+
+    // New failure (was passing, now fails)
+    if (!result.success && history.pass_rate > 80) {
+      insights.push({
+        type: 'new-failure',
+        test: result.name,
+        message: `New failure — this test had ${history.pass_rate}% pass rate over ${history.total_runs} runs.`,
+      });
+    }
+
+    // Recovered (was failing, now passes)
+    if (result.success && history.pass_rate < 50 && history.total_runs >= 3) {
+      insights.push({
+        type: 'recovered',
+        test: result.name,
+        message: `Recovered — was failing (${history.pass_rate}% pass rate over ${history.total_runs} runs).`,
+      });
+    }
+  }
+
+  // Check for unstable selectors used in this run
+  const selectorStats = d.prepare(`
+    SELECT selector, ROUND(AVG(CASE WHEN success = 0 THEN 100.0 ELSE 0.0 END), 1) AS fail_rate
+    FROM selector_learnings
+    WHERE project_id = ?
+    GROUP BY selector
+    HAVING fail_rate > 20
+  `).all(projectId);
+
+  if (selectorStats.length > 0) {
+    const selectorSet = new Set(selectorStats.map(s => s.selector));
+    const usedUnstable = [];
+
+    for (const result of report.results) {
+      if (!result.actions) continue;
+      for (const action of result.actions) {
+        if (action.selector && selectorSet.has(action.selector)) {
+          usedUnstable.push(action.selector);
+        }
+      }
+    }
+
+    const unique = [...new Set(usedUnstable)];
+    if (unique.length > 0) {
+      insights.push({
+        type: 'unstable-selectors',
+        selectors: unique.slice(0, 5),
+        message: `${unique.length} unstable selector(s) used in this run: ${unique.slice(0, 3).join(', ')}${unique.length > 3 ? '...' : ''}`,
+      });
+    }
+  }
+
+  return insights;
+}
+
+/** Drill-down: history for a specific test. */
+export function getTestHistory(projectId, testName, days = 30) {
+  const d = getDb();
+  return d.prepare(`
+    SELECT
+      tl.test_name,
+      tl.success,
+      tl.duration_ms,
+      tl.flaky,
+      tl.attempt,
+      tl.error_pattern,
+      tl.created_at,
+      r.run_id
+    FROM test_learnings tl
+    LEFT JOIN runs r ON r.id = tl.run_id
+    WHERE tl.project_id = ? AND tl.test_name = ? AND tl.created_at >= datetime('now', '-' || ? || ' days')
+    ORDER BY tl.created_at DESC
+  `).all(projectId, testName, days);
+}
+
+/** Drill-down: history for a specific page. */
+export function getPageHistory(projectId, urlPath, days = 30) {
+  const d = getDb();
+  return d.prepare(`
+    SELECT
+      url_path,
+      success,
+      load_time_ms,
+      console_errors,
+      console_warns,
+      network_errors,
+      test_name,
+      created_at
+    FROM page_learnings
+    WHERE project_id = ? AND url_path = ? AND created_at >= datetime('now', '-' || ? || ' days')
+    ORDER BY created_at DESC
+  `).all(projectId, urlPath, days);
+}
+
+/** Drill-down: history for a specific selector. */
+export function getSelectorHistory(projectId, selector, days = 30) {
+  const d = getDb();
+  return d.prepare(`
+    SELECT
+      selector,
+      action_type,
+      success,
+      page_url,
+      test_name,
+      error,
+      created_at
+    FROM selector_learnings
+    WHERE project_id = ? AND selector = ? AND created_at >= datetime('now', '-' || ? || ' days')
+    ORDER BY created_at DESC
+  `).all(projectId, selector, days);
+}