@matware/e2e-runner 1.1.0 → 1.2.1

package/src/learner-markdown.js

@@ -0,0 +1,177 @@
+/**
+ * Learnings markdown generator.
+ *
+ * Generates {cwd}/e2e/learnings.md after each run, reading from SQLite.
+ * The file is designed to be portable, versionable in git, and human-readable.
+ */
+
+import fs from 'fs';
+import path from 'path';
+import {
+  getLearningsSummary,
+  getFlakySummary,
+  getSelectorStability,
+  getPageHealth,
+  getApiHealth,
+  getErrorPatterns,
+  getTestTrends,
+} from './learner-sqlite.js';
+
+/**
+ * Generates the learnings.md file for a project.
+ * Reads from SQLite and writes to {cwd}/e2e/learnings.md.
+ */
+export function generateLearningsMarkdown(projectId, config) {
+  const days = config?.learningsDays || 30;
+  const summary = getLearningsSummary(projectId);
+  const flaky = getFlakySummary(projectId, days);
+  const selectors = getSelectorStability(projectId, days);
+  const pages = getPageHealth(projectId, days);
+  const apis = getApiHealth(projectId, days);
+  const errors = getErrorPatterns(projectId);
+  const trendsResult = getTestTrends(projectId, 7);
+  const trends = trendsResult.data || trendsResult;
+  const trendsGranularity = trendsResult.granularity || 'daily';
+
+  const lines = [];
+
+  lines.push('# E2E Test Learnings');
+  lines.push('');
+  lines.push(`> Auto-generated after each test run. Analysis window: **${days} days**.`);
+  lines.push(`> Last updated: ${summary.updatedAt || 'never'}`);
+  lines.push('');
+
+  // ── Health Overview ─────────────────────────────────────────────────────────
+  lines.push('## Health Overview');
+  lines.push('');
+  lines.push('| Metric | Value |');
+  lines.push('|--------|-------|');
+  lines.push(`| Total Runs | ${summary.totalRuns} |`);
+  lines.push(`| Total Tests | ${summary.totalTests} |`);
+  lines.push(`| Pass Rate | ${summary.overallPassRate}% |`);
+  lines.push(`| Avg Duration | ${formatDuration(summary.avgDurationMs)} |`);
+  lines.push(`| Flaky Tests | ${flaky.length} |`);
+  lines.push(`| Unstable Selectors | ${selectors.length} |`);
+
+  // Trend arrow (compare last 2 days)
+  if (trends.length >= 2) {
+    const latest = trends[trends.length - 1];
+    const prev = trends[trends.length - 2];
+    const diff = latest.pass_rate - prev.pass_rate;
+    const arrow = diff > 0 ? 'improving' : diff < 0 ? 'declining' : 'stable';
+    lines.push(`| 7-Day Trend | ${arrow} (${diff > 0 ? '+' : ''}${diff.toFixed(1)}%) |`);
+  }
+  lines.push('');
+
+  // ── Flaky Tests ─────────────────────────────────────────────────────────────
+  if (flaky.length > 0) {
+    lines.push('## Flaky Tests');
+    lines.push('');
+    lines.push('Tests that pass only after retries — potential stability issues.');
+    lines.push('');
+    lines.push('| Test | Flaky Rate | Occurrences | Total Runs | Last Flaky | Avg Attempts |');
+    lines.push('|------|-----------|-------------|------------|------------|-------------|');
+    for (const f of flaky) {
+      lines.push(`| ${f.test_name} | ${f.flaky_rate}% | ${f.flaky_count} | ${f.total_runs} | ${formatDate(f.last_flaky)} | ${f.avg_attempts} |`);
+    }
+    lines.push('');
+  }
+
+  // ── Unstable Selectors ──────────────────────────────────────────────────────
+  if (selectors.length > 0) {
+    lines.push('## Unstable Selectors');
+    lines.push('');
+    lines.push('CSS selectors that fail intermittently — candidates for improvement.');
+    lines.push('');
+    lines.push('| Selector | Action | Fail Rate | Uses | Tests | Page | Error |');
+    lines.push('|----------|--------|-----------|------|-------|------|-------|');
+    for (const s of selectors.slice(0, 20)) {
+      const selector = truncate(s.selector, 40);
+      const error = truncate(s.last_error || '-', 30);
+      lines.push(`| \`${selector}\` | ${s.action_type} | ${s.fail_rate}% | ${s.total_uses} | ${s.used_by_tests} | ${s.page_url || '-'} | ${error} |`);
+    }
+    lines.push('');
+  }
+
+  // ── Failing Pages ───────────────────────────────────────────────────────────
+  const failingPages = pages.filter(p => p.fail_rate > 0);
+  if (failingPages.length > 0) {
+    lines.push('## Failing Pages');
+    lines.push('');
+    lines.push('| Page | Fail Rate | Visits | Tests | Console Errors | Network Errors | Avg Load |');
+    lines.push('|------|-----------|--------|-------|---------------|----------------|----------|');
+    for (const p of failingPages.slice(0, 20)) {
+      lines.push(`| ${p.url_path} | ${p.fail_rate}% | ${p.total_visits} | ${p.tested_by} | ${p.console_errors} | ${p.network_errors} | ${formatDuration(p.avg_load_ms)} |`);
+    }
+    lines.push('');
+  }
+
+  // ── API Issues ──────────────────────────────────────────────────────────────
+  const apiIssues = apis.filter(a => a.error_rate > 0);
+  if (apiIssues.length > 0) {
+    lines.push('## API Issues');
+    lines.push('');
+    lines.push('| Endpoint | Error Rate | Calls | Avg Duration | Max Duration | Status Codes |');
+    lines.push('|----------|-----------|-------|-------------|-------------|-------------|');
+    for (const a of apiIssues.slice(0, 20)) {
+      lines.push(`| ${truncate(a.endpoint, 40)} | ${a.error_rate}% | ${a.total_calls} | ${formatDuration(a.avg_duration_ms)} | ${formatDuration(a.max_duration_ms)} | ${a.status_codes || '-'} |`);
+    }
+    lines.push('');
+  }
+
+  // ── Error Patterns ──────────────────────────────────────────────────────────
+  if (errors.length > 0) {
+    lines.push('## Error Patterns');
+    lines.push('');
+    lines.push('| Pattern | Category | Count | First Seen | Last Seen | Example Test |');
+    lines.push('|---------|----------|-------|------------|-----------|-------------|');
+    for (const e of errors.slice(0, 20)) {
+      lines.push(`| ${truncate(e.pattern, 50)} | ${e.category} | ${e.occurrence_count} | ${formatDate(e.first_seen)} | ${formatDate(e.last_seen)} | ${e.example_test || '-'} |`);
+    }
+    lines.push('');
+  }
+
+  // ── Recent Trend ────────────────────────────────────────────────────────────
+  if (trends.length > 0) {
+    const label = trendsGranularity === 'hourly' ? 'Recent Trend (hourly)' : 'Recent Trend (7 days)';
+    const col1 = trendsGranularity === 'hourly' ? 'Hour' : 'Date';
+    lines.push(`## ${label}`);
+    lines.push('');
+    lines.push(`| ${col1} | Pass Rate | Tests | Passed | Failed | Flaky | Avg Duration |`);
+    lines.push('|------|-----------|-------|--------|--------|-------|-------------|');
+    for (const t of trends) {
+      lines.push(`| ${t.date} | ${t.pass_rate}% | ${t.total_tests} | ${t.passed} | ${t.failed} | ${t.flaky_count} | ${formatDuration(t.avg_duration_ms)} |`);
+    }
+    lines.push('');
+  }
+
+  // Write the file
+  const cwd = config?._cwd || process.cwd();
+  const e2eDir = path.join(cwd, 'e2e');
+  if (!fs.existsSync(e2eDir)) {
+    fs.mkdirSync(e2eDir, { recursive: true });
+  }
+
+  const mdPath = path.join(e2eDir, 'learnings.md');
+  fs.writeFileSync(mdPath, lines.join('\n') + '\n');
+
+  return mdPath;
+}
+
+// ── Helpers ───────────────────────────────────────────────────────────────────
+
+function formatDuration(ms) {
+  if (ms == null || isNaN(ms)) return '-';
+  if (ms < 1000) return `${Math.round(ms)}ms`;
+  return `${(ms / 1000).toFixed(1)}s`;
+}
+
+function formatDate(dateStr) {
+  if (!dateStr) return '-';
+  return dateStr.split('T')[0] || dateStr.slice(0, 10);
+}
+
+function truncate(str, max) {
+  if (!str) return '-';
+  return str.length > max ? str.slice(0, max - 3) + '...' : str;
+}

package/src/learner-neo4j.js

@@ -0,0 +1,255 @@
+/**
+ * Neo4j knowledge graph integration for the learning system.
+ *
+ * Optional — all operations are no-ops if neo4j-driver is not installed.
+ * Enable with learningsNeo4j: true in config.
+ *
+ * Nodes: Project, Test, Page, Selector, ApiEndpoint, ErrorPattern, Run
+ * Relationships: VISITS, USES_SELECTOR, CALLS_API, FAILED_WITH, EXECUTED_IN, SELECTOR_ON
+ */
+
+let neo4j = null;
+let driver = null;
+
+/** Try to load neo4j-driver. Returns false if not available. */
+async function ensureDriver(config) {
+  if (driver) return true;
+  if (neo4j === false) return false; // already tried and failed
+
+  try {
+    neo4j = (await import('neo4j-driver')).default;
+    driver = neo4j.driver(
+      config.neo4jBoltUrl || 'bolt://localhost:7687',
+      neo4j.auth.basic(config.neo4jUser || 'neo4j', config.neo4jPassword || 'e2erunner')
+    );
+    // Verify connectivity
+    await driver.verifyConnectivity();
+    return true;
+  } catch {
+    neo4j = false;
+    driver = null;
+    return false;
+  }
+}
+
+/** Close the Neo4j driver. */
+export async function closeNeo4j() {
+  if (driver) {
+    await driver.close();
+    driver = null;
+  }
+}
+
+/**
+ * Writes learning data to Neo4j graph.
+ * Called from learnFromRun() when learningsNeo4j is enabled.
+ * All operations are no-ops if neo4j-driver is not installed or connection fails.
+ */
+export async function writeToGraph(projectId, runDbId, report, config, suiteName) {
+  if (!config?.learningsNeo4j) return;
+  const connected = await ensureDriver(config);
+  if (!connected) return;
+
+  const session = driver.session();
+  try {
+    const projectName = config.projectName || 'unknown';
+    const cwd = config._cwd || process.cwd();
+
+    // Ensure Project node
+    await session.run(
+      'MERGE (p:Project {cwd: $cwd}) SET p.name = $name, p.updatedAt = datetime()',
+      { cwd, name: projectName }
+    );
+
+    // Ensure Run node
+    await session.run(
+      `MERGE (r:Run {dbId: $runDbId})
+       SET r.total = $total, r.passed = $passed, r.failed = $failed,
+           r.passRate = $passRate, r.duration = $duration, r.suiteName = $suiteName,
+           r.createdAt = datetime()
+       WITH r
+       MATCH (p:Project {cwd: $cwd})
+       MERGE (r)-[:EXECUTED_IN]->(p)`,
+      {
+        runDbId: neo4j.int(runDbId),
+        total: neo4j.int(report.summary.total),
+        passed: neo4j.int(report.summary.passed),
+        failed: neo4j.int(report.summary.failed),
+        passRate: report.summary.passRate,
+        duration: report.summary.duration,
+        suiteName: suiteName || null,
+        cwd,
+      }
+    );
+
+    for (const result of report.results) {
+      // Test node
+      await session.run(
+        `MERGE (t:Test {name: $name, projectCwd: $cwd})
+         SET t.lastSuccess = $success, t.lastDuration = $duration, t.updatedAt = datetime()
+         WITH t
+         MATCH (r:Run {dbId: $runDbId})
+         MERGE (t)-[:EXECUTED_IN]->(r)`,
+        {
+          name: result.name,
+          cwd,
+          success: result.success,
+          duration: result.endTime && result.startTime
+            ? new Date(result.endTime) - new Date(result.startTime)
+            : 0,
+          runDbId: neo4j.int(runDbId),
+        }
+      );
+
+      // Page nodes + VISITS relationships
+      if (result.actions) {
+        for (const action of result.actions) {
+          if ((action.type === 'goto' || action.type === 'navigate') && action.value) {
+            let urlPath = action.value;
+            try { urlPath = new URL(action.value, 'http://placeholder').pathname; } catch { /* */ }
+
+            await session.run(
+              `MERGE (pg:Page {path: $path, projectCwd: $cwd})
+               SET pg.updatedAt = datetime()
+               WITH pg
+               MATCH (t:Test {name: $testName, projectCwd: $cwd})
+               MERGE (t)-[:VISITS]->(pg)`,
+              { path: urlPath, cwd, testName: result.name }
+            );
+          }
+
+          // Selector nodes + USES_SELECTOR relationships
+          if (action.selector) {
+            await session.run(
+              `MERGE (s:Selector {value: $selector, projectCwd: $cwd})
+               SET s.updatedAt = datetime()
+               WITH s
+               MATCH (t:Test {name: $testName, projectCwd: $cwd})
+               MERGE (t)-[:USES_SELECTOR {actionType: $actionType}]->(s)`,
+              { selector: action.selector, cwd, testName: result.name, actionType: action.type }
+            );
+          }
+        }
+      }
+
+      // API endpoint nodes + CALLS_API relationships
+      if (result.networkLogs?.length) {
+        for (const log of result.networkLogs) {
+          if (!log.url || !log.method) continue;
+          let urlPath;
+          try { urlPath = new URL(log.url).pathname; } catch { urlPath = log.url; }
+          urlPath = urlPath
+            .replace(/\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi, '/:uuid')
+            .replace(/\/\d+/g, '/:id');
+          const endpoint = `${log.method} ${urlPath}`;
+
+          await session.run(
+            `MERGE (a:ApiEndpoint {endpoint: $endpoint, projectCwd: $cwd})
+             SET a.updatedAt = datetime()
+             WITH a
+             MATCH (t:Test {name: $testName, projectCwd: $cwd})
+             MERGE (t)-[:CALLS_API]->(a)`,
+            { endpoint, cwd, testName: result.name }
+          );
+        }
+      }
+
+      // Error pattern nodes + FAILED_WITH relationships
+      if (result.error) {
+        const normalized = result.error.replace(/\d+ms/g, 'Nms').replace(/"[^"]+"/g, '"..."').slice(0, 200);
+        await session.run(
+          `MERGE (e:ErrorPattern {pattern: $pattern, projectCwd: $cwd})
+           SET e.count = COALESCE(e.count, 0) + 1, e.lastSeen = datetime()
+           WITH e
+           MATCH (t:Test {name: $testName, projectCwd: $cwd})
+           MERGE (t)-[:FAILED_WITH]->(e)`,
+          { pattern: normalized, cwd, testName: result.name }
+        );
+      }
+    }
+  } finally {
+    await session.close();
+  }
+}
+
+/**
+ * Query the graph for relationships — used by e2e_learnings MCP tool.
+ * Returns enriched insights about test dependencies, shared selectors, etc.
+ */
+export async function queryGraph(config, queryType, params = {}) {
+  if (!config?.learningsNeo4j) return null;
+  const connected = await ensureDriver(config);
+  if (!connected) return null;
+
+  const session = driver.session();
+  const cwd = config._cwd || process.cwd();
+
+  try {
+    switch (queryType) {
+      case 'test-dependencies': {
+        // Tests that share selectors or pages with a given test
+        const result = await session.run(
+          `MATCH (t1:Test {name: $testName, projectCwd: $cwd})-[:USES_SELECTOR]->(s)<-[:USES_SELECTOR]-(t2:Test)
+           WHERE t1 <> t2
+           RETURN DISTINCT t2.name AS related, COLLECT(DISTINCT s.value) AS sharedSelectors
+           LIMIT 20`,
+          { testName: params.testName, cwd }
+        );
+        return result.records.map(r => ({
+          test: r.get('related'),
+          sharedSelectors: r.get('sharedSelectors'),
+        }));
+      }
+
+      case 'page-impact': {
+        // All tests that visit a given page
+        const result = await session.run(
+          `MATCH (t:Test {projectCwd: $cwd})-[:VISITS]->(pg:Page {path: $path, projectCwd: $cwd})
+           RETURN t.name AS test, t.lastSuccess AS lastSuccess`,
+          { path: params.path, cwd }
+        );
+        return result.records.map(r => ({
+          test: r.get('test'),
+          lastSuccess: r.get('lastSuccess'),
+        }));
+      }
+
+      case 'error-impact': {
+        // Tests that failed with a given error pattern
+        const result = await session.run(
+          `MATCH (t:Test {projectCwd: $cwd})-[:FAILED_WITH]->(e:ErrorPattern)
+           WHERE e.pattern CONTAINS $search
+           RETURN t.name AS test, e.pattern AS pattern, e.count AS count
+           ORDER BY e.count DESC
+           LIMIT 20`,
+          { search: params.search || '', cwd }
+        );
+        return result.records.map(r => ({
+          test: r.get('test'),
+          pattern: r.get('pattern'),
+          count: r.get('count')?.toNumber?.() || r.get('count'),
+        }));
+      }
+
+      case 'selector-usage': {
+        // All tests and pages using a given selector
+        const result = await session.run(
+          `MATCH (t:Test {projectCwd: $cwd})-[r:USES_SELECTOR]->(s:Selector {value: $selector, projectCwd: $cwd})
+           OPTIONAL MATCH (t)-[:VISITS]->(pg:Page)
+           RETURN t.name AS test, r.actionType AS action, COLLECT(DISTINCT pg.path) AS pages`,
+          { selector: params.selector, cwd }
+        );
+        return result.records.map(r => ({
+          test: r.get('test'),
+          action: r.get('action'),
+          pages: r.get('pages'),
+        }));
+      }
+
+      default:
+        return null;
+    }
+  } finally {
+    await session.close();
+  }
+}