@matware/e2e-runner 1.0.3 → 1.1.1

package/src/reporter.js

@@ -5,6 +5,7 @@
 import fs from 'fs';
 import path from 'path';
 import { colors as C } from './logger.js';
+import { ensureProject, saveRun as saveRunToDb } from './db.js';
 
 function escapeXml(str) {
   return String(str)
@@ -14,6 +15,10 @@ function escapeXml(str) {
     .replace(/"/g, '"');
 }
 
+function escapeCdata(str) {
+  return String(str).replace(/\]\]>/g, ']]]]><![CDATA[>');
+}
+
 /** Generates a report object from test results */
 export function generateReport(results) {
   const passed = results.filter(r => r.success).length;
@@ -52,12 +57,12 @@ export function generateJUnitXML(report) {
 
     const logs = (result.consoleLogs || []).map(l => `[${l.type}] ${l.text}`).join('\n');
     if (logs) {
-      xml += `      <system-out><![CDATA[${logs}]]></system-out>\n`;
+      xml += `      <system-out><![CDATA[${escapeCdata(logs)}]]></system-out>\n`;
     }
 
     const netErrors = (result.networkErrors || []).map(e => `[${e.error || 'unknown'}] ${e.url}`).join('\n');
     if (netErrors) {
-      xml += `      <system-err><![CDATA[${netErrors}]]></system-err>\n`;
+      xml += `      <system-err><![CDATA[${escapeCdata(netErrors)}]]></system-err>\n`;
     }
 
     xml += '    </testcase>\n';
@@ -89,6 +94,72 @@ export function saveReport(report, screenshotsDir, config = {}) {
   return saved.length === 1 ? saved[0] : saved;
 }
 
+/** Saves a run to history */
+export function saveHistory(report, screenshotsDir, maxRuns = 100) {
+  const historyDir = path.join(screenshotsDir, 'history');
+  if (!fs.existsSync(historyDir)) {
+    fs.mkdirSync(historyDir, { recursive: true });
+  }
+
+  const runId = new Date().toISOString().replace(/:/g, '-').replace(/\.\d+Z$/, '');
+  const filename = `run-${runId}.json`;
+  const entry = { ...report, runId };
+  fs.writeFileSync(path.join(historyDir, filename), JSON.stringify(entry, null, 2));
+
+  // Auto-prune old runs
+  const files = fs.readdirSync(historyDir).filter(f => f.startsWith('run-') && f.endsWith('.json')).sort();
+  while (files.length > maxRuns) {
+    fs.unlinkSync(path.join(historyDir, files.shift()));
+  }
+
+  return runId;
+}
+
+/** Loads history summaries (newest first) */
+export function loadHistory(screenshotsDir) {
+  const historyDir = path.join(screenshotsDir, 'history');
+  if (!fs.existsSync(historyDir)) return [];
+
+  return fs.readdirSync(historyDir)
+    .filter(f => f.startsWith('run-') && f.endsWith('.json'))
+    .sort()
+    .reverse()
+    .map(f => {
+      const data = JSON.parse(fs.readFileSync(path.join(historyDir, f), 'utf-8'));
+      return { runId: data.runId, summary: data.summary, generatedAt: data.generatedAt };
+    });
+}
+
+/** Loads a full history run by runId */
+export function loadHistoryRun(screenshotsDir, runId) {
+  const historyDir = path.join(screenshotsDir, 'history');
+  const files = fs.existsSync(historyDir)
+    ? fs.readdirSync(historyDir).filter(f => f.startsWith('run-') && f.endsWith('.json'))
+    : [];
+
+  const match = files.find(f => {
+    const data = JSON.parse(fs.readFileSync(path.join(historyDir, f), 'utf-8'));
+    return data.runId === runId;
+  });
+
+  if (!match) return null;
+  return JSON.parse(fs.readFileSync(path.join(historyDir, match), 'utf-8'));
+}
+
+/** Persists a run to both filesystem history and SQLite (never throws). */
+export function persistRun(report, config, suiteName) {
+  const runId = saveHistory(report, config.screenshotsDir, config.maxHistoryRuns);
+
+  try {
+    const projectId = ensureProject(config._cwd, config.projectName, config.screenshotsDir, config.testsDir);
+    saveRunToDb(projectId, report, runId, suiteName || null, config.triggeredBy || null);
+  } catch (err) {
+    process.stderr.write(`[e2e-runner] SQLite write failed: ${err.message}\n`);
+  }
+
+  return runId;
+}
+
 /** Prints a formatted report summary to the console */
 export function printReport(report, screenshotsDir) {
   const { summary } = report;
@@ -139,6 +210,18 @@ export function printReport(report, screenshotsDir) {
     });
   }
 
+  const networkRequests = report.results.filter(r => r.networkLogs?.length > 0);
+  if (networkRequests.length > 0) {
+    console.log(`\n${C.cyan}${C.bold}NETWORK REQUESTS:${C.reset}`);
+    networkRequests.forEach(r => {
+      console.log(`  ${C.cyan}▸${C.reset} ${r.name}:`);
+      r.networkLogs.forEach(n => {
+        const statusColor = n.status < 300 ? C.green : n.status < 400 ? C.yellow : C.red;
+        console.log(`    ${C.dim}${n.method}${C.reset} ${statusColor}${n.status}${C.reset} ${n.url} ${C.dim}(${n.duration}ms)${C.reset}`);
+      });
+    });
+  }
+
   if (screenshotsDir) {
     console.log(`\n${C.dim}Report: ${path.join(screenshotsDir, 'report.json')}${C.reset}`);
     console.log(`${C.dim}Screenshots: ${screenshotsDir}${C.reset}\n`);

package/src/runner.js

@@ -7,7 +7,7 @@
 
 import fs from 'fs';
 import path from 'path';
-import { connectToPool, waitForPool } from './pool.js';
+import { connectToPool, waitForPool, getPoolStatus } from './pool.js';
 import { executeAction } from './actions.js';
 import { log, colors as C } from './logger.js';
 
@@ -47,8 +47,28 @@ function normalizeTestData(data) {
   return { tests: data.tests || [], hooks: data.hooks || {} };
 }
 
+/** Waits until the pool has capacity before connecting */
+async function waitForSlot(poolUrl, pollIntervalMs = 2000, maxWaitMs = 60000) {
+  const start = Date.now();
+  while (Date.now() - start < maxWaitMs) {
+    try {
+      const status = await getPoolStatus(poolUrl);
+      if (status.available && status.running < status.maxConcurrent) {
+        return;
+      }
+      log('⏳', `${C.dim}Pool at capacity (${status.running}/${status.maxConcurrent}, ${status.queued} queued), waiting for slot...${C.reset}`);
+    } catch {
+      // Pool unreachable, let connectToPool handle the error
+      return;
+    }
+    await sleep(pollIntervalMs);
+  }
+  // Timeout — proceed anyway and let connectToPool deal with it
+  log('⚠️', `${C.yellow}Waited ${maxWaitMs / 1000}s for pool slot, proceeding anyway${C.reset}`);
+}
+
 /** Runs a single test end-to-end */
-export async function runTest(test, config, hooks = {}) {
+export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
   let browser = null;
   let page = null;
 
@@ -60,9 +80,12 @@ export async function runTest(test, config, hooks = {}) {
     error: null,
     consoleLogs: [],
     networkErrors: [],
+    networkLogs: [],
   };
+  const pendingBodies = [];
 
   try {
+    await waitForSlot(config.poolUrl);
     browser = await connectToPool(config.poolUrl, config.connectRetries, config.connectRetryDelay);
     page = await browser.newPage();
     await page.setViewport(config.viewport);
@@ -74,32 +97,95 @@ export async function runTest(test, config, hooks = {}) {
       result.networkErrors.push({ url: req.url(), error: req.failure()?.errorText });
     });
 
+    const requestTimings = new Map();
+    page.on('request', (req) => {
+      const rt = req.resourceType();
+      if (rt === 'xhr' || rt === 'fetch') requestTimings.set(req, Date.now());
+    });
+    page.on('response', (resp) => {
+      const req = resp.request();
+      const startMs = requestTimings.get(req);
+      if (startMs !== undefined) {
+        requestTimings.delete(req);
+        const entry = {
+          url: req.url(),
+          method: req.method(),
+          status: resp.status(),
+          statusText: resp.statusText(),
+          duration: Date.now() - startMs,
+          requestHeaders: req.headers(),
+          requestBody: null,
+          responseHeaders: resp.headers(),
+          responseBody: null,
+        };
+        try { entry.requestBody = req.postData() || null; } catch { /* */ }
+        result.networkLogs.push(entry);
+        // Read response body async — collect promise for later flush
+        const bodyPromise = resp.text().then(body => {
+          entry.responseBody = body && body.length > 51200 ? body.slice(0, 51200) + '\n...[truncated]' : body;
+        }).catch(() => { /* response may be unavailable */ });
+        pendingBodies.push(bodyPromise);
+      }
+    });
+
     // Run beforeEach hook
     if (hooks.beforeEach?.length) {
       await executeHookActions(page, hooks.beforeEach, config);
     }
 
-    for (const action of test.actions) {
+    for (let i = 0; i < test.actions.length; i++) {
+      const action = test.actions[i];
       const actionStart = Date.now();
       try {
-        const actionResult = await executeAction(page, action, config);
+        let actionResult;
+        if (action.type === 'assert_no_network_errors') {
+          // Handled inline — needs access to result.networkErrors
+          if (result.networkErrors.length > 0) {
+            const summary = result.networkErrors.map(e => `${e.url} (${e.error})`).join(', ');
+            throw new Error(`assert_no_network_errors failed: ${result.networkErrors.length} error(s): ${summary}`);
+          }
+          actionResult = null;
+        } else {
+          actionResult = await executeAction(page, action, config);
+        }
+        const actionDuration = Date.now() - actionStart;
         result.actions.push({
           ...action,
           success: true,
-          duration: Date.now() - actionStart,
+          duration: actionDuration,
           result: actionResult,
         });
+        progressFn({ event: 'test:action', name: test.name, action, actionIndex: i, totalActions: test.actions.length, success: true, duration: actionDuration, screenshotPath: actionResult?.screenshot || null });
       } catch (error) {
+        const actionDuration = Date.now() - actionStart;
         result.actions.push({
           ...action,
           success: false,
-          duration: Date.now() - actionStart,
+          duration: actionDuration,
           error: error.message,
         });
+        progressFn({ event: 'test:action', name: test.name, action, actionIndex: i, totalActions: test.actions.length, success: false, duration: actionDuration, error: error.message });
         throw error;
       }
     }
 
+    // Fail the test if failOnNetworkError is enabled and network errors occurred
+    if (config.failOnNetworkError && result.networkErrors.length > 0) {
+      const summary = result.networkErrors.map(e => `${e.url} (${e.error})`).join(', ');
+      throw new Error(`Network errors detected (failOnNetworkError=true): ${result.networkErrors.length} error(s): ${summary}`);
+    }
+
+    // Auto-capture verification screenshot if test has "expect"
+    if (test.expect && page) {
+      result.expect = test.expect;
+      try {
+        const safeName = test.name.replace(/[^a-zA-Z0-9_\-. ]/g, '_');
+        const verifyPath = path.join(config.screenshotsDir, `verify-${safeName}-${Date.now()}.png`);
+        await page.screenshot({ path: verifyPath, fullPage: true });
+        result.verificationScreenshot = verifyPath;
+      } catch { /* page may be dead */ }
+    }
+
     // Run afterEach hook (success path)
     if (hooks.afterEach?.length) {
       await executeHookActions(page, hooks.afterEach, config);
@@ -115,12 +201,17 @@ export async function runTest(test, config, hooks = {}) {
 
     if (page) {
       try {
-        const errorScreenshot = path.join(config.screenshotsDir, `error-${test.name}-${Date.now()}.png`);
+        const safeName = test.name.replace(/[^a-zA-Z0-9_\-. ]/g, '_');
+        const errorScreenshot = path.join(config.screenshotsDir, `error-${safeName}-${Date.now()}.png`);
         await page.screenshot({ path: errorScreenshot, fullPage: true });
         result.errorScreenshot = errorScreenshot;
       } catch { /* page may be dead */ }
     }
   } finally {
+    // Flush pending response body reads before disconnecting
+    if (pendingBodies.length > 0) {
+      try { await Promise.allSettled(pendingBodies); } catch { /* */ }
+    }
     result.endTime = new Date().toISOString();
     if (page) {
       try { result.finalUrl = page.url(); } catch { /* */ }
@@ -155,6 +246,14 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
     }
   }
 
+  const concurrency = config.concurrency || 3;
+  const _runId = Date.now().toString(36) + Math.random().toString(36).slice(2, 6);
+  const _proj = config.projectName || null;
+  const _cwd = config._cwd || null;
+  const _triggeredBy = config.triggeredBy || 'unknown';
+  const _progress = (data) => config.onProgress && config.onProgress({ ...data, runId: _runId, project: _proj, cwd: _cwd, triggeredBy: _triggeredBy });
+  _progress({ event: 'run:start', total: tests.length, concurrency, timestamp: new Date().toISOString() });
+
   const results = [];
   const queue = [...tests];
   let activeCount = 0;
@@ -164,6 +263,7 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
       const test = queue.shift();
       activeCount++;
       log('▶▶▶', `${C.cyan}${test.name}${C.reset} ${C.dim}(${activeCount} active)${C.reset}`);
+      _progress({ event: 'test:start', name: test.name, activeCount, queueRemaining: queue.length });
 
       const maxAttempts = (test.retries ?? config.retries ?? 0) + 1;
       const testTimeout = test.timeout ?? config.testTimeout ?? 60000;
@@ -176,7 +276,7 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
         });
 
         try {
-          result = await Promise.race([runTest(test, config, hooks), timeoutPromise]);
+          result = await Promise.race([runTest(test, config, hooks, _progress), timeoutPromise]);
         } catch (error) {
           result = {
             name: test.name,
@@ -187,6 +287,7 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
             error: error.message,
             consoleLogs: [],
             networkErrors: [],
+            networkLogs: [],
           };
         }
 
@@ -195,12 +296,16 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
 
         if (result.success || attempt === maxAttempts) break;
         log('🔄', `${C.yellow}${test.name}${C.reset} failed, retrying (${attempt}/${maxAttempts})...`);
+        _progress({ event: 'test:retry', name: test.name, attempt, maxAttempts });
         await sleep(config.retryDelay || 1000);
       }
 
       results.push(result);
       activeCount--;
 
+      const screenshots = result.actions.filter(a => a.result?.screenshot).map(a => a.result.screenshot);
+      _progress({ event: 'test:complete', name: test.name, success: result.success, duration: timeDiff(result.startTime, result.endTime), error: result.error, consoleLogs: result.consoleLogs, networkErrors: result.networkErrors, networkLogs: result.networkLogs, errorScreenshot: result.errorScreenshot, screenshots });
+
       if (result.success) {
         const flaky = result.attempt > 1 ? ` ${C.yellow}(flaky, passed on attempt ${result.attempt}/${result.maxAttempts})${C.reset}` : '';
         log('✅', `${C.green}${test.name}${C.reset} ${C.dim}(${timeDiff(result.startTime, result.endTime)})${C.reset}${flaky}`);
@@ -219,7 +324,6 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
     }
   };
 
-  const concurrency = config.concurrency || 3;
   const workers = [];
   for (let i = 0; i < Math.min(concurrency, tests.length); i++) {
     workers.push(worker());
@@ -243,6 +347,12 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
     }
   }
 
+  {
+    const passed = results.filter(r => r.success).length;
+    const failed = results.filter(r => !r.success).length;
+    _progress({ event: 'run:complete', summary: { total: results.length, passed, failed } });
+  }
+
   return results;
 }
 

package/src/verify.js

@@ -0,0 +1,65 @@
+/**
+ * Bug Verification Orchestrator
+ *
+ * Combines issue fetch + AI test generation + test execution into a single pipeline.
+ * Tests assert CORRECT behavior: failures = bug confirmed, all pass = not reproducible.
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { fetchIssue } from './issues.js';
+import { generateTests } from './ai-generate.js';
+import { waitForPool } from './pool.js';
+import { runTestsParallel } from './runner.js';
+import { generateReport, saveReport, persistRun } from './reporter.js';
+
+/**
+ * Fetches an issue, generates tests via Claude API, runs them, and reports whether the bug is confirmed.
+ *
+ * @param {string} url - GitHub or GitLab issue URL
+ * @param {object} config - Loaded config (must include anthropicApiKey or ANTHROPIC_API_KEY env)
+ * @returns {Promise<{ issue: object, report: object, bugConfirmed: boolean, tests: object[], suiteName: string }>}
+ */
+export async function verifyIssue(url, config) {
+  // 1. Fetch issue
+  const issue = fetchIssue(url);
+
+  // 2. Generate tests via Claude API
+  const { tests, suiteName } = await generateTests(issue, config);
+
+  // 3. Save tests to a temp file (underscore prefix for cleanup identification)
+  const testFile = path.join(config.testsDir, `_verify-${suiteName}.json`);
+  if (!fs.existsSync(config.testsDir)) {
+    fs.mkdirSync(config.testsDir, { recursive: true });
+  }
+  fs.writeFileSync(testFile, JSON.stringify(tests, null, 2));
+
+  try {
+    // 4. Build hooks (inject auth if provided)
+    const hooks = {};
+    if (config.authToken) {
+      const storageKey = config.authStorageKey || 'accessToken';
+      hooks.beforeEach = [
+        { type: 'goto', value: '/' },
+        { type: 'evaluate', value: `localStorage.setItem('${storageKey}', '${config.authToken}')` },
+        { type: 'goto', value: '/' },
+        { type: 'wait', value: '1000' },
+      ];
+    }
+
+    // 5. Wait for pool and run
+    await waitForPool(config.poolUrl);
+    const results = await runTestsParallel(tests, config, hooks);
+    const report = generateReport(results);
+    saveReport(report, config.screenshotsDir, config);
+    persistRun(report, config, suiteName);
+
+    // 6. Interpret results
+    const bugConfirmed = report.summary.failed > 0;
+
+    return { issue, report, bugConfirmed, tests, suiteName };
+  } finally {
+    // 7. Clean up temp file
+    try { fs.unlinkSync(testFile); } catch { /* already gone */ }
+  }
+}

package/src/websocket.js

@@ -0,0 +1,177 @@
+/**
+ * Minimal WebSocket server — RFC 6455 text frames only, no external deps.
+ *
+ * Usage:
+ *   const wss = createWebSocketServer(httpServer);
+ *   wss.broadcast(JSON.stringify({ event: 'hello' }));
+ */
+
+import crypto from 'crypto';
+
+const GUID = '258EAFA5-E914-47DA-95CA-C5AB0DC85B11';
+const OP_TEXT = 0x01;
+const OP_CLOSE = 0x08;
+const OP_PING = 0x09;
+const OP_PONG = 0x0a;
+
+function acceptKey(key) {
+  return crypto.createHash('sha1').update(key + GUID).digest('base64');
+}
+
+function encodeFrame(data, opcode = OP_TEXT) {
+  const buf = Buffer.from(data, 'utf-8');
+  const len = buf.length;
+  let header;
+
+  if (len < 126) {
+    header = Buffer.alloc(2);
+    header[0] = 0x80 | opcode;
+    header[1] = len;
+  } else if (len < 65536) {
+    header = Buffer.alloc(4);
+    header[0] = 0x80 | opcode;
+    header[1] = 126;
+    header.writeUInt16BE(len, 2);
+  } else {
+    header = Buffer.alloc(10);
+    header[0] = 0x80 | opcode;
+    header[1] = 127;
+    header.writeBigUInt64BE(BigInt(len), 2);
+  }
+
+  return Buffer.concat([header, buf]);
+}
+
+function decodeFrame(buffer) {
+  if (buffer.length < 2) return null;
+
+  const firstByte = buffer[0];
+  const opcode = firstByte & 0x0f;
+  const secondByte = buffer[1];
+  const masked = (secondByte & 0x80) !== 0;
+  let payloadLen = secondByte & 0x7f;
+  let offset = 2;
+
+  if (payloadLen === 126) {
+    if (buffer.length < 4) return null;
+    payloadLen = buffer.readUInt16BE(2);
+    offset = 4;
+  } else if (payloadLen === 127) {
+    if (buffer.length < 10) return null;
+    payloadLen = Number(buffer.readBigUInt64BE(2));
+    offset = 10;
+  }
+
+  if (masked) {
+    if (buffer.length < offset + 4 + payloadLen) return null;
+    const mask = buffer.slice(offset, offset + 4);
+    offset += 4;
+    const payload = Buffer.alloc(payloadLen);
+    for (let i = 0; i < payloadLen; i++) {
+      payload[i] = buffer[offset + i] ^ mask[i % 4];
+    }
+    return { opcode, payload, totalLength: offset + payloadLen };
+  }
+
+  if (buffer.length < offset + payloadLen) return null;
+  return { opcode, payload: buffer.slice(offset, offset + payloadLen), totalLength: offset + payloadLen };
+}
+
+export function createWebSocketServer(httpServer, options = {}) {
+  const clients = new Set();
+
+  httpServer.on('upgrade', (req, socket, head) => {
+    // Validate Origin to prevent cross-site WebSocket hijacking
+    const origin = req.headers.origin;
+    if (origin && options.allowedOrigins) {
+      if (!options.allowedOrigins.includes(origin)) {
+        socket.write('HTTP/1.1 403 Forbidden\r\n\r\n');
+        socket.destroy();
+        return;
+      }
+    }
+
+    const key = req.headers['sec-websocket-key'];
+    if (!key) {
+      socket.destroy();
+      return;
+    }
+
+    const accept = acceptKey(key);
+    socket.write(
+      'HTTP/1.1 101 Switching Protocols\r\n' +
+      'Upgrade: websocket\r\n' +
+      'Connection: Upgrade\r\n' +
+      `Sec-WebSocket-Accept: ${accept}\r\n` +
+      '\r\n'
+    );
+
+    clients.add(socket);
+    let recvBuffer = Buffer.alloc(0);
+    const MAX_BUFFER = 1024 * 1024; // 1MB max receive buffer
+
+    socket.on('data', (data) => {
+      recvBuffer = Buffer.concat([recvBuffer, data]);
+      if (recvBuffer.length > MAX_BUFFER) {
+        clients.delete(socket);
+        try { socket.write(encodeFrame('', OP_CLOSE)); } catch { /* */ }
+        socket.destroy();
+        return;
+      }
+
+      while (recvBuffer.length > 0) {
+        const frame = decodeFrame(recvBuffer);
+        if (!frame) break;
+
+        recvBuffer = recvBuffer.slice(frame.totalLength);
+
+        switch (frame.opcode) {
+          case OP_TEXT:
+            // We don't process client messages for now
+            break;
+          case OP_PING:
+            try { socket.write(encodeFrame(frame.payload.toString(), OP_PONG)); } catch { /* */ }
+            break;
+          case OP_CLOSE:
+            try { socket.write(encodeFrame('', OP_CLOSE)); } catch { /* */ }
+            socket.end();
+            break;
+        }
+      }
+    });
+
+    socket.on('close', () => clients.delete(socket));
+    socket.on('error', () => clients.delete(socket));
+
+    // Send connected event
+    try {
+      socket.write(encodeFrame(JSON.stringify({ event: 'connected', timestamp: new Date().toISOString() })));
+    } catch { /* */ }
+
+    // Notify listener of new connection (for replaying state)
+    if (options.onConnect) {
+      try { options.onConnect(socket); } catch { /* */ }
+    }
+  });
+
+  return {
+    broadcast(data) {
+      const frame = encodeFrame(data);
+      for (const client of clients) {
+        try { client.write(frame); } catch { clients.delete(client); }
+      }
+    },
+    sendTo(socket, data) {
+      try { socket.write(encodeFrame(data)); } catch { /* */ }
+    },
+    get clientCount() {
+      return clients.size;
+    },
+    close() {
+      for (const client of clients) {
+        try { client.write(encodeFrame('', OP_CLOSE)); client.end(); } catch { /* */ }
+      }
+      clients.clear();
+    },
+  };
+}