@matware/e2e-runner 1.0.2 → 1.1.0

package/src/runner.js

@@ -7,7 +7,7 @@
 
 import fs from 'fs';
 import path from 'path';
-import { connectToPool, waitForPool } from './pool.js';
+import { connectToPool, waitForPool, getPoolStatus } from './pool.js';
 import { executeAction } from './actions.js';
 import { log, colors as C } from './logger.js';
 
@@ -47,8 +47,28 @@ function normalizeTestData(data) {
   return { tests: data.tests || [], hooks: data.hooks || {} };
 }
 
+/** Waits until the pool has capacity before connecting */
+async function waitForSlot(poolUrl, pollIntervalMs = 2000, maxWaitMs = 60000) {
+  const start = Date.now();
+  while (Date.now() - start < maxWaitMs) {
+    try {
+      const status = await getPoolStatus(poolUrl);
+      if (status.available && status.running < status.maxConcurrent) {
+        return;
+      }
+      log('⏳', `${C.dim}Pool at capacity (${status.running}/${status.maxConcurrent}, ${status.queued} queued), waiting for slot...${C.reset}`);
+    } catch {
+      // Pool unreachable, let connectToPool handle the error
+      return;
+    }
+    await sleep(pollIntervalMs);
+  }
+  // Timeout — proceed anyway and let connectToPool deal with it
+  log('⚠️', `${C.yellow}Waited ${maxWaitMs / 1000}s for pool slot, proceeding anyway${C.reset}`);
+}
+
 /** Runs a single test end-to-end */
-export async function runTest(test, config, hooks = {}) {
+export async function runTest(test, config, hooks = {}, progressFn = () => {}) {
   let browser = null;
   let page = null;
 
@@ -63,6 +83,7 @@ export async function runTest(test, config, hooks = {}) {
   };
 
   try {
+    await waitForSlot(config.poolUrl);
     browser = await connectToPool(config.poolUrl, config.connectRetries, config.connectRetryDelay);
     page = await browser.newPage();
     await page.setViewport(config.viewport);
@@ -79,23 +100,28 @@ export async function runTest(test, config, hooks = {}) {
       await executeHookActions(page, hooks.beforeEach, config);
     }
 
-    for (const action of test.actions) {
+    for (let i = 0; i < test.actions.length; i++) {
+      const action = test.actions[i];
       const actionStart = Date.now();
       try {
         const actionResult = await executeAction(page, action, config);
+        const actionDuration = Date.now() - actionStart;
         result.actions.push({
           ...action,
           success: true,
-          duration: Date.now() - actionStart,
+          duration: actionDuration,
           result: actionResult,
         });
+        progressFn({ event: 'test:action', name: test.name, action, actionIndex: i, totalActions: test.actions.length, success: true, duration: actionDuration, screenshotPath: actionResult?.screenshot || null });
       } catch (error) {
+        const actionDuration = Date.now() - actionStart;
         result.actions.push({
           ...action,
           success: false,
-          duration: Date.now() - actionStart,
+          duration: actionDuration,
           error: error.message,
         });
+        progressFn({ event: 'test:action', name: test.name, action, actionIndex: i, totalActions: test.actions.length, success: false, duration: actionDuration, error: error.message });
         throw error;
       }
     }
@@ -115,7 +141,8 @@ export async function runTest(test, config, hooks = {}) {
 
     if (page) {
       try {
-        const errorScreenshot = path.join(config.screenshotsDir, `error-${test.name}-${Date.now()}.png`);
+        const safeName = test.name.replace(/[^a-zA-Z0-9_\-. ]/g, '_');
+        const errorScreenshot = path.join(config.screenshotsDir, `error-${safeName}-${Date.now()}.png`);
         await page.screenshot({ path: errorScreenshot, fullPage: true });
         result.errorScreenshot = errorScreenshot;
       } catch { /* page may be dead */ }
@@ -155,6 +182,13 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
     }
   }
 
+  const concurrency = config.concurrency || 3;
+  const _runId = Date.now().toString(36) + Math.random().toString(36).slice(2, 6);
+  const _proj = config.projectName || null;
+  const _cwd = config._cwd || null;
+  const _progress = (data) => config.onProgress && config.onProgress({ ...data, runId: _runId, project: _proj, cwd: _cwd });
+  _progress({ event: 'run:start', total: tests.length, concurrency, timestamp: new Date().toISOString() });
+
   const results = [];
   const queue = [...tests];
   let activeCount = 0;
@@ -164,6 +198,7 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
       const test = queue.shift();
       activeCount++;
       log('▶▶▶', `${C.cyan}${test.name}${C.reset} ${C.dim}(${activeCount} active)${C.reset}`);
+      _progress({ event: 'test:start', name: test.name, activeCount, queueRemaining: queue.length });
 
       const maxAttempts = (test.retries ?? config.retries ?? 0) + 1;
       const testTimeout = test.timeout ?? config.testTimeout ?? 60000;
@@ -176,7 +211,7 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
         });
 
         try {
-          result = await Promise.race([runTest(test, config, hooks), timeoutPromise]);
+          result = await Promise.race([runTest(test, config, hooks, _progress), timeoutPromise]);
         } catch (error) {
           result = {
             name: test.name,
@@ -195,12 +230,16 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
 
         if (result.success || attempt === maxAttempts) break;
         log('🔄', `${C.yellow}${test.name}${C.reset} failed, retrying (${attempt}/${maxAttempts})...`);
+        _progress({ event: 'test:retry', name: test.name, attempt, maxAttempts });
         await sleep(config.retryDelay || 1000);
       }
 
       results.push(result);
       activeCount--;
 
+      const screenshots = result.actions.filter(a => a.result?.screenshot).map(a => a.result.screenshot);
+      _progress({ event: 'test:complete', name: test.name, success: result.success, duration: timeDiff(result.startTime, result.endTime), error: result.error, consoleLogs: result.consoleLogs, networkErrors: result.networkErrors, errorScreenshot: result.errorScreenshot, screenshots });
+
       if (result.success) {
         const flaky = result.attempt > 1 ? ` ${C.yellow}(flaky, passed on attempt ${result.attempt}/${result.maxAttempts})${C.reset}` : '';
         log('✅', `${C.green}${test.name}${C.reset} ${C.dim}(${timeDiff(result.startTime, result.endTime)})${C.reset}${flaky}`);
@@ -208,10 +247,17 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
         const attempts = result.maxAttempts > 1 ? ` (${result.maxAttempts} attempts)` : '';
         log('❌', `${C.red}${test.name}${C.reset}: ${result.error}${attempts}`);
       }
+
+      const consoleIssues = result.consoleLogs?.filter(l => l.type === 'error' || l.type === 'warning').length || 0;
+      if (consoleIssues > 0) {
+        log('⚠️', `${C.yellow}${test.name}: ${consoleIssues} console ${consoleIssues === 1 ? 'issue' : 'issues'}${C.reset}`);
+      }
+      if (result.networkErrors?.length > 0) {
+        log('⚠️', `${C.yellow}${test.name}: ${result.networkErrors.length} network ${result.networkErrors.length === 1 ? 'error' : 'errors'}${C.reset}`);
+      }
     }
   };
 
-  const concurrency = config.concurrency || 3;
   const workers = [];
   for (let i = 0; i < Math.min(concurrency, tests.length); i++) {
     workers.push(worker());
@@ -235,6 +281,12 @@ export async function runTestsParallel(tests, config, suiteHooks = {}) {
     }
   }
 
+  {
+    const passed = results.filter(r => r.success).length;
+    const failed = results.filter(r => !r.success).length;
+    _progress({ event: 'run:complete', summary: { total: results.length, passed, failed } });
+  }
+
   return results;
 }
 

package/src/verify.js

@@ -0,0 +1,53 @@
+/**
+ * Bug Verification Orchestrator
+ *
+ * Combines issue fetch + AI test generation + test execution into a single pipeline.
+ * Tests assert CORRECT behavior: failures = bug confirmed, all pass = not reproducible.
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { fetchIssue } from './issues.js';
+import { generateTests } from './ai-generate.js';
+import { waitForPool } from './pool.js';
+import { runTestsParallel } from './runner.js';
+import { generateReport, saveReport, persistRun } from './reporter.js';
+
+/**
+ * Fetches an issue, generates tests via Claude API, runs them, and reports whether the bug is confirmed.
+ *
+ * @param {string} url - GitHub or GitLab issue URL
+ * @param {object} config - Loaded config (must include anthropicApiKey or ANTHROPIC_API_KEY env)
+ * @returns {Promise<{ issue: object, report: object, bugConfirmed: boolean, tests: object[], suiteName: string }>}
+ */
+export async function verifyIssue(url, config) {
+  // 1. Fetch issue
+  const issue = fetchIssue(url);
+
+  // 2. Generate tests via Claude API
+  const { tests, suiteName } = await generateTests(issue, config);
+
+  // 3. Save tests to a temp file (underscore prefix for cleanup identification)
+  const testFile = path.join(config.testsDir, `_verify-${suiteName}.json`);
+  if (!fs.existsSync(config.testsDir)) {
+    fs.mkdirSync(config.testsDir, { recursive: true });
+  }
+  fs.writeFileSync(testFile, JSON.stringify(tests, null, 2));
+
+  try {
+    // 4. Wait for pool and run
+    await waitForPool(config.poolUrl);
+    const results = await runTestsParallel(tests, config, {});
+    const report = generateReport(results);
+    saveReport(report, config.screenshotsDir, config);
+    persistRun(report, config, suiteName);
+
+    // 5. Interpret results
+    const bugConfirmed = report.summary.failed > 0;
+
+    return { issue, report, bugConfirmed, tests, suiteName };
+  } finally {
+    // 6. Clean up temp file
+    try { fs.unlinkSync(testFile); } catch { /* already gone */ }
+  }
+}

package/src/websocket.js

@@ -0,0 +1,177 @@
+/**
+ * Minimal WebSocket server — RFC 6455 text frames only, no external deps.
+ *
+ * Usage:
+ *   const wss = createWebSocketServer(httpServer);
+ *   wss.broadcast(JSON.stringify({ event: 'hello' }));
+ */
+
+import crypto from 'crypto';
+
+const GUID = '258EAFA5-E914-47DA-95CA-C5AB0DC85B11';
+const OP_TEXT = 0x01;
+const OP_CLOSE = 0x08;
+const OP_PING = 0x09;
+const OP_PONG = 0x0a;
+
+function acceptKey(key) {
+  return crypto.createHash('sha1').update(key + GUID).digest('base64');
+}
+
+function encodeFrame(data, opcode = OP_TEXT) {
+  const buf = Buffer.from(data, 'utf-8');
+  const len = buf.length;
+  let header;
+
+  if (len < 126) {
+    header = Buffer.alloc(2);
+    header[0] = 0x80 | opcode;
+    header[1] = len;
+  } else if (len < 65536) {
+    header = Buffer.alloc(4);
+    header[0] = 0x80 | opcode;
+    header[1] = 126;
+    header.writeUInt16BE(len, 2);
+  } else {
+    header = Buffer.alloc(10);
+    header[0] = 0x80 | opcode;
+    header[1] = 127;
+    header.writeBigUInt64BE(BigInt(len), 2);
+  }
+
+  return Buffer.concat([header, buf]);
+}
+
+function decodeFrame(buffer) {
+  if (buffer.length < 2) return null;
+
+  const firstByte = buffer[0];
+  const opcode = firstByte & 0x0f;
+  const secondByte = buffer[1];
+  const masked = (secondByte & 0x80) !== 0;
+  let payloadLen = secondByte & 0x7f;
+  let offset = 2;
+
+  if (payloadLen === 126) {
+    if (buffer.length < 4) return null;
+    payloadLen = buffer.readUInt16BE(2);
+    offset = 4;
+  } else if (payloadLen === 127) {
+    if (buffer.length < 10) return null;
+    payloadLen = Number(buffer.readBigUInt64BE(2));
+    offset = 10;
+  }
+
+  if (masked) {
+    if (buffer.length < offset + 4 + payloadLen) return null;
+    const mask = buffer.slice(offset, offset + 4);
+    offset += 4;
+    const payload = Buffer.alloc(payloadLen);
+    for (let i = 0; i < payloadLen; i++) {
+      payload[i] = buffer[offset + i] ^ mask[i % 4];
+    }
+    return { opcode, payload, totalLength: offset + payloadLen };
+  }
+
+  if (buffer.length < offset + payloadLen) return null;
+  return { opcode, payload: buffer.slice(offset, offset + payloadLen), totalLength: offset + payloadLen };
+}
+
+export function createWebSocketServer(httpServer, options = {}) {
+  const clients = new Set();
+
+  httpServer.on('upgrade', (req, socket, head) => {
+    // Validate Origin to prevent cross-site WebSocket hijacking
+    const origin = req.headers.origin;
+    if (origin && options.allowedOrigins) {
+      if (!options.allowedOrigins.includes(origin)) {
+        socket.write('HTTP/1.1 403 Forbidden\r\n\r\n');
+        socket.destroy();
+        return;
+      }
+    }
+
+    const key = req.headers['sec-websocket-key'];
+    if (!key) {
+      socket.destroy();
+      return;
+    }
+
+    const accept = acceptKey(key);
+    socket.write(
+      'HTTP/1.1 101 Switching Protocols\r\n' +
+      'Upgrade: websocket\r\n' +
+      'Connection: Upgrade\r\n' +
+      `Sec-WebSocket-Accept: ${accept}\r\n` +
+      '\r\n'
+    );
+
+    clients.add(socket);
+    let recvBuffer = Buffer.alloc(0);
+    const MAX_BUFFER = 1024 * 1024; // 1MB max receive buffer
+
+    socket.on('data', (data) => {
+      recvBuffer = Buffer.concat([recvBuffer, data]);
+      if (recvBuffer.length > MAX_BUFFER) {
+        clients.delete(socket);
+        try { socket.write(encodeFrame('', OP_CLOSE)); } catch { /* */ }
+        socket.destroy();
+        return;
+      }
+
+      while (recvBuffer.length > 0) {
+        const frame = decodeFrame(recvBuffer);
+        if (!frame) break;
+
+        recvBuffer = recvBuffer.slice(frame.totalLength);
+
+        switch (frame.opcode) {
+          case OP_TEXT:
+            // We don't process client messages for now
+            break;
+          case OP_PING:
+            try { socket.write(encodeFrame(frame.payload.toString(), OP_PONG)); } catch { /* */ }
+            break;
+          case OP_CLOSE:
+            try { socket.write(encodeFrame('', OP_CLOSE)); } catch { /* */ }
+            socket.end();
+            break;
+        }
+      }
+    });
+
+    socket.on('close', () => clients.delete(socket));
+    socket.on('error', () => clients.delete(socket));
+
+    // Send connected event
+    try {
+      socket.write(encodeFrame(JSON.stringify({ event: 'connected', timestamp: new Date().toISOString() })));
+    } catch { /* */ }
+
+    // Notify listener of new connection (for replaying state)
+    if (options.onConnect) {
+      try { options.onConnect(socket); } catch { /* */ }
+    }
+  });
+
+  return {
+    broadcast(data) {
+      const frame = encodeFrame(data);
+      for (const client of clients) {
+        try { client.write(frame); } catch { clients.delete(client); }
+      }
+    },
+    sendTo(socket, data) {
+      try { socket.write(encodeFrame(data)); } catch { /* */ }
+    },
+    get clientCount() {
+      return clients.size;
+    },
+    close() {
+      for (const client of clients) {
+        try { client.write(encodeFrame('', OP_CLOSE)); client.end(); } catch { /* */ }
+      }
+      clients.clear();
+    },
+  };
+}