@matware/e2e-runner 1.0.0

package/src/index.js

@@ -0,0 +1,75 @@
+/**
+ * @matware/e2e-runner — Programmatic API
+ *
+ * Usage:
+ *   import { createRunner } from '@matware/e2e-runner';
+ *   const runner = await createRunner({ baseUrl: 'http://localhost:3000' });
+ *   const report = await runner.runAll();
+ */
+
+export { loadConfig } from './config.js';
+export { waitForPool, connectToPool, startPool, stopPool, restartPool, getPoolStatus } from './pool.js';
+export { executeAction } from './actions.js';
+export { runTest, runTestsParallel, loadTestFile, loadTestSuite, loadAllSuites, listSuites } from './runner.js';
+export { generateReport, generateJUnitXML, saveReport, printReport } from './reporter.js';
+
+import { loadConfig } from './config.js';
+import { waitForPool } from './pool.js';
+import { runTestsParallel, loadTestFile, loadTestSuite, loadAllSuites } from './runner.js';
+import { generateReport, saveReport, printReport } from './reporter.js';
+
+/**
+ * Creates a runner instance with custom configuration
+ * @param {object} userConfig - Configuration overrides
+ * @returns {object} Runner with runAll, runSuite, runTests, runFile methods
+ */
+export async function createRunner(userConfig = {}) {
+  const config = await loadConfig(userConfig);
+
+  return {
+    config,
+
+    /** Runs all test suites from the tests directory */
+    async runAll() {
+      await waitForPool(config.poolUrl);
+      const { tests, hooks } = loadAllSuites(config.testsDir);
+      const results = await runTestsParallel(tests, config, hooks);
+      const report = generateReport(results);
+      saveReport(report, config.screenshotsDir, config);
+      printReport(report, config.screenshotsDir);
+      return report;
+    },
+
+    /** Runs a single suite by name */
+    async runSuite(name) {
+      await waitForPool(config.poolUrl);
+      const { tests, hooks } = loadTestSuite(name, config.testsDir);
+      const results = await runTestsParallel(tests, config, hooks);
+      const report = generateReport(results);
+      saveReport(report, config.screenshotsDir, config);
+      printReport(report, config.screenshotsDir);
+      return report;
+    },
+
+    /** Runs an array of test objects */
+    async runTests(tests) {
+      await waitForPool(config.poolUrl);
+      const results = await runTestsParallel(tests, config);
+      const report = generateReport(results);
+      saveReport(report, config.screenshotsDir, config);
+      printReport(report, config.screenshotsDir);
+      return report;
+    },
+
+    /** Runs tests from a JSON file path */
+    async runFile(filePath) {
+      await waitForPool(config.poolUrl);
+      const { tests, hooks } = loadTestFile(filePath);
+      const results = await runTestsParallel(tests, config, hooks);
+      const report = generateReport(results);
+      saveReport(report, config.screenshotsDir, config);
+      printReport(report, config.screenshotsDir);
+      return report;
+    },
+  };
+}

package/src/logger.js

@@ -0,0 +1,19 @@
+/**
+ * ANSI color logger with timestamps
+ */
+
+export const colors = {
+  reset: '\x1b[0m',
+  green: '\x1b[32m',
+  red: '\x1b[31m',
+  yellow: '\x1b[33m',
+  blue: '\x1b[34m',
+  cyan: '\x1b[36m',
+  dim: '\x1b[2m',
+  bold: '\x1b[1m',
+};
+
+export function log(icon, msg) {
+  const ts = new Date().toLocaleTimeString('en-US', { hour12: false });
+  console.log(`${colors.dim}${ts}${colors.reset} ${icon} ${msg}`);
+}

package/src/mcp-server.js

@@ -0,0 +1,345 @@
+/**
+ * MCP Server for @matware/e2e-runner
+ *
+ * Exposes E2E test runner capabilities as MCP tools so Claude Code
+ * (and any MCP-compatible client) can run tests, list suites,
+ * create test files, and manage the Chrome pool.
+ *
+ * Install once for all Claude Code sessions:
+ *   claude mcp add --transport stdio --scope user e2e-runner -- npx -y -p @matware/e2e-runner e2e-runner-mcp
+ */
+
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import {
+  ListToolsRequestSchema,
+  CallToolRequestSchema,
+} from '@modelcontextprotocol/sdk/types.js';
+
+import fs from 'fs';
+import path from 'path';
+
+import { loadConfig } from './config.js';
+import { waitForPool, getPoolStatus, startPool, stopPool } from './pool.js';
+import { runTestsParallel, loadTestFile, loadTestSuite, loadAllSuites, listSuites } from './runner.js';
+import { generateReport, saveReport } from './reporter.js';
+
+// ── Redirect console.log to stderr so it doesn't corrupt the MCP stdio protocol ──
+console.log = (...args) => process.stderr.write(args.join(' ') + '\n');
+
+// ── Tool definitions ──────────────────────────────────────────────────────────
+
+const TOOLS = [
+  {
+    name: 'e2e_run',
+    description:
+      'Run E2E browser tests. Specify "all" to run every suite, "suite" for a specific suite, or "file" for a JSON file path. Returns structured results with pass/fail status, duration, and error details.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        all: {
+          type: 'boolean',
+          description: 'Run all test suites from the tests directory',
+        },
+        suite: {
+          type: 'string',
+          description: 'Suite name to run (e.g. "auth", "01-login"). Matches with or without numeric prefix.',
+        },
+        file: {
+          type: 'string',
+          description: 'Absolute or relative path to a JSON test file',
+        },
+        concurrency: {
+          type: 'number',
+          description: 'Number of parallel workers (default from config)',
+        },
+        baseUrl: {
+          type: 'string',
+          description: 'Override the base URL for this run',
+        },
+        retries: {
+          type: 'number',
+          description: 'Number of retries for failed tests',
+        },
+      },
+    },
+  },
+  {
+    name: 'e2e_list',
+    description:
+      'List all available E2E test suites with their test names and counts.',
+    inputSchema: {
+      type: 'object',
+      properties: {},
+    },
+  },
+  {
+    name: 'e2e_create_test',
+    description:
+      'Create a new E2E test JSON file. Provide the suite name and an array of test objects, each with a name and actions array.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        name: {
+          type: 'string',
+          description: 'Suite file name without .json extension (e.g. "login", "05-checkout")',
+        },
+        tests: {
+          type: 'array',
+          description: 'Array of test objects with { name, actions }',
+          items: {
+            type: 'object',
+            properties: {
+              name: { type: 'string', description: 'Test name' },
+              actions: {
+                type: 'array',
+                description: 'Sequential browser actions',
+                items: {
+                  type: 'object',
+                  properties: {
+                    type: {
+                      type: 'string',
+                      description: 'Action type: goto, click, type, wait, assert_text, assert_url, assert_visible, assert_count, screenshot, select, clear, press, scroll, hover, evaluate',
+                    },
+                    selector: { type: 'string', description: 'CSS selector' },
+                    value: { type: 'string', description: 'Value for the action' },
+                    text: { type: 'string', description: 'Text content to match' },
+                  },
+                  required: ['type'],
+                },
+              },
+            },
+            required: ['name', 'actions'],
+          },
+        },
+        hooks: {
+          type: 'object',
+          description: 'Optional hooks: beforeAll, afterAll, beforeEach, afterEach (each an array of actions)',
+          properties: {
+            beforeAll: { type: 'array', items: { type: 'object' } },
+            afterAll: { type: 'array', items: { type: 'object' } },
+            beforeEach: { type: 'array', items: { type: 'object' } },
+            afterEach: { type: 'array', items: { type: 'object' } },
+          },
+        },
+      },
+      required: ['name', 'tests'],
+    },
+  },
+  {
+    name: 'e2e_pool_status',
+    description:
+      'Get the status of the Chrome pool (browserless/chrome). Shows availability, running sessions, capacity, and queued requests.',
+    inputSchema: {
+      type: 'object',
+      properties: {},
+    },
+  },
+  {
+    name: 'e2e_pool_start',
+    description:
+      'Start the Chrome pool (browserless/chrome Docker container). Requires Docker to be running.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        port: {
+          type: 'number',
+          description: 'Port for the pool (default 3333)',
+        },
+        maxSessions: {
+          type: 'number',
+          description: 'Max concurrent Chrome sessions (default 10)',
+        },
+      },
+    },
+  },
+  {
+    name: 'e2e_pool_stop',
+    description: 'Stop the Chrome pool Docker container.',
+    inputSchema: {
+      type: 'object',
+      properties: {},
+    },
+  },
+];
+
+// ── Tool handlers ─────────────────────────────────────────────────────────────
+
+async function handleRun(args) {
+  const configOverrides = {};
+  if (args.concurrency) configOverrides.concurrency = args.concurrency;
+  if (args.baseUrl) configOverrides.baseUrl = args.baseUrl;
+  if (args.retries !== undefined) configOverrides.retries = args.retries;
+
+  const config = await loadConfig(configOverrides);
+
+  await waitForPool(config.poolUrl);
+
+  let tests, hooks;
+
+  if (args.all) {
+    ({ tests, hooks } = loadAllSuites(config.testsDir));
+  } else if (args.suite) {
+    ({ tests, hooks } = loadTestSuite(args.suite, config.testsDir));
+  } else if (args.file) {
+    const filePath = path.isAbsolute(args.file) ? args.file : path.resolve(args.file);
+    ({ tests, hooks } = loadTestFile(filePath));
+  } else {
+    return errorResult('Provide one of: all (true), suite (name), or file (path)');
+  }
+
+  if (tests.length === 0) {
+    return errorResult('No tests found');
+  }
+
+  const results = await runTestsParallel(tests, config, hooks || {});
+  const report = generateReport(results);
+  saveReport(report, config.screenshotsDir, config);
+
+  const failures = report.results
+    .filter(r => !r.success)
+    .map(r => ({
+      name: r.name,
+      error: r.error,
+      errorScreenshot: r.errorScreenshot || null,
+    }));
+
+  const flaky = report.results
+    .filter(r => r.success && r.attempt > 1)
+    .map(r => ({ name: r.name, attempts: r.attempt }));
+
+  const summary = {
+    ...report.summary,
+    reportPath: path.join(config.screenshotsDir, 'report.json'),
+  };
+
+  if (flaky.length > 0) summary.flaky = flaky;
+  if (failures.length > 0) summary.failures = failures;
+
+  return textResult(JSON.stringify(summary, null, 2));
+}
+
+async function handleList() {
+  const config = await loadConfig({});
+  const suites = listSuites(config.testsDir);
+
+  if (suites.length === 0) {
+    return textResult('No test suites found in ' + config.testsDir);
+  }
+
+  const lines = suites.map(s =>
+    `${s.name} (${s.testCount} tests): ${s.tests.join(', ')}`
+  );
+
+  return textResult(lines.join('\n'));
+}
+
+async function handleCreateTest(args) {
+  const config = await loadConfig({});
+
+  if (!fs.existsSync(config.testsDir)) {
+    fs.mkdirSync(config.testsDir, { recursive: true });
+  }
+
+  const filename = args.name.endsWith('.json') ? args.name : `${args.name}.json`;
+  const filePath = path.join(config.testsDir, filename);
+
+  if (fs.existsSync(filePath)) {
+    return errorResult(`File already exists: ${filePath}`);
+  }
+
+  let content;
+  if (args.hooks && Object.values(args.hooks).some(h => h?.length > 0)) {
+    content = { hooks: args.hooks, tests: args.tests };
+  } else {
+    content = args.tests;
+  }
+
+  fs.writeFileSync(filePath, JSON.stringify(content, null, 2) + '\n');
+  return textResult(`Created test file: ${filePath}\n\n${args.tests.length} test(s) defined.`);
+}
+
+async function handlePoolStatus() {
+  const config = await loadConfig({});
+  const status = await getPoolStatus(config.poolUrl);
+
+  const lines = [
+    `Available: ${status.available ? 'yes' : 'no'}`,
+    `Running:   ${status.running}/${status.maxConcurrent}`,
+    `Queued:    ${status.queued}`,
+    `Sessions:  ${status.sessions.length}`,
+  ];
+
+  if (status.error) {
+    lines.push(`Error:     ${status.error}`);
+  }
+
+  return textResult(lines.join('\n'));
+}
+
+async function handlePoolStart(args) {
+  const overrides = {};
+  if (args.port) overrides.poolPort = args.port;
+  if (args.maxSessions) overrides.maxSessions = args.maxSessions;
+
+  const config = await loadConfig(overrides);
+  startPool(config);
+  return textResult(`Chrome pool started on port ${config.poolPort}`);
+}
+
+async function handlePoolStop() {
+  const config = await loadConfig({});
+  stopPool(config);
+  return textResult('Chrome pool stopped');
+}
+
+// ── Helpers ───────────────────────────────────────────────────────────────────
+
+function textResult(text) {
+  return { content: [{ type: 'text', text }] };
+}
+
+function errorResult(message) {
+  return { content: [{ type: 'text', text: `Error: ${message}` }], isError: true };
+}
+
+// ── Server setup ──────────────────────────────────────────────────────────────
+
+export async function startMcpServer() {
+  const server = new Server(
+    { name: 'e2e-runner', version: '1.0.0' },
+    { capabilities: { tools: {} } }
+  );
+
+  server.setRequestHandler(ListToolsRequestSchema, async () => ({
+    tools: TOOLS,
+  }));
+
+  server.setRequestHandler(CallToolRequestSchema, async (request) => {
+    const { name, arguments: args = {} } = request.params;
+
+    try {
+      switch (name) {
+        case 'e2e_run':
+          return await handleRun(args);
+        case 'e2e_list':
+          return await handleList();
+        case 'e2e_create_test':
+          return await handleCreateTest(args);
+        case 'e2e_pool_status':
+          return await handlePoolStatus();
+        case 'e2e_pool_start':
+          return await handlePoolStart(args);
+        case 'e2e_pool_stop':
+          return await handlePoolStop();
+        default:
+          return errorResult(`Unknown tool: ${name}`);
+      }
+    } catch (error) {
+      return errorResult(error.message);
+    }
+  });
+
+  const transport = new StdioServerTransport();
+  await server.connect(transport);
+}

package/src/pool.js

@@ -0,0 +1,144 @@
+/**
+ * Pool Management
+ *
+ * Connectivity to the Chrome Pool (browserless/chrome) and Docker Compose lifecycle.
+ */
+
+import puppeteer from 'puppeteer-core';
+import fs from 'fs';
+import path from 'path';
+import { execFileSync } from 'child_process';
+import { fileURLToPath } from 'url';
+import { log } from './logger.js';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/** Waits for the Chrome Pool to become available */
+export async function waitForPool(poolUrl, maxWaitMs = 30000) {
+  const poolHttpUrl = poolUrl.replace('ws://', 'http://').replace('wss://', 'https://');
+  const pressureUrl = `${poolHttpUrl}/pressure`;
+  const start = Date.now();
+
+  while (Date.now() - start < maxWaitMs) {
+    try {
+      const res = await fetch(pressureUrl);
+      if (res.ok) {
+        const data = await res.json();
+        if (data.pressure?.isAvailable) {
+          return data.pressure;
+        }
+        log('⏳', `Pool busy (${data.pressure.running}/${data.pressure.maxConcurrent}), waiting...`);
+      }
+    } catch {
+      // Pool not ready
+    }
+    await sleep(2000);
+  }
+  throw new Error(`Chrome Pool unavailable after ${maxWaitMs / 1000}s. Verify the container is running.`);
+}
+
+/** Connects to the pool with retries */
+export async function connectToPool(poolUrl, retries = 3, delay = 2000) {
+  for (let attempt = 1; attempt <= retries; attempt++) {
+    try {
+      return await puppeteer.connect({
+        browserWSEndpoint: poolUrl,
+        timeout: 30000,
+      });
+    } catch (error) {
+      if (attempt === retries) {
+        throw new Error(`Failed to connect to pool: ${error.message}`);
+      }
+      log('🔄', `Attempt ${attempt}/${retries} failed, retrying...`);
+      await sleep(delay);
+    }
+  }
+}
+
+/** Generates docker-compose.yml and starts the pool */
+export function startPool(config) {
+  const cwd = process.cwd();
+  const poolDir = path.join(cwd, '.e2e-pool');
+
+  if (!fs.existsSync(poolDir)) {
+    fs.mkdirSync(poolDir, { recursive: true });
+  }
+
+  // Read template and interpolate variables
+  const templatePath = path.join(__dirname, '..', 'templates', 'docker-compose.yml');
+  let template = fs.readFileSync(templatePath, 'utf-8');
+  template = template.replace(/\$\{PORT\}/g, String(config.poolPort || 3333));
+  template = template.replace(/\$\{MAX_SESSIONS\}/g, String(config.maxSessions || 5));
+
+  const composePath = path.join(poolDir, 'docker-compose.yml');
+  fs.writeFileSync(composePath, template);
+
+  // Add .e2e-pool/ to .gitignore if missing
+  const gitignorePath = path.join(cwd, '.gitignore');
+  if (fs.existsSync(gitignorePath)) {
+    const content = fs.readFileSync(gitignorePath, 'utf-8');
+    if (!content.includes('.e2e-pool')) {
+      fs.appendFileSync(gitignorePath, '\n.e2e-pool/\n');
+    }
+  }
+
+  log('🐳', 'Starting Chrome Pool...');
+  execFileSync('docker', ['compose', '-f', composePath, 'up', '-d'], { stdio: 'inherit' });
+  log('✅', `Chrome Pool started on port ${config.poolPort || 3333}`);
+}
+
+/** Stops the pool */
+export function stopPool(config) {
+  const composePath = path.join(process.cwd(), '.e2e-pool', 'docker-compose.yml');
+  if (!fs.existsSync(composePath)) {
+    log('⚠️', '.e2e-pool/docker-compose.yml not found');
+    return;
+  }
+
+  log('🐳', 'Stopping Chrome Pool...');
+  execFileSync('docker', ['compose', '-f', composePath, 'down'], { stdio: 'inherit' });
+  log('✅', 'Chrome Pool stopped');
+}
+
+/** Restarts the pool */
+export function restartPool(config) {
+  stopPool(config);
+  startPool(config);
+}
+
+/** Gets pool status */
+export async function getPoolStatus(poolUrl) {
+  const poolHttpUrl = poolUrl.replace('ws://', 'http://').replace('wss://', 'https://');
+
+  try {
+    const [pressureRes, sessionsRes] = await Promise.all([
+      fetch(`${poolHttpUrl}/pressure`),
+      fetch(`${poolHttpUrl}/sessions`),
+    ]);
+
+    const pressure = pressureRes.ok ? await pressureRes.json() : null;
+    const sessions = sessionsRes.ok ? await sessionsRes.json() : null;
+
+    return {
+      available: pressure?.pressure?.isAvailable ?? false,
+      running: pressure?.pressure?.running ?? 0,
+      maxConcurrent: pressure?.pressure?.maxConcurrent ?? 0,
+      queued: pressure?.pressure?.queued ?? 0,
+      sessions: sessions || [],
+    };
+  } catch (error) {
+    return {
+      available: false,
+      error: error.message,
+      running: 0,
+      maxConcurrent: 0,
+      queued: 0,
+      sessions: [],
+    };
+  }
+}

package/src/reporter.js

@@ -0,0 +1,116 @@
+/**
+ * Report generation — JSON output, JUnit XML, and formatted console output
+ */
+
+import fs from 'fs';
+import path from 'path';
+import { colors as C } from './logger.js';
+
+function escapeXml(str) {
+  return String(str)
+    .replace(/&/g, '&')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;');
+}
+
+/** Generates a report object from test results */
+export function generateReport(results) {
+  const passed = results.filter(r => r.success).length;
+  const failed = results.filter(r => !r.success).length;
+  const totalDuration = results.reduce((acc, r) => acc + (new Date(r.endTime) - new Date(r.startTime)), 0);
+
+  return {
+    summary: {
+      total: results.length,
+      passed,
+      failed,
+      passRate: `${((passed / results.length) * 100).toFixed(1)}%`,
+      duration: `${(totalDuration / 1000).toFixed(1)}s`,
+    },
+    results,
+    generatedAt: new Date().toISOString(),
+  };
+}
+
+/** Generates JUnit XML string from a report */
+export function generateJUnitXML(report) {
+  const { summary, results, generatedAt } = report;
+  const totalTime = parseFloat(summary.duration);
+
+  let xml = '<?xml version="1.0" encoding="UTF-8"?>\n';
+  xml += `<testsuites tests="${summary.total}" failures="${summary.failed}" time="${totalTime}">\n`;
+  xml += `  <testsuite name="e2e" tests="${summary.total}" failures="${summary.failed}" time="${totalTime}" timestamp="${generatedAt}">\n`;
+
+  for (const result of results) {
+    const duration = ((new Date(result.endTime) - new Date(result.startTime)) / 1000).toFixed(3);
+    xml += `    <testcase name="${escapeXml(result.name)}" classname="e2e" time="${duration}">\n`;
+
+    if (!result.success) {
+      xml += `      <failure message="${escapeXml(result.error || 'Unknown error')}">${escapeXml(result.error || 'Unknown error')}</failure>\n`;
+    }
+
+    const logs = (result.consoleLogs || []).map(l => `[${l.type}] ${l.text}`).join('\n');
+    if (logs) {
+      xml += `      <system-out><![CDATA[${logs}]]></system-out>\n`;
+    }
+
+    xml += '    </testcase>\n';
+  }
+
+  xml += '  </testsuite>\n';
+  xml += '</testsuites>\n';
+
+  return xml;
+}
+
+/** Saves the report to disk based on outputFormat */
+export function saveReport(report, screenshotsDir, config = {}) {
+  const format = config.outputFormat || 'json';
+  const saved = [];
+
+  if (format === 'json' || format === 'both') {
+    const reportPath = path.join(screenshotsDir, 'report.json');
+    fs.writeFileSync(reportPath, JSON.stringify(report, null, 2));
+    saved.push(reportPath);
+  }
+
+  if (format === 'junit' || format === 'both') {
+    const junitPath = path.join(screenshotsDir, 'junit.xml');
+    fs.writeFileSync(junitPath, generateJUnitXML(report));
+    saved.push(junitPath);
+  }
+
+  return saved.length === 1 ? saved[0] : saved;
+}
+
+/** Prints a formatted report summary to the console */
+export function printReport(report, screenshotsDir) {
+  const { summary } = report;
+  console.log('');
+  console.log(`${C.bold}${'='.repeat(50)}${C.reset}`);
+  console.log(`${C.bold}  E2E RESULTS${C.reset}`);
+  console.log(`${'='.repeat(50)}`);
+  console.log(`  Total:    ${summary.total}`);
+  console.log(`  Passed:   ${C.green}${summary.passed}${C.reset}`);
+  console.log(`  Failed:   ${summary.failed > 0 ? C.red : C.green}${summary.failed}${C.reset}`);
+  console.log(`  Rate:     ${summary.passRate}`);
+  console.log(`  Duration: ${summary.duration}`);
+  console.log(`${'='.repeat(50)}`);
+
+  const failures = report.results.filter(r => !r.success);
+  if (failures.length > 0) {
+    console.log(`\n${C.red}${C.bold}FAILURES:${C.reset}`);
+    failures.forEach(f => {
+      console.log(`  ${C.red}✗${C.reset} ${f.name}: ${f.error}`);
+      if (f.errorScreenshot) {
+        console.log(`    ${C.dim}Screenshot: ${f.errorScreenshot}${C.reset}`);
+      }
+    });
+  }
+
+  if (screenshotsDir) {
+    console.log(`\n${C.dim}Report: ${path.join(screenshotsDir, 'report.json')}${C.reset}`);
+    console.log(`${C.dim}Screenshots: ${screenshotsDir}${C.reset}\n`);
+  }
+}