codeceptjs 3.7.4 → 3.7.5-beta.10

package/README.md

@@ -64,6 +64,8 @@ You don't need to worry about asynchronous nature of NodeJS or about various API
 - Also plays nice with TypeScript.
 - </> Smart locators: use names, labels, matching text, CSS or XPath to locate elements.
 - 🌐 Interactive debugging shell: pause test at any point and try different commands in a browser.
+- ⚡ **Parallel testing** with dynamic test pooling for optimal load balancing and performance.
+- 📊 **Built-in HTML Reporter** with interactive dashboard, step-by-step execution details, and comprehensive test analytics.
 - Easily create tests, pageobjects, stepobjects with CLI generators.
 
 ## Installation
@@ -233,6 +235,49 @@ Scenario('test title', () => {
 })
 ```
 
+## HTML Reporter
+
+CodeceptJS includes a powerful built-in HTML Reporter that generates comprehensive, interactive test reports with detailed information about your test runs. The HTML reporter is **enabled by default** for all new projects and provides:
+
+### Features
+
+- **Interactive Dashboard**: Visual statistics, pie charts, and expandable test details
+- **Step-by-Step Execution**: Shows individual test steps with timing and status indicators
+- **BDD/Gherkin Support**: Full support for feature files with proper scenario formatting
+- **System Information**: Comprehensive environment details including browser versions
+- **Advanced Filtering**: Real-time filtering by status, tags, features, and test types
+- **History Tracking**: Multi-run history with trend visualization
+- **Error Details**: Clean formatting of error messages and stack traces
+- **Artifacts Support**: Display screenshots and other test artifacts
+
+### Visual Examples
+
+#### Interactive Test Dashboard
+
+The main dashboard provides a complete overview with interactive statistics and pie charts:
+
+![HTML Reporter Dashboard](docs/shared/html-reporter-main-dashboard.png)
+
+#### Detailed Test Results
+
+Each test shows comprehensive execution details with expandable step information:
+
+![HTML Reporter Test Details](docs/shared/html-reporter-test-details.png)
+
+#### Advanced Filtering Capabilities
+
+Real-time filtering allows quick navigation through test results:
+
+![HTML Reporter Filtering](docs/shared/html-reporter-filtering.png)
+
+#### BDD/Gherkin Support
+
+Full support for Gherkin scenarios with proper feature formatting:
+
+![HTML Reporter BDD Details](docs/shared/html-reporter-bdd-details.png)
+
+The HTML reporter generates self-contained reports that can be easily shared with your team. Learn more about configuration and features in the [HTML Reporter documentation](https://codecept.io/plugins/#htmlreporter).
+
 ## PageObjects
 
 CodeceptJS provides the most simple way to create and use page objects in your test.

package/bin/codecept.js

@@ -165,6 +165,7 @@ program
   .option('--no-timeouts', 'disable all timeouts')
   .option('-p, --plugins <k=v,k2=v2,...>', 'enable plugins, comma-separated')
   .option('--shuffle', 'Shuffle the order in which test files run')
+  .option('--shard <index/total>', 'run only a fraction of tests (e.g., --shard 1/4)')
 
   // mocha options
   .option('--colors', 'force enabling of colors')
@@ -196,6 +197,7 @@ program
   .option('-i, --invert', 'inverts --grep matches')
   .option('-o, --override [value]', 'override current config options')
   .option('--suites', 'parallel execution of suites not single tests')
+  .option('--by <strategy>', 'test distribution strategy: "test" (pre-assign individual tests), "suite" (pre-assign test suites), or "pool" (dynamic distribution for optimal load balancing, recommended)')
   .option(commandFlags.debug.flag, commandFlags.debug.description)
   .option(commandFlags.verbose.flag, commandFlags.verbose.description)
   .option('--features', 'run only *.feature files and skip tests')
@@ -294,6 +296,29 @@ program
 
   .action(require('../lib/command/run-rerun'))
 
+program
+  .command('run-failed-tests')
+  .description('Re-run tests that failed in the previous test run')
+  .option(commandFlags.config.flag, commandFlags.config.description)
+  .option(commandFlags.profile.flag, commandFlags.profile.description)
+  .option(commandFlags.verbose.flag, commandFlags.verbose.description)
+  .option(commandFlags.debug.flag, commandFlags.debug.description)
+  .option(commandFlags.steps.flag, commandFlags.steps.description)
+  .option('-o, --override [value]', 'override current config options')
+  .option('-f, --file [path]', 'path to failed tests file (default: ./failed-tests.json)')
+  .option('-g, --grep <pattern>', 'only run failed tests matching <pattern>')
+  .option('-p, --plugins <k=v,k2=v2,...>', 'enable plugins, comma-separated')
+  .option('--features', 'run only *.feature files and skip tests')
+  .option('--tests', 'run only JS test files and skip features')
+  .option('--colors', 'force enabling of colors')
+  .option('--no-colors', 'force disabling of colors')
+  .option('-R, --reporter <name>', 'specify the reporter to use')
+  .option('-O, --reporter-options <k=v,k2=v2,...>', 'reporter-specific options')
+  .option('--workers <number>', 'run failed tests in parallel using specified number of workers')
+  .option('--suites', 'parallel execution of suites not single tests (when using --workers)')
+  .option('--by <strategy>', 'test distribution strategy when using --workers: "test", "suite", or "pool"')
+  .action(errorHandler(require('../lib/command/run-failed-tests')))
+
 program.on('command:*', cmd => {
   console.log(`\nUnknown command ${cmd}\n`)
   program.outputHelp()

package/bin/test-server.js

@@ -0,0 +1,53 @@
+#!/usr/bin/env node
+
+/**
+ * Standalone test server script to replace json-server
+ */
+
+const path = require('path')
+const TestServer = require('../lib/test-server')
+
+// Parse command line arguments
+const args = process.argv.slice(2)
+let dbFile = path.join(__dirname, '../test/data/rest/db.json')
+let port = 8010
+let host = '0.0.0.0'
+
+// Simple argument parsing
+for (let i = 0; i < args.length; i++) {
+  const arg = args[i]
+
+  if (arg === '-p' || arg === '--port') {
+    port = parseInt(args[++i])
+  } else if (arg === '--host') {
+    host = args[++i]
+  } else if (!arg.startsWith('-')) {
+    dbFile = path.resolve(arg)
+  }
+}
+
+// Create and start server
+const server = new TestServer({ port, host, dbFile })
+
+console.log(`Starting test server with db file: ${dbFile}`)
+
+server
+  .start()
+  .then(() => {
+    console.log(`Test server is ready and listening on http://${host}:${port}`)
+  })
+  .catch(err => {
+    console.error('Failed to start test server:', err)
+    process.exit(1)
+  })
+
+// Graceful shutdown
+process.on('SIGINT', () => {
+  console.log('\nShutting down test server...')
+  server.stop().then(() => process.exit(0))
+})
+
+process.on('SIGTERM', () => {
+  console.log('\nShutting down test server...')
+  server.stop().then(() => process.exit(0))
+})

package/lib/codecept.js

@@ -111,6 +111,7 @@ class Codecept {
     runHook(require('./listener/helpers'))
     runHook(require('./listener/globalTimeout'))
     runHook(require('./listener/globalRetry'))
+    runHook(require('./listener/retryEnhancer'))
     runHook(require('./listener/exit'))
     runHook(require('./listener/emptyRun'))
 
@@ -185,6 +186,46 @@ class Codecept {
     if (this.opts.shuffle) {
       this.testFiles = shuffle(this.testFiles)
     }
+
+    if (this.opts.shard) {
+      this.testFiles = this._applySharding(this.testFiles, this.opts.shard)
+    }
+  }
+
+  /**
+   * Apply sharding to test files based on shard configuration
+   *
+   * @param {Array<string>} testFiles - Array of test file paths
+   * @param {string} shardConfig - Shard configuration in format "index/total" (e.g., "1/4")
+   * @returns {Array<string>} - Filtered array of test files for this shard
+   */
+  _applySharding(testFiles, shardConfig) {
+    const shardMatch = shardConfig.match(/^(\d+)\/(\d+)$/)
+    if (!shardMatch) {
+      throw new Error('Invalid shard format. Expected format: "index/total" (e.g., "1/4")')
+    }
+
+    const shardIndex = parseInt(shardMatch[1], 10)
+    const shardTotal = parseInt(shardMatch[2], 10)
+
+    if (shardTotal < 1) {
+      throw new Error('Shard total must be at least 1')
+    }
+
+    if (shardIndex < 1 || shardIndex > shardTotal) {
+      throw new Error(`Shard index ${shardIndex} must be between 1 and ${shardTotal}`)
+    }
+
+    if (testFiles.length === 0) {
+      return testFiles
+    }
+
+    // Calculate which tests belong to this shard
+    const shardSize = Math.ceil(testFiles.length / shardTotal)
+    const startIndex = (shardIndex - 1) * shardSize
+    const endIndex = Math.min(startIndex + shardSize, testFiles.length)
+
+    return testFiles.slice(startIndex, endIndex)
   }
 
   /**

package/lib/command/init.js

@@ -18,6 +18,11 @@ const defaultConfig = {
   output: '',
   helpers: {},
   include: {},
+  plugins: {
+    htmlReporter: {
+      enabled: true,
+    },
+  },
 }
 
 const helpers = ['Playwright', 'WebDriver', 'Puppeteer', 'REST', 'GraphQL', 'Appium', 'TestCafe']

package/lib/command/run-failed-tests.js

@@ -0,0 +1,263 @@
+const fs = require('fs')
+const path = require('path')
+const { getConfig, printError, getTestRoot, createOutputDir } = require('./utils')
+const Config = require('../config')
+const store = require('../store')
+const Codecept = require('../codecept')
+const output = require('../output')
+const Workers = require('../workers')
+const { tryOrDefault } = require('../utils')
+
+module.exports = async function (options) {
+  // registering options globally to use in config
+  if (options.profile) {
+    process.env.profile = options.profile
+  }
+  if (options.verbose || options.debug) store.debugMode = true
+
+  const configFile = options.config
+  let config = getConfig(configFile)
+  
+  if (options.override) {
+    config = Config.append(JSON.parse(options.override))
+  }
+  
+  const testRoot = getTestRoot(configFile)
+  createOutputDir(config, testRoot)
+
+  // Determine failed tests file path - respect CodeceptJS output directory
+  const failedTestsFile = options.file || 'failed-tests.json'
+  const failedTestsPath = path.isAbsolute(failedTestsFile)
+    ? failedTestsFile
+    : path.resolve(global.output_dir || './output', failedTestsFile)
+
+  // Check if failed tests file exists
+  if (!fs.existsSync(failedTestsPath)) {
+    output.error(`Failed tests file not found: ${failedTestsPath}`)
+    output.print('Run tests first to generate a failed tests file, or specify a different file with --file option')
+    process.exitCode = 1
+    return
+  }
+
+  let failedTestsData
+  try {
+    const fileContent = fs.readFileSync(failedTestsPath, 'utf8')
+    failedTestsData = JSON.parse(fileContent)
+  } catch (error) {
+    output.error(`Failed to read or parse failed tests file: ${error.message}`)
+    process.exitCode = 1
+    return
+  }
+
+  if (!failedTestsData.tests || failedTestsData.tests.length === 0) {
+    output.print('No failed tests found in the file')
+    return
+  }
+
+  output.print(`Found ${failedTestsData.tests.length} failed tests from ${failedTestsData.timestamp}`)
+
+  // Debug: Show what's in the failed tests data
+  if (options.verbose) {
+    output.print('\nFailed tests data structure:')
+    failedTestsData.tests.forEach((test, index) => {
+      output.print(`  ${index + 1}. Title: "${test.title}", UID: "${test.uid}", File: "${test.file}"`)
+    })
+    output.print('')
+  }
+
+  // Build test patterns from failed tests
+  const testPatterns = []
+  const testsByFile = new Map()
+
+  // Group tests by file for more efficient execution
+  failedTestsData.tests.forEach(test => {
+    if (test.file) {
+      if (!testsByFile.has(test.file)) {
+        testsByFile.set(test.file, [])
+      }
+      testsByFile.get(test.file).push(test)
+    }
+  })
+
+  // Build precise test selection from failed tests
+  if (testsByFile.size > 0) {
+    // Use file paths for loading tests
+    for (const [file, tests] of testsByFile) {
+      testPatterns.push(file)
+    }
+    
+    // Extract exact test UIDs for precise filtering
+    const failedTestUIDs = failedTestsData.tests
+      .map(test => test.uid)
+      .filter(Boolean)
+    
+    if (failedTestUIDs.length > 0) {
+      // Pass UIDs to options for precise test filtering
+      options.tests = failedTestUIDs
+      output.print(`Targeting ${failedTestUIDs.length} specific failed tests by UID`)
+    } else {
+      // Fallback to title-based grep if no UIDs available
+      const testTitles = failedTestsData.tests.map(test => test.title).filter(Boolean)
+      if (testTitles.length > 0) {
+        const grepPattern = testTitles.map(title => `^${title.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`).join('|')
+        options.grep = grepPattern
+        output.print(`Targeting failed tests by title pattern (no UIDs available)`)
+      }
+    }
+  } else {
+    // Fallback: use test titles with exact match grep
+    const testTitles = failedTestsData.tests.map(test => test.title).filter(Boolean)
+    if (testTitles.length > 0) {
+      const grepPattern = testTitles.map(title => `^${title.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`).join('|')
+      options.grep = grepPattern
+      output.print(`Targeting failed tests by title pattern (no file info available)`)
+    }
+  }
+
+  // Check if user wants to run with workers
+  if (options.workers) {
+    await runWithWorkers(config, options, testPatterns, failedTestsData)
+  } else {
+    await runWithoutWorkers(config, options, testPatterns, failedTestsData, testRoot)
+  }
+}
+
+async function runWithWorkers(config, options, testPatterns, failedTestsData) {
+  const numberOfWorkers = parseInt(options.workers, 10)
+  const overrideConfigs = tryOrDefault(() => JSON.parse(options.override || '{}'), {})
+  
+  // Determine test split strategy
+  let by = 'test' // default for failed tests
+  if (options.by) {
+    by = options.by
+  } else if (options.suites) {
+    by = 'suite'
+  }
+
+  // Validate the by option
+  const validStrategies = ['test', 'suite', 'pool']
+  if (!validStrategies.includes(by)) {
+    throw new Error(`Invalid --by strategy: ${by}. Valid options are: ${validStrategies.join(', ')}`)
+  }
+
+  const workerConfig = {
+    by,
+    testConfig: options.config,
+    options,
+    selectedRuns: undefined,
+  }
+
+  // If we have specific test UIDs, override the worker test selection
+  if (options.tests && options.tests.length > 0) {
+    workerConfig.by = 'test' // Force test-level distribution for precise targeting
+    output.print(`Using precise test UID targeting for ${options.tests.length} failed tests`)
+  }
+
+  output.print(`CodeceptJS v${require('../codecept').version()}`)
+  output.print(`Re-running ${failedTestsData.tests.length} failed tests in ${output.styles.bold(numberOfWorkers)} workers...`)
+  output.print()
+  store.hasWorkers = true
+
+  const workers = new Workers(numberOfWorkers, workerConfig)
+  workers.overrideConfig(overrideConfigs)
+
+  // Set up event listeners for worker output
+  workers.on('test.failed', test => {
+    output.test.failed(test)
+  })
+
+  workers.on('test.passed', test => {
+    output.test.passed(test)
+  })
+
+  workers.on('test.skipped', test => {
+    output.test.skipped(test)
+  })
+
+  workers.on('all.result', result => {
+    workers.printResults()
+  })
+
+  try {
+    if (options.verbose || options.debug) store.debugMode = true
+
+    if (options.verbose) {
+      output.print('\nFailed tests to re-run with workers:')
+      failedTestsData.tests.forEach((test, index) => {
+        output.print(`  ${index + 1}. ${test.fullTitle || test.title} (${test.file || 'unknown file'})`)
+        if (test.error && test.error.message) {
+          output.print(`     Error: ${test.error.message}`)
+        }
+      })
+      output.print('')
+      
+      const { getMachineInfo } = require('./info')
+      await getMachineInfo()
+    }
+
+    await workers.bootstrapAll()
+    await workers.run()
+  } catch (err) {
+    printError(err)
+    process.exitCode = 1
+  } finally {
+    await workers.teardownAll()
+  }
+}
+
+async function runWithoutWorkers(config, options, testPatterns, failedTestsData, testRoot) {
+  const codecept = new Codecept(config, options)
+
+  try {
+    codecept.init(testRoot)
+    await codecept.bootstrap()
+
+    // Load tests - if we have specific patterns, use them, otherwise load all and filter with grep
+    if (testPatterns.length > 0) {
+      codecept.loadTests(testPatterns.join(' '))
+    } else {
+      codecept.loadTests()
+    }
+
+    // If we have specific test UIDs, filter the loaded tests to only include those
+    if (options.tests && options.tests.length > 0) {
+      const Container = require('../container')
+      const mocha = Container.mocha()
+      
+      // Filter suites to only include tests with matching UIDs
+      for (const suite of mocha.suite.suites) {
+        suite.tests = suite.tests.filter(test => options.tests.includes(test.uid))
+      }
+      
+      // Remove empty suites
+      mocha.suite.suites = mocha.suite.suites.filter(suite => suite.tests.length > 0)
+      
+      output.print(`Filtered to ${options.tests.length} specific failed tests by UID`)
+    }
+
+    if (options.verbose) {
+      global.debugMode = true
+      const { getMachineInfo } = require('./info')
+      await getMachineInfo()
+    }
+
+    // Display information about what we're running
+    if (options.verbose) {
+      output.print('\nFailed tests to re-run:')
+      failedTestsData.tests.forEach((test, index) => {
+        output.print(`  ${index + 1}. ${test.fullTitle || test.title} (${test.file || 'unknown file'})`)
+        if (test.error && test.error.message) {
+          output.print(`     Error: ${test.error.message}`)
+        }
+      })
+      output.print('')
+    }
+
+    await codecept.run()
+  } catch (err) {
+    printError(err)
+    process.exitCode = 1
+  } finally {
+    await codecept.teardown()
+  }
+}

package/lib/command/run-workers.js

@@ -10,7 +10,22 @@ module.exports = async function (workerCount, selectedRuns, options) {
 
   const { config: testConfig, override = '' } = options
   const overrideConfigs = tryOrDefault(() => JSON.parse(override), {})
-  const by = options.suites ? 'suite' : 'test'
+
+  // Determine test split strategy
+  let by = 'test' // default
+  if (options.by) {
+    // Explicit --by option takes precedence
+    by = options.by
+  } else if (options.suites) {
+    // Legacy --suites option
+    by = 'suite'
+  }
+
+  // Validate the by option
+  const validStrategies = ['test', 'suite', 'pool']
+  if (!validStrategies.includes(by)) {
+    throw new Error(`Invalid --by strategy: ${by}. Valid options are: ${validStrategies.join(', ')}`)
+  }
   delete options.parent
   const config = {
     by,