codeceptjs 4.2.0 → 4.2.1-beta.1

package/docs/installation.md

@@ -99,6 +99,22 @@ export const config = {
 
 See [Alternative Browser Engines](/alternative-browsers) for connection modes and limitations, and the [Obscura helper reference](/helpers/Obscura) for every option.
 
+## Lightpanda (experimental)
+
+Lightpanda is a headless browser built for automation. Install it following the [official instructions](https://lightpanda.io/docs/open-source/installation), then enable the helper:
+
+```js
+export const config = {
+  helpers: {
+    Lightpanda: {
+      url: 'http://localhost:3000',
+    },
+  },
+}
+```
+
+See [Alternative Browser Engines](/alternative-browsers) for running an existing Playwright suite on Lightpanda.
+
 ## Appium (mobile)
 
 Native iOS and Android testing. Appium speaks the WebDriver protocol, so CodeceptJS drives it through `webdriverio`:

package/docs/mcp.md

@@ -27,7 +27,13 @@ Two ways to launch the server:
 
 ## Configuration
 
-Set up the MCP server in your client (Claude Desktop, Cursor, Continue, etc.):
+For Claude Code, Codex, Cursor, or OpenCode, one command registers the server and installs the CodeceptJS skills:
+
+```bash
+npx codeceptjs init:agent claude
+```
+
+To set up the MCP server in another client (Claude Desktop, Continue, etc.), add it to the client config:
 
 ### Basic
 

package/docs/parallel.md

@@ -20,7 +20,7 @@ For anything more specific — your own grouping, a config per group, several br
 npx codeceptjs run-workers 4
 ```
 
-Steps are not streamed to the console in this mode — output from separate threads can't be interleaved cleanly. While workers run, CodeceptJS sets `process.env.RUNS_WITH_WORKERS=true`, so plugins and helpers can branch on it. All `run` options work here too: `--grep "@smoke"`, `-c codecept.conf.js`, `--debug`, and the rest.
+Steps are not streamed to the console in this mode — output from separate threads can't be interleaved cleanly. While workers run, CodeceptJS sets `process.env.RUNS_WITH_WORKERS=true`, so plugins and helpers can branch on it. All `run` options work here too: `--grep "@smoke"`, `-c codecept.conf.js`, `--debug`, and the rest. With `--grep`, only matching tests are spread across workers, so a narrow pattern may start fewer workers than requested.
 
 By default, workers are created with a staggered delay of 200ms to prevent CPU spikes and stagger browser initializations. You can adjust this via `workerInitializationDelay` in your configuration.
 
@@ -40,6 +40,40 @@ npx codeceptjs run-workers 4 --by pool
 
 `--suites` is shorthand for `--by suite`.
 
+### Serial tests
+
+Some tests can't share the stage: they change global settings, reset data, or toggle a feature flag that other tests read. Mark them with a tag and set the `serial` pattern in config or pass it with `--serial`:
+
+```js
+// codecept.conf.js
+export const config = {
+  serial: '@serial',
+}
+```
+
+```js
+Scenario('change account timezone @serial', ({ I }) => {
+  // ...
+})
+```
+
+`run-workers` keeps matching tests out of the parallel workers. When those workers finish, one more worker starts and runs the serial tests one by one. It is still a single run: `bootstrapAll`, `teardownAll` and `event.workers.result` fire once, and the exit code covers both parts.
+
+```
+Test distribution:
+  Worker 1: 14 tests
+  Worker 2: 13 tests
+  Worker 3: 4 tests (serial)
+```
+
+Pass `--serial` to set or override the pattern for a single run:
+
+```sh
+npx codeceptjs run-workers 3 --serial "@serial"
+```
+
+`serial` is matched against the full test title the same way `--grep` is, so a tag on a `Feature` makes all its scenarios serial. It works with every `--by` strategy and combines with `--grep`: only serial tests that also match grep are run. With [multiple browsers](#multiple-browsers) each profile gets its own serial worker, started one after another.
+
 ### Multiple browsers
 
 Define browser profiles under `multiple` in `codecept.conf.js`:

package/docs/plugins/pause.md

@@ -25,7 +25,8 @@ plugins: {
 
 #### `on=` modes
 
-*   **fail** — pause when a step fails (default)
+*   **fail** — pause when a step fails (default). Failures inside `tryTo` and
+    non-final `retryTo` attempts are ignored
 *   **test** — pause after each test
 *   **step** — pause before the first step (interactive walk-through)
 *   **file** — pause when execution reaches `path=...[;line=...]`

package/docs/plugins/sitemap.md

@@ -0,0 +1,48 @@
+---
+permalink: /plugins/sitemap
+editLink: false
+sidebar: auto
+title: sitemap
+---
+
+<!-- Generated by documentation.js. Update this documentation by updating the source code. -->
+
+## sitemap
+
+Collects all pages visited while tests are running and saves them as a [sitemap][1] XML file.
+
+The plugin listens to browser navigation events, so it doesn't slow down test steps.
+Page loads and SPA route changes (`history.pushState`) of the top-level page are recorded, iframes are ignored.
+Every unique `http(s)` URL is added to the sitemap. URL fragments (`#...`) are dropped.
+
+Supported helpers: Playwright, Puppeteer, and WebDriver with BiDi protocol enabled (default).
+The sitemap is written once all tests finish.
+
+#### Configuration
+
+```js
+"plugins": {
+   "sitemap": {
+     "enabled": true
+   }
+ }
+```
+
+Possible config options:
+
+*   `file`: path to the sitemap file, relative to the output directory. Default: `sitemap.xml`, saved as `output/sitemap.xml`.
+*   `stripQuery`: remove query strings so `/posts?page=1` and `/posts?page=2` become one `/posts` entry. Set to `false` to keep them. Default: true.
+
+CLI examples:
+
+    npx codeceptjs run -p sitemap
+    npx codeceptjs run -p sitemap:file=visited.xml
+    npx codeceptjs run -p sitemap:stripQuery=false
+
+Works with `run-workers`: URLs from all workers are merged into a single sitemap.
+
+### Parameters
+
+*   `config` **any**  
+
+[1]: https://www.sitemaps.org/protocol.html

package/docs/plugins.md

@@ -81,6 +81,10 @@ Saves screenshots from the browser at points triggered by `on=`.
 
 Saves a screenshot when a test fails.
 
+## [sitemap](/plugins/sitemap)
+
+Collects all pages visited while tests are running and saves them as a [sitemap][1] XML file.
+
 ## [stepTimeout](/plugins/stepTimeout)
 
 Set timeout for test steps globally.

package/lib/ai.js

@@ -3,7 +3,7 @@ const debug = debugModule('codeceptjs:ai')
 import output from './output.js'
 import event from './event.js'
 import { removeNonInteractiveElements, minifyHtml, splitByChunks } from './html.js'
-import { generateText } from 'ai'
+import { generateText, experimental_decide as decide, APICallError } from 'ai'
 import { fileURLToPath } from 'url'
 import path from 'path'
 import { fileExists, resolveImportModulePath } from './utils.js'
@@ -59,7 +59,7 @@ class AiAssistant {
     debug('Enabling AI assistant')
     this.isEnabled = true
 
-    const { html, prompts, ...aiConfig } = config
+    const { html, prompts, decisionModel, ...aiConfig } = config
 
     this.config = Object.assign(this.config, aiConfig)
     this.htmlConfig = Object.assign(defaultHtmlConfig, html)
@@ -271,4 +271,83 @@ function parseCodeBlocks(response) {
   return modifiedSnippets.filter(snippet => !!snippet)
 }
 
+const defaultDecisionConfig = {
+  confidence: 0.7,
+  timeout: 15000,
+  maxLength: 12000,
+}
+
+class DecisionConnectionError extends Error {}
+
+class DecisionAI {
+  constructor(config = {}) {
+    this.config = { ...defaultDecisionConfig, ...config }
+
+    const { confidence } = this.config
+    if (!(confidence > 0 && confidence < 1)) throw new Error(`ai.decisionModel.confidence must be between 0 and 1, got ${confidence}`)
+  }
+
+  checkModel(option) {
+    if (this.config[option]) return
+
+    const noModelErrorMessage = `
+     No decision model is set in ai.decisionModel.${option} config.
+
+     [!] Configure decision model with OpenRouter provider:
+
+     import { createOpenRouter } from '@openrouter/ai-sdk-provider'
+
+     const openrouter = createOpenRouter({
+       apiKey: process.env.OPENROUTER_API_KEY,
+     })
+
+     ai: {
+       decisionModel: {
+         model: openrouter.evaluationModel('typesafe/jev-1.13'),
+         visualModel: openrouter.evaluationModel('cloudflare/clef'),
+       }
+     }
+
+     See https://openrouter.ai/models?output_modalities=decisions for all decision models.
+    `.trim()
+
+    throw new Error(noModelErrorMessage)
+  }
+
+  async decide(option, state, statements) {
+    this.checkModel(option)
+    const model = this.config[option]
+
+    const questions = Object.fromEntries(statements.map((statement, i) => [`q${i}`, { type: 'boolean', instructions: statement }]))
+    debug('Decision request', model.modelId, statements)
+
+    const controller = new AbortController()
+    const timer = setTimeout(() => controller.abort(), this.config.timeout)
+
+    let result
+    try {
+      result = await decide({
+        model,
+        state,
+        questions,
+        maxRetries: 0,
+        abortSignal: controller.signal,
+      })
+    } catch (err) {
+      if (controller.signal.aborted) throw new DecisionConnectionError(`Decision model ${model.modelId} did not respond in ${this.config.timeout}ms`)
+      if (APICallError.isInstance(err) && !err.statusCode) throw new DecisionConnectionError(`Decision model ${model.modelId} request failed: ${err.message}`)
+      if (APICallError.isInstance(err)) throw new Error(`Decision model ${model.modelId} responded with ${err.statusCode}: ${err.responseBody || err.message}`)
+      throw err
+    } finally {
+      clearTimeout(timer)
+    }
+
+    debug('Decision response', result.answers, result.usage)
+
+    return Object.keys(questions).map(id => result.answers[id].probability)
+  }
+}
+
+export { DecisionAI, DecisionConnectionError }
+
 export default new AiAssistant()

package/lib/codecept.js

@@ -6,6 +6,7 @@ import { resolve } from 'path'
 import { fileURLToPath, pathToFileURL } from 'url'
 import { dirname } from 'path'
 import { createRequire } from 'module'
+import { isMainThread } from 'worker_threads'
 
 const __filename = fileURLToPath(import.meta.url)
 const __dirname = dirname(__filename)
@@ -23,6 +24,8 @@ import loadTests from './mocha/loadTests.js'
 import { validateTypeScriptSetup, getTSNodeESMWarning } from './utils/loaderCheck.js'
 import recorder from './recorder.js'
 import store from './store.js'
+import Result from './result.js'
+import WebServer from './webServer.js'
 
 import storeListener from './listener/store.js'
 import stepsListener from './listener/steps.js'
@@ -159,6 +162,7 @@ class Codecept {
    * @returns {Promise<void>}
    */
   async bootstrap() {
+    if (isMainThread && !this.opts?.child) await WebServer.startAll(this.config.webServer)
     return runHook(this.config.bootstrap, 'bootstrap')
   }
 
@@ -168,7 +172,11 @@ class Codecept {
    * @returns {Promise<void>}
    */
   async teardown() {
-    return runHook(this.config.teardown, 'teardown')
+    try {
+      return await runHook(this.config.teardown, 'teardown')
+    } finally {
+      await WebServer.stopAll()
+    }
   }
 
   /**
@@ -264,6 +272,21 @@ class Codecept {
     return testFiles.slice(startIndex, endIndex)
   }
 
+  /**
+   * Keeps only tests that failed in the previous run when `--last-failed` is set.
+   *
+   * @param {Object} mocha Mocha instance with loaded tests
+   */
+  filterLastFailedTests(mocha) {
+    if (!this.opts.lastFailed) return
+    const failedTestUids = new Set(Result.loadFailedTests().map(test => test.uid))
+    const filterSuite = suite => {
+      suite.tests = suite.tests.filter(test => failedTestUids.has(test.uid))
+      suite.suites.forEach(filterSuite)
+    }
+    filterSuite(mocha.suite)
+  }
+
   /**
    * Run a specific test or all loaded tests.
    *
@@ -314,7 +337,13 @@ class Codecept {
       })
     }
 
-    await loadTests(mocha)
+    // Worker threads pre-load and filter tests before run(); reloading duplicates Gherkin suites.
+    if (!mocha.suite.total()) {
+      await loadTests(mocha)
+      this.filterLastFailedTests(mocha)
+    } else {
+      mocha.lazyLoadFiles(true)
+    }
 
     return new Promise((resolve, reject) => {
       const done = async (failures) => {
@@ -322,6 +351,7 @@ class Codecept {
         event.emit(event.all.after, this)
         // Wait for any recorder tasks added by event.all.after handlers
         await recorder.promise()
+        if (!this.opts.child && !store.dryRun && container.result().tests.length) container.result().save()
         // Set exit code based on test failures
         if (failures) {
           process.exitCode = 1

package/lib/command/init.js

@@ -12,6 +12,7 @@ import { fileExists, beautify, installedLocally } from '../utils.js'
 import { getTestRoot } from './utils.js'
 import generateDefinitions from './definitions.js'
 import { test as generateTest } from './generate.js'
+import initAgent, { agents } from './initAgent.js'
 const isLocal = installedLocally()
 
 const defaultConfig = {
@@ -318,6 +319,27 @@ export default async function (initPath, options = {}) {
       print(generateDefinitionsManually)
     }
 
+    let agentName = options.agent
+    if (!agentName && !skipPrompts) {
+      const answer = await inquirer.prompt([
+        {
+          name: 'agent',
+          type: 'list',
+          message: 'Set up an AI coding agent?',
+          choices: [...Object.keys(agents), 'none'],
+          default: 'none',
+        },
+      ])
+      agentName = answer?.agent
+    }
+
+    if (agentName && agentName !== 'none') {
+      await initAgent(agentName, { yes: true })
+    } else {
+      print()
+      print(`Using an AI agent? Run: ${colors.green('npx codeceptjs init:agent')}`)
+    }
+
     print('')
     success(' Almost ready... Next step:')
 

package/lib/command/initAgent.js

@@ -0,0 +1,78 @@
+import fs from 'fs'
+import path from 'path'
+import inquirer from 'inquirer'
+import { execSync } from 'child_process'
+
+import output from '../output.js'
+
+const { print, success, error } = output
+
+export const agents = {
+  claude: {
+    skills: 'claude-code',
+    mcp: 'claude mcp add codeceptjs -- npx codeceptjs-mcp',
+  },
+  codex: {
+    skills: 'codex',
+    mcp: 'codex mcp add codeceptjs -- npx codeceptjs-mcp',
+  },
+  cursor: {
+    skills: 'cursor',
+    file: '.cursor/mcp.json',
+    key: 'mcpServers',
+    server: { command: 'npx', args: ['codeceptjs-mcp'] },
+  },
+  opencode: {
+    skills: 'opencode',
+    file: 'opencode.json',
+    key: 'mcp',
+    server: { type: 'local', command: ['npx', 'codeceptjs-mcp'], enabled: true },
+  },
+}
+
+export default async function (agentName, options = {}) {
+  if (!agentName && !options.yes) {
+    const answer = await inquirer.prompt([{ name: 'agent', type: 'list', message: 'Which coding agent do you use?', choices: Object.keys(agents) }])
+    agentName = answer.agent
+  }
+
+  const agent = agents[agentName]
+  if (!agent) {
+    error(`Agent must be one of: ${Object.keys(agents).join(', ')}`)
+    process.exitCode = 1
+    return
+  }
+
+  let ready = true
+  try {
+    if (agent.mcp) {
+      execSync(agent.mcp, { stdio: 'inherit' })
+    } else {
+      addServerToFile(agent)
+      print(`Added codeceptjs MCP server to ${agent.file}`)
+    }
+  } catch (err) {
+    ready = false
+    error(`Could not register the codeceptjs MCP server: ${err.message}`)
+    if (agent.mcp) print(`Run: ${agent.mcp}`)
+    if (agent.file) print(`Add to "${agent.key}" in ${agent.file}: ${JSON.stringify({ codeceptjs: agent.server })}`)
+  }
+
+  const skills = `npx skills add codeceptjs/skills -a ${agent.skills} -y`
+  try {
+    execSync(skills, { stdio: 'inherit' })
+  } catch (err) {
+    ready = false
+    error(`Could not install skills. Run: ${skills}`)
+  }
+
+  if (ready) success(`Start ${agentName} in this project. CodeceptJS skills and MCP are ready.`)
+}
+
+function addServerToFile(agent) {
+  let config = {}
+  if (fs.existsSync(agent.file)) config = JSON.parse(fs.readFileSync(agent.file, 'utf8'))
+  config[agent.key] = { ...config[agent.key], codeceptjs: agent.server }
+  fs.mkdirSync(path.dirname(agent.file), { recursive: true })
+  fs.writeFileSync(agent.file, `${JSON.stringify(config, null, 2)}\n`)
+}

package/lib/command/run-multiple.js

@@ -4,6 +4,7 @@ import crypto from 'crypto'
 import { fileURLToPath } from 'url'
 
 import runHook from '../hooks.js'
+import WebServer from '../webServer.js'
 import event from '../event.js'
 import { createRuns } from './run-multiple/collection.js'
 import { clearString, replaceValueDeep } from '../utils.js'
@@ -67,6 +68,7 @@ export default async function (selectedRuns, options) {
     fail('No runs provided. Use --all option to run all configured runs')
   }
 
+  await WebServer.startAll(config.webServer)
   await runHook(config.bootstrapAll, 'bootstrapAll')
 
   event.emit(event.multiple.before, null)
@@ -110,6 +112,7 @@ export default async function (selectedRuns, options) {
   return childProcessesPromise.then(async () => {
     // fire hook
     await runHook(config.teardownAll, 'teardownAll')
+    await WebServer.stopAll()
     event.emit(event.multiple.after, null)
   })
 }

package/lib/command/run-workers.js

@@ -5,7 +5,9 @@ import store from '../store.js'
 import event from '../event.js'
 import Workers from '../workers.js'
 import Codecept from '../codecept.js'
+import Config from '../config.js'
 import { getMachineInfo } from './info.js'
+import { getConfig, getTestRoot, createOutputDir, hasLastFailedTests } from './utils.js'
 
 export default async function (workerCount, selectedRuns, options) {
   process.env.profile = options.profile
@@ -36,10 +38,17 @@ export default async function (workerCount, selectedRuns, options) {
     selectedRuns,
   }
 
+  if (options.lastFailed) {
+    await getConfig(testConfig)
+    const outputDir = createOutputDir(Config.append(overrideConfigs), getTestRoot(testConfig))
+    if (!(await hasLastFailedTests(outputDir))) return
+  }
+
   const numberOfWorkers = parseInt(workerCount, 10)
 
   output.print(`CodeceptJS v${Codecept.version()} ${output.standWithUkraine()}`)
   output.print(`Running tests in ${output.styles.bold(numberOfWorkers)} workers...`)
+  if (options.verbose || options.debug) store.debugMode = true
   store.hasWorkers = true
   store.workerMode = true
   process.env.RUNS_WITH_WORKERS = 'true'
@@ -55,7 +64,7 @@ export default async function (workerCount, selectedRuns, options) {
   output.print('Test distribution:')
   workerObjects.forEach((worker, index) => {
     const testCount = worker.tests.length
-    output.print(`  Worker ${index + 1}: ${testCount} test${testCount !== 1 ? 's' : ''}`)
+    output.print(`  Worker ${index + 1}: ${testCount} test${testCount !== 1 ? 's' : ''}${worker.serial ? ' (serial)' : ''}`)
   })
   output.print()
 
@@ -76,8 +85,6 @@ export default async function (workerCount, selectedRuns, options) {
   })
 
   try {
-    if (options.verbose || options.debug) store.debugMode = true
-
     if (options.verbose) {
       await getMachineInfo()
     }

package/lib/command/run.js

@@ -1,4 +1,4 @@
-import { getConfig, printError, getTestRoot, createOutputDir, autoExit } from './utils.js'
+import { getConfig, printError, getTestRoot, createOutputDir, autoExit, hasLastFailedTests } from './utils.js'
 import Config from '../config.js'
 import store from '../store.js'
 import Codecept from '../codecept.js'
@@ -21,7 +21,8 @@ export default async function (test, options) {
     config = Config.append(JSON.parse(options.override))
   }
   const testRoot = getTestRoot(configFile)
-  createOutputDir(config, testRoot)
+  const outputDir = createOutputDir(config, testRoot)
+  if (options.lastFailed && !(await hasLastFailedTests(outputDir))) return
 
   const codecept = new Codecept(config, options)
 

package/lib/command/utils.js

@@ -105,6 +105,21 @@ export const createOutputDir = (config, testRoot) => {
     output.print(`creating output directory: ${outputDir}`)
     mkdirp.sync(outputDir)
   }
+  return outputDir
+}
+
+export async function hasLastFailedTests(outputDir) {
+  const { default: Result } = await import('../result.js')
+  let failedTests
+  try {
+    failedTests = Result.loadFailedTests(outputDir)
+  } catch (err) {
+    output.error(err.message)
+    process.exitCode = 1
+    return false
+  }
+  if (!failedTests.length) output.print('No failed tests in the last run')
+  return failedTests.length > 0
 }
 
 export async function autoExit() {

package/lib/command/workers/runTests.js

@@ -17,7 +17,7 @@ let stdout = ''
 
 const stderr = ''
 
-const { options, tests, testRoot, workerIndex, poolMode } = workerData
+const { options, tests, skipTests = [], testRoot, workerIndex, poolMode } = workerData
 
 // Global error handlers to catch critical errors but not test failures
 process.on('uncaughtException', (err) => {
@@ -287,6 +287,8 @@ async function runPoolTests() {
             // Load only the assigned test file
             mocha.files = [testIdentifier]
             await loadTests(mocha)
+            codecept.filterLastFailedTests(mocha)
+            filterSuites(mocha.suite.suites, test => !skipTests.includes(test.uid))
 
             if (mocha.suite.total() > 0) {
               // Run only the tests in the current mocha suite
@@ -423,16 +425,14 @@ async function filterTests() {
   mocha.files = files
   await loadTests(mocha)
 
-  // Recursively filter tests in all suites (including nested ones)
-  const filterSuiteTests = (suite) => {
-    suite.tests = suite.tests.filter(test => tests.indexOf(test.uid) >= 0)
-    for (const childSuite of suite.suites) {
-      filterSuiteTests(childSuite)
-    }
-  }
+  filterSuites(mocha.suite.suites, test => tests.indexOf(test.uid) >= 0)
+}
 
-  for (const suite of mocha.suite.suites) {
-    filterSuiteTests(suite)
+// Recursively filter tests in all suites (including nested ones)
+function filterSuites(suites, keep) {
+  for (const suite of suites) {
+    suite.tests = suite.tests.filter(keep)
+    filterSuites(suite.suites, keep)
   }
 }
 

package/lib/container.js

@@ -778,8 +778,8 @@ async function loadGherkinStepsAsync(paths) {
   // Import BDD module to access step file tracking functions and step DSL
   const bddModule = await import('./mocha/bdd.js')
 
-  global.Before = fn => event.dispatcher.on(event.test.started, fn)
-  global.After = fn => event.dispatcher.on(event.test.finished, fn)
+  global.Before = global.GherkinBefore = fn => event.dispatcher.on(event.test.started, fn)
+  global.After = global.GherkinAfter = fn => event.dispatcher.on(event.test.finished, fn)
   global.Fail = fn => event.dispatcher.on(event.test.failed, fn)
 
   // Scope-inject Given/When/Then/And while loading step files so they work
@@ -818,7 +818,9 @@ async function loadGherkinStepsAsync(paths) {
   }
 
   delete global.Before
+  delete global.GherkinBefore
   delete global.After
+  delete global.GherkinAfter
   delete global.Fail
   if (injectStepDsl) {
     delete global.Given