codeceptjs 4.0.2-beta.9 → 4.0.2

package/lib/mocha/factory.js

@@ -7,6 +7,8 @@ import gherkinParser, { loadTranslations } from './gherkin.js'
 import output from '../output.js'
 import scenarioUiFunction from './ui.js'
 import { initMochaGlobals } from '../globals.js'
+import { fixErrorStack } from '../utils/typescript.js'
+import container from '../container.js'
 
 const __filename = fileURLToPath(import.meta.url)
 const __dirname = fsPath.dirname(__filename)
@@ -15,7 +17,11 @@ let mocha
 
 class MochaFactory {
   static create(config, opts) {
-    mocha = new Mocha(Object.assign(config, opts))
+    const merged = Object.assign({}, config, opts)
+    mocha = new Mocha(merged)
+    if (merged.cleanReferencesAfterRun !== true) {
+      mocha.cleanReferencesAfterRun(false)
+    }
     output.process(opts.child)
     mocha.ui(scenarioUiFunction)
 
@@ -34,6 +40,10 @@ class MochaFactory {
           // Handle ECONNREFUSED without dynamic import for now
           err = new Error('Connection refused: ' + err.toString())
         }
+        const fileMapping = container?.tsFileMapping?.()
+        if (fileMapping) {
+          fixErrorStack(err, fileMapping)
+        }
         output.error(err)
         output.print(err.stack)
         process.exit(1)

package/lib/mocha/inject.js

@@ -5,7 +5,7 @@ const getInjectedArguments = async (fn, test, suite) => {
   const container = containerModule.default || containerModule
 
   const testArgs = {}
-  const params = getParams(fn) || []
+  const params = getParams(fn, { warnOnLegacyFormat: true }) || []
   const objects = container.support()
 
   for (const key of params) {

package/lib/mocha/scenarioConfig.js

@@ -1,4 +1,5 @@
 import { isAsyncFunction } from '../utils.js'
+import store from '../store.js'
 
 /** @class */
 class ScenarioConfig {
@@ -40,7 +41,7 @@ class ScenarioConfig {
    * @returns {this}
    */
   retry(retries) {
-    if (process.env.SCENARIO_ONLY) retries = -retries
+    if (store.scenarioOnly) retries = -retries
     this.test.retries(retries)
     return this
   }

package/lib/mocha/ui.js

@@ -9,13 +9,12 @@ import { HookConfig, AfterSuiteHook, AfterHook, BeforeSuiteHook, BeforeHook } fr
 import { initMochaGlobals } from '../globals.js'
 import common from 'mocha/lib/interfaces/common.js'
 import container from '../container.js'
+import store from '../store.js'
 
 const setContextTranslation = context => {
-  // Try global container first, then local container instance
-  const containerToUse = global.container || container
-  if (!containerToUse) return
+  if (!container) return
 
-  const translation = containerToUse.translation?.() || containerToUse.translation
+  const translation = container.translation?.() || container.translation
   const contexts = translation?.value?.('contexts')
 
   if (contexts) {
@@ -119,7 +118,7 @@ export default function (suite) {
     context.Feature.only = function (title, opts) {
       const reString = `^${escapeRe(`${title}:`)}`
       mocha.grep(new RegExp(reString))
-      process.env.FEATURE_ONLY = true
+      store.featureOnly = true
       return context.Feature(title, opts)
     }
 
@@ -171,7 +170,7 @@ export default function (suite) {
     context.Scenario.only = function (title, opts, fn) {
       const reString = `^${escapeRe(`${suites[0].title}: ${title}`.replace(/( \| {.+})?$/g, ''))}`
       mocha.grep(new RegExp(reString))
-      process.env.SCENARIO_ONLY = true
+      store.scenarioOnly = true
       return addScenario(title, opts, fn)
     }
 

package/lib/parser.js

@@ -14,11 +14,11 @@ export const getParamsToString = function (fn) {
   return getParams(newFn).join(', ')
 }
 
-function getParams(fn) {
+function getParams(fn, { warnOnLegacyFormat = false } = {}) {
   if (fn.isSinonProxy) return []
   try {
     const reflected = parser.parse(fn)
-    if (reflected.args.length > 1 || reflected.args[0] === 'I') {
+    if (warnOnLegacyFormat && (reflected.args.length > 1 || reflected.args[0] === 'I')) {
       output.error('Error: old CodeceptJS v2 format detected. Upgrade your project to the new format -> https://bit.ly/codecept3Up')
     }
     if (reflected.destructuredArgs.length > 0) reflected.args = [...reflected.destructuredArgs]

package/lib/pause.js

@@ -18,6 +18,8 @@ let nextStep
 let finish
 let next
 let registeredVariables = {}
+let externalHandler = null
+
 /**
  * Pauses test execution and starts interactive shell
  * @param {Object<string, *>} [passedObject]
@@ -37,10 +39,10 @@ const pause = function (passedObject = {}) {
   })
 
   event.dispatcher.on(event.test.finished, () => {
-    finish()
+    if (typeof finish === 'function') finish()
     recorder.session.restore('pause')
-    rl.close()
-    history.save()
+    if (rl) rl.close()
+    if (!externalHandler) history.save()
   })
 
   recorder.add('Start new session', () => pauseSession(passedObject))
@@ -49,6 +51,15 @@ const pause = function (passedObject = {}) {
 function pauseSession(passedObject = {}) {
   registeredVariables = passedObject
   recorder.session.start('pause')
+
+  if (externalHandler) {
+    store.onPause = true
+    return externalHandler({ registeredVariables }).then(() => {
+      store.onPause = false
+      recorder.session.restore('pause')
+    })
+  }
+
   if (!next) {
     let vars = Object.keys(registeredVariables).join(', ')
     if (vars) vars = `(vars: ${vars})`
@@ -234,5 +245,28 @@ function registerVariable(name, value) {
   registeredVariables[name] = value
 }
 
+/**
+ * Hook for external pause drivers (e.g. the MCP server). When set, pauseSession
+ * delegates to the handler instead of opening a readline REPL. The handler
+ * receives `{ registeredVariables }` and returns a Promise that resolves when
+ * the driver decides to continue (resume) or step.
+ *
+ * The driver controls step-vs-resume by mutating `next` via setNextStep before
+ * resolving its Promise.
+ */
+function setPauseHandler(handler) {
+  externalHandler = handler
+}
+
+/**
+ * Trigger a one-shot pause from outside the test (e.g. the MCP server,
+ * pausing the test at a specific step index without modifying the test).
+ * Schedules pauseSession through the recorder so it slots between steps.
+ */
+function pauseNow(passedObject = {}) {
+  if (store.dryRun) return
+  recorder.add('Triggered pause', () => pauseSession(passedObject))
+}
+
 export default pause
-export { registerVariable }
+export { registerVariable, setPauseHandler, pauseNow }

package/lib/plugin/aiTrace.js

@@ -0,0 +1,457 @@
+import fs from 'fs'
+import { mkdirp } from 'mkdirp'
+import path from 'path'
+
+import store from '../store.js'
+import Container from '../container.js'
+import recorder from '../recorder.js'
+import event from '../event.js'
+import output from '../output.js'
+import { deleteDir, clearString } from '../utils.js'
+import { captureSnapshot, pickActingHelper, traceDirFor, artifactLinks } from '../utils/trace.js'
+import {
+  parsePluginArgs,
+  resolveTrigger,
+  matchStepFile,
+  matchUrl,
+} from '../utils/pluginParser.js'
+
+const defaultConfig = {
+  on: 'step',
+  deleteSuccessful: false,
+  fullPageScreenshots: false,
+  output: store.outputDir,
+  captureHTML: true,
+  captureARIA: true,
+  captureBrowserLogs: true,
+  captureHTTP: true,
+  captureDebugOutput: true,
+  ignoreSteps: [],
+}
+
+/**
+ *
+ * Generates AI-friendly trace files for debugging with AI agents.
+ * This plugin creates a markdown file with test execution logs and links to all artifacts
+ * (screenshots, HTML, ARIA snapshots, browser logs, HTTP requests) for each step.
+ *
+ * #### Configuration
+ *
+ * ```js
+ * "plugins": {
+ *    "aiTrace": {
+ *      "enabled": true
+ *    }
+ *  }
+ * ```
+ *
+ * Possible config options:
+ *
+ * * `deleteSuccessful`: delete traces for successfully executed tests. Default: false.
+ * * `fullPageScreenshots`: should full page screenshots be used. Default: false.
+ * * `output`: a directory where traces should be stored. Default: `output`.
+ * * `captureHTML`: capture HTML for each step. Default: true.
+ * * `captureARIA`: capture ARIA snapshot for each step. Default: true.
+ * * `captureBrowserLogs`: capture browser console logs. Default: true.
+ * * `captureHTTP`: capture HTTP requests (requires `trace` or `recordHar` enabled in helper config). Default: true.
+ * * `captureDebugOutput`: capture CodeceptJS debug output. Default: true.
+ * * `ignoreSteps`: steps to ignore in trace. Array of RegExps is expected.
+ * * `on`: trigger mode — `step` (default), `fail`, `test`, `file`, `url`.
+ *
+ * #### `on=` modes
+ *
+ * * **step** — persist every step (default)
+ * * **fail** — persist only the failed step
+ * * **test** — persist only the last step of each test
+ * * **file** — persist steps from `path=...[;line=...]`
+ * * **url** — persist when the current URL matches `pattern=...`
+ *
+ * @param {*} config
+ */
+export default function (config = {}) {
+  const cliArgs = parsePluginArgs(config._args)
+  const trigger = resolveTrigger(cliArgs, config, { on: defaultConfig.on }, { name: 'aiTrace' })
+  if (!trigger) return
+
+  config = Object.assign(defaultConfig, config)
+
+  const helper = pickActingHelper(Container.helpers())
+
+  if (!helper) {
+    output.warn('aiTrace plugin: No supported helper found (Playwright, Puppeteer, WebDriver). Plugin disabled.')
+    return
+  }
+
+  let dir
+  let stepNum
+  let steps = []
+  let debugOutput = []
+  let error
+  let savedSteps = new Set()
+  let currentTest = null
+  let testStartTime
+  let currentUrl = null
+  let testFailed = false
+  let pendingArtifactCapture = null
+  let firstFailedStepSaved = false
+
+  const reportDir = config.output ? path.resolve(store.codeceptDir, config.output) : defaultConfig.output
+
+  if (config.captureDebugOutput) {
+    const originalDebug = output.debug
+    output.debug = function (...args) {
+      debugOutput.push(args.join(' '))
+      originalDebug.apply(output, args)
+    }
+  }
+
+  event.dispatcher.on(event.suite.before, suite => {
+    stepNum = -1
+  })
+
+  event.dispatcher.on(event.test.before, test => {
+    let title
+    try {
+      title = test.fullTitle ? test.fullTitle() : test.title
+    } catch (err) {
+      title = test.title
+    }
+    dir = traceDirFor(test.file, title, reportDir)
+    mkdirp.sync(dir)
+    deleteDir(dir)
+    mkdirp.sync(dir)
+    stepNum = 0
+    error = null
+    steps = []
+    debugOutput = []
+    savedSteps.clear()
+    currentTest = test
+    testStartTime = Date.now()
+    currentUrl = null
+    testFailed = false
+    firstFailedStepSaved = false
+    pendingArtifactCapture = null
+  })
+
+  event.dispatcher.on(event.step.after, step => {
+    if (!currentTest) return
+    if (step.status === 'failed') {
+      testFailed = true
+    }
+    if (step.status === 'queued' && testFailed) {
+      output.debug(`aiTrace: Skipping queued step "${step.toString()}" - testFailed: ${testFailed}`)
+      return
+    }
+    if (step.status === 'failed' && firstFailedStepSaved) {
+      output.debug(`aiTrace: Skipping failed step "${step.toString()}" - already handled by step.failed event`)
+      return
+    }
+
+    // on= filtering
+    if (trigger.on === 'fail') return // failed steps handled by step.failed
+    if (trigger.on === 'file' && !matchStepFile(step, trigger.path, trigger.line)) return
+    if (trigger.on === 'url') {
+      recorder.add('aiTrace:url check', async () => {
+        try {
+          if (!helper.grabCurrentUrl) return
+          const url = await helper.grabCurrentUrl()
+          if (!matchUrl(url, trigger.pattern)) return
+          await persistStep(step)
+        } catch (err) {
+          output.debug(`aiTrace: Error in url-mode step persistence: ${err.message}`)
+        }
+      }, true)
+      return
+    }
+
+    recorder.add(`aiTrace step persistence: ${step.toString()}`, () => persistStep(step).catch(err => {
+      output.debug(`aiTrace: Error saving step: ${err.message}`)
+    }), true)
+  })
+
+  event.dispatcher.on(event.step.failed, step => {
+    if (!currentTest) return
+    if (step.status === 'queued' && testFailed) {
+      output.debug(`aiTrace: Skipping queued failed step "${step.toString()}" - testFailed: ${testFailed}`)
+      return
+    }
+    if (firstFailedStepSaved) {
+      output.debug(`aiTrace: Skipping subsequent failed step "${step.toString()}" - already saved first failed step`)
+      return
+    }
+
+    const stepKey = step.toString()
+    if (savedSteps.has(stepKey)) {
+      const existingStep = steps.find(s => s.step === stepKey)
+      if (!existingStep) {
+        output.debug(`aiTrace: Step "${stepKey}" marked as saved but not found in steps array`)
+        return
+      }
+      existingStep.status = 'failed'
+
+      pendingArtifactCapture = captureArtifactsForStep(step, existingStep, existingStep.prefix).catch(err => {
+        output.debug(`aiTrace: Error updating failed step: ${err.message}`)
+      })
+    } else {
+      if (stepNum === -1) return
+      if (isStepIgnored(step)) return
+      if (step.metaStep && step.metaStep.title === 'BeforeSuite') return
+
+      const stepPrefix = generateStepPrefix(step, stepNum)
+      stepNum++
+
+      const stepData = {
+        step: stepKey,
+        status: 'failed',
+        prefix: stepPrefix,
+        artifacts: {},
+        meta: {},
+        debugOutput: [],
+      }
+
+      if (step.startTime && step.endTime) {
+        stepData.meta.duration = ((step.endTime - step.startTime) / 1000).toFixed(2) + 's'
+      }
+
+      savedSteps.add(stepKey)
+      steps.push(stepData)
+      firstFailedStepSaved = true
+
+      pendingArtifactCapture = captureArtifactsForStep(step, stepData, stepPrefix).catch(err => {
+        output.debug(`aiTrace: Error capturing failed step artifacts: ${err.message}`)
+      })
+    }
+  })
+
+  event.dispatcher.on(event.test.passed, test => {
+    if (config.deleteSuccessful) {
+      deleteDir(dir)
+      return
+    }
+    persist(test, 'passed')
+  })
+
+  event.dispatcher.on(event.test.failed, (test, _err, hookName) => {
+    if (hookName === 'BeforeSuite' || hookName === 'AfterSuite') {
+      return
+    }
+    recorder.add('aiTrace:persist failed', async () => {
+      if (pendingArtifactCapture) {
+        await pendingArtifactCapture
+        pendingArtifactCapture = null
+      }
+      persist(test, 'failed')
+    }, true)
+  })
+
+  async function persistStep(step) {
+    if (stepNum === -1) return
+    if (isStepIgnored(step)) return
+    if (step.metaStep && step.metaStep.title === 'BeforeSuite') return
+
+    const stepKey = step.toString()
+
+    if (savedSteps.has(stepKey)) {
+      const existingStep = steps.find(s => s.step === stepKey)
+      if (existingStep && step.status === 'failed') {
+        existingStep.status = 'failed'
+        step.artifacts = {}
+        await captureArtifactsForStep(step, existingStep, existingStep.prefix)
+      }
+      return
+    }
+    savedSteps.add(stepKey)
+
+    const stepPrefix = generateStepPrefix(step, stepNum)
+    stepNum++
+
+    const stepData = {
+      step: step.toString(),
+      status: step.status,
+      prefix: stepPrefix,
+      artifacts: {},
+      meta: {},
+      debugOutput: [],
+    }
+
+    if (step.startTime && step.endTime) {
+      stepData.meta.duration = ((step.endTime - step.startTime) / 1000).toFixed(2) + 's'
+    }
+
+    if (config.captureDebugOutput && debugOutput.length > 0) {
+      stepData.debugOutput = [...debugOutput]
+      debugOutput = []
+    }
+
+    await captureArtifactsForStep(step, stepData, stepPrefix)
+    steps.push(stepData)
+  }
+
+  async function captureArtifactsForStep(step, stepData, stepPrefix) {
+    if (!step.artifacts) {
+      step.artifacts = {}
+    }
+
+    let browserAvailable = true
+
+    try {
+      try {
+        if (helper.grabCurrentUrl) {
+          const url = await helper.grabCurrentUrl()
+          stepData.meta.url = url
+          currentUrl = url
+        }
+      } catch (err) {
+        browserAvailable = false
+        output.debug(`aiTrace: Browser unavailable, partial artifact capture: ${err.message}`)
+      }
+
+      let preExistingScreenshot = false
+      if (step.artifacts?.screenshot) {
+        const screenshotPath = path.isAbsolute(step.artifacts.screenshot)
+          ? step.artifacts.screenshot
+          : path.resolve(dir, step.artifacts.screenshot)
+        const screenshotFile = path.basename(screenshotPath)
+        stepData.artifacts.screenshot = screenshotFile
+        step.artifacts.screenshot = screenshotPath
+        preExistingScreenshot = true
+
+        if (!fs.existsSync(screenshotPath)) {
+          try {
+            await helper.saveScreenshot(screenshotPath, config.fullPageScreenshots)
+          } catch (err) {
+            output.debug(`aiTrace: Could not save screenshot: ${err.message}`)
+          }
+        }
+      }
+
+      const captured = await captureSnapshot(helper, {
+        dir,
+        prefix: stepPrefix,
+        fullPage: config.fullPageScreenshots,
+        captureHTML: config.captureHTML && browserAvailable,
+        captureARIA: config.captureARIA && browserAvailable,
+        captureBrowserLogs: config.captureBrowserLogs && browserAvailable,
+        captureStorage: false,
+      })
+
+      if (!preExistingScreenshot && captured.screenshot) {
+        stepData.artifacts.screenshot = captured.screenshot
+        step.artifacts.screenshot = path.join(dir, captured.screenshot)
+      }
+      if (step.artifacts?.html) {
+        stepData.artifacts.html = step.artifacts.html
+      } else if (captured.html) {
+        stepData.artifacts.html = captured.html
+      }
+      if (captured.aria) stepData.artifacts.aria = captured.aria
+      if (captured.console) {
+        stepData.artifacts.console = captured.console
+        stepData.meta.consoleCount = captured.consoleCount
+      }
+    } catch (err) {
+      output.plugin(`aiTrace: Can't save step artifacts: ${err}`)
+    }
+  }
+
+  function persist(test, status) {
+    if (!steps.length) {
+      output.debug('aiTrace: No steps to save in trace')
+      return
+    }
+
+    // on=test: only render the last step in markdown; artifacts of earlier steps
+    // remain on disk unreferenced.
+    if (trigger.on === 'test') {
+      steps = steps.slice(-1)
+    }
+
+    const testDuration = ((Date.now() - testStartTime) / 1000).toFixed(2)
+
+    let markdown = `file: ${test.file || 'unknown'}\n`
+    markdown += `name: ${test.title}\n`
+    markdown += `time: ${testDuration}s\n`
+    markdown += `---\n\n`
+
+    if (status === 'failed') {
+      if (test.art && test.art.message) {
+        markdown += `Error: ${test.art.message}\n\n`
+      }
+      if (test.art && test.art.stack) {
+        markdown += `${test.art.stack}\n\n`
+      }
+      markdown += `---\n\n`
+    }
+
+    if (config.captureDebugOutput && debugOutput.length > 0) {
+      markdown += `CodeceptJS Debug Output:\n\n`
+      debugOutput.forEach(line => {
+        markdown += `> ${line}\n`
+      })
+      markdown += `\n---\n\n`
+    }
+
+    steps.forEach((stepData, index) => {
+      const stepAnchor = clearString(stepData.step).replace(/[^a-zA-Z0-9_-]/g, '_').slice(0, 50)
+      markdown += `### Step ${index + 1}: ${stepData.step}\n`
+      markdown += `<a id="${stepAnchor}"></a>\n`
+
+      if (stepData.meta.duration) {
+        markdown += `  > duration: ${stepData.meta.duration}\n`
+      }
+
+      if (stepData.meta.url) {
+        markdown += `  > navigated to ${stepData.meta.url}\n`
+      }
+
+      if (config.captureDebugOutput && stepData.debugOutput && stepData.debugOutput.length > 0) {
+        stepData.debugOutput.forEach(line => {
+          markdown += `  > ${line}\n`
+        })
+      }
+
+      const links = artifactLinks(stepData.artifacts, { consoleCount: stepData.meta.consoleCount })
+      if (links) markdown += links + '\n'
+
+      if (config.captureHTTP) {
+        if (test.artifacts && test.artifacts.har) {
+          const harPath = path.relative(reportDir, test.artifacts.har)
+          markdown += `  > HTTP: see [HAR file](../${harPath}) for network requests\n`
+        } else if (test.artifacts && test.artifacts.trace) {
+          const tracePath = path.relative(reportDir, test.artifacts.trace)
+          markdown += `  > HTTP: see [Playwright trace](../${tracePath}) for network requests\n`
+        }
+      }
+
+      markdown += `\n`
+    })
+
+    const traceFile = path.join(dir, 'trace.md')
+    fs.writeFileSync(traceFile, markdown)
+
+    output.print(`Trace Saved: file://${traceFile}`)
+
+    if (!test.artifacts) test.artifacts = {}
+    test.artifacts.aiTrace = traceFile
+  }
+
+  function isStepIgnored(step) {
+    if (!config.ignoreSteps) return false
+    if (!step.title) return false
+    for (const pattern of config.ignoreSteps || []) {
+      if (step.title.match(pattern)) return true
+    }
+    return false
+  }
+
+  function generateStepPrefix(step, index) {
+    const stepName = step.toString()
+    const cleanedName = clearString(stepName)
+      .replace(/[^a-zA-Z0-9_-]/g, '_')
+      .replace(/_{2,}/g, '_')
+      .slice(0, 80)
+      .trim()
+
+    return `${String(index).padStart(4, '0')}_${cleanedName}`
+  }
+}

package/lib/plugin/analyze.js

@@ -12,6 +12,7 @@ const ai = aiModule.default || aiModule
 import colors from 'chalk'
 import ora from 'ora'
 import event from '../event.js'
+import recorder from '../recorder.js'
 
 import output from '../output.js'
 
@@ -154,10 +155,9 @@ const defaultConfig = {
       if (config.vision && test.artifacts.screenshot) {
         debug('Adding screenshot to prompt')
         messages[0].content.push({
-          type: 'image_url',
-          image_url: {
-            url: 'data:image/png;base64,' + base64EncodeFile(test.artifacts.screenshot),
-          },
+          type: 'image',
+          image: base64EncodeFile(test.artifacts.screenshot),
+          mediaType: 'image/png',
         })
       }
 
@@ -227,14 +227,14 @@ export default function (config = {}) {
     console.log('Enabled AI analysis')
   })
 
-  event.dispatcher.on(event.all.result, async result => {
+  event.dispatcher.on(event.all.result, result => {
     if (!isMainThread) return // run only on main thread
     if (!ai.isEnabled) {
       console.log('AI is disabled, no analysis will be performed. Run tests with --ai flag to enable it.')
       return
     }
 
-    printReport(result)
+    recorder.add('analyze:print-ai-report', () => printReport(result), true)
   })
 
   event.dispatcher.on(event.workers.result, async result => {
@@ -248,7 +248,7 @@ export default function (config = {}) {
       return
     }
 
-    printReport(result)
+    await printReport(result)
   })
 
   async function printReport(result) {
@@ -294,7 +294,7 @@ export default function (config = {}) {
       console.error('Error analyzing failed tests', err)
     }
 
-    if (!Object.keys(container.plugins()).includes('pageInfo')) {
+    if (!Object.keys(Container.plugins()).includes('pageInfo')) {
       console.log('To improve analysis, enable pageInfo plugin to get more context for failed tests.')
     }
   }
@@ -353,7 +353,7 @@ function serializeError(error) {
     errorMessage +=
       '\n' +
       error.stack
-        .replace(global.codecept_dir || '', '.')
+        .replace(store.codeceptDir || '', '.')
         .split('\n')
         .map(line => line.replace(ansiRegExp(), ''))
         .slice(0, 5)