codeceptjs 4.0.1-beta.9 → 4.0.1

package/lib/plugin/aiTrace.js

@@ -0,0 +1,457 @@
+import fs from 'fs'
+import { mkdirp } from 'mkdirp'
+import path from 'path'
+
+import store from '../store.js'
+import Container from '../container.js'
+import recorder from '../recorder.js'
+import event from '../event.js'
+import output from '../output.js'
+import { deleteDir, clearString } from '../utils.js'
+import { captureSnapshot, pickActingHelper, traceDirFor, artifactLinks } from '../utils/trace.js'
+import {
+  parsePluginArgs,
+  resolveTrigger,
+  matchStepFile,
+  matchUrl,
+} from '../utils/pluginParser.js'
+
+const defaultConfig = {
+  on: 'step',
+  deleteSuccessful: false,
+  fullPageScreenshots: false,
+  output: store.outputDir,
+  captureHTML: true,
+  captureARIA: true,
+  captureBrowserLogs: true,
+  captureHTTP: true,
+  captureDebugOutput: true,
+  ignoreSteps: [],
+}
+
+/**
+ *
+ * Generates AI-friendly trace files for debugging with AI agents.
+ * This plugin creates a markdown file with test execution logs and links to all artifacts
+ * (screenshots, HTML, ARIA snapshots, browser logs, HTTP requests) for each step.
+ *
+ * #### Configuration
+ *
+ * ```js
+ * "plugins": {
+ *    "aiTrace": {
+ *      "enabled": true
+ *    }
+ *  }
+ * ```
+ *
+ * Possible config options:
+ *
+ * * `deleteSuccessful`: delete traces for successfully executed tests. Default: false.
+ * * `fullPageScreenshots`: should full page screenshots be used. Default: false.
+ * * `output`: a directory where traces should be stored. Default: `output`.
+ * * `captureHTML`: capture HTML for each step. Default: true.
+ * * `captureARIA`: capture ARIA snapshot for each step. Default: true.
+ * * `captureBrowserLogs`: capture browser console logs. Default: true.
+ * * `captureHTTP`: capture HTTP requests (requires `trace` or `recordHar` enabled in helper config). Default: true.
+ * * `captureDebugOutput`: capture CodeceptJS debug output. Default: true.
+ * * `ignoreSteps`: steps to ignore in trace. Array of RegExps is expected.
+ * * `on`: trigger mode — `step` (default), `fail`, `test`, `file`, `url`.
+ *
+ * #### `on=` modes
+ *
+ * * **step** — persist every step (default)
+ * * **fail** — persist only the failed step
+ * * **test** — persist only the last step of each test
+ * * **file** — persist steps from `path=...[;line=...]`
+ * * **url** — persist when the current URL matches `pattern=...`
+ *
+ * @param {*} config
+ */
+export default function (config = {}) {
+  const cliArgs = parsePluginArgs(config._args)
+  const trigger = resolveTrigger(cliArgs, config, { on: defaultConfig.on }, { name: 'aiTrace' })
+  if (!trigger) return
+
+  config = Object.assign(defaultConfig, config)
+
+  const helper = pickActingHelper(Container.helpers())
+
+  if (!helper) {
+    output.warn('aiTrace plugin: No supported helper found (Playwright, Puppeteer, WebDriver). Plugin disabled.')
+    return
+  }
+
+  let dir
+  let stepNum
+  let steps = []
+  let debugOutput = []
+  let error
+  let savedSteps = new Set()
+  let currentTest = null
+  let testStartTime
+  let currentUrl = null
+  let testFailed = false
+  let pendingArtifactCapture = null
+  let firstFailedStepSaved = false
+
+  const reportDir = config.output ? path.resolve(store.codeceptDir, config.output) : defaultConfig.output
+
+  if (config.captureDebugOutput) {
+    const originalDebug = output.debug
+    output.debug = function (...args) {
+      debugOutput.push(args.join(' '))
+      originalDebug.apply(output, args)
+    }
+  }
+
+  event.dispatcher.on(event.suite.before, suite => {
+    stepNum = -1
+  })
+
+  event.dispatcher.on(event.test.before, test => {
+    let title
+    try {
+      title = test.fullTitle ? test.fullTitle() : test.title
+    } catch (err) {
+      title = test.title
+    }
+    dir = traceDirFor(test.file, title, reportDir)
+    mkdirp.sync(dir)
+    deleteDir(dir)
+    mkdirp.sync(dir)
+    stepNum = 0
+    error = null
+    steps = []
+    debugOutput = []
+    savedSteps.clear()
+    currentTest = test
+    testStartTime = Date.now()
+    currentUrl = null
+    testFailed = false
+    firstFailedStepSaved = false
+    pendingArtifactCapture = null
+  })
+
+  event.dispatcher.on(event.step.after, step => {
+    if (!currentTest) return
+    if (step.status === 'failed') {
+      testFailed = true
+    }
+    if (step.status === 'queued' && testFailed) {
+      output.debug(`aiTrace: Skipping queued step "${step.toString()}" - testFailed: ${testFailed}`)
+      return
+    }
+    if (step.status === 'failed' && firstFailedStepSaved) {
+      output.debug(`aiTrace: Skipping failed step "${step.toString()}" - already handled by step.failed event`)
+      return
+    }
+
+    // on= filtering
+    if (trigger.on === 'fail') return // failed steps handled by step.failed
+    if (trigger.on === 'file' && !matchStepFile(step, trigger.path, trigger.line)) return
+    if (trigger.on === 'url') {
+      recorder.add('aiTrace:url check', async () => {
+        try {
+          if (!helper.grabCurrentUrl) return
+          const url = await helper.grabCurrentUrl()
+          if (!matchUrl(url, trigger.pattern)) return
+          await persistStep(step)
+        } catch (err) {
+          output.debug(`aiTrace: Error in url-mode step persistence: ${err.message}`)
+        }
+      }, true)
+      return
+    }
+
+    recorder.add(`aiTrace step persistence: ${step.toString()}`, () => persistStep(step).catch(err => {
+      output.debug(`aiTrace: Error saving step: ${err.message}`)
+    }), true)
+  })
+
+  event.dispatcher.on(event.step.failed, step => {
+    if (!currentTest) return
+    if (step.status === 'queued' && testFailed) {
+      output.debug(`aiTrace: Skipping queued failed step "${step.toString()}" - testFailed: ${testFailed}`)
+      return
+    }
+    if (firstFailedStepSaved) {
+      output.debug(`aiTrace: Skipping subsequent failed step "${step.toString()}" - already saved first failed step`)
+      return
+    }
+
+    const stepKey = step.toString()
+    if (savedSteps.has(stepKey)) {
+      const existingStep = steps.find(s => s.step === stepKey)
+      if (!existingStep) {
+        output.debug(`aiTrace: Step "${stepKey}" marked as saved but not found in steps array`)
+        return
+      }
+      existingStep.status = 'failed'
+
+      pendingArtifactCapture = captureArtifactsForStep(step, existingStep, existingStep.prefix).catch(err => {
+        output.debug(`aiTrace: Error updating failed step: ${err.message}`)
+      })
+    } else {
+      if (stepNum === -1) return
+      if (isStepIgnored(step)) return
+      if (step.metaStep && step.metaStep.title === 'BeforeSuite') return
+
+      const stepPrefix = generateStepPrefix(step, stepNum)
+      stepNum++
+
+      const stepData = {
+        step: stepKey,
+        status: 'failed',
+        prefix: stepPrefix,
+        artifacts: {},
+        meta: {},
+        debugOutput: [],
+      }
+
+      if (step.startTime && step.endTime) {
+        stepData.meta.duration = ((step.endTime - step.startTime) / 1000).toFixed(2) + 's'
+      }
+
+      savedSteps.add(stepKey)
+      steps.push(stepData)
+      firstFailedStepSaved = true
+
+      pendingArtifactCapture = captureArtifactsForStep(step, stepData, stepPrefix).catch(err => {
+        output.debug(`aiTrace: Error capturing failed step artifacts: ${err.message}`)
+      })
+    }
+  })
+
+  event.dispatcher.on(event.test.passed, test => {
+    if (config.deleteSuccessful) {
+      deleteDir(dir)
+      return
+    }
+    persist(test, 'passed')
+  })
+
+  event.dispatcher.on(event.test.failed, (test, _err, hookName) => {
+    if (hookName === 'BeforeSuite' || hookName === 'AfterSuite') {
+      return
+    }
+    recorder.add('aiTrace:persist failed', async () => {
+      if (pendingArtifactCapture) {
+        await pendingArtifactCapture
+        pendingArtifactCapture = null
+      }
+      persist(test, 'failed')
+    }, true)
+  })
+
+  async function persistStep(step) {
+    if (stepNum === -1) return
+    if (isStepIgnored(step)) return
+    if (step.metaStep && step.metaStep.title === 'BeforeSuite') return
+
+    const stepKey = step.toString()
+
+    if (savedSteps.has(stepKey)) {
+      const existingStep = steps.find(s => s.step === stepKey)
+      if (existingStep && step.status === 'failed') {
+        existingStep.status = 'failed'
+        step.artifacts = {}
+        await captureArtifactsForStep(step, existingStep, existingStep.prefix)
+      }
+      return
+    }
+    savedSteps.add(stepKey)
+
+    const stepPrefix = generateStepPrefix(step, stepNum)
+    stepNum++
+
+    const stepData = {
+      step: step.toString(),
+      status: step.status,
+      prefix: stepPrefix,
+      artifacts: {},
+      meta: {},
+      debugOutput: [],
+    }
+
+    if (step.startTime && step.endTime) {
+      stepData.meta.duration = ((step.endTime - step.startTime) / 1000).toFixed(2) + 's'
+    }
+
+    if (config.captureDebugOutput && debugOutput.length > 0) {
+      stepData.debugOutput = [...debugOutput]
+      debugOutput = []
+    }
+
+    await captureArtifactsForStep(step, stepData, stepPrefix)
+    steps.push(stepData)
+  }
+
+  async function captureArtifactsForStep(step, stepData, stepPrefix) {
+    if (!step.artifacts) {
+      step.artifacts = {}
+    }
+
+    let browserAvailable = true
+
+    try {
+      try {
+        if (helper.grabCurrentUrl) {
+          const url = await helper.grabCurrentUrl()
+          stepData.meta.url = url
+          currentUrl = url
+        }
+      } catch (err) {
+        browserAvailable = false
+        output.debug(`aiTrace: Browser unavailable, partial artifact capture: ${err.message}`)
+      }
+
+      let preExistingScreenshot = false
+      if (step.artifacts?.screenshot) {
+        const screenshotPath = path.isAbsolute(step.artifacts.screenshot)
+          ? step.artifacts.screenshot
+          : path.resolve(dir, step.artifacts.screenshot)
+        const screenshotFile = path.basename(screenshotPath)
+        stepData.artifacts.screenshot = screenshotFile
+        step.artifacts.screenshot = screenshotPath
+        preExistingScreenshot = true
+
+        if (!fs.existsSync(screenshotPath)) {
+          try {
+            await helper.saveScreenshot(screenshotPath, config.fullPageScreenshots)
+          } catch (err) {
+            output.debug(`aiTrace: Could not save screenshot: ${err.message}`)
+          }
+        }
+      }
+
+      const captured = await captureSnapshot(helper, {
+        dir,
+        prefix: stepPrefix,
+        fullPage: config.fullPageScreenshots,
+        captureHTML: config.captureHTML && browserAvailable,
+        captureARIA: config.captureARIA && browserAvailable,
+        captureBrowserLogs: config.captureBrowserLogs && browserAvailable,
+        captureStorage: false,
+      })
+
+      if (!preExistingScreenshot && captured.screenshot) {
+        stepData.artifacts.screenshot = captured.screenshot
+        step.artifacts.screenshot = path.join(dir, captured.screenshot)
+      }
+      if (step.artifacts?.html) {
+        stepData.artifacts.html = step.artifacts.html
+      } else if (captured.html) {
+        stepData.artifacts.html = captured.html
+      }
+      if (captured.aria) stepData.artifacts.aria = captured.aria
+      if (captured.console) {
+        stepData.artifacts.console = captured.console
+        stepData.meta.consoleCount = captured.consoleCount
+      }
+    } catch (err) {
+      output.plugin(`aiTrace: Can't save step artifacts: ${err}`)
+    }
+  }
+
+  function persist(test, status) {
+    if (!steps.length) {
+      output.debug('aiTrace: No steps to save in trace')
+      return
+    }
+
+    // on=test: only render the last step in markdown; artifacts of earlier steps
+    // remain on disk unreferenced.
+    if (trigger.on === 'test') {
+      steps = steps.slice(-1)
+    }
+
+    const testDuration = ((Date.now() - testStartTime) / 1000).toFixed(2)
+
+    let markdown = `file: ${test.file || 'unknown'}\n`
+    markdown += `name: ${test.title}\n`
+    markdown += `time: ${testDuration}s\n`
+    markdown += `---\n\n`
+
+    if (status === 'failed') {
+      if (test.art && test.art.message) {
+        markdown += `Error: ${test.art.message}\n\n`
+      }
+      if (test.art && test.art.stack) {
+        markdown += `${test.art.stack}\n\n`
+      }
+      markdown += `---\n\n`
+    }
+
+    if (config.captureDebugOutput && debugOutput.length > 0) {
+      markdown += `CodeceptJS Debug Output:\n\n`
+      debugOutput.forEach(line => {
+        markdown += `> ${line}\n`
+      })
+      markdown += `\n---\n\n`
+    }
+
+    steps.forEach((stepData, index) => {
+      const stepAnchor = clearString(stepData.step).replace(/[^a-zA-Z0-9_-]/g, '_').slice(0, 50)
+      markdown += `### Step ${index + 1}: ${stepData.step}\n`
+      markdown += `<a id="${stepAnchor}"></a>\n`
+
+      if (stepData.meta.duration) {
+        markdown += `  > duration: ${stepData.meta.duration}\n`
+      }
+
+      if (stepData.meta.url) {
+        markdown += `  > navigated to ${stepData.meta.url}\n`
+      }
+
+      if (config.captureDebugOutput && stepData.debugOutput && stepData.debugOutput.length > 0) {
+        stepData.debugOutput.forEach(line => {
+          markdown += `  > ${line}\n`
+        })
+      }
+
+      const links = artifactLinks(stepData.artifacts, { consoleCount: stepData.meta.consoleCount })
+      if (links) markdown += links + '\n'
+
+      if (config.captureHTTP) {
+        if (test.artifacts && test.artifacts.har) {
+          const harPath = path.relative(reportDir, test.artifacts.har)
+          markdown += `  > HTTP: see [HAR file](../${harPath}) for network requests\n`
+        } else if (test.artifacts && test.artifacts.trace) {
+          const tracePath = path.relative(reportDir, test.artifacts.trace)
+          markdown += `  > HTTP: see [Playwright trace](../${tracePath}) for network requests\n`
+        }
+      }
+
+      markdown += `\n`
+    })
+
+    const traceFile = path.join(dir, 'trace.md')
+    fs.writeFileSync(traceFile, markdown)
+
+    output.print(`Trace Saved: file://${traceFile}`)
+
+    if (!test.artifacts) test.artifacts = {}
+    test.artifacts.aiTrace = traceFile
+  }
+
+  function isStepIgnored(step) {
+    if (!config.ignoreSteps) return false
+    if (!step.title) return false
+    for (const pattern of config.ignoreSteps || []) {
+      if (step.title.match(pattern)) return true
+    }
+    return false
+  }
+
+  function generateStepPrefix(step, index) {
+    const stepName = step.toString()
+    const cleanedName = clearString(stepName)
+      .replace(/[^a-zA-Z0-9_-]/g, '_')
+      .replace(/_{2,}/g, '_')
+      .slice(0, 80)
+      .trim()
+
+    return `${String(index).padStart(4, '0')}_${cleanedName}`
+  }
+}

package/lib/plugin/analyze.js

@@ -12,6 +12,7 @@ const ai = aiModule.default || aiModule
 import colors from 'chalk'
 import ora from 'ora'
 import event from '../event.js'
+import recorder from '../recorder.js'
 
 import output from '../output.js'
 
@@ -154,10 +155,9 @@ const defaultConfig = {
       if (config.vision && test.artifacts.screenshot) {
         debug('Adding screenshot to prompt')
         messages[0].content.push({
-          type: 'image_url',
-          image_url: {
-            url: 'data:image/png;base64,' + base64EncodeFile(test.artifacts.screenshot),
-          },
+          type: 'image',
+          image: base64EncodeFile(test.artifacts.screenshot),
+          mediaType: 'image/png',
         })
       }
 
@@ -227,14 +227,14 @@ export default function (config = {}) {
     console.log('Enabled AI analysis')
   })
 
-  event.dispatcher.on(event.all.result, async result => {
+  event.dispatcher.on(event.all.result, result => {
     if (!isMainThread) return // run only on main thread
     if (!ai.isEnabled) {
       console.log('AI is disabled, no analysis will be performed. Run tests with --ai flag to enable it.')
       return
     }
 
-    printReport(result)
+    recorder.add('analyze:print-ai-report', () => printReport(result), true)
   })
 
   event.dispatcher.on(event.workers.result, async result => {
@@ -248,7 +248,7 @@ export default function (config = {}) {
       return
     }
 
-    printReport(result)
+    await printReport(result)
   })
 
   async function printReport(result) {
@@ -294,7 +294,7 @@ export default function (config = {}) {
       console.error('Error analyzing failed tests', err)
     }
 
-    if (!Object.keys(container.plugins()).includes('pageInfo')) {
+    if (!Object.keys(Container.plugins()).includes('pageInfo')) {
       console.log('To improve analysis, enable pageInfo plugin to get more context for failed tests.')
     }
   }
@@ -353,7 +353,7 @@ function serializeError(error) {
     errorMessage +=
       '\n' +
       error.stack
-        .replace(global.codecept_dir || '', '.')
+        .replace(store.codeceptDir || '', '.')
         .split('\n')
         .map(line => line.replace(ansiRegExp(), ''))
         .slice(0, 5)

package/lib/plugin/auth.js

@@ -321,7 +321,7 @@ export default function (config) {
       }
       if (config.saveToFile) {
         output.debug(`Saved user session into file for ${name}`)
-        fs.writeFileSync(path.join(global.output_dir, `${name}_session.json`), JSON.stringify(cookies))
+        fs.writeFileSync(path.join(store.outputDir, `${name}_session.json`), JSON.stringify(cookies))
       }
       store[`${name}_session`] = cookies
     }
@@ -350,7 +350,8 @@ export default function (config) {
           recorder.session.restore('auto login')
           recorder.session.restore('check login')
           section.end()
-          recorder.throw(err)
+          // Use regular throw instead of recorder.throw to avoid promise chaining cycle
+          throw err
         })
     })
     recorder.add(() => {
@@ -376,7 +377,7 @@ export default function (config) {
       }
 
       if (!config.saveToFile) return
-      const cookieFile = path.join(global.output_dir, `${name}_session.json`)
+      const cookieFile = path.join(store.outputDir, `${name}_session.json`)
 
       if (!fileExists(cookieFile)) {
         return
@@ -411,7 +412,7 @@ export default function (config) {
 
 function loadCookiesFromFile(config) {
   for (const name in config.users) {
-    const fileName = path.join(global.output_dir, `${name}_session.json`)
+    const fileName = path.join(store.outputDir, `${name}_session.json`)
     if (!fileExists(fileName)) continue
     const data = fs.readFileSync(fileName).toString()
     try {

package/lib/plugin/browser.js

@@ -0,0 +1,77 @@
+import output from '../output.js'
+
+/**
+ * Overrides browser helper config from the command line. Works for all browser helpers
+ * (Playwright, Puppeteer, WebDriver, Appium) without touching `codecept.conf`.
+ *
+ * Enable it via `-p` option with one or more colon-chained args:
+ *
+ * ```
+ * npx codeceptjs run -p browser:show
+ * npx codeceptjs run -p browser:hide
+ * npx codeceptjs run -p browser:browser=firefox
+ * npx codeceptjs run -p browser:windowSize=1024x768:video=false
+ * npx codeceptjs run -p browser:hide:browser=webkit:windowSize=800x600
+ * ```
+ *
+ * #### Args
+ *
+ * * **show** — force visible browser
+ * * **hide** — force headless (also injects `--headless` into WebDriver chrome/firefox capability args)
+ * * **`<key>=<value>`** — set `helpers.<eachBrowserHelper>.<key> = <value>`. Three keys
+ *   get per-helper translation via `setBrowserConfig`:
+ *     * `browser=<name>` — Puppeteer receives `product`, Playwright receives `browser`
+ *     * `windowSize=WxH` — also adds `--window-size=W,H` chromium/chrome args
+ *     * `show=true|false` — toggles `show` on Playwright/Puppeteer; injects/strips
+ *       `--headless` in WebDriver chrome/firefox capability args
+ *
+ * Values stay as strings. `true` / `false` are coerced to booleans.
+ *
+ * Requires `@codeceptjs/configure` to be installed; if missing, the plugin
+ * logs a hint and skips the override.
+ */
+export default async function (config = {}) {
+  const { _args, enabled, ...rest } = config
+  const opts = { ...rest, ...parseArgs(_args || []) }
+  if (Object.keys(opts).length === 0) return
+
+  const configure = await tryImportConfigure()
+  if (!configure) return
+
+  configure.setBrowserConfig(opts)
+  output.debug(`browser plugin: applied ${formatOpts(opts)}`)
+}
+
+async function tryImportConfigure() {
+  try {
+    return await import('@codeceptjs/configure')
+  } catch (err) {
+    output.error("browser plugin: '@codeceptjs/configure' is not installed; CLI overrides are skipped. Run `npm i @codeceptjs/configure` to enable.")
+    return null
+  }
+}
+
+function parseArgs(args) {
+  return args.filter(Boolean).reduce((acc, arg) => Object.assign(acc, parseArg(arg)), {})
+}
+
+function parseArg(arg) {
+  if (arg === 'show') return { show: true }
+  if (arg === 'hide') return { show: false }
+  if (arg.includes('=')) {
+    const [key, ...rest] = arg.split('=')
+    return { [key]: parseValue(rest.join('=')) }
+  }
+  output.error(`browser plugin: unknown arg "${arg}"`)
+  return {}
+}
+
+function parseValue(v) {
+  if (v === 'true') return true
+  if (v === 'false') return false
+  return v
+}
+
+function formatOpts(opts) {
+  return Object.entries(opts).map(([k, v]) => `${k}=${v}`).join(', ')
+}