codeceptjs 3.7.0-beta.7 → 3.7.0-beta.8

package/lib/plugin/analyze.js

@@ -0,0 +1,351 @@
+const debug = require('debug')('codeceptjs:analyze')
+const { isMainThread } = require('node:worker_threads')
+const { arrowRight } = require('figures')
+const container = require('../container')
+const ai = require('../ai')
+const colors = require('chalk')
+const ora = require('ora-classic')
+const event = require('../event')
+const output = require('../output')
+const { ansiRegExp, base64EncodeFile, markdownToAnsi } = require('../utils')
+
+const MAX_DATA_LENGTH = 5000
+
+const defaultConfig = {
+  clusterize: 2,
+  analyze: 3,
+  vision: false,
+  categories: [
+    'Browser connection error / browser crash',
+    'Network errors (server error, timeout, etc)',
+    'HTML / page elements (not found, not visible, etc)',
+    'Navigation errors (404, etc)',
+    'Code errors (syntax error, JS errors, etc)',
+    'Library & framework errors (CodeceptJS internal errors, user-defined libraries, etc)',
+    'Data errors (password incorrect, no options in select, invalid format, etc)',
+    'Assertion failures',
+    'Other errors',
+  ],
+  prompts: {
+    clusterize: (tests, config) => {
+      const serializedFailedTests = tests
+        .map((test, index) => {
+          if (!test || !test.err) return
+          return `
+          #${index + 1}: ${serializeTest(test)}
+          ${serializeError(test.err).slice(0, MAX_DATA_LENGTH / tests.length)}`.trim()
+        })
+        .join('\n\n--------\n\n')
+
+      const messages = [
+        {
+          role: 'user',
+          content: `
+        I am test analyst analyzing failed tests in CodeceptJS testing framework.
+
+        Please analyze the following failed tests and classify them into groups by their cause.
+        If there is no groups detected, say: "No common groups found".
+
+        Provide a short description of the group and a list of failed tests that belong to this group.
+        Use percent sign to indicate the percentage of failed tests in the group if this percentage is greater than 30%.
+
+        Here are failed tests:
+
+        ${serializedFailedTests}
+
+        Common categories of failures by order of priority:
+
+        ${config.categories.join('\n- ')}
+
+        If there is no groups of tests, say: "No patterns found"
+        Preserve error messages but cut them if they are too long.
+        Respond clearly and directly, without introductory words or phrases like ‘Of course,’ ‘Here is the answer,’ etc.
+        Do not list more than 3 errors in the group.
+        If you identify that all tests in the group have the same tag, add this tag to the group report, otherwise ignore TAG section.
+        If you identify that all tests in the group have the same suite, add this suite to the group report, otherwise ignore SUITE section.
+        Pick different emojis for each group.
+        Do not include group into report if it has only one test in affected tests section.
+
+        Provide list of groups in following format:
+
+        _______________________________
+
+        ## Group <group_number>
+
+        * CATEGORY <category_of_failure>
+        * ERROR <error_message_1>, <error_message_2>, ...
+        * SUMMARY <summary_of_errors>
+        * STEP <step_of_failure> (use CodeceptJS format I.click(), I.see(), etc; if all failures happend on the same step)
+        * SUITE <suite_title>, <suite_title> (if SUITE is present, and if all tests in the group have the same suite or suites)
+        * TAG <tag> (if TAG is present, and if all tests in the group have the same tag)
+        * AFFECTED TESTS (<total number of tests>):
+            x <test1 title>
+            x <test2 title>
+            x <test3 title>
+            x ...
+        `,
+        },
+        {
+          role: 'assistant',
+          content: `## '
+        `,
+        },
+      ]
+      return messages
+    },
+    analyze: (test, config) => {
+      const testMessage = serializeTest(test)
+      const errorMessage = serializeError(test.err)
+
+      const messages = [
+        {
+          role: 'user',
+          content: [
+            {
+              type: 'text',
+              text: `
+        I am qa engineer analyzing failed tests in CodeceptJS testing framework.
+        Please analyze the following failed test and error its error and explain it.
+
+        Pick one of the categories of failures and explain it.
+
+        Categories of failures in order of priority:
+
+        ${config.categories.join('\n- ')}
+
+        Here is the test and error:
+
+        ------- TEST -------
+        ${testMessage}
+
+        ------- ERROR -------
+        ${errorMessage}
+
+        ------ INSTRUCTIONS ------
+
+        Do not get to details, be concise.
+        If there is failed step, just write it in STEPS section.
+        If you have suggestions for the test, write them in SUMMARY section.
+        Inside SUMMARY write exact values, if you have suggestions, explain which information you used to suggest.
+        Be concise, each section should not take more than one sentence.
+
+        Response format:
+
+        * CATEGORY <category_of_failure>
+        * STEPS <step_of_failure>
+        * SUMMARY <explanation_of_failure>
+
+        Do not add any other sections or explanations. Only CATEGORY, SUMMARY, STEPS.
+        ${config.vision ? 'Also a screenshot of the page is attached to the prompt.' : ''}
+      `,
+            },
+          ],
+        },
+      ]
+
+      if (config.vision && test.artifacts.screenshot) {
+        debug('Adding screenshot to prompt')
+        messages[0].content.push({
+          type: 'image_url',
+          image_url: {
+            url: 'data:image/png;base64,' + base64EncodeFile(test.artifacts.screenshot),
+          },
+        })
+      }
+
+      messages.push({
+        role: 'assistant',
+        content: `## `,
+      })
+
+      return messages
+    },
+  },
+}
+
+/**
+ *
+ * @param {*} config
+ * @returns
+ */
+module.exports = function (config = {}) {
+  config = Object.assign(defaultConfig, config)
+
+  event.dispatcher.on(event.workers.before, () => {
+    if (!ai.isEnabled) return
+    console.log('Enabled AI analysis')
+  })
+
+  event.dispatcher.on(event.all.result, async result => {
+    if (!isMainThread) return // run only on main thread
+    if (!ai.isEnabled) {
+      console.log('AI is disabled, no analysis will be performed. Run tests with --ai flag to enable it.')
+      return
+    }
+
+    printReport(result)
+  })
+
+  event.dispatcher.on(event.workers.result, async result => {
+    if (!result.hasFailed) {
+      console.log('Everything is fine, skipping AI analysis')
+      return
+    }
+
+    if (!ai.isEnabled) {
+      console.log('AI is disabled, no analysis will be performed. Run tests with --ai flag to enable it.')
+      return
+    }
+
+    printReport(result)
+  })
+
+  async function printReport(result) {
+    const failedTestsAndErrors = result.tests.filter(t => t.err)
+
+    if (!failedTestsAndErrors.length) return
+
+    debug(failedTestsAndErrors.map(t => serializeTest(t) + '\n' + serializeError(t.err)))
+
+    try {
+      if (failedTestsAndErrors.length >= config.clusterize) {
+        const response = await clusterize(failedTestsAndErrors)
+        printHeader()
+        console.log(response)
+        return
+      }
+
+      output.plugin('analyze', `Analyzing first ${config.analyze} failed tests...`)
+
+      // we pick only unique errors to not repeat answers
+      const uniqueErrors = failedTestsAndErrors.filter((item, index, array) => {
+        return array.findIndex(t => t.err?.message === item.err?.message) === index
+      })
+
+      for (let i = 0; i < config.analyze; i++) {
+        if (!uniqueErrors[i]) break
+
+        const response = await analyze(uniqueErrors[i])
+        if (!response) {
+          break
+        }
+
+        printHeader()
+        console.log()
+        console.log('--------------------------------')
+        console.log(arrowRight, colors.bold.white(uniqueErrors[i].fullTitle()), config.vision ? '👀' : '')
+        console.log()
+        console.log()
+        console.log(response)
+        console.log()
+      }
+    } catch (err) {
+      console.error('Error analyzing failed tests', err)
+    }
+
+    if (!Object.keys(container.plugins()).includes('pageInfo')) {
+      console.log('To improve analysis, enable pageInfo plugin to get more context for failed tests.')
+    }
+  }
+
+  let hasPrintedHeader = false
+
+  function printHeader() {
+    if (!hasPrintedHeader) {
+      console.log()
+      console.log(colors.bold.white('🪄 AI REPORT:'))
+      hasPrintedHeader = true
+    }
+  }
+
+  async function clusterize(failedTestsAndErrors) {
+    const spinner = ora('Clusterizing failures...').start()
+    const prompt = config.prompts.clusterize(failedTestsAndErrors, config)
+    try {
+      const response = await ai.createCompletion(prompt)
+      spinner.stop()
+      return formatResponse(response)
+    } catch (err) {
+      spinner.stop()
+      console.error('Error clusterizing failures', err.message)
+    }
+  }
+
+  async function analyze(failedTestAndError) {
+    const spinner = ora('Analyzing failure...').start()
+    const prompt = config.prompts.analyze(failedTestAndError, config)
+    try {
+      const response = await ai.createCompletion(prompt)
+      spinner.stop()
+      return formatResponse(response)
+    } catch (err) {
+      spinner.stop()
+      console.error('Error analyzing failure:', err.message)
+    }
+  }
+}
+
+function serializeError(error) {
+  if (typeof error === 'string') {
+    return error
+  }
+
+  if (!error) return
+
+  let errorMessage = 'ERROR: ' + error.message
+
+  if (error.inspect) {
+    errorMessage = 'ERROR: ' + error.inspect()
+  }
+
+  if (error.stack) {
+    errorMessage +=
+      '\n' +
+      error.stack
+        .replace(global.codecept_dir || '', '.')
+        .split('\n')
+        .map(line => line.replace(ansiRegExp(), ''))
+        .slice(0, 5)
+        .join('\n')
+  }
+  if (error.steps) {
+    errorMessage += '\n STEPS: ' + error.steps.map(s => s.toCode()).join('\n')
+  }
+  return errorMessage
+}
+
+function serializeTest(test) {
+  if (!test.uid) return
+
+  let testMessage = 'TEST TITLE: ' + test.title
+
+  if (test.suite) {
+    testMessage += '\n SUITE: ' + test.suite.title
+  }
+  if (test.parent) {
+    testMessage += '\n SUITE: ' + test.parent.title
+  }
+
+  if (test.steps?.length) {
+    const failedSteps = test.steps
+    if (failedSteps.length) testMessage += '\n STEP: ' + failedSteps.map(s => s.toCode()).join('; ')
+  }
+
+  const pageInfo = test.notes.find(n => n.type === 'pageInfo')
+  if (pageInfo) {
+    testMessage += '\n PAGE INFO: ' + pageInfo.text
+  }
+
+  return testMessage
+}
+
+function formatResponse(response) {
+  if (!response.startsWith('##')) response = '## ' + response
+  return response
+    .split('\n')
+    .map(line => line.trim())
+    .filter(line => !/^[A-Z\s]+$/.test(line))
+    .map(line => markdownToAnsi(line))
+    .map(line => line.replace(/^x /gm, `    ${colors.red.bold('x')} `))
+    .join('\n')
+}

package/lib/plugin/commentStep.js

@@ -7,6 +7,8 @@ let currentCommentStep
 const defaultGlobalName = '__'
 
 /**
+ * @deprecated
+ *
  * Add descriptive nested steps for your tests:
  *
  * ```js
@@ -100,6 +102,9 @@ const defaultGlobalName = '__'
  * ```
  */
 module.exports = function (config) {
+  console.log('commentStep is deprecated, disable it and use Section instead')
+  console.log('const { Section: __ } = require("codeceptjs/steps")')
+
   event.dispatcher.on(event.test.started, () => {
     currentCommentStep = null
   })

package/lib/plugin/customReporter.js

@@ -0,0 +1,52 @@
+const event = require('../event')
+
+/**
+ * Sample custom reporter for CodeceptJS.
+ */
+module.exports = function (config) {
+  event.dispatcher.on(event.hook.finished, hook => {
+    if (config.onHookFinished) {
+      config.onHookFinished(hook)
+    }
+  })
+
+  event.dispatcher.on(event.test.before, test => {
+    if (config.onTestBefore) {
+      config.onTestBefore(test)
+    }
+  })
+
+  event.dispatcher.on(event.test.failed, (test, err) => {
+    if (config.onTestFailed) {
+      config.onTestFailed(test, err)
+    }
+  })
+
+  event.dispatcher.on(event.test.passed, test => {
+    if (config.onTestPassed) {
+      config.onTestPassed(test)
+    }
+  })
+
+  event.dispatcher.on(event.test.skipped, test => {
+    if (config.onTestSkipped) {
+      config.onTestSkipped(test)
+    }
+  })
+
+  event.dispatcher.on(event.test.finished, test => {
+    if (config.onTestFinished) {
+      config.onTestFinished(test)
+    }
+  })
+
+  event.dispatcher.on(event.all.result, result => {
+    if (config.onResult) {
+      config.onResult(result)
+    }
+
+    if (config.save) {
+      result.save()
+    }
+  })
+}

package/lib/plugin/heal.js

@@ -117,10 +117,10 @@ module.exports = function (config = {}) {
     }
   })
 
-  event.dispatcher.on(event.workers.result, ({ tests }) => {
+  event.dispatcher.on(event.workers.result, result => {
     const { print } = output
 
-    const healedTests = Object.values(tests)
+    const healedTests = Object.values(result.tests)
       .flat()
       .filter(test => test.notes.some(note => note.type === 'heal'))
     if (!healedTests.length) return

package/lib/plugin/pageInfo.js

@@ -0,0 +1,143 @@
+const path = require('path')
+const fs = require('fs')
+const Container = require('../container')
+const recorder = require('../recorder')
+const event = require('../event')
+const supportedHelpers = require('./standardActingHelpers')
+const { scanForErrorMessages } = require('../html')
+const { output } = require('..')
+const { humanizeString, ucfirst } = require('../utils')
+const { testToFileName } = require('../mocha/test')
+const defaultConfig = {
+  errorClasses: ['error', 'warning', 'alert', 'danger'],
+  browserLogs: ['error'],
+}
+
+/**
+ * Collects information from web page after each failed test and adds it to the test as an artifact.
+ * It is suggested to enable this plugin if you run tests on CI and you need to debug failed tests.
+ * This plugin can be paired with `analyze` plugin to provide more context.
+ *
+ * It collects URL, HTML errors (by classes), and browser logs.
+ *
+ * Enable this plugin in config:
+ *
+ * ```js
+ * plugins: {
+ *  pageInfo: {
+ *   enabled: true,
+ * }
+ * ```
+ *
+ * Additional config options:
+ *
+ * * `errorClasses` - list of classes to search for errors (default: `['error', 'warning', 'alert', 'danger']`)
+ * * `browserLogs` - list of types of errors to search for in browser logs (default: `['error']`)
+ *
+ */
+module.exports = function (config = {}) {
+  const helpers = Container.helpers()
+  let helper
+
+  config = Object.assign(defaultConfig, config)
+
+  for (const helperName of supportedHelpers) {
+    if (Object.keys(helpers).indexOf(helperName) > -1) {
+      helper = helpers[helperName]
+    }
+  }
+
+  if (!helper) return // no helpers for screenshot
+
+  event.dispatcher.on(event.test.failed, test => {
+    const pageState = {}
+
+    recorder.add('URL of failed test', async () => {
+      try {
+        const url = await helper.grabCurrentUrl()
+        pageState.url = url
+      } catch (err) {
+        // not really needed
+      }
+    })
+    recorder.add('HTML snapshot failed test', async () => {
+      try {
+        const currentOutputLevel = output.level()
+        output.level(0)
+        const html = await helper.grabHTMLFrom('body')
+        output.level(currentOutputLevel)
+
+        if (!html) return
+
+        const errors = scanForErrorMessages(html, config.errorClasses)
+        if (errors.length) {
+          output.debug('Detected errors in HTML code')
+          errors.forEach(error => output.debug(error))
+          pageState.htmlErrors = errors
+        }
+      } catch (err) {
+        // not really needed
+      }
+    })
+
+    recorder.add('Browser logs for failed test', async () => {
+      try {
+        const logs = await helper.grabBrowserLogs()
+
+        if (!logs) return
+
+        pageState.browserErrors = getBrowserErrors(logs, config.browserLogs)
+      } catch (err) {
+        // not really needed
+      }
+    })
+
+    recorder.add('Save page info', () => {
+      test.addNote('pageInfo', pageStateToMarkdown(pageState))
+
+      const pageStateFileName = path.join(global.output_dir, `${testToFileName(test)}.pageInfo.md`)
+      fs.writeFileSync(pageStateFileName, pageStateToMarkdown(pageState))
+      test.artifacts.pageInfo = pageStateFileName
+      return pageState
+    })
+  })
+}
+
+function pageStateToMarkdown(pageState) {
+  let markdown = ''
+
+  for (const [key, value] of Object.entries(pageState)) {
+    if (!value) continue
+    let result = ''
+
+    if (Array.isArray(value)) {
+      result = value.map(v => `- ${JSON.stringify(v, null, 2)}`).join('\n')
+    } else if (typeof value === 'string') {
+      result = `${value}`
+    } else {
+      result = JSON.stringify(value, null, 2)
+    }
+
+    if (!result.trim()) continue
+
+    markdown += `### ${ucfirst(humanizeString(key))}\n\n`
+    markdown += result
+    markdown += '\n\n'
+  }
+
+  return markdown
+}
+
+function getBrowserErrors(logs, type = ['error']) {
+  // Playwright & WebDriver console messages
+  let errors = logs
+    .map(log => {
+      if (typeof log === 'string') return log
+      if (!log.type) return null
+      return { type: log.type(), text: log.text() }
+    })
+    .filter(l => l && (typeof l === 'string' || type.includes(l.type)))
+    .map(l => (typeof l === 'string' ? l : l.text))
+
+  return errors
+}

package/lib/plugin/retryTo.js

@@ -1,6 +1,8 @@
-module.exports = function () {
+const { retryTo } = require('../effects')
+
+module.exports = function (config) {
   console.log(`
-Deprecated Warning: 'retryTo' has been moved to the effects module.
+Deprecation Warning: 'retryTo' has been moved to the effects module.
 You should update your tests to use it as follows:
 
 \`\`\`javascript
@@ -16,4 +18,10 @@ await retryTo((tryNum) => {
 
 For more details, refer to the documentation.
   `)
+
+  if (config.registerGlobal) {
+    global.retryTo = retryTo
+  }
+
+  return retryTo
 }

package/lib/plugin/screenshotOnFail.js

@@ -5,8 +5,9 @@ const Container = require('../container')
 const recorder = require('../recorder')
 const event = require('../event')
 const output = require('../output')
-const { fileExists, clearString } = require('../utils')
+const { fileExists } = require('../utils')
 const Codeceptjs = require('../index')
+const { testToFileName } = require('../mocha/test')
 
 const defaultConfig = {
   uniqueScreenshotNames: false,
@@ -79,13 +80,10 @@ module.exports = function (config) {
     recorder.add(
       'screenshot of failed test',
       async () => {
-        let fileName = clearString(test.title)
         const dataType = 'image/png'
         // This prevents data driven to be included in the failed screenshot file name
-        if (fileName.indexOf('{') !== -1) {
-          fileName = fileName.substr(0, fileName.indexOf('{') - 3).trim()
-        }
-        if (test.ctx && test.ctx.test && test.ctx.test.type === 'hook') fileName = clearString(`${test.title}_${test.ctx.test.title}`)
+        let fileName = testToFileName(test)
+
         if (options.uniqueScreenshotNames && test) {
           const uuid = _getUUID(test)
           fileName = `${fileName.substring(0, 10)}_${uuid}.failed.png`

package/lib/plugin/stepTimeout.js

@@ -1,5 +1,5 @@
 const event = require('../event')
-const { TIMEOUT_ORDER } = require('../step/timeout')
+const { TIMEOUT_ORDER } = require('../timeout')
 
 const defaultConfig = {
   timeout: 150,

package/lib/plugin/tryTo.js

@@ -1,4 +1,6 @@
-module.exports = function () {
+const { tryTo } = require('../effects')
+
+module.exports = function (config) {
   console.log(`
 Deprecated Warning: 'tryTo' has been moved to the effects module.
 You should update your tests to use it as follows:
@@ -14,4 +16,10 @@ await tryTo(() => {
 
 For more details, refer to the documentation.
   `)
+
+  if (config.registerGlobal) {
+    global.tryTo = tryTo
+  }
+
+  return tryTo
 }

package/lib/recorder.js

@@ -3,7 +3,7 @@ const promiseRetry = require('promise-retry')
 const chalk = require('chalk')
 const { printObjectProperties } = require('./utils')
 const { log } = require('./output')
-
+const { TimeoutError } = require('./timeout')
 const MAX_TASKS = 100
 
 let promise
@@ -191,13 +191,13 @@ module.exports = {
         .slice(-1)
         .pop()
       // no retries or unnamed tasks
+      debug(`${currentQueue()} Running | ${taskName} | Timeout: ${timeout || 'None'}`)
+
       if (!retryOpts || !taskName || !retry) {
         const [promise, timer] = getTimeoutPromise(timeout, taskName)
         return Promise.race([promise, Promise.resolve(res).then(fn)]).finally(() => clearTimeout(timer))
       }
 
-      debug(`${currentQueue()} Running | ${taskName}`)
-
       const retryRules = this.retries.slice().reverse()
       return promiseRetry(Object.assign(defaultRetryOptions, retryOpts), (retry, number) => {
         if (number > 1) log(`${currentQueue()}Retrying... Attempt #${number}`)
@@ -386,7 +386,7 @@ function getTimeoutPromise(timeoutMs, taskName) {
   return [
     new Promise((done, reject) => {
       timer = setTimeout(() => {
-        reject(new Error(`Action ${taskName} was interrupted on step timeout ${timeoutMs}ms`))
+        reject(new TimeoutError(`Action ${taskName} was interrupted on timeout ${timeoutMs}ms`))
       }, timeoutMs || 2e9)
     }),
     timer,