codeceptjs 3.6.10 → 3.7.0-beta.10

package/lib/plugin/analyze.js

@@ -0,0 +1,349 @@
+const debug = require('debug')('codeceptjs:analyze')
+const { isMainThread } = require('node:worker_threads')
+const { arrowRight } = require('figures')
+const container = require('../container')
+const ai = require('../ai')
+const colors = require('chalk')
+const ora = require('ora-classic')
+const event = require('../event')
+const output = require('../output')
+const { ansiRegExp, base64EncodeFile, markdownToAnsi } = require('../utils')
+
+const MAX_DATA_LENGTH = 5000
+
+const defaultConfig = {
+  clusterize: 5,
+  analyze: 2,
+  vision: false,
+  categories: [
+    'Browser connection error / browser crash',
+    'Network errors (server error, timeout, etc)',
+    'HTML / page elements (not found, not visible, etc)',
+    'Navigation errors (404, etc)',
+    'Code errors (syntax error, JS errors, etc)',
+    'Library & framework errors (CodeceptJS internal errors, user-defined libraries, etc)',
+    'Data errors (password incorrect, no options in select, invalid format, etc)',
+    'Assertion failures',
+    'Other errors',
+  ],
+  prompts: {
+    clusterize: (tests, config) => {
+      const serializedFailedTests = tests
+        .map((test, index) => {
+          if (!test || !test.err) return
+          return `
+          #${index + 1}: ${serializeTest(test)}
+          ${serializeError(test.err).slice(0, MAX_DATA_LENGTH / tests.length)}`.trim()
+        })
+        .join('\n\n--------\n\n')
+
+      const messages = [
+        {
+          role: 'user',
+          content: `
+        I am test analyst analyzing failed tests in CodeceptJS testing framework.
+
+        Please analyze the following failed tests and classify them into groups by their cause.
+        If there is no groups detected, say: "No common groups found".
+
+        Provide a short description of the group and a list of failed tests that belong to this group.
+        Use percent sign to indicate the percentage of failed tests in the group if this percentage is greater than 30%.
+
+        Here are failed tests:
+
+        ${serializedFailedTests}
+
+        Common categories of failures by order of priority:
+
+        ${config.categories.join('\n- ')}
+
+        If there is no groups of tests, say: "No patterns found"
+        Preserve error messages but cut them if they are too long.
+        Respond clearly and directly, without introductory words or phrases like ‘Of course,’ ‘Here is the answer,’ etc.
+        Do not list more than 3 errors in the group.
+        If you identify that all tests in the group have the same tag, add this tag to the group report, otherwise ignore TAG section.
+        If you identify that all tests in the group have the same suite, add this suite to the group report, otherwise ignore SUITE section.
+        Pick different emojis for each group.
+        Order groups by the number of tests in the group.
+        If group has one test, skip that group.
+
+        Provide list of groups in following format:
+
+        _______________________________
+
+        ## Group <group_number> <emoji>
+
+        * SUMMARY <summary_of_errors>
+        * CATEGORY <category_of_failure>
+        * ERROR <error_message_1>, <error_message_2>, ...
+        * STEP <step_of_failure> (use CodeceptJS format I.click(), I.see(), etc; if all failures happend on the same step)
+        * SUITE <suite_title>, <suite_title> (if SUITE is present, and if all tests in the group have the same suite or suites)
+        * TAG <tag> (if TAG is present, and if all tests in the group have the same tag)
+        * AFFECTED TESTS (<total number of tests>):
+            x <test1 title>
+            x <test2 title>
+            x <test3 title>
+            x ...
+        `,
+        },
+        {
+          role: 'assistant',
+          content: `## '
+        `,
+        },
+      ]
+      return messages
+    },
+    analyze: (test, config) => {
+      const testMessage = serializeTest(test)
+      const errorMessage = serializeError(test.err)
+
+      const messages = [
+        {
+          role: 'user',
+          content: [
+            {
+              type: 'text',
+              text: `
+        I am qa engineer analyzing failed tests in CodeceptJS testing framework.
+        Please analyze the following failed test and error its error and explain it.
+
+        Pick one of the categories of failures and explain it.
+
+        Categories of failures in order of priority:
+
+        ${config.categories.join('\n- ')}
+
+        Here is the test and error:
+
+        ------- TEST -------
+        ${testMessage}
+
+        ------- ERROR -------
+        ${errorMessage}
+
+        ------ INSTRUCTIONS ------
+
+        Do not get to details, be concise.
+        If there is failed step, just write it in STEPS section.
+        If you have suggestions for the test, write them in SUMMARY section.
+        Do not be too technical in SUMMARY section.
+        Inside SUMMARY write exact values, if you have suggestions, explain which information you used to suggest.
+        Be concise, each section should not take more than one sentence.
+
+        Response format:
+
+        * SUMMARY <explanation_of_failure>
+        * ERROR <error_message_1>, <error_message_2>, ...
+        * CATEGORY <category_of_failure>
+        * STEPS <step_of_failure>
+
+        Do not add any other sections or explanations. Only CATEGORY, SUMMARY, STEPS.
+        ${config.vision ? 'Also a screenshot of the page is attached to the prompt.' : ''}
+      `,
+            },
+          ],
+        },
+      ]
+
+      if (config.vision && test.artifacts.screenshot) {
+        debug('Adding screenshot to prompt')
+        messages[0].content.push({
+          type: 'image_url',
+          image_url: {
+            url: 'data:image/png;base64,' + base64EncodeFile(test.artifacts.screenshot),
+          },
+        })
+      }
+
+      return messages
+    },
+  },
+}
+
+/**
+ *
+ * @param {*} config
+ * @returns
+ */
+module.exports = function (config = {}) {
+  config = Object.assign(defaultConfig, config)
+
+  event.dispatcher.on(event.workers.before, () => {
+    if (!ai.isEnabled) return
+    console.log('Enabled AI analysis')
+  })
+
+  event.dispatcher.on(event.all.result, async result => {
+    if (!isMainThread) return // run only on main thread
+    if (!ai.isEnabled) {
+      console.log('AI is disabled, no analysis will be performed. Run tests with --ai flag to enable it.')
+      return
+    }
+
+    printReport(result)
+  })
+
+  event.dispatcher.on(event.workers.result, async result => {
+    if (!result.hasFailed) {
+      console.log('Everything is fine, skipping AI analysis')
+      return
+    }
+
+    if (!ai.isEnabled) {
+      console.log('AI is disabled, no analysis will be performed. Run tests with --ai flag to enable it.')
+      return
+    }
+
+    printReport(result)
+  })
+
+  async function printReport(result) {
+    const failedTestsAndErrors = result.tests.filter(t => t.err)
+
+    if (!failedTestsAndErrors.length) return
+
+    debug(failedTestsAndErrors.map(t => serializeTest(t) + '\n' + serializeError(t.err)))
+
+    try {
+      if (failedTestsAndErrors.length >= config.clusterize) {
+        const response = await clusterize(failedTestsAndErrors)
+        printHeader()
+        console.log(response)
+        return
+      }
+
+      output.plugin('analyze', `Analyzing first ${config.analyze} failed tests...`)
+
+      // we pick only unique errors to not repeat answers
+      const uniqueErrors = failedTestsAndErrors.filter((item, index, array) => {
+        return array.findIndex(t => t.err?.message === item.err?.message) === index
+      })
+
+      for (let i = 0; i < config.analyze; i++) {
+        if (!uniqueErrors[i]) break
+
+        const response = await analyze(uniqueErrors[i])
+        if (!response) {
+          break
+        }
+
+        printHeader()
+        console.log()
+        console.log('--------------------------------')
+        console.log(arrowRight, colors.bold.white(uniqueErrors[i].fullTitle()), config.vision ? '👀' : '')
+        console.log()
+        console.log()
+        console.log(response)
+        console.log()
+      }
+    } catch (err) {
+      console.error('Error analyzing failed tests', err)
+    }
+
+    if (!Object.keys(container.plugins()).includes('pageInfo')) {
+      console.log('To improve analysis, enable pageInfo plugin to get more context for failed tests.')
+    }
+  }
+
+  let hasPrintedHeader = false
+
+  function printHeader() {
+    if (!hasPrintedHeader) {
+      console.log()
+      console.log(colors.bold.white('🪄 AI REPORT:'))
+      hasPrintedHeader = true
+    }
+  }
+
+  async function clusterize(failedTestsAndErrors) {
+    const spinner = ora('Clusterizing failures...').start()
+    const prompt = config.prompts.clusterize(failedTestsAndErrors, config)
+    try {
+      const response = await ai.createCompletion(prompt)
+      spinner.stop()
+      return formatResponse(response)
+    } catch (err) {
+      spinner.stop()
+      console.error('Error clusterizing failures', err.message)
+    }
+  }
+
+  async function analyze(failedTestAndError) {
+    const spinner = ora('Analyzing failure...').start()
+    const prompt = config.prompts.analyze(failedTestAndError, config)
+    try {
+      const response = await ai.createCompletion(prompt)
+      spinner.stop()
+      return formatResponse(response)
+    } catch (err) {
+      spinner.stop()
+      console.error('Error analyzing failure:', err.message)
+    }
+  }
+}
+
+function serializeError(error) {
+  if (typeof error === 'string') {
+    return error
+  }
+
+  if (!error) return
+
+  let errorMessage = 'ERROR: ' + error.message
+
+  if (error.inspect) {
+    errorMessage = 'ERROR: ' + error.inspect()
+  }
+
+  if (error.stack) {
+    errorMessage +=
+      '\n' +
+      error.stack
+        .replace(global.codecept_dir || '', '.')
+        .split('\n')
+        .map(line => line.replace(ansiRegExp(), ''))
+        .slice(0, 5)
+        .join('\n')
+  }
+  if (error.steps) {
+    errorMessage += '\n STEPS: ' + error.steps.map(s => s.toCode()).join('\n')
+  }
+  return errorMessage
+}
+
+function serializeTest(test) {
+  if (!test.uid) return
+
+  let testMessage = 'TEST TITLE: ' + test.title
+
+  if (test.suite) {
+    testMessage += '\n SUITE: ' + test.suite.title
+  }
+  if (test.parent) {
+    testMessage += '\n SUITE: ' + test.parent.title
+  }
+
+  if (test.steps?.length) {
+    const failedSteps = test.steps
+    if (failedSteps.length) testMessage += '\n STEP: ' + failedSteps.map(s => s.toCode()).join('; ')
+  }
+
+  const pageInfo = test.notes.find(n => n.type === 'pageInfo')
+  if (pageInfo) {
+    testMessage += '\n PAGE INFO: ' + pageInfo.text
+  }
+
+  return testMessage
+}
+
+function formatResponse(response) {
+  if (!response.startsWith('##')) response = '## ' + response
+  return response
+    .split('\n')
+    .map(line => line.trim())
+    .filter(line => !/^[A-Z\s]+$/.test(line))
+    .map(line => markdownToAnsi(line))
+    .map(line => line.replace(/^x /gm, `    ${colors.red.bold('x')} `))
+    .join('\n')
+}

package/lib/plugin/autoDelay.js

@@ -2,8 +2,8 @@ const Container = require('../container')
 const store = require('../store')
 const recorder = require('../recorder')
 const event = require('../event')
-const log = require('../output').log
-const supportedHelpers = require('./standardActingHelpers').slice()
+const { log } = require('../output')
+const standardActingHelpers = Container.STANDARD_ACTING_HELPERS
 
 const methodsToDelay = ['click', 'fillField', 'checkOption', 'pressKey', 'doubleClick', 'rightClick']
 
@@ -66,25 +66,25 @@ module.exports = function (config) {
 
   if (!helper) return // no helpers for auto-delay
 
-  event.dispatcher.on(event.step.before, (step) => {
+  event.dispatcher.on(event.step.before, step => {
     if (config.methods.indexOf(step.helperMethod) < 0) return // skip non-actions
 
     recorder.add('auto-delay', async () => {
       if (store.debugMode) return // no need to delay in debug
       log(`Delaying for ${config.delayBefore}ms`)
-      return new Promise((resolve) => {
+      return new Promise(resolve => {
         setTimeout(resolve, config.delayBefore)
       })
     })
   })
 
-  event.dispatcher.on(event.step.after, (step) => {
+  event.dispatcher.on(event.step.after, step => {
     if (config.methods.indexOf(step.helperMethod) < 0) return // skip non-actions
 
     recorder.add('auto-delay', async () => {
       if (store.debugMode) return // no need to delay in debug
       log(`Delaying for ${config.delayAfter}ms`)
-      return new Promise((resolve) => {
+      return new Promise(resolve => {
         setTimeout(resolve, config.delayAfter)
       })
     })

package/lib/plugin/autoLogin.js

@@ -8,7 +8,7 @@ const { debug } = require('../output')
 const isAsyncFunction = require('../utils').isAsyncFunction
 
 const defaultUser = {
-  fetch: (I) => I.grabCookie(),
+  fetch: I => I.grabCookie(),
   check: () => {},
   restore: (I, cookies) => {
     I.amOnPage('/') // open a page
@@ -253,7 +253,7 @@ const defaultConfig = {
 module.exports = function (config) {
   config = Object.assign(defaultConfig, config)
   Object.keys(config.users).map(
-    (u) =>
+    u =>
       (config.users[u] = {
         ...defaultUser,
         ...config.users[u],
@@ -275,12 +275,11 @@ module.exports = function (config) {
     }
   }
 
-  const loginFunction = async (name) => {
+  const loginFunction = async name => {
     const userSession = config.users[name]
     const I = container.support('I')
     const cookies = store[`${name}_session`]
-    const shouldAwait =
-      isAsyncFunction(userSession.login) || isAsyncFunction(userSession.restore) || isAsyncFunction(userSession.check)
+    const shouldAwait = isAsyncFunction(userSession.login) || isAsyncFunction(userSession.restore) || isAsyncFunction(userSession.check)
 
     const loginAndSave = async () => {
       if (shouldAwait) {
@@ -311,7 +310,7 @@ module.exports = function (config) {
       userSession.restore(I, cookies)
       userSession.check(I, cookies)
     }
-    recorder.session.catch((err) => {
+    recorder.session.catch(err => {
       debug(`Failed auto login for ${name} due to ${err}`)
       debug('Logging in again')
       recorder.session.start('auto login')
@@ -320,7 +319,7 @@ module.exports = function (config) {
           recorder.add(() => recorder.session.restore('auto login'))
           recorder.catch(() => debug('continue'))
         })
-        .catch((err) => {
+        .catch(err => {
           recorder.session.restore('auto login')
           recorder.session.restore('check login')
           recorder.throw(err)

package/lib/plugin/commentStep.js

@@ -7,6 +7,8 @@ let currentCommentStep
 const defaultGlobalName = '__'
 
 /**
+ * @deprecated
+ *
  * Add descriptive nested steps for your tests:
  *
  * ```js
@@ -100,11 +102,14 @@ const defaultGlobalName = '__'
  * ```
  */
 module.exports = function (config) {
+  console.log('commentStep is deprecated, disable it and use Section instead')
+  console.log('const { Section: __ } = require("codeceptjs/steps")')
+
   event.dispatcher.on(event.test.started, () => {
     currentCommentStep = null
   })
 
-  event.dispatcher.on(event.step.started, (step) => {
+  event.dispatcher.on(event.step.started, step => {
     if (currentCommentStep) {
       const metaStep = getRootMetaStep(step)
 

package/lib/plugin/coverage.js

@@ -15,7 +15,7 @@ const supportedHelpers = ['Puppeteer', 'Playwright', 'WebDriver']
 
 const v8CoverageHelpers = {
   Playwright: {
-    startCoverage: async (page) => {
+    startCoverage: async page => {
       await Promise.all([
         page.coverage.startJSCoverage({
           resetOnNavigation: false,
@@ -26,16 +26,13 @@ const v8CoverageHelpers = {
       ])
     },
     takeCoverage: async (page, coverageReport) => {
-      const [jsCoverage, cssCoverage] = await Promise.all([
-        page.coverage.stopJSCoverage(),
-        page.coverage.stopCSSCoverage(),
-      ])
+      const [jsCoverage, cssCoverage] = await Promise.all([page.coverage.stopJSCoverage(), page.coverage.stopCSSCoverage()])
       const coverageList = [...jsCoverage, ...cssCoverage]
       await coverageReport.add(coverageList)
     },
   },
   Puppeteer: {
-    startCoverage: async (page) => {
+    startCoverage: async page => {
       await Promise.all([
         page.coverage.startJSCoverage({
           resetOnNavigation: false,
@@ -47,13 +44,10 @@ const v8CoverageHelpers = {
       ])
     },
     takeCoverage: async (page, coverageReport) => {
-      const [jsCoverage, cssCoverage] = await Promise.all([
-        page.coverage.stopJSCoverage(),
-        page.coverage.stopCSSCoverage(),
-      ])
+      const [jsCoverage, cssCoverage] = await Promise.all([page.coverage.stopJSCoverage(), page.coverage.stopCSSCoverage()])
       // to raw V8 script coverage
       const coverageList = [
-        ...jsCoverage.map((it) => {
+        ...jsCoverage.map(it => {
           return {
             source: it.text,
             ...it.rawScriptCoverage,
@@ -65,7 +59,7 @@ const v8CoverageHelpers = {
     },
   },
   WebDriver: {
-    startCoverage: async (page) => {
+    startCoverage: async page => {
       await Promise.all([
         page.coverage.startJSCoverage({
           resetOnNavigation: false,
@@ -77,13 +71,10 @@ const v8CoverageHelpers = {
       ])
     },
     takeCoverage: async (page, coverageReport) => {
-      const [jsCoverage, cssCoverage] = await Promise.all([
-        page.coverage.stopJSCoverage(),
-        page.coverage.stopCSSCoverage(),
-      ])
+      const [jsCoverage, cssCoverage] = await Promise.all([page.coverage.stopJSCoverage(), page.coverage.stopCSSCoverage()])
       // to raw V8 script coverage
       const coverageList = [
-        ...jsCoverage.map((it) => {
+        ...jsCoverage.map(it => {
           return {
             source: it.text,
             ...it.rawScriptCoverage,
@@ -131,7 +122,7 @@ module.exports = function (config) {
   let coverageRunning = false
 
   const v8Names = Object.keys(v8CoverageHelpers)
-  const helperName = Object.keys(helpers).find((it) => v8Names.includes(it))
+  const helperName = Object.keys(helpers).find(it => v8Names.includes(it))
   if (!helperName) {
     console.error(`Coverage is only supported in ${supportedHelpers.join(' or ')}`)
     // no helpers for screenshot
@@ -180,7 +171,7 @@ module.exports = function (config) {
   })
 
   // Save coverage data after every test run
-  event.dispatcher.on(event.test.after, (test) => {
+  event.dispatcher.on(event.test.after, test => {
     recorder.add(
       'take coverage',
       async () => {

package/lib/plugin/customLocator.js

@@ -111,7 +111,7 @@ const defaultConfig = {
  * I.click('=sign-up'); // matches => [data-qa=sign-up],[data-test=sign-up]
  * ```
  */
-module.exports = (config) => {
+module.exports = config => {
   config = { ...defaultConfig, ...config }
 
   Locator.addFilter((value, locatorObj) => {
@@ -125,7 +125,7 @@ module.exports = (config) => {
     if (config.strategy.toLowerCase() === 'xpath') {
       locatorObj.value = `.//*[${[]
         .concat(config.attribute)
-        .map((attr) => `@${attr}=${xpathLocator.literal(val)}`)
+        .map(attr => `@${attr}=${xpathLocator.literal(val)}`)
         .join(' or ')}]`
       locatorObj.type = 'xpath'
     }
@@ -133,7 +133,7 @@ module.exports = (config) => {
     if (config.strategy.toLowerCase() === 'css') {
       locatorObj.value = []
         .concat(config.attribute)
-        .map((attr) => `[${attr}=${val}]`)
+        .map(attr => `[${attr}=${val}]`)
         .join(',')
       locatorObj.type = 'css'
     }

package/lib/plugin/customReporter.js

@@ -0,0 +1,52 @@
+const event = require('../event')
+
+/**
+ * Sample custom reporter for CodeceptJS.
+ */
+module.exports = function (config) {
+  event.dispatcher.on(event.hook.finished, hook => {
+    if (config.onHookFinished) {
+      config.onHookFinished(hook)
+    }
+  })
+
+  event.dispatcher.on(event.test.before, test => {
+    if (config.onTestBefore) {
+      config.onTestBefore(test)
+    }
+  })
+
+  event.dispatcher.on(event.test.failed, (test, err) => {
+    if (config.onTestFailed) {
+      config.onTestFailed(test, err)
+    }
+  })
+
+  event.dispatcher.on(event.test.passed, test => {
+    if (config.onTestPassed) {
+      config.onTestPassed(test)
+    }
+  })
+
+  event.dispatcher.on(event.test.skipped, test => {
+    if (config.onTestSkipped) {
+      config.onTestSkipped(test)
+    }
+  })
+
+  event.dispatcher.on(event.test.finished, test => {
+    if (config.onTestFinished) {
+      config.onTestFinished(test)
+    }
+  })
+
+  event.dispatcher.on(event.all.result, result => {
+    if (config.onResult) {
+      config.onResult(result)
+    }
+
+    if (config.save) {
+      result.save()
+    }
+  })
+}

package/lib/plugin/eachElement.js

@@ -72,7 +72,7 @@ function eachElement(purpose, locator, fn) {
   if (store.dryRun) return
   const helpers = Object.values(container.helpers())
 
-  const helper = helpers.filter((h) => !!h._locate)[0]
+  const helper = helpers.filter(h => !!h._locate)[0]
 
   if (!helper) {
     throw new Error('No helper enabled with _locate method with returns a list of elements.')

package/lib/plugin/fakerTransform.js

@@ -40,7 +40,7 @@ const transform = require('../transform')
  *
  */
 module.exports = function (config) {
-  transform.addTransformerBeforeAll('gherkin.examples', (value) => {
+  transform.addTransformerBeforeAll('gherkin.examples', value => {
     if (typeof value === 'string' && value.length > 0) {
       return faker.helpers.fake(value)
     }