codeceptjs 4.0.0-beta.4 → 4.0.0-beta.6.esm-aria

package/lib/plugin/analyze.js

@@ -0,0 +1,403 @@
+import debugFactory from 'debug'
+const debug = debugFactory('codeceptjs:analyze')
+import { isMainThread } from 'node:worker_threads'
+import figures from 'figures'
+const { arrowRight } = figures
+import Container from '../container.js'
+// Container already imported correctly above
+import store from '../store.js'
+
+import aiModule from '../ai.js'
+const ai = aiModule.default || aiModule
+import colors from 'chalk'
+import ora from 'ora'
+import event from '../event.js'
+
+import output from '../output.js'
+
+import { ansiRegExp, base64EncodeFile, markdownToAnsi } from '../utils.js'
+
+const MAX_DATA_LENGTH = 5000
+
+const defaultConfig = {
+  clusterize: 5,
+  analyze: 2,
+  vision: false,
+  categories: [
+    'Browser connection error / browser crash',
+    'Network errors (server error, timeout, etc)',
+    'HTML / page elements (not found, not visible, etc)',
+    'Navigation errors (404, etc)',
+    'Code errors (syntax error, JS errors, etc)',
+    'Library & framework errors (CodeceptJS internal errors, user-defined libraries, etc)',
+    'Data errors (password incorrect, no options in select, invalid format, etc)',
+    'Assertion failures',
+    'Other errors',
+  ],
+  prompts: {
+    clusterize: (tests, config) => {
+      const serializedFailedTests = tests
+        .map((test, index) => {
+          if (!test || !test.err) return
+          return `
+          #${index + 1}: ${serializeTest(test)}
+          ${serializeError(test.err).slice(0, MAX_DATA_LENGTH / tests.length)}`.trim()
+        })
+        .join('\n\n--------\n\n')
+
+      const messages = [
+        {
+          role: 'user',
+          content: `
+        I am test analyst analyzing failed tests in CodeceptJS testing framework.
+
+        Please analyze the following failed tests and classify them into groups by their cause.
+        If there is no groups detected, say: "No common groups found".
+
+        Provide a short description of the group and a list of failed tests that belong to this group.
+        Use percent sign to indicate the percentage of failed tests in the group if this percentage is greater than 30%.
+
+        Here are failed tests:
+
+        ${serializedFailedTests}
+
+        Common categories of failures by order of priority:
+
+        ${config.categories.join('\n- ')}
+
+        If there is no groups of tests, say: "No patterns found"
+        Preserve error messages but cut them if they are too long.
+        Respond clearly and directly, without introductory words or phrases like 'Of course,' 'Here is the answer,' etc.
+        Do not list more than 3 errors in the group.
+        If you identify that all tests in the group have the same tag, add this tag to the group report, otherwise ignore TAG section.
+        If you identify that all tests in the group have the same suite, add this suite to the group report, otherwise ignore SUITE section.
+        Pick different emojis for each group.
+        Order groups by the number of tests in the group.
+        If group has one test, skip that group.
+
+        Provide list of groups in following format:
+
+        _______________________________
+
+        ## Group <group_number> <emoji>
+
+        * SUMMARY <summary_of_errors>
+        * CATEGORY <category_of_failure>
+        * URL <url_of_failure_if_any>
+        * ERROR <error_message_1>, <error_message_2>, ...
+        * STEP <step_of_failure> (use CodeceptJS format I.click(), I.see(), etc; if all failures happend on the same step)
+        * SUITE <suite_title>, <suite_title> (if SUITE is present, and if all tests in the group have the same suite or suites)
+        * TAG <tag> (if TAG is present, and if all tests in the group have the same tag)
+        * AFFECTED TESTS (<total number of tests>):
+            x <test1 title>
+            x <test2 title>
+            x <test3 title>
+            x ...
+        `,
+        },
+      ]
+      return messages
+    },
+    analyze: (test, config) => {
+      const testMessage = serializeTest(test)
+      const errorMessage = serializeError(test.err)
+
+      const messages = [
+        {
+          role: 'user',
+          content: [
+            {
+              type: 'text',
+              text: `
+        I am qa engineer analyzing failed tests in CodeceptJS testing framework.
+        Please analyze the following failed test and error its error and explain it.
+
+        Pick one of the categories of failures and explain it.
+
+        Categories of failures in order of priority:
+
+        ${config.categories.join('\n- ')}
+
+        Here is the test and error:
+
+        ------- TEST -------
+        ${testMessage}
+
+        ------- ERROR -------
+        ${errorMessage}
+
+        ------ INSTRUCTIONS ------
+
+        Do not get to details, be concise.
+        If there is failed step, just write it in STEPS section.
+        If you have suggestions for the test, write them in SUMMARY section.
+        Do not be too technical in SUMMARY section.
+        Inside SUMMARY write exact values, if you have suggestions, explain which information you used to suggest.
+        Be concise, each section should not take more than one sentence.
+
+        Response format:
+
+        * SUMMARY <explanation_of_failure>
+        * ERROR <error_message_1>, <error_message_2>, ...
+        * CATEGORY <category_of_failure>
+        * STEPS <step_of_failure>
+        * URL <url_of_failure_if_any>
+
+        Do not add any other sections or explanations. Only CATEGORY, SUMMARY, STEPS.
+        ${config.vision ? 'Also a screenshot of the page is attached to the prompt.' : ''}
+      `,
+            },
+          ],
+        },
+      ]
+
+      if (config.vision && test.artifacts.screenshot) {
+        debug('Adding screenshot to prompt')
+        messages[0].content.push({
+          type: 'image_url',
+          image_url: {
+            url: 'data:image/png;base64,' + base64EncodeFile(test.artifacts.screenshot),
+          },
+        })
+      }
+
+      return messages
+    },
+  },
+}
+
+/**
+ *
+ * Uses AI to analyze test failures and provide insights
+ *
+ * This plugin analyzes failed tests using AI to provide detailed explanations and group similar failures.
+ * When enabled with --ai flag, it generates reports after test execution.
+ *
+ * #### Usage
+ *
+ * ```js
+ * // in codecept.conf.js
+ * exports.config = {
+ *   plugins: {
+ *     analyze: {
+ *       enabled: true,
+ *       clusterize: 5,
+ *       analyze: 2,
+ *       vision: false
+ *     }
+ *   }
+ * }
+ * ```
+ *
+ * #### Configuration
+ *
+ * * `clusterize` (number) - minimum number of failures to trigger clustering analysis. Default: 5
+ * * `analyze` (number) - maximum number of individual test failures to analyze in detail. Default: 2
+ * * `vision` (boolean) - enables visual analysis of test screenshots. Default: false
+ * * `categories` (array) - list of failure categories for classification. Defaults to:
+ *   - Browser connection error / browser crash
+ *   - Network errors (server error, timeout, etc)
+ *   - HTML / page elements (not found, not visible, etc)
+ *   - Navigation errors (404, etc)
+ *   - Code errors (syntax error, JS errors, etc)
+ *   - Library & framework errors
+ *   - Data errors (password incorrect, invalid format, etc)
+ *   - Assertion failures
+ *   - Other errors
+ * * `prompts` (object) - customize AI prompts for analysis
+ *   - `clusterize` - prompt for clustering analysis
+ *   - `analyze` - prompt for individual test analysis
+ *
+ * #### Features
+ *
+ * * Groups similar failures when number of failures >= clusterize value
+ * * Provides detailed analysis of individual failures
+ * * Analyzes screenshots if vision=true and screenshots are available
+ * * Classifies failures into predefined categories
+ * * Suggests possible causes and solutions
+ *
+ * @param {Object} config - Plugin configuration
+ * @returns {void}
+ */
+export default function (config = {}) {
+  config = Object.assign(defaultConfig, config)
+
+  event.dispatcher.on(event.workers.before, () => {
+    if (!ai.isEnabled) return
+    console.log('Enabled AI analysis')
+  })
+
+  event.dispatcher.on(event.all.result, async result => {
+    if (!isMainThread) return // run only on main thread
+    if (!ai.isEnabled) {
+      console.log('AI is disabled, no analysis will be performed. Run tests with --ai flag to enable it.')
+      return
+    }
+
+    printReport(result)
+  })
+
+  event.dispatcher.on(event.workers.result, async result => {
+    if (!result.hasFailed) {
+      console.log('Everything is fine, skipping AI analysis')
+      return
+    }
+
+    if (!ai.isEnabled) {
+      console.log('AI is disabled, no analysis will be performed. Run tests with --ai flag to enable it.')
+      return
+    }
+
+    printReport(result)
+  })
+
+  async function printReport(result) {
+    const failedTestsAndErrors = result.tests.filter(t => t.err)
+
+    if (!failedTestsAndErrors.length) return
+
+    debug(failedTestsAndErrors.map(t => serializeTest(t) + '\n' + serializeError(t.err)))
+
+    try {
+      if (failedTestsAndErrors.length >= config.clusterize) {
+        const response = await clusterize(failedTestsAndErrors)
+        printHeader()
+        console.log(response)
+        return
+      }
+
+      output.plugin('analyze', `Analyzing first ${config.analyze} failed tests...`)
+
+      // we pick only unique errors to not repeat answers
+      const uniqueErrors = failedTestsAndErrors.filter((item, index, array) => {
+        return array.findIndex(t => t.err?.message === item.err?.message) === index
+      })
+
+      for (let i = 0; i < config.analyze; i++) {
+        if (!uniqueErrors[i]) break
+
+        const response = await analyze(uniqueErrors[i])
+        if (!response) {
+          break
+        }
+
+        printHeader()
+        console.log()
+        console.log('--------------------------------')
+        console.log(arrowRight, colors.bold.white(uniqueErrors[i].fullTitle()), config.vision ? '👀' : '')
+        console.log()
+        console.log()
+        console.log(response)
+        console.log()
+      }
+    } catch (err) {
+      console.error('Error analyzing failed tests', err)
+    }
+
+    if (!Object.keys(container.plugins()).includes('pageInfo')) {
+      console.log('To improve analysis, enable pageInfo plugin to get more context for failed tests.')
+    }
+  }
+
+  let hasPrintedHeader = false
+
+  function printHeader() {
+    if (!hasPrintedHeader) {
+      console.log()
+      console.log(colors.bold.white('🪄 AI REPORT:'))
+      hasPrintedHeader = true
+    }
+  }
+
+  async function clusterize(failedTestsAndErrors) {
+    const spinner = ora('Clusterizing failures...').start()
+    const prompt = config.prompts.clusterize(failedTestsAndErrors, config)
+    try {
+      const response = await ai.createCompletion(prompt)
+      spinner.stop()
+      return formatResponse(response)
+    } catch (err) {
+      spinner.stop()
+      console.error('Error clusterizing failures', err.message)
+    }
+  }
+
+  async function analyze(failedTestAndError) {
+    const spinner = ora('Analyzing failure...').start()
+    const prompt = config.prompts.analyze(failedTestAndError, config)
+    try {
+      const response = await ai.createCompletion(prompt)
+      spinner.stop()
+      return formatResponse(response)
+    } catch (err) {
+      spinner.stop()
+      console.error('Error analyzing failure:', err.message)
+    }
+  }
+}
+
+function serializeError(error) {
+  if (typeof error === 'string') {
+    return error
+  }
+
+  if (!error) return
+
+  let errorMessage = 'ERROR: ' + error.message
+
+  if (error.inspect) {
+    errorMessage = 'ERROR: ' + error.inspect()
+  }
+
+  if (error.stack) {
+    errorMessage +=
+      '\n' +
+      error.stack
+        .replace(global.codecept_dir || '', '.')
+        .split('\n')
+        .map(line => line.replace(ansiRegExp(), ''))
+        .slice(0, 5)
+        .join('\n')
+  }
+  if (error.steps) {
+    errorMessage += '\n STEPS: ' + error.steps.map(s => s.toCode()).join('\n')
+  }
+  return errorMessage
+}
+
+function serializeTest(test) {
+  if (!test.uid) return
+
+  let testMessage = 'TEST TITLE: ' + test.title
+
+  if (test.suite) {
+    testMessage += '\n SUITE: ' + test.suite.title
+  }
+  if (test.parent) {
+    testMessage += '\n SUITE: ' + test.parent.title
+  }
+
+  if (test.steps?.length) {
+    const failedSteps = test.steps
+    if (failedSteps.length) testMessage += '\n STEP: ' + failedSteps.map(s => s.toCode()).join('; ')
+  }
+
+  const pageInfo = test.notes.find(n => n.type === 'pageInfo')
+  if (pageInfo) {
+    testMessage += '\n PAGE INFO: ' + pageInfo.text
+  }
+
+  return testMessage
+}
+
+function formatResponse(response) {
+  return response
+    .replace(/<think>([\s\S]*?)<\/think>/g, store.debugMode ? colors.cyan('$1') : '')
+    .split('\n')
+    .map(line => line.trim())
+    .filter(line => !/^[A-Z\s]+$/.test(line))
+    .map(line => markdownToAnsi(line))
+    .map(line => line.replace(/^x /gm, `    ${colors.red.bold('x')} `))
+    .join('\n')
+    .trim()
+}

package/lib/plugin/{autoLogin.js → auth.js}

@@ -1,14 +1,16 @@
-const fs = require('fs')
-const path = require('path')
-const { fileExists } = require('../utils')
-const container = require('../container')
-const store = require('../store')
-const recorder = require('../recorder')
-const { debug } = require('../output')
-const isAsyncFunction = require('../utils').isAsyncFunction
+import fs from 'fs'
+import path from 'path'
+import { fileExists, isAsyncFunction } from '../utils.js'
+import CommentStep from '../step/comment.js'
+import Section from '../step/section.js'
+import container from '../container.js'
+import store from '../store.js'
+import event from '../event.js'
+import recorder from '../recorder.js'
+import output from '../output.js'
 
 const defaultUser = {
-  fetch: (I) => I.grabCookie(),
+  fetch: I => I.grabCookie(),
   check: () => {},
   restore: (I, cookies) => {
     I.amOnPage('/') // open a page
@@ -72,7 +74,7 @@ const defaultConfig = {
  * #### Example: Simple login
  *
  * ```js
- * autoLogin: {
+ * auth: {
  *   enabled: true,
  *   saveToFile: true,
  *   inject: 'login',
@@ -93,7 +95,7 @@ const defaultConfig = {
  * #### Example: Multiple users
  *
  * ```js
- * autoLogin: {
+ * auth: {
  *   enabled: true,
  *   saveToFile: true,
  *   inject: 'loginAs', // use `loginAs` instead of login
@@ -140,7 +142,7 @@ const defaultConfig = {
  *    }
  * },
  * plugins: {
- *    autoLogin: {
+ *    auth: {
  *      users: {
  *        admin: {
  *          login: (I) => {
@@ -167,7 +169,7 @@ const defaultConfig = {
  *
  * ```js
  * plugins: {
- *    autoLogin: {
+ *    auth: {
  *     admin: {
  *       login: (I) => I.loginAsAdmin(),
  *       check: (I) => I.see('Admin', '.navbar'),
@@ -183,18 +185,18 @@ const defaultConfig = {
  * }
  * ```
  *
- * #### Tips: Using async function in the autoLogin
+ * #### Tips: Using async function in the auth
  *
- * If you use async functions in the autoLogin plugin, login function should be used with `await` keyword.
+ * If you use async functions in the auth plugin, login function should be used with `await` keyword.
  *
  * ```js
- * autoLogin: {
+ * auth: {
  *   enabled: true,
  *   saveToFile: true,
  *   inject: 'login',
  *   users: {
  *     admin: {
- *       login: async (I) => {  // If you use async function in the autoLogin plugin
+ *       login: async (I) => {  // If you use async function in the auth plugin
  *          const phrase = await I.grabTextFrom('#phrase')
  *          I.fillField('username', 'admin'),
  *          I.fillField('password', 'password')
@@ -220,13 +222,13 @@ const defaultConfig = {
  * Instead of asserting on page elements for the current user in `check`, you can use the `session` you saved in `fetch`
  *
  * ```js
- * autoLogin: {
+ * auth: {
  *   enabled: true,
  *   saveToFile: true,
  *   inject: 'login',
  *   users: {
  *     admin: {
- *       login: async (I) => {  // If you use async function in the autoLogin plugin
+ *       login: async (I) => {  // If you use async function in the auth plugin
  *          const phrase = await I.grabTextFrom('#phrase')
  *          I.fillField('username', 'admin'),
  *          I.fillField('password', 'password')
@@ -250,10 +252,10 @@ const defaultConfig = {
  *
  *
  */
-module.exports = function (config) {
+export default function (config) {
   config = Object.assign(defaultConfig, config)
   Object.keys(config.users).map(
-    (u) =>
+    u =>
       (config.users[u] = {
         ...defaultUser,
         ...config.users[u],
@@ -262,25 +264,47 @@ module.exports = function (config) {
 
   if (config.saveToFile) {
     // loading from file
-    for (const name in config.users) {
-      const fileName = path.join(global.output_dir, `${name}_session.json`)
-      if (!fileExists(fileName)) continue
-      const data = fs.readFileSync(fileName).toString()
-      try {
-        store[`${name}_session`] = JSON.parse(data)
-      } catch (err) {
-        throw new Error(`Could not load session from ${fileName}\n${err}`)
-      }
-      debug(`Loaded user session for ${name}`)
-    }
+    loadCookiesFromFile(config)
   }
 
-  const loginFunction = async (name) => {
-    const userSession = config.users[name]
+  const loginFunction = async name => {
     const I = container.support('I')
+    const userSession = config.users[name]
+
+    if (!userSession) {
+      throw new Error(`User '${name}' was not configured for authorization in auth plugin. Add it to the plugin config`)
+    }
+
+    const test = store.currentTest
+
+    // we are in BeforeSuite hook
+    if (!test) {
+      enableAuthBeforeEachTest(name)
+      return
+    }
+
+    const section = new Section(`I am logged in as ${name}`)
+
+    if (config.saveToFile && !store[`${name}_session`]) {
+      loadCookiesFromFile(config)
+    }
+
+    if (isPlaywrightSession() && test?.opts?.cookies) {
+      if (test.opts.user == name) {
+        output.debug(`Cookies already loaded for ${name}`)
+
+        alreadyLoggedIn(name)
+        return
+      } else {
+        output.debug(`Cookies already loaded for ${test.opts.user}, but not for ${name}`)
+        await I.deleteCookie()
+      }
+    }
+
+    section.start()
+
     const cookies = store[`${name}_session`]
-    const shouldAwait =
-      isAsyncFunction(userSession.login) || isAsyncFunction(userSession.restore) || isAsyncFunction(userSession.check)
+    const shouldAwait = isAsyncFunction(userSession.login) || isAsyncFunction(userSession.restore) || isAsyncFunction(userSession.check)
 
     const loginAndSave = async () => {
       if (shouldAwait) {
@@ -289,13 +313,14 @@ module.exports = function (config) {
         userSession.login(I)
       }
 
+      section.end()
       const cookies = await userSession.fetch(I)
       if (!cookies) {
-        debug("Cannot save user session with empty cookies from auto login's fetch method")
+        output.debug("Cannot save user session with empty cookies from auto login's fetch method")
         return
       }
       if (config.saveToFile) {
-        debug(`Saved user session into file for ${name}`)
+        output.debug(`Saved user session into file for ${name}`)
         fs.writeFileSync(path.join(global.output_dir, `${name}_session.json`), JSON.stringify(cookies))
       }
       store[`${name}_session`] = cookies
@@ -311,18 +336,20 @@ module.exports = function (config) {
       userSession.restore(I, cookies)
       userSession.check(I, cookies)
     }
-    recorder.session.catch((err) => {
-      debug(`Failed auto login for ${name} due to ${err}`)
-      debug('Logging in again')
+    section.end()
+    recorder.session.catch(err => {
+      output.debug(`Failed auto login for ${name} due to ${err}`)
+      output.debug('Logging in again')
       recorder.session.start('auto login')
       return loginAndSave()
         .then(() => {
           recorder.add(() => recorder.session.restore('auto login'))
-          recorder.catch(() => debug('continue'))
+          recorder.catch(() => output.debug('continue'))
         })
-        .catch((err) => {
+        .catch(err => {
           recorder.session.restore('auto login')
           recorder.session.restore('check login')
+          section.end()
           recorder.throw(err)
         })
     })
@@ -333,8 +360,75 @@ module.exports = function (config) {
     return recorder.promise()
   }
 
+  function enableAuthBeforeEachTest(name) {
+    const suite = store.currentSuite
+    if (!suite) return
+
+    output.debug(`enabling auth as ${name} for each test of suite ${suite.title}`)
+
+    // we are setting test opts so they can be picked up by Playwright if it starts browser for this test
+    suite.eachTest(test => {
+      // preload from store
+      if (store[`${name}_session`]) {
+        test.opts.cookies = store[`${name}_session`]
+        test.opts.user = name
+        return
+      }
+
+      if (!config.saveToFile) return
+      const cookieFile = path.join(global.output_dir, `${name}_session.json`)
+
+      if (!fileExists(cookieFile)) {
+        return
+      }
+
+      const context = fs.readFileSync(cookieFile).toString()
+      test.opts.cookies = JSON.parse(context)
+      test.opts.user = name
+    })
+
+    function runLoginFunctionForTest(test) {
+      if (!suite.tests.includes(test)) return
+      // let's call this function to ensure that authorization happened
+      // if no cookies, it will login and save them
+      loginFunction(name)
+    }
+
+    // we are in BeforeSuite hook
+    event.dispatcher.on(event.test.started, runLoginFunctionForTest)
+    event.dispatcher.on(event.suite.after, () => {
+      event.dispatcher.off(event.test.started, runLoginFunctionForTest)
+    })
+  }
+
   // adding this to DI container
   const support = {}
   support[config.inject] = loginFunction
   container.append({ support })
+
+  return loginFunction
+}
+
+function loadCookiesFromFile(config) {
+  for (const name in config.users) {
+    const fileName = path.join(global.output_dir, `${name}_session.json`)
+    if (!fileExists(fileName)) continue
+    const data = fs.readFileSync(fileName).toString()
+    try {
+      store[`${name}_session`] = JSON.parse(data)
+    } catch (err) {
+      throw new Error(`Could not load session from ${fileName}\n${err}`)
+    }
+    output.debug(`Loaded user session for ${name}`)
+  }
+}
+
+function isPlaywrightSession() {
+  return !!container.helpers('Playwright')
+}
+
+function alreadyLoggedIn(name) {
+  const step = new CommentStep('am logged in as')
+  step.actor = 'I'
+  return step.addToRecorder([name])
 }