codeceptjs 3.4.0 → 3.5.0

package/lib/plugin/debugErrors.js

@@ -0,0 +1,67 @@
+const Container = require('../container');
+const recorder = require('../recorder');
+const event = require('../event');
+const supportedHelpers = require('./standardActingHelpers');
+const { scanForErrorMessages } = require('../html');
+const { output } = require('..');
+
+const defaultConfig = {
+  errorClasses: ['error', 'warning', 'alert', 'danger'],
+};
+
+/**
+ * Prints errors found in HTML code after each failed test.
+ *
+ * It scans HTML and searches for elements with error classes.
+ * If an element found prints a text from it to console and adds as artifact to the test.
+ *
+ * Enable this plugin in config:
+ *
+ * ```js
+ * plugins: {
+ *  debugErrors: {
+ *   enabled: true,
+ * }
+ * ```
+ *
+ * Additional config options:
+ *
+ * * `errorClasses` - list of classes to search for errors (default: `['error', 'warning', 'alert', 'danger']`)
+ *
+ */
+module.exports = function (config = {}) {
+  const helpers = Container.helpers();
+  let helper;
+
+  config = Object.assign(defaultConfig, config);
+
+  for (const helperName of supportedHelpers) {
+    if (Object.keys(helpers).indexOf(helperName) > -1) {
+      helper = helpers[helperName];
+    }
+  }
+
+  if (!helper) return; // no helpers for screenshot
+
+  event.dispatcher.on(event.test.failed, (test) => {
+    recorder.add('HTML snapshot failed test', async () => {
+      try {
+        const currentOutputLevel = output.level();
+        output.level(0);
+        const html = await helper.grabHTMLFrom('body');
+        output.level(currentOutputLevel);
+
+        if (!html) return;
+
+        const errors = scanForErrorMessages(html, config.errorClasses);
+        if (errors.length) {
+          output.debug('Detected errors in HTML code');
+          errors.forEach((error) => output.debug(error));
+          test.artifacts.errors = errors;
+        }
+      } catch (err) {
+        // not really needed
+      }
+    });
+  });
+};

package/lib/plugin/fakerTransform.js

@@ -2,20 +2,18 @@ const { faker } = require('@faker-js/faker');
 const transform = require('../transform');
 
 /**
- * Use the [faker.js](https://www.npmjs.com/package/faker) package to generate fake data inside examples on your gherkin tests
- *
- * ![Faker.js](https://raw.githubusercontent.com/Marak/faker.js/master/logo.png)
+ * Use the `@faker-js/faker` package to generate fake data inside examples on your gherkin tests
  *
  * #### Usage
  *
- * To start please install `faker.js` package
+ * To start please install `@faker-js/faker` package
  *
  * ```
- * npm install -D faker
+ * npm install -D @faker-js/faker
  * ```
  *
  * ```
- * yarn add -D faker
+ * yarn add -D @faker-js/faker
  * ```
  *
  * Add this plugin to config file:

package/lib/plugin/heal.js

@@ -0,0 +1,179 @@
+const debug = require('debug')('codeceptjs:heal');
+const colors = require('chalk');
+const Container = require('../container');
+const AiAssistant = require('../ai');
+const recorder = require('../recorder');
+const event = require('../event');
+const output = require('../output');
+const supportedHelpers = require('./standardActingHelpers');
+
+const defaultConfig = {
+  healLimit: 2,
+  healSteps: [
+    'click',
+    'fillField',
+    'appendField',
+    'selectOption',
+    'attachFile',
+    'checkOption',
+    'uncheckOption',
+    'doubleClick',
+  ],
+};
+
+/**
+ * Self-healing tests with OpenAI.
+ *
+ * This plugin is experimental and requires OpenAI API key.
+ *
+ * To use it you need to set OPENAI_API_KEY env variable and enable plugin inside condig.
+ *
+ * ```js
+ * plugins: {
+ *   heal: {
+ *    enabled: true,
+ *   }
+ * }
+ * ```
+ *
+ * More config options are available:
+ *
+ * * `healLimit` - how many steps can be healed in a single test (default: 2)
+ * * `healSteps` - which steps can be healed (default: all steps that interact with UI, see list below)
+ *
+ * Steps to heal:
+ *
+ * * `click`
+ * * `fillField`
+ * * `appendField`
+ * * `selectOption`
+ * * `attachFile`
+ * * `checkOption`
+ * * `uncheckOption`
+ * * `doubleClick`
+ *
+ */
+module.exports = function (config = {}) {
+  const aiAssistant = new AiAssistant();
+
+  let currentTest = null;
+  let currentStep = null;
+  let healedSteps = 0;
+
+  const healSuggestions = [];
+
+  config = Object.assign(defaultConfig, config);
+
+  event.dispatcher.on(event.test.before, (test) => {
+    currentTest = test;
+    healedSteps = 0;
+  });
+
+  event.dispatcher.on(event.step.started, step => currentStep = step);
+
+  event.dispatcher.on(event.step.before, () => {
+    const store = require('../store');
+    if (store.debugMode) return;
+
+    recorder.catchWithoutStop(async (err) => {
+      if (!aiAssistant.isEnabled) throw err;
+      if (!currentStep) throw err;
+      if (!config.healSteps.includes(currentStep.name)) throw err;
+      const test = currentTest;
+
+      if (healedSteps >= config.healLimit) {
+        output.print(colors.bold.red(`Can't heal more than ${config.healLimit} steps in a test`));
+        output.print('Entire flow can be broken, please check it manually');
+        output.print('or increase healing limit in heal plugin config');
+
+        throw err;
+      }
+
+      recorder.session.start('heal');
+      const helpers = Container.helpers();
+      let helper;
+
+      for (const helperName of supportedHelpers) {
+        if (Object.keys(helpers).indexOf(helperName) > -1) {
+          helper = helpers[helperName];
+        }
+      }
+
+      if (!helper) throw err; // no helpers for html
+
+      const step = test.steps[test.steps.length - 1];
+      debug('Self-healing started', step.toCode());
+
+      const currentOutputLevel = output.level();
+      output.level(0);
+      const html = await helper.grabHTMLFrom('body');
+      output.level(currentOutputLevel);
+
+      if (!html) throw err;
+
+      aiAssistant.setHtmlContext(html);
+      await tryToHeal(step, err);
+      recorder.session.restore();
+    });
+  });
+
+  event.dispatcher.on(event.all.result, () => {
+    if (!healSuggestions.length) return;
+
+    const { print } = output;
+
+    print('');
+    print('===================');
+    print(colors.bold.green('Self-Healing Report:'));
+
+    print(`${colors.bold(healSuggestions.length)} steps were healed by AI`);
+
+    let i = 1;
+    print('');
+    print('Suggested changes:');
+    print('');
+
+    for (const suggestion of healSuggestions) {
+      print(`${i}. To fix ${colors.bold.blue(suggestion.test.title)}`);
+      print('Replace the failed code with:');
+      print(colors.red(`- ${suggestion.step.toCode()}`));
+      print(colors.green(`+ ${suggestion.snippet}`));
+      print(suggestion.step.line());
+      print('');
+      i++;
+    }
+  });
+
+  async function tryToHeal(failedStep, err) {
+    output.debug(`Running OpenAPI to heal ${failedStep.toCode()} step`);
+
+    const I = Container.support('I');
+
+    const codeSnippets = await aiAssistant.healFailedStep(
+      failedStep, err, currentTest,
+    );
+
+    output.debug(`Received ${codeSnippets.length} proposals from OpenAI`);
+
+    for (const codeSnippet of codeSnippets) {
+      try {
+        debug('Executing', codeSnippet);
+        await eval(codeSnippet); // eslint-disable-line
+
+        healSuggestions.push({
+          test: currentTest,
+          step: failedStep,
+          snippet: codeSnippet,
+        });
+
+        output.print(colors.bold.green('  Code healed successfully'));
+        healedSteps++;
+        return;
+      } catch (err) {
+        debug('Failed to execute code', err);
+      }
+    }
+
+    output.debug(`Couldn't heal the code for ${failedStep.toCode()}`);
+  }
+};

package/lib/plugin/screenshotOnFail.js

@@ -75,7 +75,8 @@ module.exports = function (config) {
   event.dispatcher.on(event.test.failed, (test) => {
     recorder.add('screenshot of failed test', async () => {
       let fileName = clearString(test.title);
-      // This prevent data driven to be included in the failed screenshot file name
+      const dataType = 'image/png';
+      // This prevents data driven to be included in the failed screenshot file name
       if (fileName.indexOf('{') !== -1) {
         fileName = fileName.substr(0, (fileName.indexOf('{') - 3)).trim();
       }
@@ -106,7 +107,15 @@ module.exports = function (config) {
 
         const allureReporter = Container.plugins('allure');
         if (allureReporter) {
-          allureReporter.addAttachment('Last Seen Screenshot', fs.readFileSync(path.join(global.output_dir, fileName)), 'image/png');
+          allureReporter.addAttachment('Main session - Last Seen Screenshot', fs.readFileSync(path.join(global.output_dir, fileName)), dataType);
+
+          if (helper.activeSessionName) {
+            for (const sessionName in helper.sessionPages) {
+              const screenshotFileName = `${sessionName}_${fileName}`;
+              test.artifacts[`${sessionName.replace(/ /g, '_')}_screenshot`] = path.join(global.output_dir, screenshotFileName);
+              allureReporter.addAttachment(`${sessionName} - Last Seen Screenshot`, fs.readFileSync(path.join(global.output_dir, screenshotFileName)), dataType);
+            }
+          }
         }
 
         const cucumberReporter = Container.plugins('cucumberJsonReporter');

package/lib/plugin/wdio.js

@@ -1,6 +1,4 @@
 const debug = require('debug')('codeceptjs:plugin:wdio');
-const path = require('path');
-const fs = require('fs');
 
 const container = require('../container');
 const mainConfig = require('../config');
@@ -100,21 +98,15 @@ module.exports = (config) => {
   const launchers = [];
 
   for (const name of config.services) {
-    // webdriverio v5 style
     const Service = safeRequire(`@wdio/${name.toLowerCase()}-service`);
     if (Service) {
       if (Service.launcher && typeof Service.launcher === 'function') {
         const Launcher = Service.launcher;
 
-        const version = JSON.parse(fs.readFileSync(path.join(require.resolve('webdriverio'), '/../../', 'package.json')).toString()).version;
-        if (version.indexOf('5') === 0) {
-          launchers.push(new Launcher(config));
-        } else {
-          const options = {
-            logPath: global.output_dir, installArgs: seleniumInstallArgs, args: seleniumArgs, ...wdioOptions,
-          };
-          launchers.push(new Launcher(options, [config.capabilities], config));
-        }
+        const options = {
+          logPath: global.output_dir, installArgs: seleniumInstallArgs, args: seleniumArgs, ...wdioOptions,
+        };
+        launchers.push(new Launcher(options, [config.capabilities], config));
       }
       if (typeof Service === 'function') {
         services.push(new Service(config, config.capabilities));

package/lib/recorder.js

@@ -161,7 +161,7 @@ module.exports = {
    *     true: it will retries if `retryOpts` set.
    *     false: ignore `retryOpts` and won't retry.
    * @param {number} [timeout]
-   * @return {Promise<*> | undefined}
+   * @return {Promise<*>}
    * @inner
    */
   add(taskName, fn = undefined, force = false, retry = undefined, timeout = undefined) {
@@ -250,9 +250,9 @@ module.exports = {
         err = new Error(`[Wrapped Error] ${JSON.stringify(err)}`); // we should be prepared for them
       }
       if (customErrFn) {
-        customErrFn(err);
-      } else if (errFn) {
-        errFn(err);
+        return customErrFn(err);
+      } if (errFn) {
+        return errFn(err);
       }
     });
   },

package/lib/scenario.js

@@ -137,16 +137,18 @@ module.exports.injected = function (fn, suite, hookName) {
     const opts = suite.opts || {};
     const retries = opts[`retry${ucfirst(hookName)}`] || 0;
 
-    promiseRetry(async (retry) => {
+    promiseRetry(async (retry, number) => {
       try {
         recorder.startUnlessRunning();
         await fn.call(this, getInjectedArguments(fn));
-        await recorder.promise();
+        await recorder.promise().catch(err => retry(err));
       } catch (err) {
         retry(err);
       } finally {
-        recorder.stop();
-        recorder.start();
+        if (number < retries) {
+          recorder.stop();
+          recorder.start();
+        }
       }
     }, { retries })
       .then(() => {

package/lib/secret.js

@@ -1,6 +1,8 @@
 /* eslint-disable max-classes-per-file */
 const { deepClone } = require('./utils');
 
+const maskedString = '*****';
+
 /** @param {string} secret */
 class Secret {
   constructor(secret) {
@@ -13,7 +15,7 @@ class Secret {
   }
 
   getMasked() {
-    return '*****';
+    return maskedString;
   }
 
   /**
@@ -36,12 +38,11 @@ function secretObject(obj, fieldsToHide = []) {
       if (prop === 'toString') {
         return function () {
           const maskedObject = deepClone(obj);
-          fieldsToHide.forEach(f => maskedObject[f] = '****');
+          fieldsToHide.forEach(f => maskedObject[f] = maskedString);
           return JSON.stringify(maskedObject);
         };
       }
-
-      return obj[prop];
+      return fieldsToHide.includes(prop) ? new Secret(obj[prop]) : obj[prop];
     },
   };
 

package/lib/step.js

@@ -172,7 +172,12 @@ class Step {
       } else if (arg.toString && arg.toString() !== '[object Object]') {
         return arg.toString();
       } else if (typeof arg === 'object') {
-        return JSON.stringify(arg);
+        const returnedArg = {};
+        for (const [key, value] of Object.entries(arg)) {
+          returnedArg[key] = value;
+          if (value instanceof Secret) returnedArg[key] = value.getMasked();
+        }
+        return JSON.stringify(returnedArg);
       }
       return arg;
     }).join(', ');

package/lib/ui.js

@@ -23,10 +23,11 @@ const setContextTranslation = (context) => {
 /**
  * Codecept-style interface:
  *
- * Feature('login')
- * Scenario('login as regular user', (I) {
+ * Feature('login');
+ *
+ * Scenario('login as regular user', ({I}) {
  *   I.fillField();
- *   I.click()
+ *   I.click();
  *   I.see('Hello, '+data.login);
  * });
  *

package/lib/utils.js

@@ -455,3 +455,7 @@ module.exports.isNotSet = function (obj) {
   if (obj === undefined) return true;
   return false;
 };
+
+module.exports.emptyFolder = async (directoryPath) => {
+  require('child_process').execSync(`rm -rf ${directoryPath}/*`);
+};

package/lib/workers.js

@@ -10,12 +10,14 @@ const MochaFactory = require('./mochaFactory');
 const Container = require('./container');
 const { getTestRoot } = require('./command/utils');
 const { isFunction, fileExists } = require('./utils');
+const { replaceValueDeep, deepClone } = require('./utils');
 const mainConfig = require('./config');
 const output = require('./output');
 const event = require('./event');
 const recorder = require('./recorder');
 const runHook = require('./hooks');
 const WorkerStorage = require('./workerStorage');
+const collection = require('./command/run-multiple/collection');
 
 const pathToWorker = path.join(__dirname, 'command', 'workers', 'runTests.js');
 
@@ -79,15 +81,39 @@ const repackTest = (test) => {
   return test;
 };
 
-const createWorkerObjects = (testGroups, config, testRoot, options) => {
-  return testGroups.map((tests, index) => {
-    const workerObj = new WorkerObject(index);
-    workerObj.addConfig(config);
-    workerObj.addTests(tests);
-    workerObj.setTestRoot(testRoot);
-    workerObj.addOptions(options);
-    return workerObj;
+const createWorkerObjects = (testGroups, config, testRoot, options, selectedRuns) => {
+  selectedRuns = options && options.all && config.multiple ? Object.keys(config.multiple) : selectedRuns;
+  if (selectedRuns === undefined || !selectedRuns.length || config.multiple === undefined) {
+    return testGroups.map((tests, index) => {
+      const workerObj = new WorkerObject(index);
+      workerObj.addConfig(config);
+      workerObj.addTests(tests);
+      workerObj.setTestRoot(testRoot);
+      workerObj.addOptions(options);
+      return workerObj;
+    });
+  }
+  const workersToExecute = [];
+  collection.createRuns(selectedRuns, config).forEach((worker) => {
+    const workerName = worker.getOriginalName() || worker.getName();
+    const workerConfig = worker.getConfig();
+    workersToExecute.push(getOverridenConfig(workerName, workerConfig, config));
+  });
+  const workers = [];
+  let index = 0;
+  testGroups.forEach((tests) => {
+    const testWorkerArray = [];
+    workersToExecute.forEach((finalConfig) => {
+      const workerObj = new WorkerObject(index++);
+      workerObj.addConfig(finalConfig);
+      workerObj.addTests(tests);
+      workerObj.setTestRoot(testRoot);
+      workerObj.addOptions(options);
+      testWorkerArray.push(workerObj);
+    });
+    workers.push(...testWorkerArray);
   });
+  return workers;
 };
 
 const indexOfSmallestElement = (groups) => {
@@ -115,6 +141,28 @@ const convertToMochaTests = (testGroup) => {
   return group;
 };
 
+const getOverridenConfig = (workerName, workerConfig, config) => {
+  // clone config
+  const overriddenConfig = deepClone(config);
+
+  // get configuration
+  const browserConfig = workerConfig.browser;
+
+  for (const key in browserConfig) {
+    overriddenConfig.helpers = replaceValueDeep(overriddenConfig.helpers, key, browserConfig[key]);
+  }
+
+  // override tests configuration
+  if (overriddenConfig.tests) {
+    overriddenConfig.tests = workerConfig.tests;
+  }
+
+  if (overriddenConfig.gherkin && workerConfig.gherkin && workerConfig.gherkin.features) {
+    overriddenConfig.gherkin.features = workerConfig.gherkin.features;
+  }
+  return overriddenConfig;
+};
+
 class WorkerObject {
   /**
    * @param {Number} workerIndex - Unique ID for worker
@@ -183,7 +231,7 @@ class Workers extends EventEmitter {
 
   _initWorkers(numberOfWorkers, config) {
     this.splitTestsByGroups(numberOfWorkers, config);
-    this.workers = createWorkerObjects(this.testGroups, this.codecept.config, config.testConfig, config.options);
+    this.workers = createWorkerObjects(this.testGroups, this.codecept.config, config.testConfig, config.options, config.selectedRuns);
     this.numberOfWorkers = this.workers.length;
   }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codeceptjs",
-  "version": "3.4.0",
+  "version": "3.5.0",
   "description": "Supercharged End 2 End Testing Framework for NodeJS",
   "keywords": [
     "acceptance",
@@ -45,8 +45,14 @@
     "test": "npm run test:unit && npm run test:runner",
     "test:appium-quick": "mocha test/helper/Appium_test.js --grep 'quick'",
     "test:appium-other": "mocha test/helper/Appium_test.js --grep 'second'",
+    "test-app:start": "php -S 127.0.0.1:8000 -t test/data/app",
+    "test-app:stop": "kill -9 $(lsof -t -i:8000)",
+    "test:unit:webbapi:playwright": "mocha test/helper/Playwright_test.js",
+    "test:unit:webbapi:puppeteer": "mocha test/helper/Puppeteer_test.js",
+    "test:unit:webbapi:webDriver": "mocha test/helper/WebDriver_test.js",
+    "test:unit:webbapi:testCafe": "mocha test/helper/TestCafe_test.js",
     "def": "./runok.js def",
-    "dev:graphql": "nodemon test/data/graphql/index.js",
+    "dev:graphql": "node test/data/graphql/index.js",
     "publish:site": "./runok.js publish:site",
     "update-contributor-faces": "contributor-faces .",
     "dtslint": "dtslint typings --localTs './node_modules/typescript/lib'",
@@ -58,13 +64,14 @@
     "@cucumber/cucumber-expressions": "^16",
     "@cucumber/gherkin": "^26",
     "@cucumber/messages": "^21.0.1",
+    "@xmldom/xmldom": "^0.7.9",
     "acorn": "^7.4.1",
     "arrify": "^2.0.1",
     "axios": "^1.3.3",
     "chai": "^4.3.6",
     "chai-deep-match": "^1.2.1",
     "chalk": "^4.1.2",
-    "commander": "^2.20.3",
+    "commander": "^11.0.0",
     "cross-spawn": "^7.0.3",
     "css-to-xpath": "^0.1.0",
     "envinfo": "^7.8.1",
@@ -73,16 +80,22 @@
     "fn-args": "^4.0.0",
     "fs-extra": "^8.1.0",
     "glob": "^6.0.1",
+    "html-minifier": "^4.0.0",
+    "i": "^0.3.7",
     "inquirer": "^6.5.2",
     "joi": "^17.6.0",
     "js-beautify": "^1.14.0",
     "lodash.clonedeep": "^4.5.0",
     "lodash.merge": "^4.6.2",
     "mkdirp": "^1.0.4",
-    "mocha": "^8.2.0",
+    "mocha": "^10.2.0",
     "mocha-junit-reporter": "^1.23.3",
     "ms": "^2.1.3",
+    "npm": "^9.6.7",
+    "openai": "^3.2.1",
+    "ora-classic": "^5.4.2",
     "parse-function": "^5.6.4",
+    "parse5": "^7.1.2",
     "promise-retry": "^1.1.1",
     "resq": "^1.10.2",
     "sprintf-js": "^1.1.1",
@@ -92,14 +105,14 @@
     "@codeceptjs/detox-helper": "^1.0.2",
     "@codeceptjs/mock-request": "^0.3.1",
     "@faker-js/faker": "^7.6.0",
-    "@pollyjs/adapter-puppeteer": "^5.1.0",
+    "@pollyjs/adapter-puppeteer": "^6.0.5",
     "@pollyjs/core": "^5.1.0",
     "@types/inquirer": "^0.0.35",
     "@types/node": "^8.10.66",
-    "@wdio/sauce-service": "^5.22.5",
-    "@wdio/selenium-standalone-service": "^5.16.10",
-    "@wdio/utils": "^5.23.0",
-    "apollo-server-express": "^2.25.3",
+    "@wdio/sauce-service": "^8.3.8",
+    "@wdio/selenium-standalone-service": "^8.3.2",
+    "@wdio/utils": "^8.3.0",
+    "apollo-server-express": "2.25.3",
     "chai-as-promised": "^7.1.1",
     "chai-subset": "^1.6.0",
     "contributor-faces": "^1.0.3",
@@ -120,11 +133,10 @@
     "jsdoc-typeof-plugin": "^1.0.0",
     "json-server": "^0.10.1",
     "nightmare": "^3.0.2",
-    "nodemon": "^1.19.4",
-    "playwright": "^1.23.2",
+    "playwright": "^1.35.1",
     "puppeteer": "^10.4.0",
     "qrcode-terminal": "^0.12.0",
-    "rosie": "^1.6.0",
+    "rosie": "^2.1.0",
     "runok": "^0.9.2",
     "sinon": "^9.2.4",
     "sinon-chai": "^3.7.0",
@@ -138,11 +150,11 @@
     "wdio-docker-service": "^1.5.0",
     "webdriverio": "^8.3.8",
     "xml2js": "^0.4.23",
-    "xmldom": "^0.1.31",
+    "xmldom": "^0.5.0",
     "xpath": "0.0.27"
   },
   "engines": {
-    "node": ">=8.9.1",
+    "node": ">=16.0",
     "npm": ">=5.6.0"
   },
   "es6": true