codeceptjs 3.4.1 → 3.5.1-2.beta.7

package/lib/pause.js

@@ -1,9 +1,12 @@
 const colors = require('chalk');
 const readline = require('readline');
+const ora = require('ora-classic');
+const debug = require('debug')('codeceptjs:pause');
 
 const container = require('./container');
 const history = require('./history');
 const store = require('./store');
+const AiAssistant = require('./ai');
 const recorder = require('./recorder');
 const event = require('./event');
 const output = require('./output');
@@ -15,8 +18,10 @@ let nextStep;
 let finish;
 let next;
 let registeredVariables = {};
+let aiAssistant;
 /**
  * Pauses test execution and starts interactive shell
+ * @param {Object<string, *>} [passedObject]
  */
 const pause = function (passedObject = {}) {
   if (store.dryRun) return;
@@ -39,12 +44,22 @@ function pauseSession(passedObject = {}) {
     let vars = Object.keys(registeredVariables).join(', ');
     if (vars) vars = `(vars: ${vars})`;
 
+    aiAssistant = AiAssistant.getInstance();
+
     output.print(colors.yellow(' Interactive shell started'));
     output.print(colors.yellow(' Use JavaScript syntax to try steps in action'));
     output.print(colors.yellow(` - Press ${colors.bold('ENTER')} to run the next step`));
     output.print(colors.yellow(` - Press ${colors.bold('TAB')} twice to see all available commands`));
     output.print(colors.yellow(` - Type ${colors.bold('exit')} + Enter to exit the interactive shell`));
     output.print(colors.yellow(` - Prefix ${colors.bold('=>')} to run js commands ${colors.bold(vars)}`));
+
+    if (aiAssistant.isEnabled) {
+      output.print(colors.blue(` ${colors.bold('OpenAI is enabled! (experimental)')} Write what you want and make OpenAI run it`));
+      output.print(colors.blue(' Please note, only HTML fragments with interactive elements are sent to OpenAI'));
+      output.print(colors.blue(' Ideas: ask it to fill forms for you or to click'));
+    } else {
+      output.print(colors.blue(` Enable OpenAI assistant by setting ${colors.bold('OPENAI_API_KEY')} env variable`));
+    }
   }
   rl = readline.createInterface(process.stdin, process.stdout, completer);
 
@@ -54,15 +69,16 @@ function pauseSession(passedObject = {}) {
   });
   return new Promise(((resolve) => {
     finish = resolve;
+    // eslint-disable-next-line
     return askForStep();
   }));
 }
 
 /* eslint-disable */
-function parseInput(cmd) {
+async function parseInput(cmd) {
   rl.pause();
   next = false;
-  store.debugMode = false;
+  recorder.session.start('pause');
   if (cmd === '') next = true;
   if (!cmd || cmd === 'resume' || cmd === 'exit') {
     finish();
@@ -74,37 +90,79 @@ function parseInput(cmd) {
   for (const k of Object.keys(registeredVariables)) {
     eval(`var ${k} = registeredVariables['${k}'];`); // eslint-disable-line no-eval
   }
-  store.debugMode = true;
+
+  let executeCommand = Promise.resolve();
+
+  const getCmd = () => {
+    debug('Command:', cmd)
+    return cmd;
+  };
+
   let isCustomCommand = false;
   let lastError = null;
+  let isAiCommand = false;
+  let $res;
   try {
+    // eslint-disable-next-line
     const locate = global.locate; // enable locate in this context
+    // eslint-disable-next-line
     const I = container.support('I');
     if (cmd.trim().startsWith('=>')) {
       isCustomCommand = true;
       cmd = cmd.trim().substring(2, cmd.length);
+    } else if (aiAssistant.isEnabled && !cmd.match(/^\w+\(/) && cmd.includes(' ')) {
+      const currentOutputLevel = output.level();
+      output.level(0);
+      const res = I.grabSource();
+      isAiCommand = true;
+      executeCommand = executeCommand.then(async () => {
+        try {
+          const html = await res;
+          await aiAssistant.setHtmlContext(html);
+        } catch (err) {
+          output.print(output.styles.error(' ERROR '), 'Can\'t get HTML context', err.stack);
+          return;
+        } finally {
+          output.level(currentOutputLevel);
+        }
+        // aiAssistant.mockResponse("```js\nI.click('Sign in');\n```");
+        const spinner = ora("Processing OpenAI request...").start();
+        cmd = await aiAssistant.writeSteps(cmd);
+        spinner.stop();
+        output.print('');
+        output.print(colors.blue(aiAssistant.getResponse()));
+        output.print('');
+        return cmd;
+      })
     } else {
       cmd = `I.${cmd}`;
     }
-    const executeCommand = eval(cmd); // eslint-disable-line no-eval
-
-    const result = executeCommand instanceof Promise ? executeCommand : Promise.resolve(executeCommand);
-    result.then((val) => {
-      if (isCustomCommand) {
-        console.log(val);
-        return;
-      }
-      if (cmd.startsWith('I.see') || cmd.startsWith('I.dontSee')) {
-        output.print(output.styles.success('  OK  '), cmd);
-        return;
-      }
-      if (cmd.startsWith('I.grab')) {
-        output.print(output.styles.debug(val));
-      }
+    executeCommand = executeCommand.then(async () => {
+      const cmd = getCmd();
+      if (!cmd) return;
+      return eval(cmd); // eslint-disable-line no-eval
     }).catch((err) => {
+      debug(err);
+      if (isAiCommand) return;
       if (!lastError) output.print(output.styles.error(' ERROR '), err.message);
+      debug(err.stack)
+
       lastError = err.message;
-    });
+    })
+
+    const val = await executeCommand;
+
+    if (isCustomCommand) {
+      if (val !== undefined) console.log('Result', '$res=', val); // eslint-disable-line
+      $res = val;
+    }
+
+    if (cmd?.startsWith('I.see') || cmd?.startsWith('I.dontSee')) {
+      output.print(output.styles.success('  OK  '), cmd);
+    }
+    if (cmd?.startsWith('I.grab')) {
+      output.print(output.styles.debug(val));
+    }
 
     history.push(cmd); // add command to history when successful
   } catch (err) {
@@ -117,6 +175,7 @@ function parseInput(cmd) {
     // pop latest command from history because it failed
     history.pop();
 
+    if (isAiCommand) return;
     if (!lastError) output.print(output.styles.error(' FAIL '), msg);
     lastError = err.message;
   });

package/lib/plugin/autoLogin.js

@@ -9,6 +9,7 @@ const isAsyncFunction = require('../utils').isAsyncFunction;
 
 const defaultUser = {
   fetch: I => I.grabCookie(),
+  check: () => {},
   restore: (I, cookies) => {
     I.amOnPage('/'); // open a page
     I.setCookie(cookies);
@@ -37,12 +38,14 @@ const defaultConfig = {
  * ```js
  * // inside a test file
  * // use login to inject auto-login function
+ * Feature('Login');
+ *
  * Before(({ login }) => {
  *    login('user'); // login using user session
  * });
  *
- * // Alternatively log in for one scenario
- * Scenario('log me in', ( {I, login} ) => {
+ * // Alternatively log in for one scenario.
+ * Scenario('log me in', ( { I, login } ) => {
  *    login('admin');
  *    I.see('I am logged in');
  * });
@@ -61,7 +64,7 @@ const defaultConfig = {
  * #### How It Works
  *
  * 1. `restore` method is executed. It should open a page and set credentials.
- * 2. `check` method is executed. It should reload a page (so cookies are applied) and check that this page belongs to logged in user.
+ * 2. `check` method is executed. It should reload a page (so cookies are applied) and check that this page belongs to logged-in user. When you pass the second args `session`, you could perform the validation using passed session.
  * 3. If `restore` and `check` were not successful, `login` is executed
  * 4. `login` should fill in login form
  * 5. After successful login, `fetch` is executed to save cookies into memory or file.
@@ -212,6 +215,38 @@ const defaultConfig = {
  * })
  * ```
  *
+ * #### Tips: Using session to validate user
+ *
+ * Instead of asserting on page elements for the current user in `check`, you can use the `session` you saved in `fetch`
+ *
+ * ```js
+ * autoLogin: {
+ *   enabled: true,
+ *   saveToFile: true,
+ *   inject: 'login',
+ *   users: {
+ *     admin: {
+ *       login: async (I) => {  // If you use async function in the autoLogin plugin
+ *          const phrase = await I.grabTextFrom('#phrase')
+ *          I.fillField('username', 'admin'),
+ *          I.fillField('password', 'password')
+ *          I.fillField('phrase', phrase)
+ *       },
+ *       check: (I, session) => {
+ *          // Throwing an error in `check` will make CodeceptJS perform the login step for the user
+ *          if (session.profile.email !== the.email.you.expect@some-mail.com) {
+ *               throw new Error ('Wrong user signed in');
+ *         }
+ *       },
+ *     }
+ *   }
+ * }
+ * ```
+ *
+ * ```js
+ * Scenario('login', async ( {I, login} ) => {
+ *   await login('admin') // you should use `await`
+ * })
  *
  *
 */
@@ -251,27 +286,28 @@ module.exports = function (config) {
       } else {
         userSession.login(I);
       }
-      store.debugMode = true;
+
       const cookies = await userSession.fetch(I);
+      if (!cookies) {
+        debug('Cannot save user session with empty cookies from auto login\'s fetch method');
+        return;
+      }
       if (config.saveToFile) {
         debug(`Saved user session into file for ${name}`);
         fs.writeFileSync(path.join(global.output_dir, `${name}_session.json`), JSON.stringify(cookies));
       }
       store[`${name}_session`] = cookies;
-      store.debugMode = false;
     };
 
     if (!cookies) return loginAndSave();
 
-    store.debugMode = true;
-
     recorder.session.start('check login');
     if (shouldAwait) {
       await userSession.restore(I, cookies);
-      await userSession.check(I);
+      await userSession.check(I, cookies);
     } else {
       userSession.restore(I, cookies);
-      userSession.check(I);
+      userSession.check(I, cookies);
     }
     recorder.session.catch((err) => {
       debug(`Failed auto login for ${name} due to ${err}`);
@@ -287,7 +323,6 @@ module.exports = function (config) {
       });
     });
     recorder.add(() => {
-      store.debugMode = false;
       recorder.session.restore('check login');
     });
 

package/lib/plugin/debugErrors.js

@@ -0,0 +1,67 @@
+const Container = require('../container');
+const recorder = require('../recorder');
+const event = require('../event');
+const supportedHelpers = require('./standardActingHelpers');
+const { scanForErrorMessages } = require('../html');
+const { output } = require('..');
+
+const defaultConfig = {
+  errorClasses: ['error', 'warning', 'alert', 'danger'],
+};
+
+/**
+ * Prints errors found in HTML code after each failed test.
+ *
+ * It scans HTML and searches for elements with error classes.
+ * If an element found prints a text from it to console and adds as artifact to the test.
+ *
+ * Enable this plugin in config:
+ *
+ * ```js
+ * plugins: {
+ *  debugErrors: {
+ *   enabled: true,
+ * }
+ * ```
+ *
+ * Additional config options:
+ *
+ * * `errorClasses` - list of classes to search for errors (default: `['error', 'warning', 'alert', 'danger']`)
+ *
+ */
+module.exports = function (config = {}) {
+  const helpers = Container.helpers();
+  let helper;
+
+  config = Object.assign(defaultConfig, config);
+
+  for (const helperName of supportedHelpers) {
+    if (Object.keys(helpers).indexOf(helperName) > -1) {
+      helper = helpers[helperName];
+    }
+  }
+
+  if (!helper) return; // no helpers for screenshot
+
+  event.dispatcher.on(event.test.failed, (test) => {
+    recorder.add('HTML snapshot failed test', async () => {
+      try {
+        const currentOutputLevel = output.level();
+        output.level(0);
+        const html = await helper.grabHTMLFrom('body');
+        output.level(currentOutputLevel);
+
+        if (!html) return;
+
+        const errors = scanForErrorMessages(html, config.errorClasses);
+        if (errors.length) {
+          output.debug('Detected errors in HTML code');
+          errors.forEach((error) => output.debug(error));
+          test.artifacts.errors = errors;
+        }
+      } catch (err) {
+        // not really needed
+      }
+    });
+  });
+};

package/lib/plugin/fakerTransform.js

@@ -2,20 +2,18 @@ const { faker } = require('@faker-js/faker');
 const transform = require('../transform');
 
 /**
- * Use the [faker.js](https://www.npmjs.com/package/faker) package to generate fake data inside examples on your gherkin tests
- *
- * ![Faker.js](https://raw.githubusercontent.com/Marak/faker.js/master/logo.png)
+ * Use the `@faker-js/faker` package to generate fake data inside examples on your gherkin tests
  *
  * #### Usage
  *
- * To start please install `faker.js` package
+ * To start please install `@faker-js/faker` package
  *
  * ```
- * npm install -D faker
+ * npm install -D @faker-js/faker
  * ```
  *
  * ```
- * yarn add -D faker
+ * yarn add -D @faker-js/faker
  * ```
  *
  * Add this plugin to config file:

package/lib/plugin/heal.js

@@ -0,0 +1,209 @@
+const debug = require('debug')('codeceptjs:heal');
+const colors = require('chalk');
+const Container = require('../container');
+const AiAssistant = require('../ai');
+const recorder = require('../recorder');
+const event = require('../event');
+const output = require('../output');
+const supportedHelpers = require('./standardActingHelpers');
+
+const defaultConfig = {
+  healTries: 1,
+  healLimit: 2,
+  healSteps: [
+    'click',
+    'fillField',
+    'appendField',
+    'selectOption',
+    'attachFile',
+    'checkOption',
+    'uncheckOption',
+    'doubleClick',
+  ],
+};
+
+/**
+ * Self-healing tests with OpenAI.
+ *
+ * This plugin is experimental and requires OpenAI API key.
+ *
+ * To use it you need to set OPENAI_API_KEY env variable and enable plugin inside the config.
+ *
+ * ```js
+ * plugins: {
+ *   heal: {
+ *    enabled: true,
+ *   }
+ * }
+ * ```
+ *
+ * More config options are available:
+ *
+ * * `healLimit` - how many steps can be healed in a single test (default: 2)
+ * * `healSteps` - which steps can be healed (default: all steps that interact with UI, see list below)
+ *
+ * Steps to heal:
+ *
+ * * `click`
+ * * `fillField`
+ * * `appendField`
+ * * `selectOption`
+ * * `attachFile`
+ * * `checkOption`
+ * * `uncheckOption`
+ * * `doubleClick`
+ *
+ */
+module.exports = function (config = {}) {
+  const aiAssistant = AiAssistant.getInstance();
+
+  let currentTest = null;
+  let currentStep = null;
+  let healedSteps = 0;
+  let caughtError;
+  let healTries = 0;
+  let isHealing = false;
+
+  const healSuggestions = [];
+
+  config = Object.assign(defaultConfig, config);
+
+  event.dispatcher.on(event.test.before, (test) => {
+    currentTest = test;
+    healedSteps = 0;
+    caughtError = null;
+  });
+
+  event.dispatcher.on(event.step.started, step => currentStep = step);
+
+  event.dispatcher.on(event.step.after, (step) => {
+    if (isHealing) return;
+    const store = require('../store');
+    if (store.debugMode) return;
+    recorder.catchWithoutStop(async (err) => {
+      isHealing = true;
+      if (caughtError === err) throw err; // avoid double handling
+      caughtError = err;
+      if (!aiAssistant.isEnabled) {
+        output.print(colors.yellow('Heal plugin can\'t operate, AI assistant is disabled. Please set OPENAI_API_KEY env variable to enable it.'));
+        throw err;
+      }
+      if (!currentStep) throw err;
+      if (!config.healSteps.includes(currentStep.name)) throw err;
+      const test = currentTest;
+
+      if (healTries >= config.healTries) {
+        output.print(colors.bold.red(`Healing failed for ${config.healTries} time(s)`));
+        output.print('AI couldn\'t identify the correct solution');
+        output.print('Probably the entire flow has changed and the test should be updated');
+
+        throw err;
+      }
+
+      if (healedSteps >= config.healLimit) {
+        output.print(colors.bold.red(`Can't heal more than ${config.healLimit} step(s) in a test`));
+        output.print('Entire flow can be broken, please check it manually');
+        output.print('or increase healing limit in heal plugin config');
+
+        throw err;
+      }
+
+      recorder.session.start('heal');
+      const helpers = Container.helpers();
+      let helper;
+
+      for (const helperName of supportedHelpers) {
+        if (Object.keys(helpers).indexOf(helperName) > -1) {
+          helper = helpers[helperName];
+        }
+      }
+
+      if (!helper) throw err; // no helpers for html
+
+      const step = test.steps[test.steps.length - 1];
+      debug('Self-healing started', step.toCode());
+
+      const currentOutputLevel = output.level();
+      output.level(0);
+      const html = await helper.grabHTMLFrom('body');
+      output.level(currentOutputLevel);
+
+      if (!html) throw err;
+
+      healTries++;
+      await aiAssistant.setHtmlContext(html);
+      await tryToHeal(step, err);
+
+      recorder.add('close healing session', () => {
+        recorder.session.restore('heal');
+        recorder.ignoreErr(err);
+      });
+      await recorder.promise();
+
+      isHealing = false;
+    });
+  });
+
+  event.dispatcher.on(event.all.result, () => {
+    if (!healSuggestions.length) return;
+
+    const { print } = output;
+
+    print('');
+    print('===================');
+    print(colors.bold.green('Self-Healing Report:'));
+
+    print(`${colors.bold(healSuggestions.length)} step(s) were healed by AI`);
+
+    let i = 1;
+    print('');
+    print('Suggested changes:');
+    print('');
+
+    for (const suggestion of healSuggestions) {
+      print(`${i}. To fix ${colors.bold.blue(suggestion.test.title)}`);
+      print('Replace the failed code with:');
+      print(colors.red(`- ${suggestion.step.toCode()}`));
+      print(colors.green(`+ ${suggestion.snippet}`));
+      print(suggestion.step.line());
+      print('');
+      i++;
+    }
+  });
+
+  async function tryToHeal(failedStep, err) {
+    output.debug(`Running OpenAI to heal ${failedStep.toCode()} step`);
+
+    const codeSnippets = await aiAssistant.healFailedStep(failedStep, err, currentTest);
+
+    output.debug(`Received ${codeSnippets.length} suggestions from OpenAI`);
+    const I = Container.support('I'); // eslint-disable-line
+
+    for (const codeSnippet of codeSnippets) {
+      try {
+        debug('Executing', codeSnippet);
+        recorder.catch((e) => {
+          console.log(e);
+        });
+        await eval(codeSnippet); // eslint-disable-line
+
+        healSuggestions.push({
+          test: currentTest,
+          step: failedStep,
+          snippet: codeSnippet,
+        });
+
+        recorder.add('healed', () => output.print(colors.bold.green('  Code healed successfully')));
+        healedSteps++;
+        return;
+      } catch (err) {
+        debug('Failed to execute code', err);
+        recorder.ignoreErr(err); // healing ded not help
+        // recorder.catch(() => output.print(colors.bold.red('  Failed healing code')));
+      }
+    }
+
+    output.debug(`Couldn't heal the code for ${failedStep.toCode()}`);
+  }
+  return recorder.promise();
+};

package/lib/plugin/retryFailedStep.js

@@ -1,5 +1,7 @@
 const event = require('../event');
 const recorder = require('../recorder');
+const container = require('../container');
+const { log } = require('../output');
 
 const defaultConfig = {
   retries: 3,
@@ -42,7 +44,7 @@ const defaultConfig = {
  * * `factor` - The exponential factor to use. Default is 1.5.
  * * `minTimeout` - The number of milliseconds before starting the first retry. Default is 1000.
  * * `maxTimeout` - The maximum number of milliseconds between two retries. Default is Infinity.
- * * `randomize` - Randomizes the timeouts by multiplying with a factor between 1 to 2. Default is false.
+ * * `randomize` - Randomizes the timeouts by multiplying with a factor from 1 to 2. Default is false.
  * * `defaultIgnoredSteps` - an array of steps to be ignored for retry. Includes:
  *     * `amOnPage`
  *     * `wait*`
@@ -98,6 +100,11 @@ module.exports = (config) => {
   config.when = when;
 
   event.dispatcher.on(event.step.started, (step) => {
+    if (process.env.TRY_TO === 'true') {
+      log('Info: RetryFailedStep plugin is disabled inside tryTo block');
+      return;
+    }
+
     // if a step is ignored - return
     for (const ignored of config.ignoredSteps) {
       if (step.name === ignored) return;
@@ -114,6 +121,8 @@ module.exports = (config) => {
 
   event.dispatcher.on(event.test.before, (test) => {
     if (test && test.disableRetryFailedStep) return; // disable retry when a test is not active
+    // this env var is used to set the retries inside _before() block of helpers
+    process.env.FAILED_STEP_RETRIES = config.retries;
     recorder.retry(config);
   });
 };

package/lib/plugin/retryTo.js

@@ -83,16 +83,15 @@ module.exports = function (config) {
   return retryTo;
 
   function retryTo(callback, maxTries, pollInterval = undefined) {
-    const mode = store.debugMode;
     let tries = 1;
     if (!pollInterval) pollInterval = config.pollInterval;
 
     let err = null;
 
     return new Promise((done) => {
-      const tryBlock = () => {
+      const tryBlock = async () => {
         recorder.session.start(`retryTo ${tries}`);
-        callback(tries);
+        await callback(tries);
         recorder.add(() => {
           recorder.session.restore(`retryTo ${tries}`);
           done(null);
@@ -113,7 +112,6 @@ module.exports = function (config) {
       };
 
       recorder.add('retryTo', async () => {
-        store.debugMode = true;
         tryBlock();
       });
     }).then(() => {

package/lib/plugin/screenshotOnFail.js

@@ -75,7 +75,8 @@ module.exports = function (config) {
   event.dispatcher.on(event.test.failed, (test) => {
     recorder.add('screenshot of failed test', async () => {
       let fileName = clearString(test.title);
-      // This prevent data driven to be included in the failed screenshot file name
+      const dataType = 'image/png';
+      // This prevents data driven to be included in the failed screenshot file name
       if (fileName.indexOf('{') !== -1) {
         fileName = fileName.substr(0, (fileName.indexOf('{') - 3)).trim();
       }
@@ -106,7 +107,15 @@ module.exports = function (config) {
 
         const allureReporter = Container.plugins('allure');
         if (allureReporter) {
-          allureReporter.addAttachment('Last Seen Screenshot', fs.readFileSync(path.join(global.output_dir, fileName)), 'image/png');
+          allureReporter.addAttachment('Main session - Last Seen Screenshot', fs.readFileSync(path.join(global.output_dir, fileName)), dataType);
+
+          if (helper.activeSessionName) {
+            for (const sessionName in helper.sessionPages) {
+              const screenshotFileName = `${sessionName}_${fileName}`;
+              test.artifacts[`${sessionName.replace(/ /g, '_')}_screenshot`] = path.join(global.output_dir, screenshotFileName);
+              allureReporter.addAttachment(`${sessionName} - Last Seen Screenshot`, fs.readFileSync(path.join(global.output_dir, screenshotFileName)), dataType);
+            }
+          }
         }
 
         const cucumberReporter = Container.plugins('cucumberJsonReporter');

package/lib/plugin/selenoid.js

@@ -58,7 +58,12 @@ let seleniumUrl = 'http://localhost:$port$';
 const supportedHelpers = ['WebDriver'];
 const SELENOID_START_TIMEOUT = 2000;
 const SELENOID_STOP_TIMEOUT = 10000;
-const wait = time => new Promise((res) => setTimeout(() => res(), time));
+const wait = time => new Promise((res) => {
+  setTimeout(() => {
+    // @ts-ignore
+    res();
+  }, time);
+});
 
 /**
  * [Selenoid](https://aerokube.com/selenoid/) plugin automatically starts browsers and video recording.