codeceptjs 3.1.2 → 3.2.2

package/lib/listener/steps.js

@@ -1,5 +1,7 @@
+const debug = require('debug')('codeceptjs:steps');
 const event = require('../event');
 const store = require('../store');
+const output = require('../output');
 
 let currentTest;
 let currentHook;
@@ -9,6 +11,7 @@ let currentHook;
  */
 module.exports = function () {
   event.dispatcher.on(event.test.before, (test) => {
+    test.startedAt = +new Date();
     test.artifacts = {};
   });
 
@@ -19,17 +22,23 @@ module.exports = function () {
     else currentTest.retryNum += 1;
   });
 
-  event.dispatcher.on(event.test.after, () => {
+  event.dispatcher.on(event.test.after, (test) => {
     currentTest = null;
   });
 
-  event.dispatcher.on(event.hook.passed, () => {
-    currentHook = null;
+  event.dispatcher.on(event.test.finished, (test) => {
   });
 
   event.dispatcher.on(event.hook.started, (suite) => {
     currentHook = suite.ctx.test;
     currentHook.steps = [];
+
+    if (suite.ctx && suite.ctx.test) output.log(`--- STARTED ${suite.ctx.test.title} ---`);
+  });
+
+  event.dispatcher.on(event.hook.passed, (suite) => {
+    currentHook = null;
+    if (suite.ctx && suite.ctx.test) output.log(`--- ENDED ${suite.ctx.test.title} ---`);
   });
 
   event.dispatcher.on(event.test.failed, () => {
@@ -51,6 +60,7 @@ module.exports = function () {
   });
 
   event.dispatcher.on(event.test.passed, () => {
+    if (!currentTest) return;
     // To be sure that passed test will be passed in report
     delete currentTest.err;
     currentTest.state = 'passed';
@@ -58,10 +68,18 @@ module.exports = function () {
 
   event.dispatcher.on(event.step.started, (step) => {
     if (store.debugMode) return;
+    step.startedAt = +new Date();
     if (currentHook && Array.isArray(currentHook.steps)) {
       return currentHook.steps.push(step);
     }
     if (!currentTest || !currentTest.steps) return;
     currentTest.steps.push(step);
   });
+
+  event.dispatcher.on(event.step.finished, (step) => {
+    if (store.debugMode) return;
+    step.finishedAt = +new Date();
+    if (step.startedAt) step.duration = step.finishedAt - step.startedAt;
+    debug(`Step '${step}' finished; Duration: ${step.duration}ms`);
+  });
 };

package/lib/listener/timeout.js

@@ -0,0 +1,72 @@
+const event = require('../event');
+const output = require('../output');
+const recorder = require('../recorder');
+const Config = require('../config');
+const { timeouts } = require('../store');
+const TIMEOUT_ORDER = require('../step').TIMEOUT_ORDER;
+
+module.exports = function () {
+  let timeout;
+  let timeoutStack = [];
+  let currentTest;
+  let currentTimeout;
+
+  if (!timeouts) {
+    console.log('Timeouts were disabled');
+    return;
+  }
+
+  event.dispatcher.on(event.suite.before, (suite) => {
+    timeoutStack = [];
+    const globalTimeout = Config.get('timeout');
+    if (globalTimeout) {
+      if (globalTimeout >= 1000) {
+        console.log(`Warning: Timeout was set to ${globalTimeout}secs.\nGlobal timeout should be specified in seconds.`);
+      }
+      timeoutStack.push(globalTimeout);
+    }
+    if (suite.totalTimeout) timeoutStack.push(suite.totalTimeout);
+    output.log(`Timeouts: ${timeoutStack}`);
+  });
+
+  event.dispatcher.on(event.test.before, (test) => {
+    currentTest = test;
+    timeout = test.totalTimeout || timeoutStack[timeoutStack.length - 1];
+    if (!timeout) return;
+    currentTimeout = timeout;
+    output.debug(`Test Timeout: ${timeout}s`);
+    timeout *= 1000;
+  });
+
+  event.dispatcher.on(event.test.passed, (test) => {
+    currentTest = null;
+  });
+
+  event.dispatcher.on(event.test.failed, (test) => {
+    currentTest = null;
+  });
+
+  event.dispatcher.on(event.step.before, (step) => {
+    if (typeof timeout !== 'number') return;
+
+    if (timeout < 0) {
+      step.setTimeout(0.01, TIMEOUT_ORDER.testOrSuite);
+    } else {
+      step.setTimeout(timeout, TIMEOUT_ORDER.testOrSuite);
+    }
+  });
+
+  event.dispatcher.on(event.step.finished, (step) => {
+    if (typeof timeout === 'number' && !Number.isNaN(timeout)) timeout -= step.duration;
+
+    if (typeof timeout === 'number' && timeout <= 0 && recorder.isRunning()) {
+      if (currentTest && currentTest.callback) {
+        recorder.reset();
+        // replace mocha timeout with custom timeout
+        currentTest.timeout(0);
+        currentTest.callback(new Error(`Timeout ${currentTimeout}s exceeded (with Before hook)`));
+        currentTest.timedOut = true;
+      }
+    }
+  });
+};

package/lib/locator.js

@@ -313,11 +313,14 @@ Locator.build = (locator) => {
 
 /**
  * Filters to modify locators
+ * @type {Array<function(CodeceptJS.LocatorOrString, Locator): void>}
  */
 Locator.filters = [];
 
 /**
  * Appends new `Locator` filter to an `Locator.filters` array, and returns the new length of the array.
+ * @param {function(CodeceptJS.LocatorOrString, Locator): void} fn
+ * @returns {number}
  */
 Locator.addFilter = fn => Locator.filters.push(fn);
 

package/lib/mochaFactory.js

@@ -2,7 +2,7 @@ const Mocha = require('mocha');
 const fsPath = require('path');
 const fs = require('fs');
 const reporter = require('./cli');
-const gherkinParser = require('./interfaces/gherkin.js');
+const gherkinParser = require('./interfaces/gherkin');
 const output = require('./output');
 const { genTestId } = require('./utils');
 const ConnectionRefused = require('./helper/errors/ConnectionRefused');
@@ -35,8 +35,7 @@ class MochaFactory {
       if (mocha.suite.suites.length === 0) {
         mocha.files
           .filter(file => file.match(/\.feature$/))
-          .map(file => fs.readFileSync(file, 'utf8'))
-          .forEach(content => mocha.suite.addSuite(gherkinParser(content)));
+          .forEach(file => mocha.suite.addSuite(gherkinParser(fs.readFileSync(file, 'utf8'), file)));
 
         // remove feature files
         mocha.files = mocha.files.filter(file => !file.match(/\.feature$/));

package/lib/plugin/allure.js

@@ -283,7 +283,12 @@ module.exports = (config) => {
   let maxLevel;
   function finishMetastep(level) {
     const metaStepsToFinish = currentMetaStep.splice(maxLevel - level);
-    metaStepsToFinish.forEach(() => reporter.endStep('passed'));
+    metaStepsToFinish.forEach(() => {
+      // only if the current step is of type Step, end it.
+      if (reporter.suites && reporter.suites.length && reporter.suites[0].currentStep && reporter.suites[0].currentStep.constructor.name === 'Step') {
+        reporter.endStep('passed');
+      }
+    });
   }
 
   function startMetaStep(metaStep, level = 0) {

package/lib/plugin/coverage.js

@@ -48,7 +48,7 @@ function buildFileName(test, uniqueFileName) {
  *
  * ```js
  * plugins: {
- *    playwrightCoverage: {
+ *    coverage: {
  *      enabled: true
  *    }
  * }

package/lib/plugin/retryFailedStep.js

@@ -2,7 +2,7 @@ const event = require('../event');
 const recorder = require('../recorder');
 
 const defaultConfig = {
-  retries: 5,
+  retries: 3,
   defaultIgnoredSteps: [
     'amOnPage',
     'wait*',
@@ -11,6 +11,7 @@ const defaultConfig = {
     'run*',
     'have*',
   ],
+  factor: 1.5,
   ignoredSteps: [],
 };
 
@@ -36,9 +37,9 @@ const defaultConfig = {
  *
  * #### Configuration:
  *
- * * `retries` - number of retries (by default 5),
+ * * `retries` - number of retries (by default 3),
  * * `when` - function, when to perform a retry (accepts error as parameter)
- * * `factor` - The exponential factor to use. Default is 2.
+ * * `factor` - The exponential factor to use. Default is 1.5.
  * * `minTimeout` - The number of milliseconds before starting the first retry. Default is 1000.
  * * `maxTimeout` - The maximum number of milliseconds between two retries. Default is Infinity.
  * * `randomize` - Randomizes the timeouts by multiplying with a factor between 1 to 2. Default is false.

package/lib/plugin/retryTo.js

@@ -0,0 +1,130 @@
+const recorder = require('../recorder');
+const store = require('../store');
+const { debug } = require('../output');
+
+const defaultConfig = {
+  registerGlobal: true,
+  pollInterval: 200,
+};
+
+/**
+ *
+ *
+ * Adds global `retryTo` which retries steps a few times before failing.
+ *
+ * Enable this plugin in `codecept.conf.js` (enabled by default for new setups):
+ *
+ * ```js
+ * plugins: {
+ *   retryTo: {
+ *     enabled: true
+ *   }
+ * }
+ * ```
+ *
+ * Use it in your tests:
+ *
+ * ```js
+ * // retry these steps 5 times before failing
+ * await retryTo((tryNum) => {
+ *   I.switchTo('#editor frame');
+ *   I.click('Open');
+ *   I.see('Opened')
+ * }, 5);
+ * ```
+ * Set polling interval as 3rd argument (200ms by default):
+ *
+ * ```js
+ * // retry these steps 5 times before failing
+ * await retryTo((tryNum) => {
+ *   I.switchTo('#editor frame');
+ *   I.click('Open');
+ *   I.see('Opened')
+ * }, 5, 100);
+ * ```
+ *
+ * Default polling interval can be changed in a config:
+ *
+ * ```js
+ * plugins: {
+ *   retryTo: {
+ *     enabled: true,
+ *     pollInterval: 500,
+ *   }
+ * }
+ * ```
+ *
+ * Disables retryFailedStep plugin for steps inside a block;
+ *
+ * Use this plugin if:
+ *
+ * * you need repeat a set of actions in flaky tests
+ * * iframe was not rendered and you need to retry switching to it
+ *
+ *
+ * #### Configuration
+ *
+ * * `pollInterval` - default interval between retries in ms. 200 by default.
+ * * `registerGlobal` - to register `retryTo` function globally, true by default
+ *
+ * If `registerGlobal` is false you can use retryTo from the plugin:
+ *
+ * ```js
+ * const retryTo = codeceptjs.container.plugins('retryTo');
+ * ```
+ *
+*/
+module.exports = function (config) {
+  config = Object.assign(defaultConfig, config);
+
+  if (config.registerGlobal) {
+    global.retryTo = retryTo;
+  }
+  return retryTo;
+
+  function retryTo(callback, maxTries, pollInterval = undefined) {
+    const mode = store.debugMode;
+    let tries = 1;
+    if (!pollInterval) pollInterval = config.pollInterval;
+
+    let err = null;
+
+    return new Promise((done) => {
+      const tryBlock = () => {
+        recorder.session.start(`retryTo ${tries}`);
+        callback(tries);
+        recorder.add(() => {
+          recorder.session.restore(`retryTo ${tries}`);
+          done(null);
+        });
+        recorder.session.catch((e) => {
+          err = e;
+          recorder.session.restore(`retryTo ${tries}`);
+          tries++;
+          // recorder.session.restore(`retryTo`);
+          if (tries <= maxTries) {
+            debug(`Error ${err}... Retrying`);
+            err = null;
+
+            recorder.add(`retryTo ${tries}`, () => {
+              tryBlock();
+              // recorder.add(() => new Promise(done => setTimeout(done, pollInterval)));
+            });
+          } else {
+            // recorder.throw(err);
+            done(null);
+          }
+        });
+        // return recorder.promise();
+      };
+
+      recorder.add('retryTo', async () => {
+        store.debugMode = true;
+        tryBlock();
+        // recorder.add(() => recorder.session.restore(`retryTo ${tries-1}`));
+      });
+    }).then(() => {
+      if (err) recorder.throw(err);
+    });
+  }
+};

package/lib/plugin/screenshotOnFail.js

@@ -98,6 +98,7 @@ module.exports = function (config) {
         }
         await helper.saveScreenshot(fileName, options.fullPageScreenshots);
 
+        if (!test.artifacts) test.artifacts = {};
         test.artifacts.screenshot = path.join(global.output_dir, fileName);
         if (Container.mocha().options.reporterOptions['mocha-junit-reporter'] && Container.mocha().options.reporterOptions['mocha-junit-reporter'].options.attachments) {
           test.attachments = [path.join(global.output_dir, fileName)];

package/lib/plugin/stepByStepReport.js

@@ -20,6 +20,7 @@ const defaultConfig = {
   fullPageScreenshots: false,
   output: global.output_dir,
   screenshotsForAllureReport: false,
+  disableScreenshotOnFail: true,
 };
 
 const templates = {};
@@ -56,6 +57,7 @@ const templates = {};
  * * `fullPageScreenshots`: should full page screenshots be used. Default: false.
  * * `output`: a directory where reports should be stored. Default: `output`.
  * * `screenshotsForAllureReport`: If Allure plugin is enabled this plugin attaches each saved screenshot to allure report. Default: false.
+ * * `disableScreenshotOnFail : Disables the capturing of screeshots after the failed step. Default: true.
  *
  * @param {*} config
  */
@@ -80,6 +82,7 @@ module.exports = function (config) {
   let error;
   let savedStep = null;
   let currentTest = null;
+  let scenarioFailed = false;
 
   const recordedTests = {};
   const pad = '0000';
@@ -135,10 +138,14 @@ module.exports = function (config) {
     if (isStepIgnored(step)) return;
     if (savedStep === step) return; // already saved
     // Ignore steps from BeforeSuite function
+    if (scenarioFailed && config.disableScreenshotOnFail) return;
     if (step.metaStep && step.metaStep.name === 'BeforeSuite') return;
 
     const fileName = `${pad.substring(0, pad.length - stepNum.toString().length) + stepNum.toString()}.png`;
     try {
+      if (step.status === 'failed') {
+        scenarioFailed = true;
+      }
       stepNum++;
       slides[fileName] = step;
       await helper.saveScreenshot(path.relative(reportDir, path.join(dir, fileName)), config.fullPageScreenshots);

package/lib/plugin/stepTimeout.js

@@ -0,0 +1,91 @@
+const event = require('../event');
+const TIMEOUT_ORDER = require('../step').TIMEOUT_ORDER;
+
+const defaultConfig = {
+  timeout: 150,
+  overrideStepLimits: false,
+  noTimeoutSteps: [
+    'amOnPage',
+    'wait*',
+  ],
+  customTimeoutSteps: [],
+};
+
+/**
+ * Set timeout for test steps globally.
+ *
+ * Add this plugin to config file:
+ *
+ * ```js
+ * plugins: {
+ *     stepTimeout: {
+ *        enabled: true
+ *     }
+ * }
+ * ```
+ *
+ *
+ * Run tests with plugin enabled:
+ *
+ * ```
+ * npx codeceptjs run --plugins stepTimeout
+ * ```
+ *
+ * #### Configuration:
+ *
+ * * `timeout` - global step timeout, default 150 seconds
+ * * `overrideStepLimits` - whether to use timeouts set in plugin config to override step timeouts set in code with I.limitTime(x).action(...), default false
+ * * `noTimeoutSteps` - an array of steps with no timeout. Default:
+ *     * `amOnPage`
+ *     * `wait*`
+ *
+ *   you could set your own noTimeoutSteps which would replace the default one.
+ *
+ * * `customTimeoutSteps` - an array of step actions with custom timeout. Use it to override or extend noTimeoutSteps.
+ * You can use step names or step prefixes ending with `*`. As such, `wait*` will match all steps starting with `wait`.
+ *
+ * #### Example
+ *
+ * ```js
+ * plugins: {
+ *     stepTimeout: {
+ *         enabled: true,
+ *         overrideStepLimits: true,
+ *         noTimeoutSteps: [
+ *           'scroll*', // ignore all scroll steps
+ *           /Cookie/, // ignore all steps with a Cookie in it (by regexp)
+ *         ],
+ *         customTimeoutSteps: [
+ *           ['myFlakyStep*', 1],
+ *           ['scrollWhichRequiresTimeout', 5],
+ *         ]
+ *     }
+ * }
+ * ```
+ *
+ */
+module.exports = (config) => {
+  config = Object.assign(defaultConfig, config);
+  // below override rule makes sure customTimeoutSteps go first but then they override noTimeoutSteps in case of exact pattern match
+  config.customTimeoutSteps = config.customTimeoutSteps.concat(config.noTimeoutSteps).concat(config.customTimeoutSteps);
+
+  event.dispatcher.on(event.step.before, (step) => {
+    let stepTimeout;
+    for (let stepRule of config.customTimeoutSteps) {
+      let customTimeout = 0;
+      if (Array.isArray(stepRule)) {
+        if (stepRule.length > 1) customTimeout = stepRule[1];
+        stepRule = stepRule[0];
+      }
+      if (stepRule instanceof RegExp
+        ? step.name.match(stepRule)
+        : (step.name === stepRule || stepRule.indexOf('*') && step.name.startsWith(stepRule.slice(0, -1)))
+      ) {
+        stepTimeout = customTimeout;
+        break;
+      }
+    }
+    stepTimeout = stepTimeout === undefined ? config.timeout : stepTimeout;
+    step.setTimeout(stepTimeout * 1000, config.overrideStepLimits ? TIMEOUT_ORDER.stepTimeoutHard : TIMEOUT_ORDER.stepTimeoutSoft);
+  });
+};

package/lib/recorder.js

@@ -3,6 +3,8 @@ const promiseRetry = require('promise-retry');
 
 const { log } = require('./output');
 
+const MAX_TASKS = 100;
+
 let promise;
 let running = false;
 let errFn;
@@ -116,7 +118,7 @@ module.exports = {
      * @inner
      */
     start(name) {
-      log(`${currentQueue()}Starting <${name}> session`);
+      debug(`${currentQueue()}Starting <${name}> session`);
       tasks.push('--->');
       oldPromises.push(promise);
       this.running = true;
@@ -130,7 +132,7 @@ module.exports = {
      */
     restore(name) {
       tasks.push('<---');
-      log(`${currentQueue()}Finalize <${name}> session`);
+      debug(`${currentQueue()}Finalize <${name}> session`);
       this.running = false;
       sessionId = null;
       this.catch(errFn);
@@ -158,10 +160,11 @@ module.exports = {
    *     undefined: `add(fn)` -> `false` and `add('step',fn)` -> `true`
    *     true: it will retries if `retryOpts` set.
    *     false: ignore `retryOpts` and won't retry.
+   * @param {number} [timeout]
    * @return {Promise<*> | undefined}
    * @inner
    */
-  add(taskName, fn = undefined, force = false, retry = undefined) {
+  add(taskName, fn = undefined, force = false, retry = undefined, timeout = undefined) {
     if (typeof taskName === 'function') {
       fn = taskName;
       taskName = fn.toString();
@@ -172,19 +175,22 @@ module.exports = {
       return;
     }
     tasks.push(taskName);
-    debug(`${currentQueue()}Queued | ${taskName}`);
+    if (process.env.DEBUG) debug(`${currentQueue()}Queued | ${taskName}`);
 
     return promise = Promise.resolve(promise).then((res) => {
-      const retryOpts = this.retries.slice(-1).pop();
+      // prefer options for non-conditional retries
+      const retryOpts = this.retries.sort((r1, r2) => r1.when && !r2.when).slice(-1).pop();
       // no retries or unnamed tasks
       if (!retryOpts || !taskName || !retry) {
-        return Promise.resolve(res).then(fn);
+        const [promise, timer] = getTimeoutPromise(timeout, taskName);
+        return Promise.race([promise, Promise.resolve(res).then(fn)]).finally(() => clearTimeout(timer));
       }
 
       const retryRules = this.retries.slice().reverse();
       return promiseRetry(Object.assign(defaultRetryOptions, retryOpts), (retry, number) => {
         if (number > 1) log(`${currentQueue()}Retrying... Attempt #${number}`);
-        return Promise.resolve(res).then(fn).catch((err) => {
+        const [promise, timer] = getTimeoutPromise(timeout, taskName);
+        return Promise.race([promise, Promise.resolve(res).then(fn)]).finally(() => clearTimeout(timer)).catch((err) => {
           for (const retryObj of retryRules) {
             if (!retryObj.when) return retry(err);
             if (retryObj.when && retryObj.when(err)) return retry(err);
@@ -296,7 +302,7 @@ module.exports = {
    * @inner
    */
   stop() {
-    debug(this.toString());
+    if (process.env.DEBUG) debug(this.toString());
     log(`${currentQueue()}Stopping recording promises`);
     running = false;
   },
@@ -318,7 +324,7 @@ module.exports = {
    * @inner
    */
   scheduled() {
-    return tasks.join('\n');
+    return tasks.slice(-MAX_TASKS).join('\n');
   },
 
   /**
@@ -341,6 +347,14 @@ module.exports = {
 
 };
 
+function getTimeoutPromise(timeoutMs, taskName) {
+  let timer;
+  if (timeoutMs) debug(`Timing out in ${timeoutMs}ms`);
+  return [new Promise((done, reject) => {
+    timer = setTimeout(() => { reject(new Error(`Action ${taskName} was interrupted on step timeout ${timeoutMs}ms`)); }, timeoutMs || 2e9);
+  }), timer];
+}
+
 function currentQueue() {
   let session = '';
   if (sessionId) session = `<${sessionId}> `;

package/lib/step.js

@@ -13,6 +13,27 @@ const STACK_LINE = 4;
  * @param {string} name
  */
 class Step {
+  static get TIMEOUT_ORDER() {
+    return {
+      /**
+       * timeouts set with order below zero only override timeouts of higher order if their value is smaller
+       */
+      testOrSuite: -5,
+      /**
+       * 0-9 - designated for override of timeouts set from code, 5 is used by stepTimeout plugin when stepTimeout.config.overrideStepLimits=true
+       */
+      stepTimeoutHard: 5,
+      /**
+       * 10-19 - designated for timeouts set from code, 15 is order of I.setTimeout(t) operation
+       */
+      codeLimitTime: 15,
+      /**
+       * 20-29 - designated for timeout settings which could be overriden in tests code, 25 is used by stepTimeout plugin when stepTimeout.config.overrideStepLimits=false
+       */
+      stepTimeoutSoft: 25,
+    };
+  }
+
   constructor(helper, name) {
     /** @member {string} */
     this.actor = 'I'; // I = actor
@@ -38,6 +59,41 @@ class Step {
     this.metaStep = undefined;
     /** @member {string} */
     this.stack = '';
+
+    const timeouts = new Map();
+    /**
+     * @method
+     * @returns {number|undefined}
+     */
+    this.getTimeout = function () {
+      let totalTimeout;
+      // iterate over all timeouts starting from highest values of order
+      new Map([...timeouts.entries()].sort().reverse()).forEach((timeout, order) => {
+        if (timeout !== undefined && (
+          // when orders >= 0 - timeout value overrides those set with higher order elements
+          order >= 0
+
+          // when `order < 0 && totalTimeout === undefined` - timeout is used when nothing is set by elements with higher order
+          || totalTimeout === undefined
+
+          // when `order < 0` - timeout overrides higher values of timeout or 'no timeout' (totalTimeout === 0) set by elements with higher order
+          || timeout > 0 && (timeout < totalTimeout || totalTimeout === 0)
+        )) {
+          totalTimeout = timeout;
+        }
+      });
+      return totalTimeout;
+    };
+    /**
+     * @method
+     * @param {number} timeout - timeout in milliseconds or 0 if no timeout
+     * @param {number} order - order defines the priority of timeout, timeouts set with lower order override those set with higher order.
+     *                         When order below 0 value of timeout only override if new value is lower
+     */
+    this.setTimeout = function (timeout, order) {
+      timeouts.set(order, timeout);
+    };
+
     this.setTrace();
   }
 
@@ -228,6 +284,8 @@ class MetaStep extends Step {
   }
 }
 
+Step.TIMEOUTS = {};
+
 /** @type {Class<MetaStep>} */
 Step.MetaStep = MetaStep;
 

package/lib/store.js

@@ -5,6 +5,8 @@
 const store = {
   /** @type {boolean} */
   debugMode: false,
+  /** @type {boolean} */
+  timeouts: true,
 };
 
 module.exports = store;