codeceptjs 3.1.1 → 3.2.1

package/lib/mochaFactory.js

@@ -2,7 +2,7 @@ const Mocha = require('mocha');
 const fsPath = require('path');
 const fs = require('fs');
 const reporter = require('./cli');
-const gherkinParser = require('./interfaces/gherkin.js');
+const gherkinParser = require('./interfaces/gherkin');
 const output = require('./output');
 const { genTestId } = require('./utils');
 const ConnectionRefused = require('./helper/errors/ConnectionRefused');
@@ -17,12 +17,6 @@ class MochaFactory {
     output.process(opts.child);
     mocha.ui(scenarioUi);
 
-    // process.on('unhandledRejection', (reason) => {
-    //   output.error('Unhandled rejection');
-    //   console.log(Error.captureStackTrace(reason));
-    //   output.error(reason);
-    // });
-
     Mocha.Runner.prototype.uncaught = function (err) {
       if (err) {
         if (err.toString().indexOf('ECONNREFUSED') >= 0) {
@@ -41,8 +35,7 @@ class MochaFactory {
       if (mocha.suite.suites.length === 0) {
         mocha.files
           .filter(file => file.match(/\.feature$/))
-          .map(file => fs.readFileSync(file, 'utf8'))
-          .forEach(content => mocha.suite.addSuite(gherkinParser(content)));
+          .forEach(file => mocha.suite.addSuite(gherkinParser(fs.readFileSync(file, 'utf8'), file)));
 
         // remove feature files
         mocha.files = mocha.files.filter(file => !file.match(/\.feature$/));
@@ -51,19 +44,30 @@ class MochaFactory {
 
         // add ids for each test and check uniqueness
         const dupes = [];
+        let missingFeatureInFile = [];
         const seenTests = [];
         mocha.suite.eachTest(test => {
           test.id = genTestId(test);
+
           const name = test.fullTitle();
           if (seenTests.includes(test.id)) {
             dupes.push(name);
           }
           seenTests.push(test.id);
+
+          if (name.slice(0, name.indexOf(':')) === '') {
+            missingFeatureInFile.push(test.file);
+          }
         });
         if (dupes.length) {
           // ideally this should be no-op and throw (breaking change)...
           output.error(`Duplicate test names detected - Feature + Scenario name should be unique:\n${dupes.join('\n')}`);
         }
+
+        if (missingFeatureInFile.length) {
+          missingFeatureInFile = [...new Set(missingFeatureInFile)];
+          output.error(`Missing Feature section in:\n${missingFeatureInFile.join('\n')}`);
+        }
       }
     };
 

package/lib/plugin/allure.js

@@ -283,7 +283,12 @@ module.exports = (config) => {
   let maxLevel;
   function finishMetastep(level) {
     const metaStepsToFinish = currentMetaStep.splice(maxLevel - level);
-    metaStepsToFinish.forEach(() => reporter.endStep('passed'));
+    metaStepsToFinish.forEach(() => {
+      // only if the current step is of type Step, end it.
+      if (reporter.suites && reporter.suites.length && reporter.suites[0].currentStep && reporter.suites[0].currentStep.constructor.name === 'Step') {
+        reporter.endStep('passed');
+      }
+    });
   }
 
   function startMetaStep(metaStep, level = 0) {

package/lib/plugin/{puppeteerCoverage.js → coverage.js}

@@ -1,4 +1,4 @@
-const debug = require('debug')('codeceptjs:plugin:puppeteerCoverage');
+const debugModule = require('debug');
 const fs = require('fs');
 const path = require('path');
 
@@ -13,7 +13,7 @@ const defaultConfig = {
   uniqueFileName: true,
 };
 
-const supportedHelpers = ['Puppeteer'];
+const supportedHelpers = ['Puppeteer', 'Playwright'];
 
 function buildFileName(test, uniqueFileName) {
   let fileName = clearString(test.title);
@@ -41,15 +41,14 @@ function buildFileName(test, uniqueFileName) {
 }
 
 /**
- * Dumps puppeteers code coverage after every test.
+ * Dumps code coverage from Playwright/Puppeteer after every test.
  *
  * #### Configuration
  *
- * Configuration can either be taken from a corresponding helper (deprecated) or a from plugin config (recommended).
  *
  * ```js
  * plugins: {
- *    puppeteerCoverage: {
+ *    coverage: {
  *      enabled: true
  *    }
  * }
@@ -59,33 +58,22 @@ function buildFileName(test, uniqueFileName) {
  *
  * * `coverageDir`: directory to dump coverage files
  * * `uniqueFileName`: generate a unique filename by adding uuid
- *
- *  First of all, your mileage may vary!
- *
- *  To work, you need the client javascript code to be NOT uglified. They need to be built in "development" mode.
- *  And the end of your tests, you'll get a directory full of coverage per test run.  Now what?
- *  You'll need to convert the coverage code to something istanbul can read.  Good news is someone wrote the code
- *  for you (see puppeteer-to-istanbul link below).  Then using istanbul you need to combine the converted
- *  coverage and create a report.  Good luck!
- *
- *  Links:
- *  * https://github.com/GoogleChrome/puppeteer/blob/v1.12.2/docs/api.md#class-coverage
- *  * https://github.com/istanbuljs/puppeteer-to-istanbul
- *  * https://github.com/gotwarlost/istanbul
  */
 module.exports = function (config) {
   const helpers = Container.helpers();
   let coverageRunning = false;
   let helper;
 
+  let debug;
   for (const helperName of supportedHelpers) {
     if (Object.keys(helpers).indexOf(helperName) > -1) {
       helper = helpers[helperName];
+      debug = debugModule(`codeceptjs:plugin:${helperName.toLowerCase()}Coverage`);
     }
   }
 
   if (!helper) {
-    console.error('Coverage is only supported in Puppeteer');
+    console.error('Coverage is only supported in Puppeteer, Playwright');
     return; // no helpers for screenshot
   }
 
@@ -102,7 +90,7 @@ module.exports = function (config) {
       'starting coverage',
       async () => {
         try {
-          if (!coverageRunning) {
+          if (!coverageRunning && helper.page && helper.page.coverage) {
             debug('--> starting coverage <--');
             coverageRunning = true;
             await helper.page.coverage.startJSCoverage();
@@ -115,13 +103,13 @@ module.exports = function (config) {
     );
   });
 
-  // Save puppeteer coverage data after every test run
+  // Save coverage data after every test run
   event.dispatcher.on(event.test.after, async (test) => {
     recorder.add(
       'saving coverage',
       async () => {
         try {
-          if (coverageRunning) {
+          if (coverageRunning && helper.page && helper.page.coverage) {
             debug('--> stopping coverage <--');
             coverageRunning = false;
             const coverage = await helper.page.coverage.stopJSCoverage();

package/lib/plugin/customLocator.js

@@ -38,7 +38,7 @@ const defaultConfig = {
  * // in codecept.conf.js
  * plugins: {
  *  customLocator: {
- *    enabled: true
+ *    enabled: true,
  *    attribute: 'data-test'
  *  }
  * }
@@ -57,7 +57,7 @@ const defaultConfig = {
  * // in codecept.conf.js
  * plugins: {
  *  customLocator: {
- *    enabled: true
+ *    enabled: true,
  *    prefix: '=',
  *    attribute: 'data-qa'
  *  }

package/lib/plugin/retryFailedStep.js

@@ -2,7 +2,7 @@ const event = require('../event');
 const recorder = require('../recorder');
 
 const defaultConfig = {
-  retries: 5,
+  retries: 3,
   defaultIgnoredSteps: [
     'amOnPage',
     'wait*',
@@ -11,6 +11,7 @@ const defaultConfig = {
     'run*',
     'have*',
   ],
+  factor: 1.5,
   ignoredSteps: [],
 };
 
@@ -36,9 +37,9 @@ const defaultConfig = {
  *
  * #### Configuration:
  *
- * * `retries` - number of retries (by default 5),
+ * * `retries` - number of retries (by default 3),
  * * `when` - function, when to perform a retry (accepts error as parameter)
- * * `factor` - The exponential factor to use. Default is 2.
+ * * `factor` - The exponential factor to use. Default is 1.5.
  * * `minTimeout` - The number of milliseconds before starting the first retry. Default is 1000.
  * * `maxTimeout` - The maximum number of milliseconds between two retries. Default is Infinity.
  * * `randomize` - Randomizes the timeouts by multiplying with a factor between 1 to 2. Default is false.

package/lib/plugin/retryTo.js

@@ -0,0 +1,130 @@
+const recorder = require('../recorder');
+const store = require('../store');
+const { debug } = require('../output');
+
+const defaultConfig = {
+  registerGlobal: true,
+  pollInterval: 200,
+};
+
+/**
+ *
+ *
+ * Adds global `retryTo` which retries steps a few times before failing.
+ *
+ * Enable this plugin in `codecept.conf.js` (enabled by default for new setups):
+ *
+ * ```js
+ * plugins: {
+ *   retryTo: {
+ *     enabled: true
+ *   }
+ * }
+ * ```
+ *
+ * Use it in your tests:
+ *
+ * ```js
+ * // retry these steps 5 times before failing
+ * await retryTo((tryNum) => {
+ *   I.switchTo('#editor frame');
+ *   I.click('Open');
+ *   I.see('Opened')
+ * }, 5);
+ * ```
+ * Set polling interval as 3rd argument (200ms by default):
+ *
+ * ```js
+ * // retry these steps 5 times before failing
+ * await retryTo((tryNum) => {
+ *   I.switchTo('#editor frame');
+ *   I.click('Open');
+ *   I.see('Opened')
+ * }, 5, 100);
+ * ```
+ *
+ * Default polling interval can be changed in a config:
+ *
+ * ```js
+ * plugins: {
+ *   retryTo: {
+ *     enabled: true,
+ *     pollInterval: 500,
+ *   }
+ * }
+ * ```
+ *
+ * Disables retryFailedStep plugin for steps inside a block;
+ *
+ * Use this plugin if:
+ *
+ * * you need repeat a set of actions in flaky tests
+ * * iframe was not rendered and you need to retry switching to it
+ *
+ *
+ * #### Configuration
+ *
+ * * `pollInterval` - default interval between retries in ms. 200 by default.
+ * * `registerGlobal` - to register `retryTo` function globally, true by default
+ *
+ * If `registerGlobal` is false you can use retryTo from the plugin:
+ *
+ * ```js
+ * const retryTo = codeceptjs.container.plugins('retryTo');
+ * ```
+ *
+*/
+module.exports = function (config) {
+  config = Object.assign(defaultConfig, config);
+
+  if (config.registerGlobal) {
+    global.retryTo = retryTo;
+  }
+  return retryTo;
+
+  function retryTo(callback, maxTries, pollInterval = undefined) {
+    const mode = store.debugMode;
+    let tries = 1;
+    if (!pollInterval) pollInterval = config.pollInterval;
+
+    let err = null;
+
+    return new Promise((done) => {
+      const tryBlock = () => {
+        recorder.session.start(`retryTo ${tries}`);
+        callback(tries);
+        recorder.add(() => {
+          recorder.session.restore(`retryTo ${tries}`);
+          done(null);
+        });
+        recorder.session.catch((e) => {
+          err = e;
+          recorder.session.restore(`retryTo ${tries}`);
+          tries++;
+          // recorder.session.restore(`retryTo`);
+          if (tries <= maxTries) {
+            debug(`Error ${err}... Retrying`);
+            err = null;
+
+            recorder.add(`retryTo ${tries}`, () => {
+              tryBlock();
+              // recorder.add(() => new Promise(done => setTimeout(done, pollInterval)));
+            });
+          } else {
+            // recorder.throw(err);
+            done(null);
+          }
+        });
+        // return recorder.promise();
+      };
+
+      recorder.add('retryTo', async () => {
+        store.debugMode = true;
+        tryBlock();
+        // recorder.add(() => recorder.session.restore(`retryTo ${tries-1}`));
+      });
+    }).then(() => {
+      if (err) recorder.throw(err);
+    });
+  }
+};

package/lib/plugin/screenshotOnFail.js

@@ -98,6 +98,7 @@ module.exports = function (config) {
         }
         await helper.saveScreenshot(fileName, options.fullPageScreenshots);
 
+        if (!test.artifacts) test.artifacts = {};
         test.artifacts.screenshot = path.join(global.output_dir, fileName);
         if (Container.mocha().options.reporterOptions['mocha-junit-reporter'] && Container.mocha().options.reporterOptions['mocha-junit-reporter'].options.attachments) {
           test.attachments = [path.join(global.output_dir, fileName)];

package/lib/plugin/stepByStepReport.js

@@ -20,6 +20,7 @@ const defaultConfig = {
   fullPageScreenshots: false,
   output: global.output_dir,
   screenshotsForAllureReport: false,
+  disableScreenshotOnFail: true,
 };
 
 const templates = {};
@@ -56,6 +57,7 @@ const templates = {};
  * * `fullPageScreenshots`: should full page screenshots be used. Default: false.
  * * `output`: a directory where reports should be stored. Default: `output`.
  * * `screenshotsForAllureReport`: If Allure plugin is enabled this plugin attaches each saved screenshot to allure report. Default: false.
+ * * `disableScreenshotOnFail : Disables the capturing of screeshots after the failed step. Default: true.
  *
  * @param {*} config
  */
@@ -80,6 +82,7 @@ module.exports = function (config) {
   let error;
   let savedStep = null;
   let currentTest = null;
+  let scenarioFailed = false;
 
   const recordedTests = {};
   const pad = '0000';
@@ -135,10 +138,14 @@ module.exports = function (config) {
     if (isStepIgnored(step)) return;
     if (savedStep === step) return; // already saved
     // Ignore steps from BeforeSuite function
+    if (scenarioFailed && config.disableScreenshotOnFail) return;
     if (step.metaStep && step.metaStep.name === 'BeforeSuite') return;
 
     const fileName = `${pad.substring(0, pad.length - stepNum.toString().length) + stepNum.toString()}.png`;
     try {
+      if (step.status === 'failed') {
+        scenarioFailed = true;
+      }
       stepNum++;
       slides[fileName] = step;
       await helper.saveScreenshot(path.relative(reportDir, path.join(dir, fileName)), config.fullPageScreenshots);

package/lib/plugin/stepTimeout.js

@@ -0,0 +1,90 @@
+const event = require('../event');
+
+const defaultConfig = {
+  timeout: 150,
+  force: false,
+  noTimeoutSteps: [
+    'amOnPage',
+    'wait*',
+  ],
+  customTimeoutSteps: [],
+};
+
+/**
+ * Set timeout for test steps globally.
+ *
+ * Add this plugin to config file:
+ *
+ * ```js
+ * plugins: {
+ *     stepTimeout: {
+ *        enabled: true
+ *     }
+ * }
+ * ```
+ *
+ *
+ * Run tests with plugin enabled:
+ *
+ * ```
+ * npx codeceptjs run --plugins stepTimeout
+ * ```
+ *
+ * #### Configuration:
+ *
+ * * `timeout` - global step timeout, default 150 seconds
+ * * `force` - whether to use timeouts set in plugin config to override step timeouts set in code with I.limitTime(x).action(...), default false
+ * * `noTimeoutSteps` - an array of steps with no timeout. Default:
+ *     * `amOnPage`
+ *     * `wait*`
+ *
+ *   you could set your own noTimeoutSteps which would replace the default one.
+ *
+ * * `customTimeoutSteps` - an array of step actions with custom timeout. Use it to override or extend noTimeoutSteps.
+ * You can use step names or step prefixes ending with `*`. As such, `wait*` will match all steps starting with `wait`.
+ *
+ * #### Example
+ *
+ * ```js
+ * plugins: {
+ *     stepTimeout: {
+ *         enabled: true,
+ *         force: true,
+ *         noTimeoutSteps: [
+ *           'scroll*', // ignore all scroll steps
+ *           /Cookie/, // ignore all steps with a Cookie in it (by regexp)
+ *         ],
+ *         customTimeoutSteps: [
+ *           ['myFlakyStep*', 1],
+ *           ['scrollWhichRequiresTimeout', 5],
+ *         ]
+ *     }
+ * }
+ * ```
+ *
+ */
+module.exports = (config) => {
+  config = Object.assign(defaultConfig, config);
+  // below override rule makes sure customTimeoutSteps go first but then they override noTimeoutSteps in case of exact pattern match
+  config.customTimeoutSteps = config.customTimeoutSteps.concat(config.noTimeoutSteps).concat(config.customTimeoutSteps);
+
+  event.dispatcher.on(event.step.before, (step) => {
+    let stepTimeout;
+    for (let stepRule of config.customTimeoutSteps) {
+      let customTimeout = 0;
+      if (Array.isArray(stepRule)) {
+        if (stepRule.length > 1) customTimeout = stepRule[1];
+        stepRule = stepRule[0];
+      }
+      if (stepRule instanceof RegExp
+        ? step.name.match(stepRule)
+        : (step.name === stepRule || stepRule.indexOf('*') && step.name.startsWith(stepRule.slice(0, -1)))
+      ) {
+        stepTimeout = customTimeout;
+        break;
+      }
+    }
+    stepTimeout = stepTimeout === undefined ? config.timeout : stepTimeout;
+    step.totalTimeout = stepTimeout * 1000;
+  });
+};

package/lib/plugin/subtitles.js

@@ -0,0 +1,88 @@
+const { v4: uuidv4 } = require('uuid');
+const fsPromise = require('fs').promises;
+const path = require('path');
+const event = require('../event');
+
+// This will convert a given timestamp in milliseconds to
+// an SRT recognized timestamp, ie HH:mm:ss,SSS
+function formatTimestamp(timestampInMs) {
+  const date = new Date(0, 0, 0, 0, 0, 0, timestampInMs);
+  const hours = date.getHours();
+  const minutes = date.getMinutes();
+  const seconds = date.getSeconds();
+  const ms = timestampInMs - (hours * 3600000 + minutes * 60000 + seconds * 1000);
+  return `${hours.toString().padStart(2, '0')}:${minutes.toString().padStart(2, '0')}:${seconds.toString().padStart(2, '0')},${ms.toString().padStart(3, '0')}`;
+}
+
+let steps = {};
+let testStartedAt;
+/**
+ * Automatically captures steps as subtitle, and saves it as an artifact when a video is found for a failed test
+ *
+ * #### Configuration
+ * ```js
+ * plugins: {
+ *  subtitles: {
+ *    enabled: true
+ *  }
+ * }
+ * ```
+ */
+module.exports = function () {
+  event.dispatcher.on(event.test.before, (_) => {
+    testStartedAt = Date.now();
+    steps = {};
+  });
+
+  event.dispatcher.on(event.step.started, (step) => {
+    const stepStartedAt = Date.now();
+    step.id = uuidv4();
+
+    let title = `${step.actor}.${step.name}(${step.args ? step.args.join(',') : ''})`;
+    if (title.length > 100) {
+      title = `${title.substring(0, 100)}...`;
+    }
+
+    steps[step.id] = {
+      start: formatTimestamp(stepStartedAt - testStartedAt),
+      startedAt: stepStartedAt,
+      title,
+    };
+  });
+
+  event.dispatcher.on(event.step.finished, (step) => {
+    if (step && step.id && steps[step.id]) {
+      steps[step.id].end = formatTimestamp(Date.now() - testStartedAt);
+    }
+  });
+
+  event.dispatcher.on(event.test.after, async (test) => {
+    if (test && test.artifacts && test.artifacts.video) {
+      const stepsSortedByStartTime = Object.values(steps);
+      stepsSortedByStartTime.sort((stepA, stepB) => {
+        return stepA.startedAt - stepB.startedAt;
+      });
+
+      let subtitle = '';
+
+      // For an SRT file, every subtitle has to be in the format as mentioned below:
+      //
+      // 1
+      // HH:mm:ss,SSS --> HH:mm:ss,SSS
+      // [title]
+      stepsSortedByStartTime.forEach((step, index) => {
+        if (step.end) {
+          subtitle = `${subtitle}${index + 1}
+${step.start} --> ${step.end}
+${step.title}
+
+`;
+        }
+      });
+
+      const { dir: artifactsDirectory, name: fileName } = path.parse(test.artifacts.video);
+      await fsPromise.writeFile(`${artifactsDirectory}/${fileName}.srt`, subtitle);
+      test.artifacts.subtitle = `${artifactsDirectory}/${fileName}.srt`;
+    }
+  });
+};

package/lib/plugin/tryTo.js

@@ -89,7 +89,7 @@ function tryTo(callback) {
     recorder.session.catch((err) => {
       result = false;
       const msg = err.inspect ? err.inspect() : err.toString();
-      debug(`Unsuccesful try > ${msg}`);
+      debug(`Unsuccessful try > ${msg}`);
       recorder.session.restore('tryTo');
       return result;
     });

package/lib/recorder.js

@@ -3,6 +3,8 @@ const promiseRetry = require('promise-retry');
 
 const { log } = require('./output');
 
+const MAX_TASKS = 100;
+
 let promise;
 let running = false;
 let errFn;
@@ -116,7 +118,7 @@ module.exports = {
      * @inner
      */
     start(name) {
-      log(`${currentQueue()}Starting <${name}> session`);
+      debug(`${currentQueue()}Starting <${name}> session`);
       tasks.push('--->');
       oldPromises.push(promise);
       this.running = true;
@@ -130,7 +132,7 @@ module.exports = {
      */
     restore(name) {
       tasks.push('<---');
-      log(`${currentQueue()}Finalize <${name}> session`);
+      debug(`${currentQueue()}Finalize <${name}> session`);
       this.running = false;
       sessionId = null;
       this.catch(errFn);
@@ -158,10 +160,11 @@ module.exports = {
    *     undefined: `add(fn)` -> `false` and `add('step',fn)` -> `true`
    *     true: it will retries if `retryOpts` set.
    *     false: ignore `retryOpts` and won't retry.
+   * @param {number} [timeout]
    * @return {Promise<*> | undefined}
    * @inner
    */
-  add(taskName, fn = undefined, force = false, retry = undefined) {
+  add(taskName, fn = undefined, force = false, retry = undefined, timeout = undefined) {
     if (typeof taskName === 'function') {
       fn = taskName;
       taskName = fn.toString();
@@ -172,19 +175,21 @@ module.exports = {
       return;
     }
     tasks.push(taskName);
-    debug(`${currentQueue()}Queued | ${taskName}`);
+    if (process.env.DEBUG) debug(`${currentQueue()}Queued | ${taskName}`);
 
     return promise = Promise.resolve(promise).then((res) => {
       const retryOpts = this.retries.slice(-1).pop();
       // no retries or unnamed tasks
       if (!retryOpts || !taskName || !retry) {
-        return Promise.resolve(res).then(fn);
+        const [promise, timer] = getTimeoutPromise(timeout, taskName);
+        return Promise.race([promise, Promise.resolve(res).then(fn)]).finally(() => clearTimeout(timer));
       }
 
       const retryRules = this.retries.slice().reverse();
       return promiseRetry(Object.assign(defaultRetryOptions, retryOpts), (retry, number) => {
         if (number > 1) log(`${currentQueue()}Retrying... Attempt #${number}`);
-        return Promise.resolve(res).then(fn).catch((err) => {
+        const [promise, timer] = getTimeoutPromise(timeout, taskName);
+        return Promise.race([promise, Promise.resolve(res).then(fn)]).finally(() => clearTimeout(timer)).catch((err) => {
           for (const retryObj of retryRules) {
             if (!retryObj.when) return retry(err);
             if (retryObj.when && retryObj.when(err)) return retry(err);
@@ -296,7 +301,7 @@ module.exports = {
    * @inner
    */
   stop() {
-    debug(this.toString());
+    if (process.env.DEBUG) debug(this.toString());
     log(`${currentQueue()}Stopping recording promises`);
     running = false;
   },
@@ -318,7 +323,7 @@ module.exports = {
    * @inner
    */
   scheduled() {
-    return tasks.join('\n');
+    return tasks.slice(-MAX_TASKS).join('\n');
   },
 
   /**
@@ -341,6 +346,14 @@ module.exports = {
 
 };
 
+function getTimeoutPromise(timeoutMs, taskName) {
+  let timer;
+  if (timeoutMs) debug(`Timing out in ${timeoutMs}ms`);
+  return [new Promise((done, reject) => {
+    timer = setTimeout(() => { reject(new Error(`Action ${taskName} was interrupted on step timeout ${timeoutMs}ms`)); }, timeoutMs || 2e9);
+  }), timer];
+}
+
 function currentQueue() {
   let session = '';
   if (sessionId) session = `<${sessionId}> `;