codeceptjs 3.1.0 → 3.2.0

package/lib/interfaces/gherkin.js

@@ -11,15 +11,19 @@ const transform = require('../transform');
 const parser = new Parser();
 parser.stopAtFirstError = false;
 
-module.exports = (text) => {
+module.exports = (text, file) => {
   const ast = parser.parse(text);
 
+  if (!ast.feature) {
+    throw new Error(`No 'Features' available in Gherkin '${file}' provided!`);
+  }
   const suite = new Suite(ast.feature.name, new Context());
   const tags = ast.feature.tags.map(t => t.name);
   suite.title = `${suite.title} ${tags.join(' ')}`.trim();
   suite.tags = tags || [];
   suite.comment = ast.feature.description;
   suite.feature = ast.feature;
+  suite.file = file;
   suite.timeout(0);
 
   suite.beforeEach('codeceptjs.before', () => scenario.setup(suite));
@@ -95,6 +99,7 @@ module.exports = (text) => {
           const title = `${child.name} ${JSON.stringify(current)} ${tags.join(' ')}`.trim();
           const test = new Test(title, async () => runSteps(addExampleInTable(exampleSteps, current)));
           test.tags = suite.tags.concat(tags);
+          test.file = file;
           suite.addTest(scenario.test(test));
         }
       }
@@ -104,6 +109,7 @@ module.exports = (text) => {
     const title = `${child.name} ${tags.join(' ')}`.trim();
     const test = new Test(title, async () => runSteps(child.steps));
     test.tags = suite.tags.concat(tags);
+    test.file = file;
     suite.addTest(scenario.test(test));
   }
 

package/lib/interfaces/scenarioConfig.js

@@ -45,6 +45,10 @@ class ScenarioConfig {
    * @returns {this}
    */
   timeout(timeout) {
+    console.log(`Scenario('${this.test.title}', () => {}).timeout(${timeout}) is deprecated!`);
+    console.log(`Please use Scenario('${this.test.title}', { timeout: ${timeout / 1000} }, () => {}) instead`);
+    console.log('Timeout should be set in seconds');
+
     this.test.timeout(timeout);
     return this;
   }

package/lib/listener/helpers.js

@@ -1,3 +1,4 @@
+const path = require('path');
 const event = require('../event');
 const container = require('../container');
 const recorder = require('../recorder');

package/lib/listener/steps.js

@@ -1,5 +1,7 @@
+const debug = require('debug')('codeceptjs:steps');
 const event = require('../event');
 const store = require('../store');
+const output = require('../output');
 
 let currentTest;
 let currentHook;
@@ -9,6 +11,7 @@ let currentHook;
  */
 module.exports = function () {
   event.dispatcher.on(event.test.before, (test) => {
+    test.startedAt = +new Date();
     test.artifacts = {};
   });
 
@@ -19,17 +22,23 @@ module.exports = function () {
     else currentTest.retryNum += 1;
   });
 
-  event.dispatcher.on(event.test.after, () => {
+  event.dispatcher.on(event.test.after, (test) => {
     currentTest = null;
   });
 
-  event.dispatcher.on(event.hook.passed, () => {
-    currentHook = null;
+  event.dispatcher.on(event.test.finished, (test) => {
   });
 
   event.dispatcher.on(event.hook.started, (suite) => {
     currentHook = suite.ctx.test;
     currentHook.steps = [];
+
+    if (suite.ctx && suite.ctx.test) output.log(`--- STARTED ${suite.ctx.test.title} ---`);
+  });
+
+  event.dispatcher.on(event.hook.passed, (suite) => {
+    currentHook = null;
+    if (suite.ctx && suite.ctx.test) output.log(`--- ENDED ${suite.ctx.test.title} ---`);
   });
 
   event.dispatcher.on(event.test.failed, () => {
@@ -51,6 +60,7 @@ module.exports = function () {
   });
 
   event.dispatcher.on(event.test.passed, () => {
+    if (!currentTest) return;
     // To be sure that passed test will be passed in report
     delete currentTest.err;
     currentTest.state = 'passed';
@@ -58,10 +68,18 @@ module.exports = function () {
 
   event.dispatcher.on(event.step.started, (step) => {
     if (store.debugMode) return;
+    step.startedAt = +new Date();
     if (currentHook && Array.isArray(currentHook.steps)) {
       return currentHook.steps.push(step);
     }
     if (!currentTest || !currentTest.steps) return;
     currentTest.steps.push(step);
   });
+
+  event.dispatcher.on(event.step.finished, (step) => {
+    if (store.debugMode) return;
+    step.finishedAt = +new Date();
+    if (step.startedAt) step.duration = step.finishedAt - step.startedAt;
+    debug(`Step '${step}' finished; Duration: ${step.duration}ms`);
+  });
 };

package/lib/listener/timeout.js

@@ -0,0 +1,71 @@
+const event = require('../event');
+const output = require('../output');
+const recorder = require('../recorder');
+const Config = require('../config');
+const { timeouts } = require('../store');
+
+module.exports = function () {
+  let timeout;
+  let timeoutStack = [];
+  let currentTest;
+  let currentTimeout;
+
+  if (!timeouts) {
+    console.log('Timeouts were disabled');
+    return;
+  }
+
+  event.dispatcher.on(event.suite.before, (suite) => {
+    timeoutStack = [];
+    const globalTimeout = Config.get('timeout');
+    if (globalTimeout) {
+      if (globalTimeout >= 1000) {
+        console.log(`Warning: Timeout was set to ${globalTimeout}secs.\nGlobal timeout should be specified in seconds.`);
+      }
+      timeoutStack.push(globalTimeout);
+    }
+    if (suite.totalTimeout) timeoutStack.push(suite.totalTimeout);
+    output.log(`Timeouts: ${timeoutStack}`);
+  });
+
+  event.dispatcher.on(event.test.before, (test) => {
+    currentTest = test;
+    timeout = test.totalTimeout || timeoutStack[timeoutStack.length - 1];
+    if (!timeout) return;
+    currentTimeout = timeout;
+    output.debug(`Test Timeout: ${timeout}s`);
+    timeout *= 1000;
+  });
+
+  event.dispatcher.on(event.test.passed, (test) => {
+    currentTest = null;
+  });
+
+  event.dispatcher.on(event.test.failed, (test) => {
+    currentTest = null;
+  });
+
+  event.dispatcher.on(event.step.before, (step) => {
+    if (typeof timeout !== 'number') return;
+
+    if (timeout < 0) {
+      step.totalTimeout = 0.01;
+    } else {
+      step.totalTimeout = timeout;
+    }
+  });
+
+  event.dispatcher.on(event.step.finished, (step) => {
+    timeout -= step.duration;
+
+    if (typeof timeout === 'number' && timeout <= 0 && recorder.isRunning()) {
+      if (currentTest && currentTest.callback) {
+        recorder.reset();
+        // replace mocha timeout with custom timeout
+        currentTest.timeout(0);
+        currentTest.callback(new Error(`Timeout ${currentTimeout}s exceeded (with Before hook)`));
+        currentTest.timedOut = true;
+      }
+    }
+  });
+};

package/lib/locator.js

@@ -313,11 +313,14 @@ Locator.build = (locator) => {
 
 /**
  * Filters to modify locators
+ * @type {Array<function(CodeceptJS.LocatorOrString, Locator): void>}
  */
 Locator.filters = [];
 
 /**
  * Appends new `Locator` filter to an `Locator.filters` array, and returns the new length of the array.
+ * @param {function(CodeceptJS.LocatorOrString, Locator): void} fn
+ * @returns {number}
  */
 Locator.addFilter = fn => Locator.filters.push(fn);
 

package/lib/mochaFactory.js

@@ -2,7 +2,7 @@ const Mocha = require('mocha');
 const fsPath = require('path');
 const fs = require('fs');
 const reporter = require('./cli');
-const gherkinParser = require('./interfaces/gherkin.js');
+const gherkinParser = require('./interfaces/gherkin');
 const output = require('./output');
 const { genTestId } = require('./utils');
 const ConnectionRefused = require('./helper/errors/ConnectionRefused');
@@ -17,12 +17,6 @@ class MochaFactory {
     output.process(opts.child);
     mocha.ui(scenarioUi);
 
-    // process.on('unhandledRejection', (reason) => {
-    //   output.error('Unhandled rejection');
-    //   console.log(Error.captureStackTrace(reason));
-    //   output.error(reason);
-    // });
-
     Mocha.Runner.prototype.uncaught = function (err) {
       if (err) {
         if (err.toString().indexOf('ECONNREFUSED') >= 0) {
@@ -41,8 +35,7 @@ class MochaFactory {
       if (mocha.suite.suites.length === 0) {
         mocha.files
           .filter(file => file.match(/\.feature$/))
-          .map(file => fs.readFileSync(file, 'utf8'))
-          .forEach(content => mocha.suite.addSuite(gherkinParser(content)));
+          .forEach(file => mocha.suite.addSuite(gherkinParser(fs.readFileSync(file, 'utf8'), file)));
 
         // remove feature files
         mocha.files = mocha.files.filter(file => !file.match(/\.feature$/));
@@ -51,19 +44,30 @@ class MochaFactory {
 
         // add ids for each test and check uniqueness
         const dupes = [];
+        let missingFeatureInFile = [];
         const seenTests = [];
         mocha.suite.eachTest(test => {
           test.id = genTestId(test);
+
           const name = test.fullTitle();
           if (seenTests.includes(test.id)) {
             dupes.push(name);
           }
           seenTests.push(test.id);
+
+          if (name.slice(0, name.indexOf(':')) === '') {
+            missingFeatureInFile.push(test.file);
+          }
         });
         if (dupes.length) {
           // ideally this should be no-op and throw (breaking change)...
           output.error(`Duplicate test names detected - Feature + Scenario name should be unique:\n${dupes.join('\n')}`);
         }
+
+        if (missingFeatureInFile.length) {
+          missingFeatureInFile = [...new Set(missingFeatureInFile)];
+          output.error(`Missing Feature section in:\n${missingFeatureInFile.join('\n')}`);
+        }
       }
     };
 

package/lib/plugin/allure.js

@@ -283,7 +283,12 @@ module.exports = (config) => {
   let maxLevel;
   function finishMetastep(level) {
     const metaStepsToFinish = currentMetaStep.splice(maxLevel - level);
-    metaStepsToFinish.forEach(() => reporter.endStep('passed'));
+    metaStepsToFinish.forEach(() => {
+      // only if the current step is of type Step, end it.
+      if (reporter.suites && reporter.suites.length && reporter.suites[0].currentStep && reporter.suites[0].currentStep.constructor.name === 'Step') {
+        reporter.endStep('passed');
+      }
+    });
   }
 
   function startMetaStep(metaStep, level = 0) {

package/lib/plugin/{puppeteerCoverage.js → coverage.js}

@@ -1,4 +1,4 @@
-const debug = require('debug')('codeceptjs:plugin:puppeteerCoverage');
+const debugModule = require('debug');
 const fs = require('fs');
 const path = require('path');
 
@@ -13,7 +13,7 @@ const defaultConfig = {
   uniqueFileName: true,
 };
 
-const supportedHelpers = ['Puppeteer'];
+const supportedHelpers = ['Puppeteer', 'Playwright'];
 
 function buildFileName(test, uniqueFileName) {
   let fileName = clearString(test.title);
@@ -41,15 +41,14 @@ function buildFileName(test, uniqueFileName) {
 }
 
 /**
- * Dumps puppeteers code coverage after every test.
+ * Dumps code coverage from Playwright/Puppeteer after every test.
  *
  * #### Configuration
  *
- * Configuration can either be taken from a corresponding helper (deprecated) or a from plugin config (recommended).
  *
  * ```js
  * plugins: {
- *    puppeteerCoverage: {
+ *    coverage: {
  *      enabled: true
  *    }
  * }
@@ -59,33 +58,22 @@ function buildFileName(test, uniqueFileName) {
  *
  * * `coverageDir`: directory to dump coverage files
  * * `uniqueFileName`: generate a unique filename by adding uuid
- *
- *  First of all, your mileage may vary!
- *
- *  To work, you need the client javascript code to be NOT uglified. They need to be built in "development" mode.
- *  And the end of your tests, you'll get a directory full of coverage per test run.  Now what?
- *  You'll need to convert the coverage code to something istanbul can read.  Good news is someone wrote the code
- *  for you (see puppeteer-to-istanbul link below).  Then using istanbul you need to combine the converted
- *  coverage and create a report.  Good luck!
- *
- *  Links:
- *  * https://github.com/GoogleChrome/puppeteer/blob/v1.12.2/docs/api.md#class-coverage
- *  * https://github.com/istanbuljs/puppeteer-to-istanbul
- *  * https://github.com/gotwarlost/istanbul
  */
 module.exports = function (config) {
   const helpers = Container.helpers();
   let coverageRunning = false;
   let helper;
 
+  let debug;
   for (const helperName of supportedHelpers) {
     if (Object.keys(helpers).indexOf(helperName) > -1) {
       helper = helpers[helperName];
+      debug = debugModule(`codeceptjs:plugin:${helperName.toLowerCase()}Coverage`);
     }
   }
 
   if (!helper) {
-    console.error('Coverage is only supported in Puppeteer');
+    console.error('Coverage is only supported in Puppeteer, Playwright');
     return; // no helpers for screenshot
   }
 
@@ -102,7 +90,7 @@ module.exports = function (config) {
       'starting coverage',
       async () => {
         try {
-          if (!coverageRunning) {
+          if (!coverageRunning && helper.page && helper.page.coverage) {
             debug('--> starting coverage <--');
             coverageRunning = true;
             await helper.page.coverage.startJSCoverage();
@@ -115,13 +103,13 @@ module.exports = function (config) {
     );
   });
 
-  // Save puppeteer coverage data after every test run
+  // Save coverage data after every test run
   event.dispatcher.on(event.test.after, async (test) => {
     recorder.add(
       'saving coverage',
       async () => {
         try {
-          if (coverageRunning) {
+          if (coverageRunning && helper.page && helper.page.coverage) {
             debug('--> stopping coverage <--');
             coverageRunning = false;
             const coverage = await helper.page.coverage.stopJSCoverage();

package/lib/plugin/customLocator.js

@@ -38,7 +38,7 @@ const defaultConfig = {
  * // in codecept.conf.js
  * plugins: {
  *  customLocator: {
- *    enabled: true
+ *    enabled: true,
  *    attribute: 'data-test'
  *  }
  * }
@@ -57,7 +57,7 @@ const defaultConfig = {
  * // in codecept.conf.js
  * plugins: {
  *  customLocator: {
- *    enabled: true
+ *    enabled: true,
  *    prefix: '=',
  *    attribute: 'data-qa'
  *  }

package/lib/plugin/retryTo.js

@@ -0,0 +1,130 @@
+const recorder = require('../recorder');
+const store = require('../store');
+const { debug } = require('../output');
+
+const defaultConfig = {
+  registerGlobal: true,
+  pollInterval: 200,
+};
+
+/**
+ *
+ *
+ * Adds global `retryTo` which retries steps a few times before failing.
+ *
+ * Enable this plugin in `codecept.conf.js` (enabled by default for new setups):
+ *
+ * ```js
+ * plugins: {
+ *   retryTo: {
+ *     enabled: true
+ *   }
+ * }
+ * ```
+ *
+ * Use it in your tests:
+ *
+ * ```js
+ * // retry these steps 5 times before failing
+ * await retryTo((tryNum) => {
+ *   I.switchTo('#editor frame');
+ *   I.click('Open');
+ *   I.see('Opened')
+ * }, 5);
+ * ```
+ * Set polling interval as 3rd argument (200ms by default):
+ *
+ * ```js
+ * // retry these steps 5 times before failing
+ * await retryTo((tryNum) => {
+ *   I.switchTo('#editor frame');
+ *   I.click('Open');
+ *   I.see('Opened')
+ * }, 5, 100);
+ * ```
+ *
+ * Default polling interval can be changed in a config:
+ *
+ * ```js
+ * plugins: {
+ *   retryTo: {
+ *     enabled: true,
+ *     pollInterval: 500,
+ *   }
+ * }
+ * ```
+ *
+ * Disables retryFailedStep plugin for steps inside a block;
+ *
+ * Use this plugin if:
+ *
+ * * you need repeat a set of actions in flaky tests
+ * * iframe was not rendered and you need to retry switching to it
+ *
+ *
+ * #### Configuration
+ *
+ * * `pollInterval` - default interval between retries in ms. 200 by default.
+ * * `registerGlobal` - to register `retryTo` function globally, true by default
+ *
+ * If `registerGlobal` is false you can use retryTo from the plugin:
+ *
+ * ```js
+ * const retryTo = codeceptjs.container.plugins('retryTo');
+ * ```
+ *
+*/
+module.exports = function (config) {
+  config = Object.assign(defaultConfig, config);
+
+  if (config.registerGlobal) {
+    global.retryTo = retryTo;
+  }
+  return retryTo;
+
+  function retryTo(callback, maxTries, pollInterval = undefined) {
+    const mode = store.debugMode;
+    let tries = 1;
+    if (!pollInterval) pollInterval = config.pollInterval;
+
+    let err = null;
+
+    return new Promise((done) => {
+      const tryBlock = () => {
+        recorder.session.start(`retryTo ${tries}`);
+        callback(tries);
+        recorder.add(() => {
+          recorder.session.restore(`retryTo ${tries}`);
+          done(null);
+        });
+        recorder.session.catch((e) => {
+          err = e;
+          recorder.session.restore(`retryTo ${tries}`);
+          tries++;
+          // recorder.session.restore(`retryTo`);
+          if (tries <= maxTries) {
+            debug(`Error ${err}... Retrying`);
+            err = null;
+
+            recorder.add(`retryTo ${tries}`, () => {
+              tryBlock();
+              // recorder.add(() => new Promise(done => setTimeout(done, pollInterval)));
+            });
+          } else {
+            // recorder.throw(err);
+            done(null);
+          }
+        });
+        // return recorder.promise();
+      };
+
+      recorder.add('retryTo', async () => {
+        store.debugMode = true;
+        tryBlock();
+        // recorder.add(() => recorder.session.restore(`retryTo ${tries-1}`));
+      });
+    }).then(() => {
+      if (err) recorder.throw(err);
+    });
+  }
+};

package/lib/plugin/screenshotOnFail.js

@@ -98,6 +98,7 @@ module.exports = function (config) {
         }
         await helper.saveScreenshot(fileName, options.fullPageScreenshots);
 
+        if (!test.artifacts) test.artifacts = {};
         test.artifacts.screenshot = path.join(global.output_dir, fileName);
         if (Container.mocha().options.reporterOptions['mocha-junit-reporter'] && Container.mocha().options.reporterOptions['mocha-junit-reporter'].options.attachments) {
           test.attachments = [path.join(global.output_dir, fileName)];

package/lib/plugin/stepByStepReport.js

@@ -20,6 +20,7 @@ const defaultConfig = {
   fullPageScreenshots: false,
   output: global.output_dir,
   screenshotsForAllureReport: false,
+  disableScreenshotOnFail: true,
 };
 
 const templates = {};
@@ -56,6 +57,7 @@ const templates = {};
  * * `fullPageScreenshots`: should full page screenshots be used. Default: false.
  * * `output`: a directory where reports should be stored. Default: `output`.
  * * `screenshotsForAllureReport`: If Allure plugin is enabled this plugin attaches each saved screenshot to allure report. Default: false.
+ * * `disableScreenshotOnFail : Disables the capturing of screeshots after the failed step. Default: true.
  *
  * @param {*} config
  */
@@ -80,6 +82,7 @@ module.exports = function (config) {
   let error;
   let savedStep = null;
   let currentTest = null;
+  let scenarioFailed = false;
 
   const recordedTests = {};
   const pad = '0000';
@@ -135,10 +138,14 @@ module.exports = function (config) {
     if (isStepIgnored(step)) return;
     if (savedStep === step) return; // already saved
     // Ignore steps from BeforeSuite function
+    if (scenarioFailed && config.disableScreenshotOnFail) return;
     if (step.metaStep && step.metaStep.name === 'BeforeSuite') return;
 
     const fileName = `${pad.substring(0, pad.length - stepNum.toString().length) + stepNum.toString()}.png`;
     try {
+      if (step.status === 'failed') {
+        scenarioFailed = true;
+      }
       stepNum++;
       slides[fileName] = step;
       await helper.saveScreenshot(path.relative(reportDir, path.join(dir, fileName)), config.fullPageScreenshots);

package/lib/plugin/stepTimeout.js

@@ -0,0 +1,90 @@
+const event = require('../event');
+
+const defaultConfig = {
+  timeout: 150,
+  force: false,
+  noTimeoutSteps: [
+    'amOnPage',
+    'wait*',
+  ],
+  customTimeoutSteps: [],
+};
+
+/**
+ * Set timeout for test steps globally.
+ *
+ * Add this plugin to config file:
+ *
+ * ```js
+ * plugins: {
+ *     stepTimeout: {
+ *        enabled: true
+ *     }
+ * }
+ * ```
+ *
+ *
+ * Run tests with plugin enabled:
+ *
+ * ```
+ * npx codeceptjs run --plugins stepTimeout
+ * ```
+ *
+ * #### Configuration:
+ *
+ * * `timeout` - global step timeout, default 150 seconds
+ * * `force` - whether to use timeouts set in plugin config to override step timeouts set in code with I.limitTime(x).action(...), default false
+ * * `noTimeoutSteps` - an array of steps with no timeout. Default:
+ *     * `amOnPage`
+ *     * `wait*`
+ *
+ *   you could set your own noTimeoutSteps which would replace the default one.
+ *
+ * * `customTimeoutSteps` - an array of step actions with custom timeout. Use it to override or extend noTimeoutSteps.
+ * You can use step names or step prefixes ending with `*`. As such, `wait*` will match all steps starting with `wait`.
+ *
+ * #### Example
+ *
+ * ```js
+ * plugins: {
+ *     stepTimeout: {
+ *         enabled: true,
+ *         force: true,
+ *         noTimeoutSteps: [
+ *           'scroll*', // ignore all scroll steps
+ *           /Cookie/, // ignore all steps with a Cookie in it (by regexp)
+ *         ],
+ *         customTimeoutSteps: [
+ *           ['myFlakyStep*', 1],
+ *           ['scrollWhichRequiresTimeout', 5],
+ *         ]
+ *     }
+ * }
+ * ```
+ *
+ */
+module.exports = (config) => {
+  config = Object.assign(defaultConfig, config);
+  // below override rule makes sure customTimeoutSteps go first but then they override noTimeoutSteps in case of exact pattern match
+  config.customTimeoutSteps = config.customTimeoutSteps.concat(config.noTimeoutSteps).concat(config.customTimeoutSteps);
+
+  event.dispatcher.on(event.step.before, (step) => {
+    let stepTimeout;
+    for (let stepRule of config.customTimeoutSteps) {
+      let customTimeout = 0;
+      if (Array.isArray(stepRule)) {
+        if (stepRule.length > 1) customTimeout = stepRule[1];
+        stepRule = stepRule[0];
+      }
+      if (stepRule instanceof RegExp
+        ? step.name.match(stepRule)
+        : (step.name === stepRule || stepRule.indexOf('*') && step.name.startsWith(stepRule.slice(0, -1)))
+      ) {
+        stepTimeout = customTimeout;
+        break;
+      }
+    }
+    stepTimeout = stepTimeout === undefined ? config.timeout : stepTimeout;
+    step.totalTimeout = stepTimeout * 1000;
+  });
+};