codeceptjs 3.5.15 → 3.6.0-beta.1.ai-healers

package/lib/plugin/coverage.js

@@ -1,67 +1,44 @@
 const debugModule = require('debug');
-const { CoverageReport } = require('monocart-coverage-reports');
+const fs = require('fs');
+const path = require('path');
+
 const Container = require('../container');
 const recorder = require('../recorder');
 const event = require('../event');
 const output = require('../output');
-const { deepMerge } = require('../utils');
+const { clearString } = require('../utils');
 
 const defaultConfig = {
-  name: 'CodeceptJS Coverage Report',
-  outputDir: 'output/coverage',
+  coverageDir: 'output/coverage',
+  uniqueFileName: true,
 };
 
 const supportedHelpers = ['Puppeteer', 'Playwright'];
 
-const v8CoverageHelpers = {
-  Playwright: {
-    startCoverage: async (page) => {
-      await Promise.all([
-        page.coverage.startJSCoverage({
-          resetOnNavigation: false,
-        }),
-        page.coverage.startCSSCoverage({
-          resetOnNavigation: false,
-        }),
-      ]);
-    },
-    takeCoverage: async (page, coverageReport) => {
-      const [jsCoverage, cssCoverage] = await Promise.all([
-        page.coverage.stopJSCoverage(),
-        page.coverage.stopCSSCoverage(),
-      ]);
-      const coverageList = [...jsCoverage, ...cssCoverage];
-      await coverageReport.add(coverageList);
-    },
-  },
-  Puppeteer: {
-    startCoverage: async (page) => {
-      await Promise.all([
-        page.coverage.startJSCoverage({
-          resetOnNavigation: false,
-          includeRawScriptCoverage: true,
-        }),
-        page.coverage.startCSSCoverage({
-          resetOnNavigation: false,
-        }),
-      ]);
-    },
-    takeCoverage: async (page, coverageReport) => {
-      const [jsCoverage, cssCoverage] = await Promise.all([
-        page.coverage.stopJSCoverage(),
-        page.coverage.stopCSSCoverage(),
-      ]);
-      // to raw V8 script coverage
-      const coverageList = [...jsCoverage.map((it) => {
-        return {
-          source: it.text,
-          ...it.rawScriptCoverage,
-        };
-      }), ...cssCoverage];
-      await coverageReport.add(coverageList);
-    },
-  },
-};
+function buildFileName(test, uniqueFileName) {
+  let fileName = clearString(test.title);
+
+  // This prevent data driven to be included in the failed screenshot file name
+  if (fileName.indexOf('{') !== -1) {
+    fileName = fileName.substr(0, fileName.indexOf('{') - 3).trim();
+  }
+
+  if (test.ctx && test.ctx.test && test.ctx.test.type === 'hook') {
+    fileName = clearString(`${test.title}_${test.ctx.test.title}`);
+  }
+
+  if (uniqueFileName) {
+    const uuid = test.uuid
+      || test.ctx.test.uuid
+      || Math.floor(new Date().getTime() / 1000);
+
+    fileName = `${fileName.substring(0, 10)}_${uuid}.coverage.json`;
+  } else {
+    fileName = `${fileName}.coverage.json`;
+  }
+
+  return fileName;
+}
 
 /**
  * Dumps code coverage from Playwright/Puppeteer after every test.
@@ -72,84 +49,94 @@ const v8CoverageHelpers = {
  * ```js
  * plugins: {
  *    coverage: {
- *      enabled: true,
- *      debug: true,
- *      name: 'CodeceptJS Coverage Report',
- *      outputDir: 'output/coverage'
+ *      enabled: true
  *    }
  * }
  * ```
  *
- * Possible config options, More could be found at [monocart-coverage-reports](https://github.com/cenfun/monocart-coverage-reports?tab=readme-ov-file#default-options)
- *
- * * `debug`: debug info. By default, false.
- * * `name`: coverage report name.
- * * `outputDir`: path to coverage report.
- * * `sourceFilter`: filter the source files.
- * * `sourcePath`: option to resolve a custom path.
+ * Possible config options:
  *
+ * * `coverageDir`: directory to dump coverage files
+ * * `uniqueFileName`: generate a unique filename by adding uuid
  */
 module.exports = function (config) {
-  config = deepMerge(defaultConfig, config);
-
-  if (config.debug) config.logging = 'debug';
-
   const helpers = Container.helpers();
   let coverageRunning = false;
+  let helper;
 
-  const v8Names = Object.keys(v8CoverageHelpers);
-  const helperName = Object.keys(helpers).find((it) => v8Names.includes(it));
-  if (!helperName) {
-    console.error(`Coverage is only supported in ${supportedHelpers.join(' or ')}`);
-    // no helpers for screenshot
-    return;
+  let debug;
+  for (const helperName of supportedHelpers) {
+    if (Object.keys(helpers).indexOf(helperName) > -1) {
+      helper = helpers[helperName];
+      debug = debugModule(`codeceptjs:plugin:${helperName.toLowerCase()}Coverage`);
+    }
   }
 
-  config.name = `${config.name} - in ${helperName}`;
-  const debug = debugModule(`codeceptjs:plugin:${helperName.toLowerCase()}Coverage`);
-
-  const helper = helpers[helperName];
-  const v8Helper = v8CoverageHelpers[helperName];
+  if (!helper) {
+    console.error('Coverage is only supported in Puppeteer, Playwright');
+    return; // no helpers for screenshot
+  }
 
-  const coverageOptions = {
-    ...config,
-  };
-  const coverageReport = new CoverageReport(coverageOptions);
-  coverageReport.cleanCache();
+  const options = Object.assign(defaultConfig, helper.options, config);
 
-  event.dispatcher.on(event.all.after, async () => {
-    output.print(`writing ${coverageOptions.outputDir}`);
-    await coverageReport.generate();
+  event.dispatcher.on(event.all.before, async () => {
+    output.debug('*** Collecting coverage for tests ****');
   });
 
-  //  we're going to try to "start" coverage before each step because this is
+  //  Hack!  we're going to try to "start" coverage before each step because this is
   //  when the browser is already up and is ready to start coverage.
-  event.dispatcher.on(event.step.before, () => {
-    recorder.add('start coverage', async () => {
-      if (coverageRunning) {
-        return;
-      }
-      if (!helper.page || !helper.page.coverage) {
-        return;
-      }
-      coverageRunning = true;
-      debug('--> starting coverage <--');
-      await v8Helper.startCoverage(helper.page);
-    }, true);
+  event.dispatcher.on(event.step.before, async () => {
+    recorder.add(
+      'starting coverage',
+      async () => {
+        try {
+          if (!coverageRunning && helper.page && helper.page.coverage) {
+            debug('--> starting coverage <--');
+            coverageRunning = true;
+            await helper.page.coverage.startJSCoverage();
+          }
+        } catch (err) {
+          console.error(err);
+        }
+      },
+      true,
+    );
   });
 
   // Save coverage data after every test run
-  event.dispatcher.on(event.test.after, (test) => {
-    recorder.add('take coverage', async () => {
-      if (!coverageRunning) {
-        return;
-      }
-      if (!helper.page || !helper.page.coverage) {
-        return;
-      }
-      coverageRunning = false;
-      debug('--> stopping coverage <--');
-      await v8Helper.takeCoverage(helper.page, coverageReport);
-    }, true);
+  event.dispatcher.on(event.test.after, async (test) => {
+    recorder.add(
+      'saving coverage',
+      async () => {
+        try {
+          if (coverageRunning && helper.page && helper.page.coverage) {
+            debug('--> stopping coverage <--');
+            coverageRunning = false;
+            const coverage = await helper.page.coverage.stopJSCoverage();
+
+            const coverageDir = path.resolve(
+              process.cwd(),
+              options.coverageDir,
+            );
+
+            // Checking if coverageDir already exists, if not, create new one
+
+            if (!fs.existsSync(coverageDir)) {
+              fs.mkdirSync(coverageDir, { recursive: true });
+            }
+
+            const coveragePath = path.resolve(
+              coverageDir,
+              buildFileName(test, options.uniqueFileName),
+            );
+            output.print(`writing ${coveragePath}`);
+            fs.writeFileSync(coveragePath, JSON.stringify(coverage));
+          }
+        } catch (err) {
+          console.error(err);
+        }
+      },
+      true,
+    );
   });
 };

package/lib/plugin/heal.js

@@ -1,33 +1,19 @@
 const debug = require('debug')('codeceptjs:heal');
 const colors = require('chalk');
-const Container = require('../container');
-const AiAssistant = require('../ai');
 const recorder = require('../recorder');
 const event = require('../event');
 const output = require('../output');
-const supportedHelpers = require('./standardActingHelpers');
+const heal = require('../heal');
+const store = require('../store');
 
 const defaultConfig = {
-  healTries: 1,
   healLimit: 2,
-  healSteps: [
-    'click',
-    'fillField',
-    'appendField',
-    'selectOption',
-    'attachFile',
-    'checkOption',
-    'uncheckOption',
-    'doubleClick',
-  ],
 };
 
 /**
- * Self-healing tests with OpenAI.
+ * Self-healing tests with AI.
  *
- * This plugin is experimental and requires OpenAI API key.
- *
- * To use it you need to set OPENAI_API_KEY env variable and enable plugin inside the config.
+ * Read more about heaking in [Self-Healing Tests](https://codecept.io/heal/)
  *
  * ```js
  * plugins: {
@@ -40,22 +26,15 @@ const defaultConfig = {
  * More config options are available:
  *
  * * `healLimit` - how many steps can be healed in a single test (default: 2)
- * * `healSteps` - which steps can be healed (default: all steps that interact with UI, see list below)
- *
- * Steps to heal:
- *
- * * `click`
- * * `fillField`
- * * `appendField`
- * * `selectOption`
- * * `attachFile`
- * * `checkOption`
- * * `uncheckOption`
- * * `doubleClick`
  *
  */
 module.exports = function (config = {}) {
-  const aiAssistant = AiAssistant.getInstance();
+  if (store.debugMode && !process.env.DEBUG) {
+    event.dispatcher.on(event.test.failed, () => {
+      output.plugin('heal', 'Healing is disabled in --debug mode, use DEBUG="codeceptjs:heal" to enable it in debug mode');
+    });
+    return;
+  }
 
   let currentTest = null;
   let currentStep = null;
@@ -64,8 +43,6 @@ module.exports = function (config = {}) {
   let healTries = 0;
   let isHealing = false;
 
-  const healSuggestions = [];
-
   config = Object.assign(defaultConfig, config);
 
   event.dispatcher.on(event.test.before, (test) => {
@@ -78,63 +55,27 @@ module.exports = function (config = {}) {
 
   event.dispatcher.on(event.step.after, (step) => {
     if (isHealing) return;
-    const store = require('../store');
-    if (store.debugMode) return;
+    if (healTries >= config.healLimit) return; // out of limit
+
+    if (!heal.hasCorrespondingRecipes(step)) return;
+
     recorder.catchWithoutStop(async (err) => {
       isHealing = true;
       if (caughtError === err) throw err; // avoid double handling
       caughtError = err;
-      if (!aiAssistant.isEnabled) {
-        output.print(colors.yellow('Heal plugin can\'t operate, AI assistant is disabled. Please set OPENAI_API_KEY env variable to enable it.'));
-        throw err;
-      }
-      if (!currentStep) throw err;
-      if (!config.healSteps.includes(currentStep.name)) throw err;
-      const test = currentTest;
-
-      if (healTries >= config.healTries) {
-        output.print(colors.bold.red(`Healing failed for ${config.healTries} time(s)`));
-        output.print('AI couldn\'t identify the correct solution');
-        output.print('Probably the entire flow has changed and the test should be updated');
-
-        throw err;
-      }
 
-      if (healedSteps >= config.healLimit) {
-        output.print(colors.bold.red(`Can't heal more than ${config.healLimit} step(s) in a test`));
-        output.print('Entire flow can be broken, please check it manually');
-        output.print('or increase healing limit in heal plugin config');
-
-        throw err;
-      }
+      const test = currentTest;
 
       recorder.session.start('heal');
-      const helpers = Container.helpers();
-      let helper;
-
-      for (const helperName of supportedHelpers) {
-        if (Object.keys(helpers).indexOf(helperName) > -1) {
-          helper = helpers[helperName];
-        }
-      }
 
-      if (!helper) throw err; // no helpers for html
-
-      const step = test.steps[test.steps.length - 1];
       debug('Self-healing started', step.toCode());
 
-      const currentOutputLevel = output.level();
-      output.level(0);
-      const html = await helper.grabHTMLFrom('body');
-      output.level(currentOutputLevel);
-
-      if (!html) throw err;
+      await heal.healStep(step, err, { test });
 
       healTries++;
-      await aiAssistant.setHtmlContext(html);
-      await tryToHeal(step, err);
 
       recorder.add('close healing session', () => {
+        recorder.reset();
         recorder.session.restore('heal');
         recorder.ignoreErr(err);
       });
@@ -145,7 +86,7 @@ module.exports = function (config = {}) {
   });
 
   event.dispatcher.on(event.all.result, () => {
-    if (!healSuggestions.length) return;
+    if (!heal.fixes?.length) return;
 
     const { print } = output;
 
@@ -153,16 +94,20 @@ module.exports = function (config = {}) {
     print('===================');
     print(colors.bold.green('Self-Healing Report:'));
 
-    print(`${colors.bold(healSuggestions.length)} step(s) were healed by AI`);
+    print(`${colors.bold(heal.fixes.length)} ${heal.fixes.length === 1 ? 'step was' : 'steps were'} healed`);
+
+    const suggestions = heal.fixes.filter(fix => fix.recipe && heal.recipes[fix.recipe].suggest);
+
+    if (!suggestions.length) return;
 
     let i = 1;
     print('');
     print('Suggested changes:');
     print('');
 
-    for (const suggestion of healSuggestions) {
-      print(`${i}. To fix ${colors.bold.blue(suggestion.test.title)}`);
-      print('Replace the failed code with:');
+    for (const suggestion of suggestions) {
+      print(`${i}. To fix ${colors.bold.magenta(suggestion.test?.title)}`);
+      print('  Replace the failed code:', colors.gray(`(suggested by ${colors.bold(suggestion.recipe)})`));
       print(colors.red(`- ${suggestion.step.toCode()}`));
       print(colors.green(`+ ${suggestion.snippet}`));
       print(suggestion.step.line());
@@ -170,40 +115,4 @@ module.exports = function (config = {}) {
       i++;
     }
   });
-
-  async function tryToHeal(failedStep, err) {
-    output.debug(`Running OpenAI to heal ${failedStep.toCode()} step`);
-
-    const codeSnippets = await aiAssistant.healFailedStep(failedStep, err, currentTest);
-
-    output.debug(`Received ${codeSnippets.length} suggestions from OpenAI`);
-    const I = Container.support('I'); // eslint-disable-line
-
-    for (const codeSnippet of codeSnippets) {
-      try {
-        debug('Executing', codeSnippet);
-        recorder.catch((e) => {
-          console.log(e);
-        });
-        await eval(codeSnippet); // eslint-disable-line
-
-        healSuggestions.push({
-          test: currentTest,
-          step: failedStep,
-          snippet: codeSnippet,
-        });
-
-        recorder.add('healed', () => output.print(colors.bold.green('  Code healed successfully')));
-        healedSteps++;
-        return;
-      } catch (err) {
-        debug('Failed to execute code', err);
-        recorder.ignoreErr(err); // healing ded not help
-        // recorder.catch(() => output.print(colors.bold.red('  Failed healing code')));
-      }
-    }
-
-    output.debug(`Couldn't heal the code for ${failedStep.toCode()}`);
-  }
-  return recorder.promise();
 };

package/lib/recorder.js

@@ -1,5 +1,6 @@
 const debug = require('debug')('codeceptjs:recorder');
 const promiseRetry = require('promise-retry');
+const chalk = require('chalk');
 const { printObjectProperties } = require('./utils');
 const { log } = require('./output');
 
@@ -179,7 +180,7 @@ module.exports = {
       return;
     }
     tasks.push(taskName);
-    debug(`${currentQueue()}Queued | ${taskName}`);
+    debug(chalk.gray(`${currentQueue()} Queued | ${taskName}`));
 
     return promise = Promise.resolve(promise).then((res) => {
       // prefer options for non-conditional retries
@@ -190,11 +191,14 @@ module.exports = {
         return Promise.race([promise, Promise.resolve(res).then(fn)]).finally(() => clearTimeout(timer));
       }
 
+      debug(`${currentQueue()} Running | ${taskName}`);
+
       const retryRules = this.retries.slice().reverse();
       return promiseRetry(Object.assign(defaultRetryOptions, retryOpts), (retry, number) => {
         if (number > 1) log(`${currentQueue()}Retrying... Attempt #${number}`);
         const [promise, timer] = getTimeoutPromise(timeout, taskName);
         return Promise.race([promise, Promise.resolve(res).then(fn)]).finally(() => clearTimeout(timer)).catch((err) => {
+          if (ignoredErrs.includes(err)) return;
           for (const retryObj of retryRules) {
             if (!retryObj.when) return retry(err);
             if (retryObj.when && retryObj.when(err)) return retry(err);
@@ -229,7 +233,7 @@ module.exports = {
    */
   catch(customErrFn) {
     const fnDescription = customErrFn?.toString()?.replace(/\s{2,}/g, ' ').replace(/\n/g, ' ')?.slice(0, 50);
-    debug(`${currentQueue()}Queued | catch with error handler ${fnDescription || ''}`);
+    debug(chalk.gray(`${currentQueue()} Queued | catch with error handler ${fnDescription || ''}`));
     return promise = promise.catch((err) => {
       log(`${currentQueue()}Error | ${err} ${fnDescription}...`);
       if (!(err instanceof Error)) { // strange things may happen
@@ -253,7 +257,7 @@ module.exports = {
     const fnDescription = customErrFn?.toString()?.replace(/\s{2,}/g, ' ').replace(/\n/g, ' ')?.slice(0, 50);
     return promise = promise.catch((err) => {
       if (ignoredErrs.includes(err)) return; // already caught
-      log(`${currentQueue()}Error (Non-Terminated) | ${err} | ${fnDescription || ''}...`);
+      log(`${currentQueue()} Error (Non-Terminated) | ${err} | ${fnDescription || ''}...`);
       if (!(err instanceof Error)) { // strange things may happen
         err = new Error(`[Wrapped Error] ${JSON.stringify(err)}`); // we should be prepared for them
       }
@@ -272,7 +276,9 @@ module.exports = {
    */
 
   throw(err) {
+    if (ignoredErrs.includes(err)) return promise; // already caught
     return this.add(`throw error: ${err.message}`, () => {
+      if (ignoredErrs.includes(err)) return; // already caught
       throw err;
     });
   },
@@ -313,8 +319,8 @@ module.exports = {
    * @inner
    */
   stop() {
-    if (process.env.DEBUG) debug(this.toString());
-    log(`${currentQueue()}Stopping recording promises`);
+    debug(this.toString());
+    log(`${currentQueue()} Stopping recording promises`);
     running = false;
   },
 

package/lib/step.js

@@ -119,9 +119,7 @@ class Step {
     }
     let result;
     try {
-      if (this.helperMethod !== 'say') {
-        result = this.helper[this.helperMethod].apply(this.helper, this.args);
-      }
+      result = this.helper[this.helperMethod].apply(this.helper, this.args);
       this.setStatus('success');
     } catch (err) {
       this.setStatus('failed');

package/lib/store.js

@@ -7,6 +7,8 @@ const store = {
   debugMode: false,
   /** @type {boolean} */
   timeouts: true,
+  /** @type {boolean} */
+  dryRun: false,
 };
 
 module.exports = store;

package/lib/template/heal.js

@@ -0,0 +1,39 @@
+const { heal, ai } = require('codeceptjs');
+
+heal.addRecipe('ai', {
+  priority: 10,
+  prepare: {
+    html: ({ I }) => I.grabHTMLFrom('body'),
+  },
+  steps: [
+    'click',
+    'fillField',
+    'appendField',
+    'selectOption',
+    'attachFile',
+    'checkOption',
+    'uncheckOption',
+    'doubleClick',
+  ],
+  fn: async (args) => {
+    return ai.healFailedStep(args);
+  },
+});
+
+heal.addRecipe('clickAndType', {
+  priority: 1,
+  steps: [
+    'fillField',
+    'appendField',
+  ],
+  fn: async ({ step }) => {
+    const locator = step.args[0];
+    const text = step.args[1];
+
+    return ({ I }) => {
+      I.click(locator);
+      I.wait(1); // to open modal or something
+      I.type(text);
+    };
+  },
+});