codeceptjs 3.5.14 → 3.6.0-beta.1.ai-healers

package/lib/plugin/heal.js

@@ -1,33 +1,19 @@
 const debug = require('debug')('codeceptjs:heal');
 const colors = require('chalk');
-const Container = require('../container');
-const AiAssistant = require('../ai');
 const recorder = require('../recorder');
 const event = require('../event');
 const output = require('../output');
-const supportedHelpers = require('./standardActingHelpers');
+const heal = require('../heal');
+const store = require('../store');
 
 const defaultConfig = {
-  healTries: 1,
   healLimit: 2,
-  healSteps: [
-    'click',
-    'fillField',
-    'appendField',
-    'selectOption',
-    'attachFile',
-    'checkOption',
-    'uncheckOption',
-    'doubleClick',
-  ],
 };
 
 /**
- * Self-healing tests with OpenAI.
+ * Self-healing tests with AI.
  *
- * This plugin is experimental and requires OpenAI API key.
- *
- * To use it you need to set OPENAI_API_KEY env variable and enable plugin inside the config.
+ * Read more about heaking in [Self-Healing Tests](https://codecept.io/heal/)
  *
  * ```js
  * plugins: {
@@ -40,22 +26,15 @@ const defaultConfig = {
  * More config options are available:
  *
  * * `healLimit` - how many steps can be healed in a single test (default: 2)
- * * `healSteps` - which steps can be healed (default: all steps that interact with UI, see list below)
- *
- * Steps to heal:
- *
- * * `click`
- * * `fillField`
- * * `appendField`
- * * `selectOption`
- * * `attachFile`
- * * `checkOption`
- * * `uncheckOption`
- * * `doubleClick`
  *
  */
 module.exports = function (config = {}) {
-  const aiAssistant = AiAssistant.getInstance();
+  if (store.debugMode && !process.env.DEBUG) {
+    event.dispatcher.on(event.test.failed, () => {
+      output.plugin('heal', 'Healing is disabled in --debug mode, use DEBUG="codeceptjs:heal" to enable it in debug mode');
+    });
+    return;
+  }
 
   let currentTest = null;
   let currentStep = null;
@@ -64,8 +43,6 @@ module.exports = function (config = {}) {
   let healTries = 0;
   let isHealing = false;
 
-  const healSuggestions = [];
-
   config = Object.assign(defaultConfig, config);
 
   event.dispatcher.on(event.test.before, (test) => {
@@ -78,63 +55,27 @@ module.exports = function (config = {}) {
 
   event.dispatcher.on(event.step.after, (step) => {
     if (isHealing) return;
-    const store = require('../store');
-    if (store.debugMode) return;
+    if (healTries >= config.healLimit) return; // out of limit
+
+    if (!heal.hasCorrespondingRecipes(step)) return;
+
     recorder.catchWithoutStop(async (err) => {
       isHealing = true;
       if (caughtError === err) throw err; // avoid double handling
       caughtError = err;
-      if (!aiAssistant.isEnabled) {
-        output.print(colors.yellow('Heal plugin can\'t operate, AI assistant is disabled. Please set OPENAI_API_KEY env variable to enable it.'));
-        throw err;
-      }
-      if (!currentStep) throw err;
-      if (!config.healSteps.includes(currentStep.name)) throw err;
-      const test = currentTest;
-
-      if (healTries >= config.healTries) {
-        output.print(colors.bold.red(`Healing failed for ${config.healTries} time(s)`));
-        output.print('AI couldn\'t identify the correct solution');
-        output.print('Probably the entire flow has changed and the test should be updated');
-
-        throw err;
-      }
 
-      if (healedSteps >= config.healLimit) {
-        output.print(colors.bold.red(`Can't heal more than ${config.healLimit} step(s) in a test`));
-        output.print('Entire flow can be broken, please check it manually');
-        output.print('or increase healing limit in heal plugin config');
-
-        throw err;
-      }
+      const test = currentTest;
 
       recorder.session.start('heal');
-      const helpers = Container.helpers();
-      let helper;
-
-      for (const helperName of supportedHelpers) {
-        if (Object.keys(helpers).indexOf(helperName) > -1) {
-          helper = helpers[helperName];
-        }
-      }
 
-      if (!helper) throw err; // no helpers for html
-
-      const step = test.steps[test.steps.length - 1];
       debug('Self-healing started', step.toCode());
 
-      const currentOutputLevel = output.level();
-      output.level(0);
-      const html = await helper.grabHTMLFrom('body');
-      output.level(currentOutputLevel);
-
-      if (!html) throw err;
+      await heal.healStep(step, err, { test });
 
       healTries++;
-      await aiAssistant.setHtmlContext(html);
-      await tryToHeal(step, err);
 
       recorder.add('close healing session', () => {
+        recorder.reset();
         recorder.session.restore('heal');
         recorder.ignoreErr(err);
       });
@@ -145,7 +86,7 @@ module.exports = function (config = {}) {
   });
 
   event.dispatcher.on(event.all.result, () => {
-    if (!healSuggestions.length) return;
+    if (!heal.fixes?.length) return;
 
     const { print } = output;
 
@@ -153,16 +94,20 @@ module.exports = function (config = {}) {
     print('===================');
     print(colors.bold.green('Self-Healing Report:'));
 
-    print(`${colors.bold(healSuggestions.length)} step(s) were healed by AI`);
+    print(`${colors.bold(heal.fixes.length)} ${heal.fixes.length === 1 ? 'step was' : 'steps were'} healed`);
+
+    const suggestions = heal.fixes.filter(fix => fix.recipe && heal.recipes[fix.recipe].suggest);
+
+    if (!suggestions.length) return;
 
     let i = 1;
     print('');
     print('Suggested changes:');
     print('');
 
-    for (const suggestion of healSuggestions) {
-      print(`${i}. To fix ${colors.bold.blue(suggestion.test.title)}`);
-      print('Replace the failed code with:');
+    for (const suggestion of suggestions) {
+      print(`${i}. To fix ${colors.bold.magenta(suggestion.test?.title)}`);
+      print('  Replace the failed code:', colors.gray(`(suggested by ${colors.bold(suggestion.recipe)})`));
       print(colors.red(`- ${suggestion.step.toCode()}`));
       print(colors.green(`+ ${suggestion.snippet}`));
       print(suggestion.step.line());
@@ -170,40 +115,4 @@ module.exports = function (config = {}) {
       i++;
     }
   });
-
-  async function tryToHeal(failedStep, err) {
-    output.debug(`Running OpenAI to heal ${failedStep.toCode()} step`);
-
-    const codeSnippets = await aiAssistant.healFailedStep(failedStep, err, currentTest);
-
-    output.debug(`Received ${codeSnippets.length} suggestions from OpenAI`);
-    const I = Container.support('I'); // eslint-disable-line
-
-    for (const codeSnippet of codeSnippets) {
-      try {
-        debug('Executing', codeSnippet);
-        recorder.catch((e) => {
-          console.log(e);
-        });
-        await eval(codeSnippet); // eslint-disable-line
-
-        healSuggestions.push({
-          test: currentTest,
-          step: failedStep,
-          snippet: codeSnippet,
-        });
-
-        recorder.add('healed', () => output.print(colors.bold.green('  Code healed successfully')));
-        healedSteps++;
-        return;
-      } catch (err) {
-        debug('Failed to execute code', err);
-        recorder.ignoreErr(err); // healing ded not help
-        // recorder.catch(() => output.print(colors.bold.red('  Failed healing code')));
-      }
-    }
-
-    output.debug(`Couldn't heal the code for ${failedStep.toCode()}`);
-  }
-  return recorder.promise();
 };

package/lib/recorder.js

@@ -1,5 +1,6 @@
 const debug = require('debug')('codeceptjs:recorder');
 const promiseRetry = require('promise-retry');
+const chalk = require('chalk');
 const { printObjectProperties } = require('./utils');
 const { log } = require('./output');
 
@@ -179,7 +180,7 @@ module.exports = {
       return;
     }
     tasks.push(taskName);
-    debug(`${currentQueue()}Queued | ${taskName}`);
+    debug(chalk.gray(`${currentQueue()} Queued | ${taskName}`));
 
     return promise = Promise.resolve(promise).then((res) => {
       // prefer options for non-conditional retries
@@ -190,11 +191,14 @@ module.exports = {
         return Promise.race([promise, Promise.resolve(res).then(fn)]).finally(() => clearTimeout(timer));
       }
 
+      debug(`${currentQueue()} Running | ${taskName}`);
+
       const retryRules = this.retries.slice().reverse();
       return promiseRetry(Object.assign(defaultRetryOptions, retryOpts), (retry, number) => {
         if (number > 1) log(`${currentQueue()}Retrying... Attempt #${number}`);
         const [promise, timer] = getTimeoutPromise(timeout, taskName);
         return Promise.race([promise, Promise.resolve(res).then(fn)]).finally(() => clearTimeout(timer)).catch((err) => {
+          if (ignoredErrs.includes(err)) return;
           for (const retryObj of retryRules) {
             if (!retryObj.when) return retry(err);
             if (retryObj.when && retryObj.when(err)) return retry(err);
@@ -229,7 +233,7 @@ module.exports = {
    */
   catch(customErrFn) {
     const fnDescription = customErrFn?.toString()?.replace(/\s{2,}/g, ' ').replace(/\n/g, ' ')?.slice(0, 50);
-    debug(`${currentQueue()}Queued | catch with error handler ${fnDescription || ''}`);
+    debug(chalk.gray(`${currentQueue()} Queued | catch with error handler ${fnDescription || ''}`));
     return promise = promise.catch((err) => {
       log(`${currentQueue()}Error | ${err} ${fnDescription}...`);
       if (!(err instanceof Error)) { // strange things may happen
@@ -253,7 +257,7 @@ module.exports = {
     const fnDescription = customErrFn?.toString()?.replace(/\s{2,}/g, ' ').replace(/\n/g, ' ')?.slice(0, 50);
     return promise = promise.catch((err) => {
       if (ignoredErrs.includes(err)) return; // already caught
-      log(`${currentQueue()}Error (Non-Terminated) | ${err} | ${fnDescription || ''}...`);
+      log(`${currentQueue()} Error (Non-Terminated) | ${err} | ${fnDescription || ''}...`);
       if (!(err instanceof Error)) { // strange things may happen
         err = new Error(`[Wrapped Error] ${JSON.stringify(err)}`); // we should be prepared for them
       }
@@ -272,7 +276,9 @@ module.exports = {
    */
 
   throw(err) {
+    if (ignoredErrs.includes(err)) return promise; // already caught
     return this.add(`throw error: ${err.message}`, () => {
+      if (ignoredErrs.includes(err)) return; // already caught
       throw err;
     });
   },
@@ -313,8 +319,8 @@ module.exports = {
    * @inner
    */
   stop() {
-    if (process.env.DEBUG) debug(this.toString());
-    log(`${currentQueue()}Stopping recording promises`);
+    debug(this.toString());
+    log(`${currentQueue()} Stopping recording promises`);
     running = false;
   },
 

package/lib/step.js

@@ -119,9 +119,7 @@ class Step {
     }
     let result;
     try {
-      if (this.helperMethod !== 'say') {
-        result = this.helper[this.helperMethod].apply(this.helper, this.args);
-      }
+      result = this.helper[this.helperMethod].apply(this.helper, this.args);
       this.setStatus('success');
     } catch (err) {
       this.setStatus('failed');

package/lib/store.js

@@ -7,6 +7,8 @@ const store = {
   debugMode: false,
   /** @type {boolean} */
   timeouts: true,
+  /** @type {boolean} */
+  dryRun: false,
 };
 
 module.exports = store;

package/lib/template/heal.js

@@ -0,0 +1,39 @@
+const { heal, ai } = require('codeceptjs');
+
+heal.addRecipe('ai', {
+  priority: 10,
+  prepare: {
+    html: ({ I }) => I.grabHTMLFrom('body'),
+  },
+  steps: [
+    'click',
+    'fillField',
+    'appendField',
+    'selectOption',
+    'attachFile',
+    'checkOption',
+    'uncheckOption',
+    'doubleClick',
+  ],
+  fn: async (args) => {
+    return ai.healFailedStep(args);
+  },
+});
+
+heal.addRecipe('clickAndType', {
+  priority: 1,
+  steps: [
+    'fillField',
+    'appendField',
+  ],
+  fn: async ({ step }) => {
+    const locator = step.args[0];
+    const text = step.args[1];
+
+    return ({ I }) => {
+      I.click(locator);
+      I.wait(1); // to open modal or something
+      I.type(text);
+    };
+  },
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codeceptjs",
-  "version": "3.5.14",
+  "version": "3.6.0-beta.1.ai-healers",
   "description": "Supercharged End 2 End Testing Framework for NodeJS",
   "keywords": [
     "acceptance",
@@ -23,10 +23,10 @@
   },
   "files": [
     "bin",
+    "docs",
     "lib",
     "translations",
-    "typings/**/*.d.ts",
-    "docs/webapi/**"
+    "typings/**/*.d.ts"
   ],
   "main": "lib/index.js",
   "typings": "typings/index.d.ts",
@@ -56,7 +56,6 @@
     "test:unit:webbapi:webDriver:devtools": "mocha test/helper/WebDriver_devtools_test.js --exit",
     "test:unit:webbapi:testCafe": "mocha test/helper/TestCafe_test.js",
     "test:unit:expect": "mocha test/helper/Expect_test.js",
-    "test:unit:mockServer": "mocha test/helper/MockServer_test.js",
     "test:plugin": "mocha test/plugin/plugin_test.js",
     "def": "./runok.js def",
     "dev:graphql": "node test/data/graphql/index.js",
@@ -73,41 +72,38 @@
     "@cucumber/gherkin": "26",
     "@cucumber/messages": "24.0.1",
     "@xmldom/xmldom": "0.8.10",
-    "acorn": "8.11.3",
+    "acorn": "8.11.2",
     "arrify": "2.0.1",
-    "axios": "1.6.7",
-    "chai": "5.0.3",
+    "axios": "1.6.3",
+    "chai": "4.3.8",
     "chai-deep-match": "1.2.1",
-    "chai-exclude": "2.1.0",
-    "chai-json-schema": "1.5.1",
-    "chai-json-schema-ajv": "5.2.4",
-    "chai-match-pattern": "1.3.0",
-    "chai-string": "1.5.0",
+    "chai-exclude": "^2.1.0",
+    "chai-json-schema": "^1.5.1",
+    "chai-json-schema-ajv": "^5.2.4",
+    "chai-match-pattern": "^1.3.0",
+    "chai-string": "^1.5.0",
     "chalk": "4.1.2",
     "commander": "11.1.0",
     "cross-spawn": "7.0.3",
-    "css-to-xpath": "0.1.0",
     "csstoxpath": "1.6.0",
-    "devtools": "8.29.1",
+    "devtools": "8.27.2",
     "envinfo": "7.11.0",
     "escape-string-regexp": "4.0.0",
     "figures": "3.2.0",
     "fn-args": "4.0.0",
     "fs-extra": "11.2.0",
     "glob": "6.0.1",
-    "html-minifier-terser": "7.2.0",
+    "html-minifier-terser": "^7.2.0",
     "inquirer": "6.5.2",
-    "joi": "17.12.1",
+    "joi": "17.11.0",
     "js-beautify": "1.14.11",
     "lodash.clonedeep": "4.5.0",
     "lodash.merge": "4.6.2",
     "mkdirp": "1.0.4",
     "mocha": "10.2.0",
     "ms": "2.1.3",
-    "openai": "3.2.1",
     "ora-classic": "5.4.2",
-    "pactum": "3.6.0",
-    "parse-function": "5.6.10",
+    "parse-function": "5.6.4",
     "parse5": "7.1.2",
     "promise-retry": "1.1.1",
     "resq": "1.11.0",
@@ -122,12 +118,12 @@
     "@faker-js/faker": "7.6.0",
     "@pollyjs/adapter-puppeteer": "6.0.6",
     "@pollyjs/core": "5.1.0",
-    "@types/chai": "4.3.11",
+    "@types/chai": "^4.3.7",
     "@types/inquirer": "9.0.3",
-    "@types/node": "20.11.16",
-    "@wdio/sauce-service": "8.29.1",
+    "@types/node": "20.10.7",
+    "@wdio/sauce-service": "8.27.0",
     "@wdio/selenium-standalone-service": "8.3.2",
-    "@wdio/utils": "8.28.8",
+    "@wdio/utils": "8.27.2",
     "@xmldom/xmldom": "0.8.10",
     "apollo-server-express": "2.25.3",
     "chai-as-promised": "7.1.1",
@@ -135,7 +131,7 @@
     "contributor-faces": "1.1.0",
     "documentation": "12.3.0",
     "dtslint": "4.2.1",
-    "electron": "28.2.1",
+    "electron": "28.0.0",
     "eslint": "8.56.0",
     "eslint-config-airbnb-base": "15.0.0",
     "eslint-plugin-import": "2.29.1",
@@ -148,14 +144,14 @@
     "jsdoc": "3.6.11",
     "jsdoc-typeof-plugin": "1.0.0",
     "json-server": "0.10.1",
-    "playwright": "1.41.1",
+    "playwright": "1.40.1",
     "puppeteer": "21.1.1",
     "qrcode-terminal": "0.12.0",
     "rosie": "2.1.1",
     "runok": "0.9.3",
     "sinon": "17.0.1",
     "sinon-chai": "3.7.0",
-    "testcafe": "3.5.0",
+    "testcafe": "3.3.0",
     "ts-morph": "21.0.1",
     "ts-node": "10.9.2",
     "tsd-jsdoc": "2.5.0",
@@ -163,7 +159,7 @@
     "typedoc-plugin-markdown": "3.17.1",
     "typescript": "5.3.3",
     "wdio-docker-service": "1.5.0",
-    "webdriverio": "8.31.1",
+    "webdriverio": "8.27.2",
     "xml2js": "0.6.2",
     "xpath": "0.0.34"
   },

package/typings/index.d.ts

@@ -382,22 +382,6 @@ declare namespace CodeceptJS {
     [key: string]: any;
   };
 
-  type MockRequest = {
-    method: 'GET'|'PUT'|'POST'|'PATCH'|'DELETE'|string;
-    path: string;
-    queryParams?: object;
-  }
-
-  type MockResponse = {
-    status: number;
-    body?: object;
-  }
-
-  type MockInteraction = {
-    request: MockRequest;
-    response: MockResponse;
-  }
-
   interface PageScrollPosition {
     x: number;
     y: number;