codeceptjs 3.5.15 → 3.6.0

package/bin/codecept.js

@@ -4,6 +4,33 @@ const Codecept = require('../lib/codecept');
 const { print, error } = require('../lib/output');
 const { printError } = require('../lib/command/utils');
 
+const commandFlags = {
+  ai: {
+    flag: '--ai',
+    description: 'enable AI assistant',
+  },
+  verbose: {
+    flag: '--verbose',
+    description: 'output internal logging information',
+  },
+  debug: {
+    flag: '--debug',
+    description: 'output additional information',
+  },
+  config: {
+    flag: '-c, --config [file]',
+    description: 'configuration file to be used',
+  },
+  profile: {
+    flag: '--profile [value]',
+    description: 'configuration profile to be used',
+  },
+  steps: {
+    flag: '--steps',
+    description: 'show step-by-step execution',
+  },
+};
+
 const errorHandler = (fn) => async (...args) => {
   try {
     await fn(...args);
@@ -35,9 +62,10 @@ program.command('migrate [path]')
 program.command('shell [path]')
   .alias('sh')
   .description('Interactive shell')
-  .option('--verbose', 'output internal logging information')
-  .option('--profile [value]', 'configuration profile to be used')
-  .option('-c, --config [file]', 'configuration file to be used')
+  .option(commandFlags.verbose.flag, commandFlags.verbose.description)
+  .option(commandFlags.profile.flag, commandFlags.profile.description)
+  .option(commandFlags.ai.flag, commandFlags.ai.description)
+  .option(commandFlags.config.flag, commandFlags.config.description)
   .action(errorHandler(require('../lib/command/interactive')));
 
 program.command('list [path]')
@@ -47,27 +75,27 @@ program.command('list [path]')
 
 program.command('def [path]')
   .description('Generates TypeScript definitions for all I actions.')
-  .option('-c, --config [file]', 'configuration file to be used')
+  .option(commandFlags.config.flag, commandFlags.config.description)
   .option('-o, --output [folder]', 'target folder to paste definitions')
   .action(errorHandler(require('../lib/command/definitions')));
 
 program.command('gherkin:init [path]')
   .alias('bdd:init')
   .description('Prepare CodeceptJS to run feature files.')
-  .option('-c, --config [file]', 'configuration file to be used')
+  .option(commandFlags.config.flag, commandFlags.config.description)
   .action(errorHandler(require('../lib/command/gherkin/init')));
 
 program.command('gherkin:steps [path]')
   .alias('bdd:steps')
   .description('Prints all defined gherkin steps.')
-  .option('-c, --config [file]', 'configuration file to be used')
+  .option(commandFlags.config.flag, commandFlags.config.description)
   .action(errorHandler(require('../lib/command/gherkin/steps')));
 
 program.command('gherkin:snippets [path]')
   .alias('bdd:snippets')
   .description('Generate step definitions from steps.')
   .option('--dry-run', "don't save snippets to file")
-  .option('-c, --config [file]', 'configuration file to be used')
+  .option(commandFlags.config.flag, commandFlags.config.description)
   .option('--feature [file]', 'feature files(s) to scan')
   .option('--path [file]', 'file in which to place the new snippets')
   .action(errorHandler(require('../lib/command/gherkin/snippets')));
@@ -93,16 +121,22 @@ program.command('generate:helper [path]')
   .description('Generates a new helper')
   .action(errorHandler(require('../lib/command/generate').helper));
 
+program.command('generate:heal [path]')
+  .alias('gr')
+  .description('Generates basic heal recipes')
+  .action(errorHandler(require('../lib/command/generate').heal));
+
 program.command('run [test]')
   .description('Executes tests')
 
   // codecept-only options
-  .option('--steps', 'show step-by-step execution')
-  .option('--debug', 'output additional information')
-  .option('--verbose', 'output internal logging information')
+  .option(commandFlags.ai.flag, commandFlags.ai.description)
+  .option(commandFlags.steps.flag, commandFlags.steps.description)
+  .option(commandFlags.debug.flag, commandFlags.debug.description)
+  .option(commandFlags.verbose.flag, commandFlags.verbose.description)
   .option('-o, --override [value]', 'override current config options')
-  .option('--profile [value]', 'configuration profile to be used')
-  .option('-c, --config [file]', 'configuration file to be used')
+  .option(commandFlags.profile.flag, commandFlags.profile.description)
+  .option(commandFlags.config.flag, commandFlags.config.description)
   .option('--features', 'run only *.feature files and skip tests')
   .option('--tests', 'run only JS test files and skip features')
   .option('--no-timeouts', 'disable all timeouts')
@@ -132,16 +166,17 @@ program.command('run [test]')
 
 program.command('run-workers <workers> [selectedRuns...]')
   .description('Executes tests in workers')
-  .option('-c, --config [file]', 'configuration file to be used')
+  .option(commandFlags.config.flag, commandFlags.config.description)
   .option('-g, --grep <pattern>', 'only run tests matching <pattern>')
   .option('-i, --invert', 'inverts --grep matches')
   .option('-o, --override [value]', 'override current config options')
   .option('--suites', 'parallel execution of suites not single tests')
-  .option('--debug', 'output additional information')
-  .option('--verbose', 'output internal logging information')
+  .option(commandFlags.debug.flag, commandFlags.debug.description)
+  .option(commandFlags.verbose.flag, commandFlags.verbose.description)
   .option('--features', 'run only *.feature files and skip tests')
   .option('--tests', 'run only JS test files and skip features')
-  .option('--profile [value]', 'configuration profile to be used')
+  .option(commandFlags.profile.flag, commandFlags.profile.description)
+  .option(commandFlags.ai.flag, commandFlags.ai.description)
   .option('-p, --plugins <k=v,k2=v2,...>', 'enable plugins, comma-separated')
   .option('-O, --reporter-options <k=v,k2=v2,...>', 'reporter-specific options')
   .option('-R, --reporter <name>', 'specify the reporter to use')
@@ -149,17 +184,18 @@ program.command('run-workers <workers> [selectedRuns...]')
 
 program.command('run-multiple [suites...]')
   .description('Executes tests multiple')
-  .option('-c, --config [file]', 'configuration file to be used')
-  .option('--profile [value]', 'configuration profile to be used')
+  .option(commandFlags.config.flag, commandFlags.config.description)
+  .option(commandFlags.profile.flag, commandFlags.profile.description)
   .option('--all', 'run all suites')
   .option('--features', 'run only *.feature files and skip tests')
   .option('--tests', 'run only JS test files and skip features')
+  .option(commandFlags.ai.flag, commandFlags.ai.description)
   .option('-g, --grep <pattern>', 'only run tests matching <pattern>')
   .option('-f, --fgrep <string>', 'only run tests containing <string>')
   .option('-i, --invert', 'inverts --grep and --fgrep matches')
-  .option('--steps', 'show step-by-step execution')
-  .option('--verbose', 'output internal logging information')
-  .option('--debug', 'output additional information')
+  .option(commandFlags.steps.flag, commandFlags.steps.description)
+  .option(commandFlags.verbose.flag, commandFlags.verbose.description)
+  .option(commandFlags.debug.flag, commandFlags.debug.description)
   .option('-p, --plugins <k=v,k2=v2,...>', 'enable plugins, comma-separated')
   .option('-o, --override [value]', 'override current config options')
   .option('-O, --reporter-options <k=v,k2=v2,...>', 'reporter-specific options')
@@ -180,28 +216,28 @@ program.command('dry-run [test]')
   .description('Prints step-by-step scenario for a test without actually running it')
   .option('-p, --plugins <k=v,k2=v2,...>', 'enable plugins, comma-separated')
   .option('--bootstrap', 'enable bootstrap & teardown scripts for dry-run')
-  .option('-c, --config [file]', 'configuration file to be used')
+  .option(commandFlags.config.flag, commandFlags.config.description)
   .option('--all', 'run all suites')
   .option('--features', 'run only *.feature files and skip tests')
   .option('--tests', 'run only JS test files and skip features')
   .option('-g, --grep <pattern>', 'only run tests matching <pattern>')
   .option('-f, --fgrep <string>', 'only run tests containing <string>')
   .option('-i, --invert', 'inverts --grep and --fgrep matches')
-  .option('--steps', 'show step-by-step execution')
-  .option('--verbose', 'output internal logging information')
-  .option('--debug', 'output additional information')
+  .option(commandFlags.steps.flag, commandFlags.steps.description)
+  .option(commandFlags.verbose.flag, commandFlags.verbose.description)
+  .option(commandFlags.debug.flag, commandFlags.debug.description)
   .action(errorHandler(require('../lib/command/dryRun')));
 
 program.command('run-rerun [test]')
   .description('Executes tests in more than one test suite run')
 
   // codecept-only options
-  .option('--steps', 'show step-by-step execution')
-  .option('--debug', 'output additional information')
-  .option('--verbose', 'output internal logging information')
+  .option(commandFlags.steps.flag, commandFlags.steps.description)
+  .option(commandFlags.debug.flag, commandFlags.debug.description)
+  .option(commandFlags.verbose.flag, commandFlags.verbose.description)
   .option('-o, --override [value]', 'override current config options')
-  .option('--profile [value]', 'configuration profile to be used')
-  .option('-c, --config [file]', 'configuration file to be used')
+  .option(commandFlags.profile.flag, commandFlags.profile.description)
+  .option(commandFlags.config.flag, commandFlags.config.description)
   .option('--features', 'run only *.feature files and skip tests')
   .option('--tests', 'run only JS test files and skip features')
   .option('-p, --plugins <k=v,k2=v2,...>', 'enable plugins, comma-separated')
@@ -236,5 +272,6 @@ program.on('command:*', (cmd) => {
 
 if (process.argv.length <= 2) {
   program.outputHelp();
+} else {
+  program.parse(process.argv);
 }
-program.parse(process.argv);

package/docs/webapi/startRecordingWebSocketMessages.mustache

@@ -0,0 +1,8 @@
+Starts recording of websocket messages.
+This also resets recorded websocket messages.
+
+```js
+await I.startRecordingWebSocketMessages();
+```
+
+@returns {void} automatically synchronized promise through #recorder

package/docs/webapi/stopRecordingWebSocketMessages.mustache

@@ -0,0 +1,7 @@
+Stops recording WS messages. Recorded WS messages is not flashed.
+
+```js
+await I.stopRecordingWebSocketMessages();
+```
+
+@returns {void} automatically synchronized promise through #recorder

package/lib/ai.js

@@ -1,47 +1,118 @@
-const { Configuration, OpenAIApi } = require('openai');
 const debug = require('debug')('codeceptjs:ai');
-const config = require('./config');
 const output = require('./output');
+const event = require('./event');
 const { removeNonInteractiveElements, minifyHtml, splitByChunks } = require('./html');
 
-const defaultConfig = {
-  model: 'gpt-3.5-turbo-16k',
-  temperature: 0.1,
-};
-
-const htmlConfig = {
+const defaultHtmlConfig = {
   maxLength: 50000,
   simplify: true,
   minify: true,
   html: {},
 };
 
-const aiInstance = null;
+const defaultPrompts = {
+  writeStep: (html, input) => [{
+    role: 'user',
+    content: `I am test engineer writing test in CodeceptJS
+        I have opened web page and I want to use CodeceptJS to ${input} on this page
+        Provide me valid CodeceptJS code to accomplish it
+        Use only locators from this HTML: \n\n${html}`,
+  },
+  ],
+
+  healStep: (html, { step, error, prevSteps }) => {
+    return [{
+      role: 'user',
+      content: `As a test automation engineer I am testing web application using CodeceptJS.
+        I want to heal a test that fails. Here is the list of executed steps: ${prevSteps.map(s => s.toString()).join(', ')}
+        Propose how to adjust ${step.toCode()} step to fix the test.
+        Use locators in order of preference: semantic locator by text, CSS, XPath. Use codeblocks marked with \`\`\`
+        Here is the error message: ${error.message}
+        Here is HTML code of a page where the failure has happened: \n\n${html}`,
+    }];
+  },
+};
 
 class AiAssistant {
   constructor() {
-    this.config = config.get('ai', defaultConfig);
-    this.htmlConfig = Object.assign(htmlConfig, this.config.html);
-    delete this.config.html;
-    this.html = null;
+    this.totalTime = 0;
+    this.numTokens = 0;
+
+    this.reset();
+    this.connectToEvents();
+  }
+
+  enable(config = {}) {
+    debug('Enabling AI assistant');
+    this.isEnabled = true;
+
+    const { html, prompts, ...aiConfig } = config;
+
+    this.config = Object.assign(this.config, aiConfig);
+    this.htmlConfig = Object.assign(defaultHtmlConfig, html);
+    this.prompts = Object.assign(defaultPrompts, prompts);
+
+    debug('Config', this.config);
+  }
+
+  reset() {
+    this.numTokens = 0;
+    this.isEnabled = false;
+    this.config = {
+      maxTokens: 1000000,
+      request: null,
+      response: parseCodeBlocks,
+      // lets limit token usage to 1M
+    };
+    this.minifiedHtml = null;
     this.response = null;
+    this.totalTime = 0;
+  }
+
+  disable() {
+    this.isEnabled = false;
+  }
 
-    this.isEnabled = !!process.env.OPENAI_API_KEY;
+  connectToEvents() {
+    event.dispatcher.on(event.all.result, () => {
+      if (this.isEnabled && this.numTokens > 0) {
+        const numTokensK = Math.ceil(this.numTokens / 1000);
+        const maxTokensK = Math.ceil(this.config.maxTokens / 1000);
+        output.print(`AI assistant took ${this.totalTime}s and used ~${numTokensK}K input tokens. Tokens limit: ${maxTokensK}K`);
+      }
+    });
+  }
 
+  checkRequestFn() {
     if (!this.isEnabled) {
-      debug('No OpenAI API key provided. AI assistant is disabled.');
+      debug('AI assistant is disabled');
       return;
     }
 
-    const configuration = new Configuration({
-      apiKey: process.env.OPENAI_API_KEY,
-    });
+    if (this.config.request) return;
 
-    this.openai = new OpenAIApi(configuration);
-  }
+    const noRequestErrorMessage = `
+     No request function is set for AI assistant.  
+     Please implement your own request function and set it in the config.    
+
+     [!] AI request was decoupled from CodeceptJS. To connect to OpenAI or other AI service, please implement your own request function and set it in the config.
+
+     Example (connect to OpenAI):
+
+     ai: {
+       request: async (messages) => {
+         const OpenAI = require('openai');
+         const openai = new OpenAI({ apiKey: process.env['OPENAI_API_KEY'] })
+         const response = await openai.chat.completions.create({
+          model: 'gpt-3.5-turbo-0125',
+          messages,
+         });
+         return response?.data?.choices[0]?.message?.content;
+       }
+     }
+    `.trim();
 
-  static getInstance() {
-    return aiInstance || new AiAssistant();
+    throw new Error(noRequestErrorMessage);
   }
 
   async setHtmlContext(html) {
@@ -50,98 +121,99 @@ class AiAssistant {
     if (this.htmlConfig.simplify) {
       processedHTML = removeNonInteractiveElements(processedHTML, this.htmlConfig);
     }
+
     if (this.htmlConfig.minify) processedHTML = await minifyHtml(processedHTML);
     if (this.htmlConfig.maxLength) processedHTML = splitByChunks(processedHTML, this.htmlConfig.maxLength)[0];
 
-    debug(processedHTML);
-
-    this.html = processedHTML;
+    this.minifiedHtml = processedHTML;
   }
 
   getResponse() {
     return this.response || '';
   }
 
-  mockResponse(response) {
-    this.mockedResponse = response;
-  }
-
   async createCompletion(messages) {
-    if (!this.openai) return;
+    if (!this.isEnabled) return '';
 
-    debug(messages);
+    debug('Request', messages);
 
-    if (this.mockedResponse) return this.mockedResponse;
+    this.checkRequestFn();
 
     this.response = null;
 
-    try {
-      const completion = await this.openai.createChatCompletion({
-        ...this.config,
-        messages,
-      });
-
-      this.response = completion?.data?.choices[0]?.message?.content;
-
-      debug(this.response);
+    this.calculateTokens(messages);
 
+    try {
+      const startTime = process.hrtime();
+      this.response = await this.config.request(messages);
+      const endTime = process.hrtime(startTime);
+      const executionTimeInSeconds = endTime[0] + endTime[1] / 1e9;
+
+      this.totalTime += Math.round(executionTimeInSeconds);
+      debug('AI response time', executionTimeInSeconds);
+      debug('Response', this.response);
+      this.stopWhenReachingTokensLimit();
       return this.response;
     } catch (err) {
       debug(err.response);
       output.print('');
-      output.error(`OpenAI error: ${err.message}`);
-      output.error(err?.response?.data?.error?.code);
-      output.error(err?.response?.data?.error?.message);
+      output.error(`AI service error: ${err.message}`);
+      if (err?.response?.data?.error?.code) output.error(err?.response?.data?.error?.code);
+      if (err?.response?.data?.error?.message) output.error(err?.response?.data?.error?.message);
+      this.stopWhenReachingTokensLimit();
       return '';
     }
   }
 
-  async healFailedStep(step, err, test) {
+  async healFailedStep(failureContext) {
     if (!this.isEnabled) return [];
-    if (!this.html) throw new Error('No HTML context provided');
-
-    const messages = [
-      { role: 'user', content: 'As a test automation engineer I am testing web application using CodeceptJS.' },
-      { role: 'user', content: `I want to heal a test that fails. Here is the list of executed steps: ${test.steps.join(', ')}` },
-      { role: 'user', content: `Propose how to adjust ${step.toCode()} step to fix the test.` },
-      { role: 'user', content: 'Use locators in order of preference: semantic locator by text, CSS, XPath. Use codeblocks marked with ```.' },
-      { role: 'user', content: `Here is the error message: ${err.message}` },
-      { role: 'user', content: `Here is HTML code of a page where the failure has happened: \n\n${this.html}` },
-    ];
-
-    const response = await this.createCompletion(messages);
+    if (!failureContext.html) throw new Error('No HTML context provided');
+
+    await this.setHtmlContext(failureContext.html);
+
+    if (!this.minifiedHtml) {
+      debug('HTML context is empty after removing non-interactive elements & minification');
+      return [];
+    }
+
+    const response = await this.createCompletion(this.prompts.healStep(this.minifiedHtml, failureContext));
     if (!response) return [];
 
-    return parseCodeBlocks(response);
+    return this.config.response(response);
+  }
+
+  calculateTokens(messages) {
+    // we implement naive approach for calculating tokens with no extra requests
+    // this approach was tested via https://platform.openai.com/tokenizer
+    // we need it to display current usage tokens usage so users could analyze effectiveness of AI
+
+    const inputString = messages.map(m => m.content).join(' ').trim();
+    const numWords = (inputString.match(/[^\s\-:=]+/g) || []).length;
+
+    // 2.5 token is constant for average HTML input
+    const tokens = numWords * 2.5;
+
+    this.numTokens += tokens;
+
+    return tokens;
+  }
+
+  stopWhenReachingTokensLimit() {
+    if (this.numTokens < this.config.maxTokens) return;
+
+    output.print(`AI assistant has reached the limit of ${this.config.maxTokens} tokens in this session. It will be disabled now`);
+    this.disable();
   }
 
   async writeSteps(input) {
     if (!this.isEnabled) return;
-    if (!this.html) throw new Error('No HTML context provided');
+    if (!this.minifiedHtml) throw new Error('No HTML context provided');
 
     const snippets = [];
 
-    const messages = [
-      {
-        role: 'user',
-        content: `I am test engineer writing test in CodeceptJS
-          I have opened web page and I want to use CodeceptJS to ${input} on this page
-          Provide me valid CodeceptJS code to accomplish it
-          Use only locators from this HTML: \n\n${this.html}`,
-      },
-      { role: 'user', content: 'Propose only CodeceptJS steps code. Do not include Scenario or Feature into response' },
-
-      // old prompt
-      // { role: 'user', content: 'I want to click button Submit using CodeceptJS on this HTML page: <html><body><button>Submit</button></body></html>' },
-      // { role: 'assistant', content: '```js\nI.click("Submit");\n```' },
-      // { role: 'user', content: 'I want to click button Submit using CodeceptJS on this HTML page: <html><body><button>Login</button></body></html>' },
-      // { role: 'assistant', content: 'No suggestions' },
-      // { role: 'user', content: `Now I want to ${input} on this HTML page using CodeceptJS code` },
-      // { role: 'user', content: `Provide me with CodeceptJS code to achieve this on THIS page.` },
-    ];
-    const response = await this.createCompletion(messages);
+    const response = await this.createCompletion(this.prompts.writeStep(this.minifiedHtml, input));
     if (!response) return;
-    snippets.push(...parseCodeBlocks(response));
+    snippets.push(...this.config.response(response));
 
     debug(snippets[0]);
 
@@ -177,4 +249,4 @@ function parseCodeBlocks(response) {
   return modifiedSnippets.filter(snippet => !!snippet);
 }
 
-module.exports = AiAssistant;
+module.exports = new AiAssistant();

package/lib/cli.js

@@ -167,6 +167,7 @@ class Cli extends Base {
         }
 
         const steps = test.steps || (test.ctx && test.ctx.test.steps);
+
         if (steps && steps.length) {
           let scenarioTrace = '';
           steps.reverse().forEach((step) => {

package/lib/command/generate.js

@@ -258,3 +258,37 @@ helpers: {
     `);
   });
 };
+
+const healTemplate = fs.readFileSync(path.join(__dirname, '../template/heal.js'), 'utf8').toString();
+
+module.exports.heal = function (genPath) {
+  const testsPath = getTestRoot(genPath);
+
+  let configFile = path.join(testsPath, `codecept.conf.${extension}`);
+
+  if (!fileExists(configFile)) {
+    configFile = path.join(testsPath, `codecept.conf.${extension}`);
+    if (fileExists(configFile)) extension = 'ts';
+  }
+
+  output.print('Creating basic heal recipes');
+  output.print(`Add your own custom recipes to ./heal.${extension} file`);
+  output.print('Require this file in the config file and enable heal plugin:');
+  output.print('--------------------------');
+  output.print(`
+require('./heal')
+
+exports.config = {
+  // ... 
+  plugins: {
+    heal: {
+      enabled: true
+    }
+  }
+}
+  `);
+
+  const healFile = path.join(testsPath, `heal.${extension}`);
+  if (!safeFileWrite(healFile, healTemplate)) return;
+  output.success(`Heal recipes were created in ${healFile}`);
+};

package/lib/command/run-workers.js

@@ -1,6 +1,7 @@
 // For Node version >=10.5.0, have to use experimental flag
 const { tryOrDefault } = require('../utils');
 const output = require('../output');
+const store = require('../store');
 const event = require('../event');
 const Workers = require('../workers');
 
@@ -96,6 +97,8 @@ module.exports = async function (workerCount, selectedRuns, options) {
   });
 
   try {
+    if (options.verbose || options.debug) store.debugMode = true;
+
     if (options.verbose) {
       global.debugMode = true;
       const { getMachineInfo } = require('./info');

package/lib/command/run.js

@@ -2,16 +2,19 @@ const {
   getConfig, printError, getTestRoot, createOutputDir,
 } = require('./utils');
 const Config = require('../config');
+const store = require('../store');
 const Codecept = require('../codecept');
 
 module.exports = async function (test, options) {
   // registering options globally to use in config
   // Backward compatibility for --profile
+  // TODO: remove in CodeceptJS 4
   process.profile = options.profile;
 
   if (options.profile) {
     process.env.profile = options.profile;
   }
+  if (options.verbose || options.debug) store.debugMode = true;
 
   const configFile = options.config;
 

package/lib/container.js

@@ -8,6 +8,7 @@ const recorder = require('./recorder');
 const event = require('./event');
 const WorkerStorage = require('./workerStorage');
 const store = require('./store');
+const ai = require('./ai');
 
 let container = {
   helpers: {},
@@ -45,6 +46,7 @@ class Container {
     container.translation = loadTranslation(config.translation || null, config.vocabularies || []);
     container.support = createSupportObjects(config.include || {});
     container.plugins = createPlugins(config.plugins || {}, opts);
+    if (opts && opts.ai) ai.enable(config.ai); // enable AI Assistant
     if (config.gherkin) loadGherkinSteps(config.gherkin.steps || []);
     if (opts && typeof opts.timeouts === 'boolean') store.timeouts = opts.timeouts;
   }