codeceptjs 3.5.14 → 3.6.0-beta.1.ai-healers

package/docs/wiki/Upgrading-to-CodeceptJS-3.md

@@ -0,0 +1,153 @@
+🚀 CodeceptJS 3 is out now. Install it:
+
+```
+npm i codeceptjs
+```
+
+* [COMPLETE CHANGELOG](https://github.com/Codeception/CodeceptJS/blob/codeceptjs-v3.0/CHANGELOG.md#300-beta)
+* [Documentation](https://github.com/Codeception/CodeceptJS/tree/codeceptjs-v3.0/docs)
+
+After installing a new version, run `codecept3-upgrade` which is described below:
+
+---
+
+CodeceptJS 3.0 is a new version of CodeceptJS with some breaking changes included. 
+You should update your project following this guide to ensure that everything works correctly. 
+
+### 1️⃣ Syntax Change
+
+CodeceptJS 3.0 introduced a new syntax for declaring tests, instead of:
+
+```js
+Scenario('title', (I, loginPage) => {})
+```
+we use a new way of passing arguments into a test, via destruction:
+
+```js
+Scenario('title', ({ I, loginPage }) => {})
+```
+
+This works similarly to `inject()` command. This change was done to unify accessing [dependency injection container](https://codecept.io/pageobjects/#dependency-injection), and to provide better TypeScript support. 
+
+> If you use BDD-style project with Gherkin, no changes needed for this step. 
+
+To upgrade your project, you don't need to change manually all your tests. 
+
+💪 **Use [codecept3-upgrade tool](https://www.npmjs.com/package/codecept3-upgrade) to perform the migration**. Use it carefully, as it updates your current code.
+
+To upgrade your codebase, commit all your changes, and switch to a new branch.
+Run upgrade script (even without installing) for a directory where you have your tests:
+
+```
+npx codecept3-upgrade tests
+```
+Review the changes in Git Diff and try to execute tests using CodeceptJS 3.0
+
+
+### 2️⃣ Grabbers signature change
+
+There were confusion and inconsistency across grab* methods. What they will return if multiple elements are found? A single element or an array? This was the problem as the format was dependent on a page content. 
+
+In 3.0 we decided to make all current grab* methods to return a single value only. While we add new methods grab**FromAll that return an array.
+
+For instance, `grabTextFrom` will always return a single text value, while `grabTextFromAll` will return an array of values:
+
+```js
+await I.grabTextFrom('.username'); => 'john'
+await I.grabTextFromAll('.username'); => ['john', 'claudia', 'bill']
+```
+
+Please update parts in your project where you rely on grab* methods to return an array.
+
+
+| Single Value | Multiple Values |
+| -- | -- |
+| ✋`grabTextFrom` | 🙌 `grabTextFromAll`  |
+| ✋`grabValueFrom` | 🙌 `grabValueFromAll`  |
+| ✋ `grabAttributeFrom` | 🙌`grabAttributeFromAll`  |
+| ✋ `grabHTMLFrom` | 🙌 `grabHTMLFromAll`  |
+| ✋ `grabCssPropertyFrom` | 🙌 `grabCssPropertyFromAll` |
+
+> Single-value `grab*From` will throw error when no data was matched, while `grab*FromAll` will return array.
+
+### 3️⃣ Bootstrap / Teardown Changed
+
+`async/await` paradigm changed the way we write asynchronous code in NodeJS. 
+However, bootstrap functions were created to use old-style methods of passing `done` callback inside.
+
+In 3.0 we decided to completely change the way async bootstrap is performed and replace all it with async/await functions:
+
+```js
+// before
+bootstrap: (done) => {
+  server.start().then(done);
+},
+
+// after
+bootstrap: async () => {
+  await server.start();
+}
+```
+Passing a string as bootstrap function (to require a function from external file) is also not supported:
+
+```js
+// before
+bootstrap: './server_start.js',
+
+// after
+bootstrap: require('./server_start.js'),
+```
+
+The same rules are applied to teardown, bootstrapAll, teardownAll.
+
+### 4️⃣ Locator Detection Heuristic Change
+
+In 3.0 we added a new rule to auto-detect CSS locator. If a locator starts with `[` parser will use it as a CSS locator, without trying to match value by text.
+
+* Previous behavior: I.click('[user]') - will try to search for a link with `[user]` text, if no found - take `[user]` as CSS locator
+* Current behavior: I.click('[user]') - will check only for CSS locator `[user]`
+
+### 5️⃣ Improved Parallel Execution with Workers
+
+CodeceptJS has two parallel execution modes:
+
+* classical `run-multiple` which spawns independent CodeceptJS processes on system level.
+* threaded `run-workers` which uses NodeJS workers instead of subprocesses.
+
+Workers are faster, they can communicate with parent thread. Unfortunately, `run-workers` was not as efficient as `run-multiple` because there was no way of declaring sophisticated configs for parallel execution. For instance, running tests in 4 threads in 2 browsers could not be set.
+
+We updated workers API to make them as flexible as possible. You don't even need to put complex logic into config! You can create your own parallel runner and customize it to match your expectations:
+
+```js
+// don't initialize workers in constructor
+const workers = new Workers(null, workerConfig);
+// split tests by suites in 2 groups
+const testGroups = workers.createGroupsOfSuites(2);
+
+const browsers = ['firefox', 'chrome'];
+
+const configs = browsers.map(browser => {
+  return helpers: {
+    WebDriver: { browser }
+  }
+});
+
+for (const config of configs) {
+  for (group of groupOfTests) {
+    const worker = workers.spawn();
+    worker.addTests(group);
+    worker.addConfig(config);
+  }
+}
+
+workers.run();
+```
+[Learn more about how you can create a parallel runner](https://github.com/Codeception/CodeceptJS/blob/codeceptjs-v3.0/docs/parallel.md#custom-parallel-execution).
+
+`run-multiple` will still work but it is considered deprecated and won't be supported.
+
+### WebDriverIO helper removed
+
+WebDriverIO helper supported webdriverio v4 library, which is not maintained anymore. It should be easy to switch to WebDriver helper which supports webdriverio v6.
+
+ 

package/docs/wiki/Videos.md

@@ -0,0 +1,19 @@
+[![](http://i3.ytimg.com/vi/BRMWstiOTks/maxresdefault.jpg)](https://www.youtube.com/watch?v=BRMWstiOTks)
+
+## [An Introduction, Getting started and working with CodeceptJS & Puppeteer (EAWeekend)](https://www.youtube.com/watch?v=BRMWstiOTks)
+
+## [CodeceptJS Official YouTube Channel](https://www.youtube.com/channel/UCEs4030bmtonyDhTHEXa_2g)
+
+## [Introductory Videos](https://www.youtube.com/watch?v=FPFG1rBNJ64&list=PLcFXthgti9Lt4SjSvL1ALDg6dOeTC0TvT)
+
+Free educational videos provided by our community member **[@ontytoom](http://github.com/ontytoom)**.
+
+1. [Installation](https://www.youtube.com/watch?v=FPFG1rBNJ64)
+1. [Creating a Test](https://www.youtube.com/watch?v=mdQZjL3h9d0)
+1. [Using Page Objects](https://www.youtube.com/watch?v=s677_6VctjQ)
+
+## [Practical E2E Testing with CodeceptJS](https://www.udemy.com/practical-e2e-testing-with-codeceptjs/)
+
+Udemy course by Luke Beilharz
+
+

package/lib/actor.js

@@ -13,18 +13,15 @@ const output = require('./output');
  */
 class Actor {
   /**
-   * Print the comment on log. Also, adding a step in the `Test.steps` object
+   * add print comment method`
    * @param {string} msg
    * @param {string} color
    * @inner
    *
    * ⚠️ returns a promise which is synchronized internally by recorder
    */
-  async say(msg, color = 'cyan') {
-    const step = new Step('say', 'say');
-    step.status = 'passed';
-    return recordStep(step, [msg]).then(() => {
-      // this is backward compatibility as this event may be used somewhere
+  say(msg, color = 'cyan') {
+    return recorder.add(`say ${msg}`, () => {
       event.emit(event.step.comment, msg);
       output.say(msg, `${color}`);
     });

package/lib/ai.js

@@ -1,47 +1,118 @@
-const { Configuration, OpenAIApi } = require('openai');
 const debug = require('debug')('codeceptjs:ai');
-const config = require('./config');
 const output = require('./output');
+const event = require('./event');
 const { removeNonInteractiveElements, minifyHtml, splitByChunks } = require('./html');
 
-const defaultConfig = {
-  model: 'gpt-3.5-turbo-16k',
-  temperature: 0.1,
-};
-
-const htmlConfig = {
+const defaultHtmlConfig = {
   maxLength: 50000,
   simplify: true,
   minify: true,
   html: {},
 };
 
-const aiInstance = null;
+const defaultPrompts = {
+  writeStep: (html, input) => [{
+    role: 'user',
+    content: `I am test engineer writing test in CodeceptJS
+        I have opened web page and I want to use CodeceptJS to ${input} on this page
+        Provide me valid CodeceptJS code to accomplish it
+        Use only locators from this HTML: \n\n${html}`,
+  },
+  ],
+
+  healStep: (html, { step, error, prevSteps }) => {
+    return [{
+      role: 'user',
+      content: `As a test automation engineer I am testing web application using CodeceptJS.
+        I want to heal a test that fails. Here is the list of executed steps: ${prevSteps.map(s => s.toString()).join(', ')}
+        Propose how to adjust ${step.toCode()} step to fix the test.
+        Use locators in order of preference: semantic locator by text, CSS, XPath. Use codeblocks marked with \`\`\`
+        Here is the error message: ${error.message}
+        Here is HTML code of a page where the failure has happened: \n\n${html}`,
+    }];
+  },
+};
 
 class AiAssistant {
   constructor() {
-    this.config = config.get('ai', defaultConfig);
-    this.htmlConfig = Object.assign(htmlConfig, this.config.html);
-    delete this.config.html;
-    this.html = null;
+    this.totalTime = 0;
+    this.numTokens = 0;
+
+    this.reset();
+    this.connectToEvents();
+  }
+
+  enable(config = {}) {
+    debug('Enabling AI assistant');
+    this.isEnabled = true;
+
+    const { html, prompts, ...aiConfig } = config;
+
+    this.config = Object.assign(this.config, aiConfig);
+    this.htmlConfig = Object.assign(defaultHtmlConfig, html);
+    this.prompts = Object.assign(defaultPrompts, prompts);
+
+    debug('Config', this.config);
+  }
+
+  reset() {
+    this.numTokens = 0;
+    this.isEnabled = false;
+    this.config = {
+      maxTokens: 1000000,
+      request: null,
+      response: parseCodeBlocks,
+      // lets limit token usage to 1M
+    };
+    this.minifiedHtml = null;
     this.response = null;
+    this.totalTime = 0;
+  }
+
+  disable() {
+    this.isEnabled = false;
+  }
 
-    this.isEnabled = !!process.env.OPENAI_API_KEY;
+  connectToEvents() {
+    event.dispatcher.on(event.all.result, () => {
+      if (this.isEnabled && this.numTokens > 0) {
+        const numTokensK = Math.ceil(this.numTokens / 1000);
+        const maxTokensK = Math.ceil(this.config.maxTokens / 1000);
+        output.print(`AI assistant took ${this.totalTime}s and used ~${numTokensK}K input tokens. Tokens limit: ${maxTokensK}K`);
+      }
+    });
+  }
 
+  checkRequestFn() {
     if (!this.isEnabled) {
-      debug('No OpenAI API key provided. AI assistant is disabled.');
+      debug('AI assistant is disabled');
       return;
     }
 
-    const configuration = new Configuration({
-      apiKey: process.env.OPENAI_API_KEY,
-    });
+    if (this.config.request) return;
 
-    this.openai = new OpenAIApi(configuration);
-  }
+    const noRequestErrorMessage = `
+     No request function is set for AI assistant.  
+     Please implement your own request function and set it in the config.    
+
+     [!] AI request was decoupled from CodeceptJS. To connect to OpenAI or other AI service, please implement your own request function and set it in the config.
+
+     Example (connect to OpenAI):
+
+     ai: {
+       request: async (messages) => {
+         const OpenAI = require('openai');
+         const openai = new OpenAI({ apiKey: process.env['OPENAI_API_KEY'] })
+         const response = await openai.chat.completions.create({
+          model: 'gpt-3.5-turbo-0125',
+          messages,
+         });
+         return response?.data?.choices[0]?.message?.content;
+       }
+     }
+    `.trim();
 
-  static getInstance() {
-    return aiInstance || new AiAssistant();
+    throw new Error(noRequestErrorMessage);
   }
 
   async setHtmlContext(html) {
@@ -50,98 +121,99 @@ class AiAssistant {
     if (this.htmlConfig.simplify) {
       processedHTML = removeNonInteractiveElements(processedHTML, this.htmlConfig);
     }
+
     if (this.htmlConfig.minify) processedHTML = await minifyHtml(processedHTML);
     if (this.htmlConfig.maxLength) processedHTML = splitByChunks(processedHTML, this.htmlConfig.maxLength)[0];
 
-    debug(processedHTML);
-
-    this.html = processedHTML;
+    this.minifiedHtml = processedHTML;
   }
 
   getResponse() {
     return this.response || '';
   }
 
-  mockResponse(response) {
-    this.mockedResponse = response;
-  }
-
   async createCompletion(messages) {
-    if (!this.openai) return;
+    if (!this.isEnabled) return '';
 
-    debug(messages);
+    debug('Request', messages);
 
-    if (this.mockedResponse) return this.mockedResponse;
+    this.checkRequestFn();
 
     this.response = null;
 
-    try {
-      const completion = await this.openai.createChatCompletion({
-        ...this.config,
-        messages,
-      });
-
-      this.response = completion?.data?.choices[0]?.message?.content;
-
-      debug(this.response);
+    this.calculateTokens(messages);
 
+    try {
+      const startTime = process.hrtime();
+      this.response = await this.config.request(messages);
+      const endTime = process.hrtime(startTime);
+      const executionTimeInSeconds = endTime[0] + endTime[1] / 1e9;
+
+      this.totalTime += Math.round(executionTimeInSeconds);
+      debug('AI response time', executionTimeInSeconds);
+      debug('Response', this.response);
+      this.stopWhenReachingTokensLimit();
       return this.response;
     } catch (err) {
       debug(err.response);
       output.print('');
-      output.error(`OpenAI error: ${err.message}`);
-      output.error(err?.response?.data?.error?.code);
-      output.error(err?.response?.data?.error?.message);
+      output.error(`AI service error: ${err.message}`);
+      if (err?.response?.data?.error?.code) output.error(err?.response?.data?.error?.code);
+      if (err?.response?.data?.error?.message) output.error(err?.response?.data?.error?.message);
+      this.stopWhenReachingTokensLimit();
       return '';
     }
   }
 
-  async healFailedStep(step, err, test) {
+  async healFailedStep(failureContext) {
     if (!this.isEnabled) return [];
-    if (!this.html) throw new Error('No HTML context provided');
-
-    const messages = [
-      { role: 'user', content: 'As a test automation engineer I am testing web application using CodeceptJS.' },
-      { role: 'user', content: `I want to heal a test that fails. Here is the list of executed steps: ${test.steps.join(', ')}` },
-      { role: 'user', content: `Propose how to adjust ${step.toCode()} step to fix the test.` },
-      { role: 'user', content: 'Use locators in order of preference: semantic locator by text, CSS, XPath. Use codeblocks marked with ```.' },
-      { role: 'user', content: `Here is the error message: ${err.message}` },
-      { role: 'user', content: `Here is HTML code of a page where the failure has happened: \n\n${this.html}` },
-    ];
-
-    const response = await this.createCompletion(messages);
+    if (!failureContext.html) throw new Error('No HTML context provided');
+
+    await this.setHtmlContext(failureContext.html);
+
+    if (!this.minifiedHtml) {
+      debug('HTML context is empty after removing non-interactive elements & minification');
+      return [];
+    }
+
+    const response = await this.createCompletion(this.prompts.healStep(this.minifiedHtml, failureContext));
     if (!response) return [];
 
-    return parseCodeBlocks(response);
+    return this.config.response(response);
+  }
+
+  calculateTokens(messages) {
+    // we implement naive approach for calculating tokens with no extra requests
+    // this approach was tested via https://platform.openai.com/tokenizer
+    // we need it to display current usage tokens usage so users could analyze effectiveness of AI
+
+    const inputString = messages.map(m => m.content).join(' ').trim();
+    const numWords = (inputString.match(/[^\s\-:=]+/g) || []).length;
+
+    // 2.5 token is constant for average HTML input
+    const tokens = numWords * 2.5;
+
+    this.numTokens += tokens;
+
+    return tokens;
+  }
+
+  stopWhenReachingTokensLimit() {
+    if (this.numTokens < this.config.maxTokens) return;
+
+    output.print(`AI assistant has reached the limit of ${this.config.maxTokens} tokens in this session. It will be disabled now`);
+    this.disable();
   }
 
   async writeSteps(input) {
     if (!this.isEnabled) return;
-    if (!this.html) throw new Error('No HTML context provided');
+    if (!this.minifiedHtml) throw new Error('No HTML context provided');
 
     const snippets = [];
 
-    const messages = [
-      {
-        role: 'user',
-        content: `I am test engineer writing test in CodeceptJS
-          I have opened web page and I want to use CodeceptJS to ${input} on this page
-          Provide me valid CodeceptJS code to accomplish it
-          Use only locators from this HTML: \n\n${this.html}`,
-      },
-      { role: 'user', content: 'Propose only CodeceptJS steps code. Do not include Scenario or Feature into response' },
-
-      // old prompt
-      // { role: 'user', content: 'I want to click button Submit using CodeceptJS on this HTML page: <html><body><button>Submit</button></body></html>' },
-      // { role: 'assistant', content: '```js\nI.click("Submit");\n```' },
-      // { role: 'user', content: 'I want to click button Submit using CodeceptJS on this HTML page: <html><body><button>Login</button></body></html>' },
-      // { role: 'assistant', content: 'No suggestions' },
-      // { role: 'user', content: `Now I want to ${input} on this HTML page using CodeceptJS code` },
-      // { role: 'user', content: `Provide me with CodeceptJS code to achieve this on THIS page.` },
-    ];
-    const response = await this.createCompletion(messages);
+    const response = await this.createCompletion(this.prompts.writeStep(this.minifiedHtml, input));
     if (!response) return;
-    snippets.push(...parseCodeBlocks(response));
+    snippets.push(...this.config.response(response));
 
     debug(snippets[0]);
 
@@ -177,4 +249,4 @@ function parseCodeBlocks(response) {
   return modifiedSnippets.filter(snippet => !!snippet);
 }
 
-module.exports = AiAssistant;
+module.exports = new AiAssistant();

package/lib/cli.js

@@ -167,6 +167,7 @@ class Cli extends Base {
         }
 
         const steps = test.steps || (test.ctx && test.ctx.test.steps);
+
         if (steps && steps.length) {
           let scenarioTrace = '';
           steps.reverse().forEach((step) => {

package/lib/command/generate.js

@@ -258,3 +258,37 @@ helpers: {
     `);
   });
 };
+
+const healTemplate = fs.readFileSync(path.join(__dirname, '../template/heal.js'), 'utf8').toString();
+
+module.exports.heal = function (genPath) {
+  const testsPath = getTestRoot(genPath);
+
+  let configFile = path.join(testsPath, `codecept.conf.${extension}`);
+
+  if (!fileExists(configFile)) {
+    configFile = path.join(testsPath, `codecept.conf.${extension}`);
+    if (fileExists(configFile)) extension = 'ts';
+  }
+
+  output.print('Creating basic heal recipes');
+  output.print(`Add your own custom recipes to ./heal.${extension} file`);
+  output.print('Require this file in the config file and enable heal plugin:');
+  output.print('--------------------------');
+  output.print(`
+require('./heal')
+
+exports.config = {
+  // ... 
+  plugins: {
+    heal: {
+      enabled: true
+    }
+  }
+}
+  `);
+
+  const healFile = path.join(testsPath, `heal.${extension}`);
+  if (!safeFileWrite(healFile, healTemplate)) return;
+  output.success(`Heal recipes were created in ${healFile}`);
+};

package/lib/command/run-workers.js

@@ -1,6 +1,7 @@
 // For Node version >=10.5.0, have to use experimental flag
 const { tryOrDefault } = require('../utils');
 const output = require('../output');
+const store = require('../store');
 const event = require('../event');
 const Workers = require('../workers');
 
@@ -96,6 +97,8 @@ module.exports = async function (workerCount, selectedRuns, options) {
   });
 
   try {
+    if (options.verbose || options.debug) store.debugMode = true;
+
     if (options.verbose) {
       global.debugMode = true;
       const { getMachineInfo } = require('./info');

package/lib/command/run.js

@@ -2,16 +2,19 @@ const {
   getConfig, printError, getTestRoot, createOutputDir,
 } = require('./utils');
 const Config = require('../config');
+const store = require('../store');
 const Codecept = require('../codecept');
 
 module.exports = async function (test, options) {
   // registering options globally to use in config
   // Backward compatibility for --profile
+  // TODO: remove in CodeceptJS 4
   process.profile = options.profile;
 
   if (options.profile) {
     process.env.profile = options.profile;
   }
+  if (options.verbose || options.debug) store.debugMode = true;
 
   const configFile = options.config;
 

package/lib/container.js

@@ -8,6 +8,7 @@ const recorder = require('./recorder');
 const event = require('./event');
 const WorkerStorage = require('./workerStorage');
 const store = require('./store');
+const ai = require('./ai');
 
 let container = {
   helpers: {},
@@ -45,6 +46,7 @@ class Container {
     container.translation = loadTranslation(config.translation || null, config.vocabularies || []);
     container.support = createSupportObjects(config.include || {});
     container.plugins = createPlugins(config.plugins || {}, opts);
+    if (opts && opts.ai) ai.enable(config.ai); // enable AI Assistant
     if (config.gherkin) loadGherkinSteps(config.gherkin.steps || []);
     if (opts && typeof opts.timeouts === 'boolean') store.timeouts = opts.timeouts;
   }