codeceptjs 4.0.0-beta.2 → 4.0.0-beta.21

package/lib/ai.js

@@ -1,218 +1,202 @@
-import debug from 'debug';
-import * as event from './event.js';
-import * as output from './output.js';
-import { removeNonInteractiveElements, minifyHtml, splitByChunks } from './html.js';
-
-debug('codeceptjs:ai');
+import debugModule from 'debug'
+const debug = debugModule('codeceptjs:ai')
+import output from './output.js'
+import event from './event.js'
+import { removeNonInteractiveElements, minifyHtml, splitByChunks } from './html.js'
+import { generateText } from 'ai'
+import { fileURLToPath } from 'url'
+import path from 'path'
+import { fileExists } from './utils.js'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
 
 const defaultHtmlConfig = {
   maxLength: 50000,
   simplify: true,
   minify: true,
   html: {},
-};
-
-const defaultPrompts = {
-  writeStep: (html, input) => [{
-    role: 'user',
-    content: `I am test engineer writing test in CodeceptJS
-        I have opened web page and I want to use CodeceptJS to ${input} on this page
-        Provide me valid CodeceptJS code to accomplish it
-        Use only locators from this HTML: \n\n${html}`,
-  },
-  ],
-
-  healStep: (html, { step, error, prevSteps }) => {
-    return [{
-      role: 'user',
-      content: `As a test automation engineer I am testing web application using CodeceptJS.
-        I want to heal a test that fails. Here is the list of executed steps: ${prevSteps.map(s => s.toString()).join(', ')}
-        Propose how to adjust ${step.toCode()} step to fix the test.
-        Use locators in order of preference: semantic locator by text, CSS, XPath. Use codeblocks marked with \`\`\`
-        Here is the error message: ${error.message}
-        Here is HTML code of a page where the failure has happened: \n\n${html}`,
-    }];
-  },
-
-  generatePageObject: (html, extraPrompt = '', rootLocator = null) => [{
-    role: 'user',
-    content: `As a test automation engineer I am creating a Page Object for a web application using CodeceptJS.
-        Here is an sample page object:
-
-const { I } = inject();
-
-module.exports = {
-
-  // setting locators
-  element1: '#selector',
-  element2: '.selector',
-  element3: locate().withText('text'),
-
-  // seting methods
-  doSomethingOnPage(params) {
-    // ...
-  },  
-}        
-
-        I want to generate a Page Object for the page I provide.
-        Write JavaScript code in similar manner to list all locators on the page.
-        Use locators in order of preference: by text (use locate().withText()), label, CSS, XPath.
-        Avoid TailwindCSS, Bootstrap or React style formatting classes in locators.
-        Add methods to to interact with page when needed.
-        ${extraPrompt}
-        ${rootLocator ? `All provided elements are inside '${rootLocator}'. Declare it as root variable and for every locator use locate(...).inside(root)` : ''}
-        Add only locators from this HTML: \n\n${html}`,
-  }],
-};
+}
+
+async function loadPrompts() {
+  const prompts = {}
+  const promptNames = ['writeStep', 'healStep', 'generatePageObject']
+
+  for (const name of promptNames) {
+    let promptPath
+
+    if (global.codecept_dir) {
+      promptPath = path.join(global.codecept_dir, `prompts/${name}.js`)
+    }
+
+    if (!promptPath || !fileExists(promptPath)) {
+      promptPath = path.join(__dirname, `template/prompts/${name}.js`)
+    }
+
+    try {
+      const module = await import(promptPath)
+      prompts[name] = module.default || module
+      debug(`Loaded prompt ${name} from ${promptPath}`)
+    } catch (err) {
+      debug(`Failed to load prompt ${name}:`, err.message)
+    }
+  }
+
+  return prompts
+}
 
 class AiAssistant {
   constructor() {
-    this.totalTime = 0;
-    this.numTokens = 0;
+    this.totalTime = 0
+    this.numTokens = 0
 
-    this.reset();
-    this.connectToEvents();
+    this.reset()
+    this.connectToEvents()
   }
 
-  enable(config = {}) {
-    debug('Enabling AI assistant');
-    this.isEnabled = true;
+  async enable(config = {}) {
+    debug('Enabling AI assistant')
+    this.isEnabled = true
 
-    const { html, prompts, ...aiConfig } = config;
+    const { html, prompts, ...aiConfig } = config
 
-    this.config = Object.assign(this.config, aiConfig);
-    this.htmlConfig = Object.assign(defaultHtmlConfig, html);
-    this.prompts = Object.assign(defaultPrompts, prompts);
+    this.config = Object.assign(this.config, aiConfig)
+    this.htmlConfig = Object.assign(defaultHtmlConfig, html)
 
-    debug('Config', this.config);
+    const loadedPrompts = await loadPrompts()
+    this.prompts = Object.assign(loadedPrompts, prompts || {})
+
+    debug('Config', this.config)
   }
 
   reset() {
-    this.numTokens = 0;
-    this.isEnabled = false;
+    this.numTokens = 0
+    this.isEnabled = false
     this.config = {
       maxTokens: 1000000,
-      request: null,
+      model: null,
       response: parseCodeBlocks,
-      // lets limit token usage to 1M
-    };
-    this.minifiedHtml = null;
-    this.response = null;
-    this.totalTime = 0;
+    }
+    this.minifiedHtml = null
+    this.response = null
+    this.totalTime = 0
   }
 
   disable() {
-    this.isEnabled = false;
+    this.isEnabled = false
   }
 
   connectToEvents() {
     event.dispatcher.on(event.all.result, () => {
       if (this.isEnabled && this.numTokens > 0) {
-        const numTokensK = Math.ceil(this.numTokens / 1000);
-        const maxTokensK = Math.ceil(this.config.maxTokens / 1000);
-        output.print(`AI assistant took ${this.totalTime}s and used ~${numTokensK}K input tokens. Tokens limit: ${maxTokensK}K`);
+        const numTokensK = Math.ceil(this.numTokens / 1000)
+        const maxTokensK = Math.ceil(this.config.maxTokens / 1000)
+        output.print(`AI assistant took ${this.totalTime}s and used ${numTokensK}K tokens. Tokens limit: ${maxTokensK}K`)
       }
-    });
+    })
   }
 
-  checkRequestFn() {
+  checkModel() {
     if (!this.isEnabled) {
-      debug('AI assistant is disabled');
-      return;
+      debug('AI assistant is disabled')
+      return
     }
 
-    if (this.config.request) return;
+    if (this.config.model) return
 
-    const noRequestErrorMessage = `
-     No request function is set for AI assistant.
-     Please implement your own request function and set it in the config.
+    const noModelErrorMessage = `
+     No model is set for AI assistant.
 
-     [!] AI request was decoupled from CodeceptJS. To connect to OpenAI or other AI service, please implement your own request function and set it in the config.
+     [!] Please configure AI model using Vercel AI SDK providers.
 
      Example (connect to OpenAI):
 
+     import { openai } from '@ai-sdk/openai';
+
      ai: {
-       request: async (messages) => {
-         const OpenAI = require('openai');
-         const openai = new OpenAI({ apiKey: process.env['OPENAI_API_KEY'] })
-         const response = await openai.chat.completions.create({
-          model: 'gpt-3.5-turbo-0125',
-          messages,
-         });
-         return response?.data?.choices[0]?.message?.content;
-       }
+       model: openai('gpt-4o-mini')
      }
-    `.trim();
 
-    throw new Error(noRequestErrorMessage);
+     Example (connect to Anthropic):
+
+     import { anthropic } from '@ai-sdk/anthropic';
+
+     ai: {
+       model: anthropic('claude-3-5-sonnet-20241022')
+     }
+
+     See https://ai-sdk.dev/docs/foundations/providers-and-models for all providers.
+    `.trim()
+
+    throw new Error(noModelErrorMessage)
   }
 
   async setHtmlContext(html) {
-    let processedHTML = html;
+    let processedHTML = html
 
     if (this.htmlConfig.simplify) {
-      processedHTML = removeNonInteractiveElements(processedHTML, this.htmlConfig);
+      processedHTML = removeNonInteractiveElements(processedHTML, this.htmlConfig)
     }
 
-    if (this.htmlConfig.minify) processedHTML = await minifyHtml(processedHTML);
-    if (this.htmlConfig.maxLength) processedHTML = splitByChunks(processedHTML, this.htmlConfig.maxLength)[0];
+    if (this.htmlConfig.minify) processedHTML = await minifyHtml(processedHTML)
+    if (this.htmlConfig.maxLength) processedHTML = splitByChunks(processedHTML, this.htmlConfig.maxLength)[0]
 
-    this.minifiedHtml = processedHTML;
+    this.minifiedHtml = processedHTML
   }
 
   getResponse() {
-    return this.response || '';
+    return this.response || ''
   }
 
   async createCompletion(messages) {
-    if (!this.isEnabled) return '';
-
-    debug('Request', messages);
-
-    this.checkRequestFn();
-
-    this.response = null;
-
-    this.calculateTokens(messages);
+    if (!this.isEnabled) return ''
 
     try {
-      const startTime = process.hrtime();
-      this.response = await this.config.request(messages);
-      const endTime = process.hrtime(startTime);
-      const executionTimeInSeconds = endTime[0] + endTime[1] / 1e9;
-
-      this.totalTime += Math.round(executionTimeInSeconds);
-      debug('AI response time', executionTimeInSeconds);
-      debug('Response', this.response);
-      this.stopWhenReachingTokensLimit();
-      return this.response;
+      this.checkModel()
+      debug('Request', messages)
+
+      this.response = null
+
+      const startTime = process.hrtime()
+      const result = await generateText({
+        model: this.config.model,
+        messages,
+      })
+      const endTime = process.hrtime(startTime)
+      const executionTimeInSeconds = endTime[0] + endTime[1] / 1e9
+
+      this.response = result.text
+      this.numTokens += result.usage.totalTokens
+
+      this.totalTime += Math.round(executionTimeInSeconds)
+      debug('AI response time', executionTimeInSeconds)
+      debug('Response', this.response)
+      debug('Usage', result.usage)
+      this.stopWhenReachingTokensLimit()
+      return this.response
     } catch (err) {
-      debug(err.response);
-      output.print('');
-      output.output.error(`AI service error: ${err.message}`);
-      if (err?.response?.data?.error?.code) output.output.error(err?.response?.data?.error?.code);
-      if (err?.response?.data?.error?.message) output.output.error(err?.response?.data?.error?.message);
-      this.stopWhenReachingTokensLimit();
-
-      return '';
+      debug(err)
+      output.print('')
+      output.error(`AI service error: ${err.message}`)
+      this.stopWhenReachingTokensLimit()
+      return ''
     }
   }
 
   async healFailedStep(failureContext) {
-    if (!this.isEnabled) return [];
-    if (!failureContext.html) throw new Error('No HTML context provided');
+    if (!this.isEnabled) return []
+    if (!failureContext.html) throw new Error('No HTML context provided')
 
-    await this.setHtmlContext(failureContext.html);
+    await this.setHtmlContext(failureContext.html)
 
     if (!this.minifiedHtml) {
-      debug('HTML context is empty after removing non-interactive elements & minification');
-      return [];
+      debug('HTML context is empty after removing non-interactive elements & minification')
+      return []
     }
 
-    const response = await this.createCompletion(this.prompts.healStep(this.minifiedHtml, failureContext));
-    if (!response) return [];
+    const response = await this.createCompletion(this.prompts.healStep(this.minifiedHtml, failureContext))
+    if (!response) return []
 
-    return this.config.response(response);
+    return this.config.response(response)
   }
 
   /**
@@ -222,80 +206,67 @@ class AiAssistant {
    * @returns
    */
   async generatePageObject(extraPrompt = null, locator = null) {
-    if (!this.isEnabled) return [];
-    if (!this.minifiedHtml) throw new Error('No HTML context provided');
-
-    const response = await this.createCompletion(this.prompts.generatePageObject(this.minifiedHtml, locator, extraPrompt));
-    if (!response) return [];
-
-    return this.config.response(response);
-  }
-
-  calculateTokens(messages) {
-    // we implement naive approach for calculating tokens with no extra requests
-    // this approach was tested via https://platform.openai.com/tokenizer
-    // we need it to display current tokens usage so users could analyze effectiveness of AI
-
-    const inputString = messages.map(m => m.content).join(' ').trim();
-    const numWords = (inputString.match(/[^\s\-:=]+/g) || []).length;
-
-    // 2.5 token is constant for average HTML input
-    const tokens = numWords * 2.5;
+    if (!this.isEnabled) return []
+    if (!this.minifiedHtml) throw new Error('No HTML context provided')
 
-    this.numTokens += tokens;
+    const response = await this.createCompletion(this.prompts.generatePageObject(this.minifiedHtml, locator, extraPrompt))
+    if (!response) return []
 
-    return tokens;
+    return this.config.response(response)
   }
 
   stopWhenReachingTokensLimit() {
-    if (this.numTokens < this.config.maxTokens) return;
+    if (this.numTokens < this.config.maxTokens) return
 
-    output.print(`AI assistant has reached the limit of ${this.config.maxTokens} tokens in this session. It will be disabled now`);
-    this.disable();
+    output.print(`AI assistant has reached the limit of ${this.config.maxTokens} tokens in this session. It will be disabled now`)
+    this.disable()
   }
 
   async writeSteps(input) {
-    if (!this.isEnabled) return;
-    if (!this.minifiedHtml) throw new Error('No HTML context provided');
+    if (!this.isEnabled) return
+    if (!this.minifiedHtml) throw new Error('No HTML context provided')
 
-    const snippets = [];
+    const snippets = []
 
-    const response = await this.createCompletion(this.prompts.writeStep(this.minifiedHtml, input));
-    if (!response) return;
-    snippets.push(...this.config.response(response));
+    const response = await this.createCompletion(this.prompts.writeStep(this.minifiedHtml, input))
+    if (!response) return
+    snippets.push(...this.config.response(response))
 
-    debug(snippets[0]);
+    debug(snippets[0])
 
-    return snippets[0];
+    return snippets[0]
   }
 }
 
 function parseCodeBlocks(response) {
   // Regular expression pattern to match code snippets
-  const codeSnippetPattern = /```(?:javascript|js|typescript|ts)?\n([\s\S]+?)\n```/g;
+  const codeSnippetPattern = /```(?:javascript|js|typescript|ts)?\n([\s\S]+?)\n```/g
 
   // Array to store extracted code snippets
-  const codeSnippets = [];
+  const codeSnippets = []
 
-  response = response.split('\n').map(line => line.trim()).join('\n');
+  response = response
+    .split('\n')
+    .map(line => line.trim())
+    .join('\n')
 
   // Iterate over matches and extract code snippets
-  let match;
+  let match
   while ((match = codeSnippetPattern.exec(response)) !== null) {
-    codeSnippets.push(match[1]);
+    codeSnippets.push(match[1])
   }
 
   // Remove "Scenario", "Feature", and "require()" lines
   const modifiedSnippets = codeSnippets.map(snippet => {
-    const lines = snippet.split('\n');
+    const lines = snippet.split('\n')
 
-    const filteredLines = lines.filter(line => !line.includes('I.amOnPage') && !line.startsWith('Scenario') && !line.startsWith('Feature') && !line.includes('= require('));
+    const filteredLines = lines.filter(line => !line.includes('I.amOnPage') && !line.startsWith('Scenario') && !line.startsWith('Feature') && !line.includes('= require('))
 
-    return filteredLines.join('\n');
+    return filteredLines.join('\n')
     // remove snippets that move from current url
-  }); // .filter(snippet => !line.includes('I.amOnPage'));
+  }) // .filter(snippet => !line.includes('I.amOnPage'));
 
-  return modifiedSnippets.filter(snippet => !!snippet);
+  return modifiedSnippets.filter(snippet => !!snippet)
 }
 
-export default new AiAssistant();
+export default new AiAssistant()

package/lib/assert/empty.js

@@ -1,44 +1,42 @@
-import Assertion from '../assert.js';
-import AssertionFailedError from './error.js';
-import { template } from '../utils.js';
-import * as output from '../output.js';
+import assertionModule from '../assert.js'
+const Assertion = assertionModule.default || assertionModule
+import AssertionFailedError from './error.js'
+import { template } from '../utils.js'
+import outputModule from '../output.js'
+const output = outputModule.default || outputModule
 
 class EmptinessAssertion extends Assertion {
   constructor(params) {
-    super((value) => {
+    super(value => {
       if (Array.isArray(value)) {
-        return value.length === 0;
+        return value.length === 0
       }
-      return !value;
-    }, params);
-    this.params.type = 'to be empty';
+      return !value
+    }, params)
+    this.params.type = 'to be empty'
   }
 
   getException() {
     if (Array.isArray(this.params.value)) {
-      this.params.value = `[${this.params.value.join(', ')}]`;
+      this.params.value = `[${this.params.value.join(', ')}]`
     }
 
-    const err = new AssertionFailedError(this.params, "{{customMessage}}expected {{subject}} '{{value}}' {{type}}");
+    const err = new AssertionFailedError(this.params, "{{customMessage}}expected {{subject}} '{{value}}' {{type}}")
 
     err.cliMessage = () => {
-      const msg = err.template
-        .replace('{{value}}', output.colors.bold('{{value}}'))
-        .replace('{{subject}}', output.colors.bold('{{subject}}'));
-      return template(msg, this.params);
-    };
-    return err;
+      const msg = err.template.replace('{{value}}', output.colors.bold('{{value}}')).replace('{{subject}}', output.colors.bold('{{subject}}'))
+      return template(msg, this.params)
+    }
+    return err
   }
 
   addAssertParams() {
-    this.params.value = this.params.actual = arguments[0];
-    this.params.expected = [];
-    this.params.customMessage = arguments[1] ? `${arguments[1]}\n\n` : '';
+    this.params.value = this.params.actual = arguments[0]
+    this.params.expected = []
+    this.params.customMessage = arguments[1] ? `${arguments[1]}\n\n` : ''
   }
 }
 
-export { EmptinessAssertion as Assertion };
+export { EmptinessAssertion as Assertion }
 
-export function empty(subject) {
-  return new EmptinessAssertion({ subject });
-}
+export const empty = subject => new EmptinessAssertion({ subject })

package/lib/assert/equal.js

@@ -1,59 +1,52 @@
-import Assertion from '../assert.js';
-import AssertionFailedError from './error.js';
-import { template } from '../utils.js';
-import * as output from '../output.js';
+import Assertion from '../assert.js'
+import AssertionFailedError from './error.js'
+import { template } from '../utils.js'
+import output from '../output.js'
 
 class EqualityAssertion extends Assertion {
   constructor(params) {
     const comparator = function (a, b) {
       if (b.length === 0) {
-        b = '';
+        b = ''
       }
-      return a === b;
-    };
-    super(comparator, params);
-    this.params.type = 'to equal';
+      return a === b
+    }
+    super(comparator, params)
+    this.params.type = 'to equal'
   }
 
   getException() {
-    const params = this.params;
-    params.jar = template(params.jar, params);
-    const err = new AssertionFailedError(params, '{{customMessage}}expected {{jar}} "{{expected}}" {{type}} "{{actual}}"');
-    err.showDiff = false;
+    const params = this.params
+    params.jar = template(params.jar, params)
+    const err = new AssertionFailedError(params, '{{customMessage}}expected {{jar}} "{{expected}}" {{type}} "{{actual}}"')
+    err.showDiff = false
     if (typeof err.cliMessage === 'function') {
-      err.message = err.cliMessage();
+      err.message = err.cliMessage()
     }
     err.cliMessage = () => {
-      const msg = err.template
-        .replace('{{jar}}', output.colors.bold('{{jar}}'));
-      return template(msg, this.params);
-    };
-    return err;
+      const msg = err.template.replace('{{jar}}', output.colors.bold('{{jar}}'))
+      return template(msg, this.params)
+    }
+    return err
   }
 
   addAssertParams() {
-    this.params.expected = arguments[0];
-    this.params.actual = arguments[1];
-    this.params.customMessage = arguments[2] ? `${arguments[2]}\n\n` : '';
+    this.params.expected = arguments[0]
+    this.params.actual = arguments[1]
+    this.params.customMessage = arguments[2] ? `${arguments[2]}\n\n` : ''
   }
 }
 
-export function fileEquals(file) {
-  return new EqualityAssertion({ file, jar: 'contents of {{file}}' });
-}
-
-export { EqualityAssertion as Assertion };
-export function equals(jar) {
-  return new EqualityAssertion({ jar });
-}
-
-export function urlEquals(baseUrl) {
-  const assert = new EqualityAssertion({ jar: 'url of current page' });
+export { EqualityAssertion as Assertion }
+export const equals = jar => new EqualityAssertion({ jar })
+export const urlEquals = baseUrl => {
+  const assert = new EqualityAssertion({ jar: 'url of current page' })
   assert.comparator = function (expected, actual) {
     if (expected.indexOf('http') !== 0) {
-      actual = actual.slice(actual.indexOf(baseUrl) + baseUrl.length);
+      actual = actual.slice(actual.indexOf(baseUrl) + baseUrl.length)
     }
-    return actual === expected;
-  };
-  return assert;
+    return actual === expected
+  }
+  return assert
 }
+export const fileEquals = file => new EqualityAssertion({ file, jar: 'contents of {{file}}' })

package/lib/assert/error.js

@@ -1,4 +1,4 @@
-import { template as subs } from '../utils.js';
+import { template as subs } from '../utils.js'
 
 /**
  * Assertion errors, can provide a detailed error messages.
@@ -6,27 +6,27 @@ import { template as subs } from '../utils.js';
  * inspect() and cliMessage() added to display errors with params.
  */
 function AssertionFailedError(params, template) {
-  this.params = params;
-  this.template = template;
+  this.params = params
+  this.template = template
   // this.message = "AssertionFailedError";
   // this.showDiff = true;
 
   // @todo cut assert things nicer
-  this.showDiff = true;
+  this.showDiff = true
 
-  this.actual = this.params.actual;
-  this.expected = this.params.expected;
+  this.actual = this.params.actual
+  this.expected = this.params.expected
 
   this.inspect = () => {
-    const params = this.params || {};
-    const msg = params.customMessage || '';
-    return msg + subs(this.template, params);
-  };
+    const params = this.params || {}
+    const msg = params.customMessage || ''
+    return msg + subs(this.template, params)
+  }
 
-  this.cliMessage = () => this.inspect();
+  this.cliMessage = () => this.inspect()
 }
 
-AssertionFailedError.prototype = Object.create(Error.prototype);
-AssertionFailedError.constructor = AssertionFailedError;
+AssertionFailedError.prototype = Object.create(Error.prototype)
+AssertionFailedError.constructor = AssertionFailedError
 
-export default AssertionFailedError;
+export default AssertionFailedError