@testomatio/reporter 1.2.0-beta-4 → 1.2.0

package/lib/pipe/testomatio.js

@@ -1,14 +1,18 @@
 const debug = require('debug')('@testomatio/reporter:pipe:testomatio');
 const chalk = require('chalk');
+// Retry interceptor function
+const axiosRetry = require('axios-retry');
+// Default axios instance
 const axios = require('axios');
 const JsonCycle = require('json-cycle');
-const { APP_PREFIX, STATUS } = require('../constants');
+
+const { APP_PREFIX, STATUS, AXIOS_TIMEOUT, AXIOS_RETRY_TIMEOUT } = require('../constants');
 const { isValidUrl, foundedTestLog } = require('../utils/utils');
-const { parseFilterParams, generateFilterRequestParams, setS3Credentials, } = require('../utils/pipe_utils');
+const { parseFilterParams, generateFilterRequestParams, setS3Credentials } = require('../utils/pipe_utils');
+const config = require('../config');
 
-const { TESTOMATIO_RUN } = process.env;
-if (TESTOMATIO_RUN) {
-  process.env.runId = TESTOMATIO_RUN;
+if (process.env.TESTOMATIO_RUN) {
+  process.env.runId = process.env.TESTOMATIO_RUN;
 }
 
 /**
@@ -21,7 +25,7 @@ class TestomatioPipe {
   constructor(params, store) {
     this.isEnabled = false;
     this.url = params.testomatioUrl || process.env.TESTOMATIO_URL || 'https://app.testomat.io';
-    this.apiKey = params.apiKey || process.env.TESTOMATIO;
+    this.apiKey = params.apiKey || config.TESTOMATIO;
     debug('Testomatio Pipe: ', this.apiKey ? 'API KEY' : '*no api key*');
     if (!this.apiKey) {
       return;
@@ -33,10 +37,32 @@ class TestomatioPipe {
     this.sharedRun = !!process.env.TESTOMATIO_SHARED_RUN;
     this.groupTitle = params.groupTitle || process.env.TESTOMATIO_RUNGROUP_TITLE;
     this.env = process.env.TESTOMATIO_ENV;
-
+    this.label = process.env.TESTOMATIO_LABEL;
+    // Create a new instance of axios with a custom config
     this.axios = axios.create({
       baseURL: `${this.url.trim()}`,
-      timeout: 30000
+      timeout: AXIOS_TIMEOUT,
+    });
+    // Pass the axios instance to the retry function
+    axiosRetry(this.axios, {
+      retries: 3, // Number of retries (Defaults to 3)
+      shouldResetTimeout: true,
+      retryCondition: error => {
+        // Conditional check the error status code
+        switch (error.response.status) {
+          case 409:
+          case 429:
+          case 502:
+          case 503:
+            return true; // Retry request with response status code 409, 429, 502, 503
+          default:
+            return false; // Do not retry the others
+        }
+      },
+      retryDelay: retryCount => retryCount * AXIOS_RETRY_TIMEOUT, // sum = 15sec
+      onRetry: retryCount => {
+        debug(`Retry attempt #${retryCount} failed. Retrying again...`);
+      },
     });
 
     this.isEnabled = true;
@@ -56,7 +82,7 @@ class TestomatioPipe {
   /**
    * Asynchronously prepares and retrieves the Testomat.io test grepList based on the provided options.
    * @param {Object} opts - The options for preparing the test grepList.
-   * @returns {Promise<string[]>} - An array containing the retrieved 
+   * @returns {Promise<string[]>} - An array containing the retrieved
    * test grepList, or an empty array if no tests are found or the request is disabled.
    * @throws {Error} - Throws an error if there was a problem while making the request.
    */
@@ -69,7 +95,7 @@ class TestomatioPipe {
       const q = generateFilterRequestParams({
         type,
         id,
-        apiKey: this.apiKey.trim()
+        apiKey: this.apiKey.trim(),
       });
 
       if (!q) {
@@ -83,14 +109,10 @@ class TestomatioPipe {
         foundedTestLog(APP_PREFIX, data.tests);
         return data.tests;
       }
-      
+
       console.log(APP_PREFIX, `⛔  No tests found for your --filter --> ${type}=${id}`);
-    }
-    catch (err) {
-      console.error(
-        APP_PREFIX,
-        `🚩 Error getting Testomat.io test grepList: ${err}`,
-      );
+    } catch (err) {
+      console.error(APP_PREFIX, `🚩 Error getting Testomat.io test grepList: ${err}`);
     }
   }
 
@@ -131,6 +153,7 @@ class TestomatioPipe {
         jira_id: this.jiraId,
         env: this.env,
         title: this.title,
+        label: this.label,
         shared_run: this.sharedRun,
       }).filter(([, value]) => !!value),
     );
@@ -148,10 +171,13 @@ class TestomatioPipe {
         maxContentLength: Infinity,
         maxBodyLength: Infinity,
       });
+
       this.runId = resp.data.uid;
       this.runUrl = `${this.url}/${resp.data.url.split('/').splice(3).join('/')}`;
       this.runPublicUrl = resp.data.public_url;
+
       if (resp.data.artifacts) setS3Credentials(resp.data.artifacts);
+
       this.store.runUrl = this.runUrl;
       this.store.runPublicUrl = this.runPublicUrl;
       this.store.runId = this.runId;
@@ -168,50 +194,47 @@ class TestomatioPipe {
     debug('"createRun" function finished');
   }
 
-  /**
-   *
-   * @param testData data
-   * @returns
-   */
   addTest(data) {
-    debug('Adding test...');
     if (!this.isEnabled) return;
     if (!this.runId) return;
     data.api_key = this.apiKey;
     data.create = this.createNewTests;
     const json = JsonCycle.stringify(data);
 
-    return this.axios.post(`/api/reporter/${this.runId}/testrun`, json, {
-      maxContentLength: Infinity,
-      maxBodyLength: Infinity,
-      headers: {
-        // Overwrite Axios's automatically set Content-Type
-        'Content-Type': 'application/json',
-      },
-    })
-    .catch(err => {
-      if (err.response) {
-        if (err.response.status >= 400) {
-          const responseData = err.response.data || { message: '' };
+    debug('Adding test', json);
+
+    return this.axios
+      .post(`/api/reporter/${this.runId}/testrun`, json, {
+        maxContentLength: Infinity,
+        maxBodyLength: Infinity,
+        headers: {
+          // Overwrite Axios's automatically set Content-Type
+          'Content-Type': 'application/json',
+        },
+      })
+      .catch(err => {
+        if (err.response) {
+          if (err.response.status >= 400) {
+            const responseData = err.response.data || { message: '' };
+            console.log(
+              APP_PREFIX,
+              chalk.yellow(`Warning: ${responseData.message} (${err.response.status})`),
+              chalk.grey(data?.title || ''),
+            );
+            if (err.response.data.message.includes('could not be matched')) {
+              this.hasUnmatchedTests = true;
+            }
+            return;
+          }
           console.log(
             APP_PREFIX,
-            chalk.yellow(`Warning: ${responseData.message} (${err.response.status})`),
-            chalk.grey(data?.title || ''),
+            chalk.yellow(`Warning: ${data?.title || ''} (${err.response?.status})`),
+            `Report couldn't be processed: ${err?.response?.data?.message}`,
           );
-          if (err.response.data.message.includes('could not be matched')) {
-            this.hasUnmatchedTests = true;
-          }
-          return;
+        } else {
+          console.log(APP_PREFIX, chalk.blue(data?.title || ''), "Report couldn't be processed", err);
         }
-        console.log(
-          APP_PREFIX,
-          chalk.yellow(`Warning: ${data?.title || ''} (${err.response?.status})`),
-          `Report couldn't be processed: ${err?.response?.data?.message}`,
-        );
-      } else {
-        console.log(APP_PREFIX, chalk.blue(data?.title || ''), "Report couldn't be processed", err);
-      }
-    });
+      });
   }
 
   /**

package/lib/reporter-functions.js

@@ -0,0 +1,46 @@
+const { services } = require('./services');
+const { initPlaywrightForStorage } = require('./adapter/playwright');
+
+if (process.env.PLAYWRIGHT_TEST_BASE_URL) {
+  initPlaywrightForStorage();
+}
+
+/**
+ * Stores path to file as artifact and uploads it to the S3 storage
+ * @param {string | {path: string, type: string, name: string}} data - path to file or object with path, type and name
+ */
+function saveArtifact(data, context = null) {
+  if (!data) return;
+  services.artifacts.put(data, context);
+}
+
+/**
+ * Attach log message(s) to the test report
+ * @param  {...any} args
+ */
+function logMessage(...args) {
+  services.logger._templateLiteralLog(...args);
+}
+
+/**
+ * Similar to "log" function but marks message in report as a step
+ * @param {*} message
+ */
+function addStep(message) {
+  services.logger.step(message);
+}
+
+/**
+ * Add key-value pair(s) to the test report
+ * @param {*} keyValue
+ */
+function setKeyValue(keyValue) {
+  services.keyValues.put(keyValue);
+}
+
+module.exports = {
+  artifact: saveArtifact,
+  log: logMessage,
+  step: addStep,
+  keyValue: setKeyValue,
+};

package/lib/reporter.js

@@ -1,18 +1,19 @@
-const logger = require('./logger');
 const TestomatClient = require('./client');
 const TRConstants = require('./constants');
-const TRArtifacts = require('./_ArtifactStorageOld');
+const { services } = require('./services');
 
-const log = logger.templateLiteralLog.bind(logger);
-const step = logger.step.bind(logger);
+const reporterFunctions = require('./reporter-functions');
 
 module.exports = {
-  logger,
-  testomatioLogger: logger,
-  log,
-  step,
+  // TODO: deprecate in future; use log or testomat.log
+  testomatioLogger: services.logger,
+
+  artifact: reporterFunctions.artifact,
+  log: reporterFunctions.log,
+  logger: services.logger,
+  meta: reporterFunctions.keyValue,
+  step: reporterFunctions.step,
+
   TestomatClient,
   TRConstants,
-  TRArtifacts,
-  addArtifact: TRArtifacts.artifact,
 };

package/lib/services/artifacts.js

@@ -0,0 +1,57 @@
+const debug = require('debug')('@testomatio/reporter:services-artifacts');
+const { dataStorage } = require('../data-storage');
+
+/**
+ * Artifact storage is supposed to store file paths
+ */
+class ArtifactStorage {
+  static #instance;
+
+  /**
+   * Singleton
+   * @returns {ArtifactStorage}
+   */
+  static getInstance() {
+    if (!this.#instance) {
+      this.#instance = new ArtifactStorage();
+    }
+    return this.#instance;
+  }
+
+  /**
+   * Stores path to file as artifact and uploads it to the S3 storage
+   * @param {string | {path: string, type: string, name: string}} data - path to file or object with path, type and name
+   * @param {*} context testId or test title
+   */
+  put(data, context = null) {
+    if (!data) return;
+    debug('Save artifact:', data);
+    dataStorage.putData('artifact', data, context);
+  }
+
+  /**
+   * Returns list of artifacts to upload
+   * @param {*} context testId or test context from test runner
+   * @returns {(string | {path: string, type: string, name: string})[]}
+   */
+  get(context) {
+    let artifacts = dataStorage.getData('artifact', context);
+    if (!artifacts || !artifacts.length) return [];
+
+    artifacts = artifacts.map(artifactData => {
+      // artifact could be an object ({type, path, name} props) or string (just path)
+      let artifact;
+      try {
+        artifact = JSON.parse(artifactData);
+      } catch (e) {
+        artifact = artifactData;
+      }
+      return artifact;
+    });
+    artifacts = artifacts.filter(artifact => !!artifact);
+    debug(`Artifacts for test ${context}:`, artifacts);
+    return artifacts.length ? artifacts : [];
+  }
+}
+
+module.exports.artifactStorage = ArtifactStorage.getInstance();

package/lib/services/index.js

@@ -0,0 +1,13 @@
+const { logger } = require('./logger');
+const { artifactStorage } = require('./artifacts');
+const { keyValueStorage } = require('./key-values');
+const { dataStorage } = require('../data-storage');
+
+module.exports.services = {
+  logger,
+  artifacts: artifactStorage,
+  keyValues: keyValueStorage,
+  setContext: context => {
+    dataStorage.setContext(context);
+  },
+};

package/lib/services/key-values.js

@@ -0,0 +1,58 @@
+const debug = require('debug')('@testomatio/reporter:services-key-value');
+const { dataStorage } = require('../data-storage');
+
+class KeyValueStorage {
+  static #instance;
+
+  /**
+   *
+   * @returns {KeyValueStorage}
+   */
+  static getInstance() {
+    if (!this.#instance) {
+      this.#instance = new KeyValueStorage();
+    }
+    return this.#instance;
+  }
+
+  /**
+   * Stores key-value pair and passes it to reporter
+   * @param {{key: string}} keyValue - key-value pair(s) as object
+   * @param {*} context - full test title
+   */
+  put(keyValue, context = null) {
+    if (!keyValue) return;
+    dataStorage.putData('keyvalue', keyValue, context);
+  }
+
+  #isKeyValueObject(smth) {
+    return smth && typeof smth === 'object' && !Array.isArray(smth) && smth !== null;
+  }
+
+  /**
+   * Returns key-values pairs for the test as object
+   * @param {*} context testId or test context from test runner
+   * @returns {{[key: string]: string} | {}} key-values pairs as object, e.g. {priority: 'high', browser: 'chrome'}
+   */
+  get(context = null) {
+    const keyValuesList = dataStorage.getData('keyvalue', context);
+    if (!keyValuesList || !keyValuesList?.length) return {};
+
+    const keyValues = {};
+    for (const keyValue of keyValuesList) {
+      if (this.#isKeyValueObject(keyValue)) {
+        Object.assign(keyValues, keyValue);
+      } else if (typeof keyValue === 'string') {
+        try {
+          Object.assign(keyValues, JSON.parse(keyValue));
+        } catch (e) {
+          debug(`Error parsing key-values for test ${context}`, keyValue);
+        }
+      }
+    }
+
+    return keyValues;
+  }
+}
+
+module.exports.keyValueStorage = KeyValueStorage.getInstance();

package/lib/{logger.js → services/logger.js}

@@ -1,6 +1,6 @@
 const chalk = require('chalk');
-const debug = require('debug')('@testomatio/reporter:logger');
-const { DataStorage } = require('./dataStorage');
+const debug = require('debug')('@testomatio/reporter:services-logger');
+const { dataStorage } = require('../data-storage');
 
 const LOG_METHODS = ['assert', 'debug', 'error', 'info', 'log', 'trace', 'warn'];
 const LEVELS = {
@@ -26,32 +26,25 @@ class Logger {
   // set default logger to be used in log, warn, error, etc methods
   #originalUserLogger = { ...console };
 
-  #dataStorage = new DataStorage('log');
+  #userLoggerWithOverridenMethods;
 
-  logLevel = process?.env?.LOG_LEVEL?.toUpperCase() || 'ALL';
-
-  constructor() {
-    // intercept console by default
-    this.intercept(console);
+  static #instance;
 
-    // singleton
-    if (!Logger.instance) {
-      Logger.instance = this;
+  /**
+   *
+   * @returns {Logger}
+   */
+  static getInstance() {
+    if (!this.#instance) {
+      this.#instance = new Logger();
     }
+    return this.#instance;
+  }
 
-    // add beforeEach hook for mocha. it does not override existing hook, just add new one
-    try {
-      // @ts-ignore
-      if (!beforeEach) return;
-      // @ts-ignore-next-line
-      beforeEach(function () {
-        if (this.currentTest?.__mocha_id__) {
-          global.testTitle = this.currentTest.fullTitle();
-        }
-      });
-    } catch (e) {
-      // ignore
-    }
+  logLevel = process?.env?.LOG_LEVEL?.toUpperCase() || 'ALL';
+
+  constructor() {
+    if (!dataStorage.isFileStorage || process.env.TESTOMATIO_INTERCEPT_CONSOLE_LOGS) this.intercept(console);
   }
 
   /**
@@ -61,9 +54,6 @@ class Logger {
    * @param  {...any} values
    */
   step(strings, ...values) {
-    // get testId for mocha
-    const context = global.testTitle ?? null;
-
     let logs = '';
     for (let i = 0; i < strings.length; i++) {
       logs += strings[i];
@@ -72,17 +62,18 @@ class Logger {
       }
     }
     logs = chalk.blue(`> ${logs}`);
-    this.#dataStorage.putData(logs, context);
+    dataStorage.putData('log', logs);
   }
 
   /**
    *
-   * @param {*} context testId or test context from test runner
-   * @returns
+   * @param {string} context testId or test context from test runner
+   * @returns {string[]}
    */
   getLogs(context) {
-    const logs = this.#dataStorage.getData(context);
-    return logs || '';
+    const logs = dataStorage.getData('log', context);
+    if (!logs) return [];
+    return logs;
   }
 
   #stringifyLogs(...args) {
@@ -114,14 +105,9 @@ class Logger {
    * 2. Standard: log(`text ${someVar}`)
    * 3. Standard with multiple arguments: log('text', someVar)
    */
-  templateLiteralLog(strings, ...args) {
+  _templateLiteralLog(strings, ...args) {
     if (Array.isArray(strings)) strings = strings.filter(item => item !== '').map(item => item.trim());
     if (Array.isArray(args)) args = args.filter(item => item !== '');
-    // entity which is used to define testId
-    let context = null;
-
-    // get testId for mocha
-    context = global.testTitle ?? null;
 
     let logs;
     // this block means tagged template is used (syntax like $`text ${someVar}`)
@@ -146,7 +132,7 @@ class Logger {
       logs = this.#stringifyLogs(strings, ...args);
     }
     this.#originalUserLogger.log(logs);
-    this.#dataStorage.putData(logs, context);
+    dataStorage.putData('log', logs);
   }
 
   /**
@@ -158,18 +144,17 @@ class Logger {
   #logWrapper(argsArray, level) {
     if (!argsArray.length) return;
 
-    // entity which is used to define testId
-    let context = null;
-
-    // get context for mocha
-    context = global.testTitle ?? null;
-
     const severity = LEVELS[level].severity;
     if (severity < LEVELS[this.logLevel]?.severity) return;
 
     const logs = this.#stringifyLogs(...argsArray);
+
     const colorizedLogs = chalk[LEVELS[level].color](logs);
-    this.#dataStorage.putData(colorizedLogs, context);
+    // do not attach logs from testomatio reporter itself
+    if (!logs.includes('[TESTOMATIO]')) {
+      dataStorage.putData('log', colorizedLogs);
+    }
+
     try {
       // level.toLowerCase() represents method name (log, warn, error, etc)
       this.#originalUserLogger[level.toLowerCase()](colorizedLogs);
@@ -212,16 +197,18 @@ class Logger {
    * @param {*} userLogger
    */
   intercept(userLogger) {
-    /* prevent multiple console interceptions (cause of infinite loop)
-    actual only for "console", because its used as default output and is intercepted by default */
-    const isUserLoggerConsole = userLogger.toString?.().toLowerCase() === '[object console]';
-    if (isUserLoggerConsole && process.env.TESTOMATIO_LOGGER_CONSOLE_INTERCEPTED) {
-      debug(`Try to intercept console, but it is already intercepted`);
-      return;
+    // STEP 1: reset previously intercepted logger methods to original
+    if (this.#userLoggerWithOverridenMethods) {
+      for (const method of LOG_METHODS) {
+        this.#userLoggerWithOverridenMethods[method] = this.#originalUserLogger[method];
+      }
     }
 
-    process.env.TESTOMATIO_LOGGER_CONSOLE_INTERCEPTED = 'true';
-    debug(isUserLoggerConsole ? 'console intercepted' : 'User logger intercepted');
+    // STEP 2: intercept new logger
+    this.#originalUserLogger = { ...userLogger };
+
+    const isUserLoggerConsole = userLogger.toString?.().toLowerCase() === '[object console]';
+    debug(`Intercepting ${isUserLoggerConsole ? 'console' : 'some user'} logger}`);
 
     // override user logger (any, e.g. console) methods to intercept log messages
     for (const method of LOG_METHODS) {
@@ -234,10 +221,7 @@ class Logger {
       userLogger[method] = (...args) => this[method](...args);
     }
 
-    /* Playwright
-    Playwright intercepts console messages by default. Thus when we intercept any logger and provide output by console,
-    messages are intercepted by Playwright.
-    */
+    this.#userLoggerWithOverridenMethods = userLogger;
 
     /*
     Initial idea was to intercept any logger (tracer, pino, etc),
@@ -247,11 +231,22 @@ class Logger {
     Also its difficult to understand (actually did not find the way to do it) if logger was already intercepted or not.
     Thus, decided to intercept only console by default and provide output by default console.
     It means, if user uses his own logger, its messages will be intercepted,
-    but the output will be provided by console.
+    but the output will be always provided by console.
     TODO: try to implement the providing output to terminal by user logger
     */
   }
 
+  stopInterception() {
+    debug('Stop ntercepting logs');
+
+    // restore original user logger
+    if (this.#userLoggerWithOverridenMethods) {
+      for (const method of LOG_METHODS) {
+        this.#userLoggerWithOverridenMethods[method] = this.#originalUserLogger[method];
+      }
+    }
+  }
+
   /**
    * Allows to configure logger. Make sure you do it before the logger usage in your code.
    *
@@ -267,12 +262,7 @@ class Logger {
   }
 }
 
-Logger.instance = null;
-
-// module.exports.Logger = Logger;
-const logger = new Logger();
-
-module.exports = logger;
+module.exports.logger = Logger.getInstance();
 
 // TODO: parse passed arguments as {level: 'str', message: 'str'} because some loggers use such syntax;
 // upd: did not face such loggers, but still could be useful
@@ -289,7 +279,6 @@ I found the only ability to get any logs using .task('log', 'message') (this is
 and then intercept it with:
   on('task', {
     log (message) {
-      console.log(message)
       return null
     }
   })
@@ -317,3 +306,6 @@ Finally, in the test it will look like:
 
 Parallelization in Cypress is only available if using Cypress Dashboard??
 */
+
+// TODO: add time to logs
+// TODO: add logger name to logs?

package/lib/utils/utils.js

@@ -141,7 +141,15 @@ const fetchSourceCode = (contents, opts = {}) => {
   // remove special chars from title
   if (!lineIndex && opts.title) {
     const title = opts.title.replace(/[([@].*/g, '');
-    lineIndex = lines.findIndex(l => l.includes(title));
+    
+    if (opts.lang === 'java') {
+      lineIndex = lines.findIndex(l => l.includes(`test${title}`));
+      if (lineIndex === -1) lineIndex = lines.findIndex(l => l.includes(`@DisplayName("${title}`));
+      if (lineIndex === -1) lineIndex = lines.findIndex(l => l.includes(`public void ${title}`));
+      if (lineIndex === -1) lineIndex = lines.findIndex(l => l.includes(`${title}(`));
+    } else {
+      lineIndex = lines.findIndex(l => l.includes(title));
+    }
   }
 
   if (opts.prepend) {
@@ -290,6 +298,23 @@ function removeColorCodes(input) {
   return input.replace(/\x1b\[[0-9;]*m/g, '');
 }
 
+const testRunnerHelper = {
+  // for Jest
+  getNameOfCurrentlyRunningTest: () => {
+    if (global.testomatioTestTitle) return global.testomatioTestTitle;
+
+    if (!process.env.JEST_WORKER_ID) return null;
+    try {
+      // TODO: expect?.getState()?.testPath + ' ' + expect?.getState()?.currentTestName
+      // @ts-expect-error "expect" could only be defined inside Jest environement (forbidden to import it outside)
+      // eslint-disable-next-line no-undef
+      return expect?.getState()?.currentTestName;
+    } catch (e) {
+      return null;
+    }
+  },
+};
+
 module.exports = {
   isSameTest,
   fetchSourceCode,
@@ -306,5 +331,6 @@ module.exports = {
   parseSuite,
   humanize,
   removeColorCodes,
-  foundedTestLog
+  foundedTestLog,
+  testRunnerHelper,
 };