@testomatio/reporter 1.4.1-beta-1 → 1.4.1

package/lib/pipe/testomatio.js

@@ -1,14 +1,18 @@
 const debug = require('debug')('@testomatio/reporter:pipe:testomatio');
 const chalk = require('chalk');
+// Retry interceptor function
+const axiosRetry = require('axios-retry');
+// Default axios instance
 const axios = require('axios');
 const JsonCycle = require('json-cycle');
-const { APP_PREFIX, STATUS } = require('../constants');
+
+const { APP_PREFIX, STATUS, AXIOS_TIMEOUT, REPORTER_REQUEST_RETRIES } = require('../constants');
 const { isValidUrl, foundedTestLog } = require('../utils/utils');
-const { parseFilterParams, generateFilterRequestParams, setS3Credentials, } = require('../utils/pipe_utils');
+const { parseFilterParams, generateFilterRequestParams, setS3Credentials } = require('../utils/pipe_utils');
+const config = require('../config');
 
-const { TESTOMATIO_RUN } = process.env;
-if (TESTOMATIO_RUN) {
-  process.env.runId = TESTOMATIO_RUN;
+if (process.env.TESTOMATIO_RUN) {
+  process.env.runId = process.env.TESTOMATIO_RUN;
 }
 
 /**
@@ -19,9 +23,20 @@ if (TESTOMATIO_RUN) {
  */
 class TestomatioPipe {
   constructor(params, store) {
+    this.batch = {
+      isEnabled: params.isBatchEnabled ?? !process.env.TESTOMATIO_DISABLE_BATCH_UPLOAD ?? true,
+      intervalFunction: null, // will be created in createRun by setInterval function
+      intervalTime: 5000, // how often tests are sent
+      tests: [], // array of tests in batch
+      batchIndex: 0, // represents the current batch index (starts from 1 and increments by 1 for each batch)
+    };
+    this.retriesTimestamps = [];
+    this.reportingCanceledDueToReqFailures = false;
+    this.notReportedTestsCount = 0;
+
     this.isEnabled = false;
     this.url = params.testomatioUrl || process.env.TESTOMATIO_URL || 'https://app.testomat.io';
-    this.apiKey = params.apiKey || process.env.TESTOMATIO;
+    this.apiKey = params.apiKey || config.TESTOMATIO;
     debug('Testomatio Pipe: ', this.apiKey ? 'API KEY' : '*no api key*');
     if (!this.apiKey) {
       return;
@@ -33,10 +48,37 @@ class TestomatioPipe {
     this.sharedRun = !!process.env.TESTOMATIO_SHARED_RUN;
     this.groupTitle = params.groupTitle || process.env.TESTOMATIO_RUNGROUP_TITLE;
     this.env = process.env.TESTOMATIO_ENV;
-
+    this.label = process.env.TESTOMATIO_LABEL;
+    // Create a new instance of axios with a custom config
     this.axios = axios.create({
       baseURL: `${this.url.trim()}`,
-      timeout: 30000
+      timeout: AXIOS_TIMEOUT,
+    });
+
+    // Pass the axios instance to the retry function
+    axiosRetry(this.axios, {
+      // do not use retries for unit tests
+      retries: REPORTER_REQUEST_RETRIES.retriesPerRequest, // Number of retries
+      shouldResetTimeout: true,
+      retryCondition: error => {
+        if (!error.response) return false;
+        switch (error.response?.status) {
+          case 400: // Bad request (probably wrong API key)
+          case 404: // Test not matched
+          case 429: // Rate limit exceeded
+          case 500: // Internal server error
+            return false;
+          default:
+            break;
+        }
+        return error.response?.status >= 401; // Retry on 401+ and 5xx
+      },
+      retryDelay: () => REPORTER_REQUEST_RETRIES.retryTimeout, // sum = 15sec
+      onRetry: async (retryCount, error) => {
+        this.retriesTimestamps.push(Date.now());
+
+        debug(`${error.message || `Request failed ${error.status}`}. Retry #${retryCount} ...`);
+      },
     });
 
     this.isEnabled = true;
@@ -56,7 +98,7 @@ class TestomatioPipe {
   /**
    * Asynchronously prepares and retrieves the Testomat.io test grepList based on the provided options.
    * @param {Object} opts - The options for preparing the test grepList.
-   * @returns {Promise<string[]>} - An array containing the retrieved 
+   * @returns {Promise<string[]>} - An array containing the retrieved
    * test grepList, or an empty array if no tests are found or the request is disabled.
    * @throws {Error} - Throws an error if there was a problem while making the request.
    */
@@ -69,7 +111,7 @@ class TestomatioPipe {
       const q = generateFilterRequestParams({
         type,
         id,
-        apiKey: this.apiKey.trim()
+        apiKey: this.apiKey.trim(),
       });
 
       if (!q) {
@@ -83,23 +125,23 @@ class TestomatioPipe {
         foundedTestLog(APP_PREFIX, data.tests);
         return data.tests;
       }
-      
+
       console.log(APP_PREFIX, `⛔  No tests found for your --filter --> ${type}=${id}`);
-    }
-    catch (err) {
-      console.error(
-        APP_PREFIX,
-        `🚩 Error getting Testomat.io test grepList: ${err}`,
-      );
+    } catch (err) {
+      console.error(APP_PREFIX, `🚩 Error getting Testomat.io test grepList: ${err}`);
     }
   }
 
   /**
+   * Creates a new run on Testomat.io
+   * @param {{isBatchEnabled?: boolean}} params
    * @returns Promise<void>
    */
-  async createRun() {
+  async createRun(params = {}) {
+    this.batch.isEnabled = params.isBatchEnabled ?? this.batch.isEnabled;
     debug('Creating run...');
     if (!this.isEnabled) return;
+    if (this.batch.isEnabled) this.batch.intervalFunction = setInterval(this.#batchUpload, this.batch.intervalTime);
 
     let buildUrl = process.env.BUILD_URL || process.env.CI_JOB_URL || process.env.CIRCLE_BUILD_URL;
 
@@ -131,6 +173,7 @@ class TestomatioPipe {
         jira_id: this.jiraId,
         env: this.env,
         title: this.title,
+        label: this.label,
         shared_run: this.sharedRun,
       }).filter(([, value]) => !!value),
     );
@@ -148,10 +191,13 @@ class TestomatioPipe {
         maxContentLength: Infinity,
         maxBodyLength: Infinity,
       });
+
       this.runId = resp.data.uid;
       this.runUrl = `${this.url}/${resp.data.url.split('/').splice(3).join('/')}`;
       this.runPublicUrl = resp.data.public_url;
+
       if (resp.data.artifacts) setS3Credentials(resp.data.artifacts);
+
       this.store.runUrl = this.runUrl;
       this.store.runPublicUrl = this.runPublicUrl;
       this.store.runId = this.runId;
@@ -161,57 +207,146 @@ class TestomatioPipe {
     } catch (err) {
       console.error(
         APP_PREFIX,
-        'Error creating Testomat.io report, please check if your API key is valid. Skipping report',
-        err,
+        'Error creating Testomat.io report, please check if your API key is valid. Skipping report | ',
+        err?.response?.statusText || err?.status || err.message,
       );
+      printCreateIssue(err);
     }
     debug('"createRun" function finished');
   }
 
   /**
-   *
-   * @param testData data
-   * @returns
+   * Decides whether to skip test reporting in case of too many request failures
+   * @param {TestData} testData
+   * @returns {boolean}
    */
-  addTest(data) {
-    debug('Adding test...');
+  #cancelTestReportingInCaseOfTooManyReqFailures(testData) {
+    if (this.reportingCanceledDueToReqFailures) return true;
+
+    const retriesCountWithinTime = this.retriesTimestamps.filter(
+      timestamp => Date.now() - timestamp < REPORTER_REQUEST_RETRIES.withinTimeSeconds * 1000,
+    ).length;
+    debug(`${retriesCountWithinTime} failed requests within ${REPORTER_REQUEST_RETRIES.withinTimeSeconds}s`);
+
+    if (retriesCountWithinTime > REPORTER_REQUEST_RETRIES.maxTotalRetries) {
+      const errorMessage = chalk.yellow(
+        `${retriesCountWithinTime} requests were failed within ${REPORTER_REQUEST_RETRIES.withinTimeSeconds}s,\
+ reporting for test "${testData.title}" to Testomat is skipped`,
+      );
+      console.warn(`${APP_PREFIX} ${errorMessage}`);
+
+      this.reportingCanceledDueToReqFailures = true;
+      this.notReportedTestsCount++;
+
+      return true;
+    }
+
+    return false;
+  }
+
+  #uploadSingleTest = async data => {
     if (!this.isEnabled) return;
     if (!this.runId) return;
+    if (this.#cancelTestReportingInCaseOfTooManyReqFailures(data)) return;
+
     data.api_key = this.apiKey;
     data.create = this.createNewTests;
+
+    if (!process.env.TESTOMATIO_STACK_PASSED && data.status === STATUS.PASSED) {
+      data.stack = null;
+    }
+
     const json = JsonCycle.stringify(data);
 
-    return this.axios.post(`/api/reporter/${this.runId}/testrun`, json, {
-      maxContentLength: Infinity,
-      maxBodyLength: Infinity,
-      headers: {
-        // Overwrite Axios's automatically set Content-Type
-        'Content-Type': 'application/json',
-      },
-    })
-    .catch(err => {
-      if (err.response) {
-        if (err.response.status >= 400) {
-          const responseData = err.response.data || { message: '' };
+    debug('Adding test', json);
+
+    return this.axios
+      .post(`/api/reporter/${this.runId}/testrun`, json, axiosAddTestrunRequestConfig)
+      .catch(err => {
+        if (err.response) {
+          if (err.response.status >= 400) {
+            const responseData = err.response.data || { message: '' };
+            console.log(
+              APP_PREFIX,
+              chalk.yellow(`Warning: ${responseData.message} (${err.response.status})`),
+              chalk.grey(data?.title || ''),
+            );
+            if (err.response?.data?.message?.includes('could not be matched')) {
+              this.hasUnmatchedTests = true;
+            }
+            return;
+          }
           console.log(
             APP_PREFIX,
-            chalk.yellow(`Warning: ${responseData.message} (${err.response.status})`),
-            chalk.grey(data?.title || ''),
+            chalk.yellow(`Warning: ${data?.title || ''} (${err.response?.status})`),
+            `Report couldn't be processed: ${err?.response?.data?.message}`,
           );
-          if (err.response.data.message.includes('could not be matched')) {
-            this.hasUnmatchedTests = true;
+          printCreateIssue(err);
+        } else {
+          console.log(APP_PREFIX, chalk.blue(data?.title || ''), "Report couldn't be processed", err);
+        }
+      });
+    };
+    
+
+  /**
+   * Uploads tests as a batch (multiple tests at once). Intended to be used with a setInterval
+   */
+  #batchUpload = async () => {
+    this.batch.batchIndex++;
+    if (!this.batch.isEnabled) return;
+    if (!this.batch.tests.length) return;
+
+    // get tests from batch and clear batch
+    const testsToSend = this.batch.tests.splice(0);
+    debug('📨 Batch upload', testsToSend.length, 'tests');
+
+    return this.axios
+      .post(
+        `/api/reporter/${this.runId}/testrun`,
+        { api_key: this.apiKey, tests: testsToSend, batch_index: this.batch.batchIndex },
+        axiosAddTestrunRequestConfig,
+      )
+      .catch(err => {
+        if (err.response) {
+          if (err.response.status >= 400) {
+            const responseData = err.response.data || { message: '' };
+            console.log(
+              APP_PREFIX,
+              chalk.yellow(`Warning: ${responseData.message} (${err.response.status})`),
+              // chalk.grey(data?.title || ''),
+            );
+            if (err.response?.data?.message?.includes('could not be matched')) {
+              this.hasUnmatchedTests = true;
+            }
+            return;
           }
-          return;
+          console.log(
+            APP_PREFIX,
+            chalk.yellow(`Warning: (${err.response?.status})`),
+            `Report couldn't be processed: ${err?.response?.data?.message}`,
+          );
+          printCreateIssue(err);
+        } else {
+          console.log(APP_PREFIX, "Report couldn't be processed", err);
         }
-        console.log(
-          APP_PREFIX,
-          chalk.yellow(`Warning: ${data?.title || ''} (${err.response?.status})`),
-          `Report couldn't be processed: ${err?.response?.data?.message}`,
-        );
-      } else {
-        console.log(APP_PREFIX, chalk.blue(data?.title || ''), "Report couldn't be processed", err);
-      }
-    });
+      });
+  };
+
+  /**
+   * Adds a test to the batch uploader (or reports a single test if batch uploading is disabled)
+   */
+  addTest(data) {
+    if (!this.isEnabled) return;
+    if (!this.runId) return;
+    data.api_key = this.apiKey;
+    data.create = this.createNewTests;
+
+    if (!this.batch.isEnabled) this.#uploadSingleTest(data);
+    else this.batch.tests.push(data);
+
+    // if test is added after run already finished
+    if (!this.batch.intervalFunction) this.#batchUpload();
   }
 
   /**
@@ -219,8 +354,19 @@ class TestomatioPipe {
    * @returns
    */
   async finishRun(params) {
+    this.#batchUpload();
+    if (this.batch.intervalFunction) clearInterval(this.batch.intervalFunction);
+
     debug('Finishing run...');
     if (!this.isEnabled) return;
+    debug('Finishing run...');
+
+    if (this.reportingCanceledDueToReqFailures) {
+      const errorMessage = chalk.red(
+        `⚠️ Due to request failures, ${this.notReportedTestsCount} test(s) were not reported to Testomat.io`,
+      );
+      console.warn(`${APP_PREFIX} ${errorMessage}`);
+    }
 
     const { status, parallel } = params;
 
@@ -277,6 +423,7 @@ class TestomatioPipe {
       }
     } catch (err) {
       console.log(APP_PREFIX, 'Error updating status, skipping...', err);
+      printCreateIssue(err);
     }
     debug('Run finished');
   }
@@ -286,4 +433,36 @@ class TestomatioPipe {
   }
 }
 
+let registeredErrorHints = false;
+function printCreateIssue(err) {
+  if (registeredErrorHints) return;
+  registeredErrorHints = true;
+  process.on('exit', () => {
+    console.log();
+    console.log(APP_PREFIX, 'There was an error reporting to Testomat.io:');
+    console.log(
+      APP_PREFIX,
+      'If you think this is a bug please create an issue: https://github.com/testomatio/reporter/issues/new',
+    ); // eslint-disable-line max-len
+    console.log(APP_PREFIX, 'Provide this information:');
+    console.log('Error:', err.message || err.code);
+    if (!err.config) return;
+
+    const time = new Date().toUTCString();
+    const { data, url, baseURL, method } = err?.config || {};
+    console.log('```js');
+    console.log({ data: data?.replace(/"(tstmt_[^"]+)"/g, 'tstmt_*'), url, baseURL, method, time });
+    console.log('```');
+  });
+}
+
+const axiosAddTestrunRequestConfig = {
+  maxContentLength: Infinity,
+  maxBodyLength: Infinity,
+  headers: {
+    // Overwrite Axios's automatically set Content-Type
+    'Content-Type': 'application/json',
+  },
+};
+
 module.exports = TestomatioPipe;

package/lib/reporter-functions.js

@@ -1,6 +1,9 @@
-const logger = require('./storages/logger');
-const artifactStorage = require('./storages/artifact-storage');
-const keyValueStorage = require('./storages/key-value-storage');
+const { services } = require('./services');
+const { initPlaywrightForStorage } = require('./adapter/playwright');
+
+if (process.env.PLAYWRIGHT_TEST_BASE_URL) {
+  initPlaywrightForStorage();
+}
 
 /**
  * Stores path to file as artifact and uploads it to the S3 storage
@@ -8,7 +11,7 @@ const keyValueStorage = require('./storages/key-value-storage');
  */
 function saveArtifact(data, context = null) {
   if (!data) return;
-  artifactStorage.put(data, context);
+  services.artifacts.put(data, context);
 }
 
 /**
@@ -16,23 +19,23 @@ function saveArtifact(data, context = null) {
  * @param  {...any} args
  */
 function logMessage(...args) {
-  logger.log(...args);
+  services.logger._templateLiteralLog(...args);
 }
 
 /**
  * Similar to "log" function but marks message in report as a step
- * @param {*} message 
+ * @param {*} message
  */
 function addStep(message) {
-  logger.step(message);
+  services.logger.step(message);
 }
 
 /**
  * Add key-value pair(s) to the test report
- * @param {*} keyValue 
+ * @param {*} keyValue
  */
 function setKeyValue(keyValue) {
-  keyValueStorage.put(keyValue);
+  services.keyValues.put(keyValue);
 }
 
 module.exports = {

package/lib/reporter.js

@@ -1,23 +1,19 @@
-const logger = require('./storages/logger');
 const TestomatClient = require('./client');
 const TRConstants = require('./constants');
+const { services } = require('./services');
 
-const log = logger.templateLiteralLog.bind(logger);
-const step = logger.step.bind(logger);
 const reporterFunctions = require('./reporter-functions');
 
 module.exports = {
-  logger,
-  testomatioLogger: logger,
-  log,
-  step,
-  TestomatClient,
-  TRConstants,
-};
+  // TODO: deprecate in future; use log or testomat.log
+  testomatioLogger: services.logger,
 
-module.exports.testomat = {
   artifact: reporterFunctions.artifact,
   log: reporterFunctions.log,
-  step: reporterFunctions.step,
+  logger: services.logger,
   meta: reporterFunctions.keyValue,
+  step: reporterFunctions.step,
+
+  TestomatClient,
+  TRConstants,
 };

package/lib/services/artifacts.js

@@ -0,0 +1,57 @@
+const debug = require('debug')('@testomatio/reporter:services-artifacts');
+const { dataStorage } = require('../data-storage');
+
+/**
+ * Artifact storage is supposed to store file paths
+ */
+class ArtifactStorage {
+  static #instance;
+
+  /**
+   * Singleton
+   * @returns {ArtifactStorage}
+   */
+  static getInstance() {
+    if (!this.#instance) {
+      this.#instance = new ArtifactStorage();
+    }
+    return this.#instance;
+  }
+
+  /**
+   * Stores path to file as artifact and uploads it to the S3 storage
+   * @param {string | {path: string, type: string, name: string}} data - path to file or object with path, type and name
+   * @param {*} context testId or test title
+   */
+  put(data, context = null) {
+    if (!data) return;
+    debug('Save artifact:', data);
+    dataStorage.putData('artifact', data, context);
+  }
+
+  /**
+   * Returns list of artifacts to upload
+   * @param {*} context testId or test context from test runner
+   * @returns {(string | {path: string, type: string, name: string})[]}
+   */
+  get(context) {
+    let artifacts = dataStorage.getData('artifact', context);
+    if (!artifacts || !artifacts.length) return [];
+
+    artifacts = artifacts.map(artifactData => {
+      // artifact could be an object ({type, path, name} props) or string (just path)
+      let artifact;
+      try {
+        artifact = JSON.parse(artifactData);
+      } catch (e) {
+        artifact = artifactData;
+      }
+      return artifact;
+    });
+    artifacts = artifacts.filter(artifact => !!artifact);
+    debug(`Artifacts for test ${context}:`, artifacts);
+    return artifacts.length ? artifacts : [];
+  }
+}
+
+module.exports.artifactStorage = ArtifactStorage.getInstance();

package/lib/services/index.js

@@ -0,0 +1,13 @@
+const { logger } = require('./logger');
+const { artifactStorage } = require('./artifacts');
+const { keyValueStorage } = require('./key-values');
+const { dataStorage } = require('../data-storage');
+
+module.exports.services = {
+  logger,
+  artifacts: artifactStorage,
+  keyValues: keyValueStorage,
+  setContext: context => {
+    dataStorage.setContext(context);
+  },
+};

package/lib/{storages/key-value-storage.js → services/key-values.js}

@@ -1,24 +1,28 @@
-const debug = require('debug')('@testomatio/reporter:key-value-storage');
-const DataStorage = require('./data-storage');
+const debug = require('debug')('@testomatio/reporter:services-key-value');
+const { dataStorage } = require('../data-storage');
 
 class KeyValueStorage {
-  constructor() {
-    this.dataStorage = new DataStorage('keyvalue');
+  static #instance;
 
-    // singleton
-    if (!KeyValueStorage.instance) {
-      KeyValueStorage.instance = this;
+  /**
+   *
+   * @returns {KeyValueStorage}
+   */
+  static getInstance() {
+    if (!this.#instance) {
+      this.#instance = new KeyValueStorage();
     }
+    return this.#instance;
   }
 
   /**
    * Stores key-value pair and passes it to reporter
    * @param {{key: string}} keyValue - key-value pair(s) as object
-   * @param {*} context testId or test title
+   * @param {*} context - full test title
    */
   put(keyValue, context = null) {
     if (!keyValue) return;
-    this.dataStorage.putData(keyValue, context);
+    dataStorage.putData('keyvalue', keyValue, context);
   }
 
   #isKeyValueObject(smth) {
@@ -28,13 +32,11 @@ class KeyValueStorage {
   /**
    * Returns key-values pairs for the test as object
    * @param {*} context testId or test context from test runner
-   * @returns {Object} key-values pairs as object, e.g. {priority: 'high', browser: 'chrome'}
+   * @returns {{[key: string]: string} | {}} key-values pairs as object, e.g. {priority: 'high', browser: 'chrome'}
    */
-  get(context) {
-    if (!context) return null;
-
-    const keyValuesList = this.dataStorage.getData(context);
-    if (!keyValuesList || !keyValuesList?.length) return null;
+  get(context = null) {
+    const keyValuesList = dataStorage.getData('keyvalue', context);
+    if (!keyValuesList || !keyValuesList?.length) return {};
 
     const keyValues = {};
     for (const keyValue of keyValuesList) {
@@ -49,10 +51,8 @@ class KeyValueStorage {
       }
     }
 
-    return Object.keys(keyValues).length ? keyValues : null;
+    return keyValues;
   }
 }
 
-KeyValueStorage.instance = null;
-
-module.exports = new KeyValueStorage();
+module.exports.keyValueStorage = KeyValueStorage.getInstance();