@testomatio/reporter 1.3.6-beta → 1.4.0-beta-wdio-bdd

package/lib/pipe/testomatio.js

@@ -1,14 +1,18 @@
 const debug = require('debug')('@testomatio/reporter:pipe:testomatio');
 const chalk = require('chalk');
+// Retry interceptor function
+const axiosRetry = require('axios-retry');
+// Default axios instance
 const axios = require('axios');
 const JsonCycle = require('json-cycle');
-const { APP_PREFIX, STATUS } = require('../constants');
-const { isValidUrl } = require('../util');
-const { resetConfig } = require('../fileUploader');
 
-const { TESTOMATIO_RUN } = process.env;
-if (TESTOMATIO_RUN) {
-  process.env.runId = TESTOMATIO_RUN;
+const { APP_PREFIX, STATUS, AXIOS_TIMEOUT, REPORTER_REQUEST_RETRIES } = require('../constants');
+const { isValidUrl, foundedTestLog } = require('../utils/utils');
+const { parseFilterParams, generateFilterRequestParams, setS3Credentials } = require('../utils/pipe_utils');
+const config = require('../config');
+
+if (process.env.TESTOMATIO_RUN) {
+  process.env.runId = process.env.TESTOMATIO_RUN;
 }
 
 /**
@@ -19,9 +23,13 @@ if (TESTOMATIO_RUN) {
  */
 class TestomatioPipe {
   constructor(params, store) {
+    this.retriesTimestamps = [];
+    this.reportingCanceledDueToReqFailures = false;
+    this.notReportedTestsCount = 0;
+
     this.isEnabled = false;
     this.url = params.testomatioUrl || process.env.TESTOMATIO_URL || 'https://app.testomat.io';
-    this.apiKey = params.apiKey || process.env.TESTOMATIO;
+    this.apiKey = params.apiKey || config.TESTOMATIO;
     debug('Testomatio Pipe: ', this.apiKey ? 'API KEY' : '*no api key*');
     if (!this.apiKey) {
       return;
@@ -33,12 +41,46 @@ class TestomatioPipe {
     this.sharedRun = !!process.env.TESTOMATIO_SHARED_RUN;
     this.groupTitle = params.groupTitle || process.env.TESTOMATIO_RUNGROUP_TITLE;
     this.env = process.env.TESTOMATIO_ENV;
-    this.axios = axios.create();
+    this.label = process.env.TESTOMATIO_LABEL;
+    // Create a new instance of axios with a custom config
+    this.axios = axios.create({
+      baseURL: `${this.url.trim()}`,
+      timeout: AXIOS_TIMEOUT,
+    });
+
+    // Pass the axios instance to the retry function
+    axiosRetry(this.axios, {
+      // do not use retries for unit tests
+      retries: REPORTER_REQUEST_RETRIES.retriesPerRequest, // Number of retries
+      shouldResetTimeout: true,
+      retryCondition: error => {
+        if (!error.response) return false;
+        switch (error.response?.status) {
+          case 400: // Bad request (probably wrong API key)
+          case 404: // Test not matched
+          case 429: // Rate limit exceeded
+          case 500: // Internal server error
+            return false;
+          default:
+            break;
+        }
+        return error.response?.status >= 401; // Retry on 401+ and 5xx
+      },
+      retryDelay: () => REPORTER_REQUEST_RETRIES.retryTimeout, // sum = 15sec
+      onRetry: async (retryCount, error) => {
+        this.retriesTimestamps.push(Date.now());
+
+        debug(`${error.message || `Request failed ${error.status}`}. Retry #${retryCount} ...`);
+      },
+    });
+
     this.isEnabled = true;
     // do not finish this run (for parallel testing)
     this.proceed = process.env.TESTOMATIO_PROCEED;
+    this.jiraId = process.env.TESTOMATIO_JIRA_ID;
     this.runId = params.runId || process.env.runId;
-    this.createNewTests = !!process.env.TESTOMATIO_CREATE;
+    this.createNewTests = params.createNewTests ?? !!process.env.TESTOMATIO_CREATE;
+    this.hasUnmatchedTests = false;
 
     if (!isValidUrl(this.url.trim())) {
       this.isEnabled = false;
@@ -46,93 +88,217 @@ class TestomatioPipe {
     }
   }
 
+  /**
+   * Asynchronously prepares and retrieves the Testomat.io test grepList based on the provided options.
+   * @param {Object} opts - The options for preparing the test grepList.
+   * @returns {Promise<string[]>} - An array containing the retrieved
+   * test grepList, or an empty array if no tests are found or the request is disabled.
+   * @throws {Error} - Throws an error if there was a problem while making the request.
+   */
+  async prepareRun(opts) {
+    if (!this.isEnabled) return [];
+
+    const { type, id } = parseFilterParams(opts);
+
+    try {
+      const q = generateFilterRequestParams({
+        type,
+        id,
+        apiKey: this.apiKey.trim(),
+      });
+
+      if (!q) {
+        return;
+      }
+
+      const resp = await this.axios.get('/api/test_grep', q);
+      const { data } = resp;
+
+      if (Array.isArray(data?.tests) && data?.tests?.length > 0) {
+        foundedTestLog(APP_PREFIX, data.tests);
+        return data.tests;
+      }
+
+      console.log(APP_PREFIX, `⛔  No tests found for your --filter --> ${type}=${id}`);
+    } catch (err) {
+      console.error(APP_PREFIX, `🚩 Error getting Testomat.io test grepList: ${err}`);
+    }
+  }
+
   /**
    * @returns Promise<void>
    */
   async createRun() {
+    debug('Creating run...');
     if (!this.isEnabled) return;
 
+    let buildUrl = process.env.BUILD_URL || process.env.CI_JOB_URL || process.env.CIRCLE_BUILD_URL;
+
+    // GitHub Actions Url
+    if (!buildUrl && process.env.GITHUB_RUN_ID) {
+      // eslint-disable-next-line max-len
+      buildUrl = `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`;
+    }
+
+    // Azure DevOps Url
+    if (!buildUrl && process.env.SYSTEM_TEAMFOUNDATIONCOLLECTIONURI) {
+      const collectionUri = process.env.SYSTEM_TEAMFOUNDATIONCOLLECTIONURI;
+      const project = process.env.SYSTEM_TEAMPROJECT;
+      const buildId = process.env.BUILD_BUILDID;
+      buildUrl = `${collectionUri}/${project}/_build/results?buildId=${buildId}`;
+    }
+
+    if (buildUrl && !buildUrl.startsWith('http')) buildUrl = undefined;
+
+    const accessEvent = process.env.TESTOMATIO_PUBLISH ? 'publish' : null;
+
     const runParams = Object.fromEntries(
       Object.entries({
+        ci_build_url: buildUrl,
         parallel: this.parallel,
         api_key: this.apiKey.trim(),
         group_title: this.groupTitle,
+        access_event: accessEvent,
+        jira_id: this.jiraId,
         env: this.env,
         title: this.title,
+        label: this.label,
         shared_run: this.sharedRun,
-      }).filter(([, value]) => !!value)
+      }).filter(([, value]) => !!value),
     );
+    debug('Run params', JSON.stringify(runParams, null, 2));
 
     if (this.runId) {
-      const resp = await this.axios.put(`${this.url.trim()}/api/reporter/${this.runId}`, runParams);
+      debug(`Run with id ${this.runId} already created, updating...`);
+      const resp = await this.axios.put(`/api/reporter/${this.runId}`, runParams);
       if (resp.data.artifacts) setS3Credentials(resp.data.artifacts);
       return;
     }
 
     try {
-      const resp = await this.axios.post(`${this.url.trim()}/api/reporter`, runParams, {
+      const resp = await this.axios.post(`/api/reporter`, runParams, {
         maxContentLength: Infinity,
         maxBodyLength: Infinity,
       });
+
       this.runId = resp.data.uid;
       this.runUrl = `${this.url}/${resp.data.url.split('/').splice(3).join('/')}`;
+      this.runPublicUrl = resp.data.public_url;
+
       if (resp.data.artifacts) setS3Credentials(resp.data.artifacts);
+
       this.store.runUrl = this.runUrl;
+      this.store.runPublicUrl = this.runPublicUrl;
       this.store.runId = this.runId;
       console.log(APP_PREFIX, '📊 Report created. Report ID:', this.runId);
       process.env.runId = this.runId;
+      debug('Run created', this.runId);
     } catch (err) {
       console.error(
         APP_PREFIX,
-        'Error creating Testomat.io report, please check if your API key is valid. Skipping report',
+        'Error creating Testomat.io report, please check if your API key is valid. Skipping report | ',
+        err?.response?.statusText || err?.status || err.message,
       );
+      printCreateIssue(err);
     }
+    debug('"createRun" function finished');
   }
 
   /**
-   * 
-   * @param testData data 
-   * @returns 
+   * Decides whether to skip test reporting in case of too many request failures
+   * @param {TestData} testData
+   * @returns {boolean}
    */
+  #cancelTestReportingInCaseOfTooManyReqFailures(testData) {
+    if (this.reportingCanceledDueToReqFailures) return true;
+
+    const retriesCountWithinTime = this.retriesTimestamps.filter(
+      timestamp => Date.now() - timestamp < REPORTER_REQUEST_RETRIES.withinTimeSeconds * 1000,
+    ).length;
+    debug(`${retriesCountWithinTime} failed requests within ${REPORTER_REQUEST_RETRIES.withinTimeSeconds}s`);
+
+    if (retriesCountWithinTime > REPORTER_REQUEST_RETRIES.maxTotalRetries) {
+      const errorMessage = chalk.yellow(
+        `${retriesCountWithinTime} requests were failed within ${REPORTER_REQUEST_RETRIES.withinTimeSeconds}s,\
+ reporting for test "${testData.title}" to Testomat is skipped`,
+      );
+      console.warn(`${APP_PREFIX} ${errorMessage}`);
+
+      this.reportingCanceledDueToReqFailures = true;
+      this.notReportedTestsCount++;
+
+      return true;
+    }
+
+    return false;
+  }
+
   addTest(data) {
     if (!this.isEnabled) return;
     if (!this.runId) return;
+    if (this.#cancelTestReportingInCaseOfTooManyReqFailures(data)) return;
+
     data.api_key = this.apiKey;
     data.create = this.createNewTests;
+
+    if (!process.env.TESTOMATIO_STACK_PASSED && data.status === STATUS.PASSED) {
+      data.stack = null;
+    }
+
     const json = JsonCycle.stringify(data);
 
-    return this.axios.post(`${this.url}/api/reporter/${this.runId}/testrun`, json, {
-      maxContentLength: Infinity,
-      maxBodyLength: Infinity,
-      headers: {
-        // Overwrite Axios's automatically set Content-Type
-        'Content-Type': 'application/json',
-      },
-    })
-    .catch((err) => {
-      if (err.response) {
-        if (err.response.status >= 400) {
-          const responseData = err.response.data || { message: '' };
+    debug('Adding test', json);
+
+
+    return this.axios
+      .post(`/api/reporter/${this.runId}/testrun`, json, {
+        maxContentLength: Infinity,
+        maxBodyLength: Infinity,
+        headers: {
+          // Overwrite Axios's automatically set Content-Type
+          'Content-Type': 'application/json',
+        },
+      })
+      .catch(err => {
+        if (err.response) {
+          if (err.response.status >= 400) {
+            const responseData = err.response.data || { message: '' };
+            console.log(
+              APP_PREFIX,
+              chalk.yellow(`Warning: ${responseData.message} (${err.response.status})`),
+              chalk.grey(data?.title || ''),
+            );
+            if (err.response?.data?.message?.includes('could not be matched')) {
+              this.hasUnmatchedTests = true;
+            }
+            return;
+          }
           console.log(
             APP_PREFIX,
-            chalk.blue(this.title),
-            `Report couldn't be processed: (${err.response.status}) ${responseData.message}`,
+            chalk.yellow(`Warning: ${data?.title || ''} (${err.response?.status})`),
+            `Report couldn't be processed: ${err?.response?.data?.message}`,
           );
-          return;
+          printCreateIssue(err);
+        } else {
+          console.log(APP_PREFIX, chalk.blue(data?.title || ''), "Report couldn't be processed", err);
         }
-        console.log(APP_PREFIX, chalk.blue(this.title), `Report couldn't be processed: ${err.response.data.message}`);
-      } else {
-        console.log(APP_PREFIX, chalk.blue(this.title), "Report couldn't be processed", err);
-      }
-    });
+      });
   }
 
   /**
-   * @param {import('../../types').RunData} params 
-   * @returns 
+   * @param {import('../../types').RunData} params
+   * @returns
    */
   async finishRun(params) {
     if (!this.isEnabled) return;
+    debug('Finishing run...');
+
+    if (this.reportingCanceledDueToReqFailures) {
+      const errorMessage = chalk.red(
+        `⚠️ Due to request failures, ${this.notReportedTestsCount} test(s) were not reported to Testomat.io`,
+      );
+      console.warn(`${APP_PREFIX} ${errorMessage}`);
+    }
 
     const { status, parallel } = params;
 
@@ -145,7 +311,7 @@ class TestomatioPipe {
 
     try {
       if (this.runId && !this.proceed) {
-        await this.axios.put(`${this.url}/api/reporter/${this.runId}`, {
+        await this.axios.put(`/api/reporter/${this.runId}`, {
           api_key: this.apiKey,
           status_event,
           tests: params.tests,
@@ -153,15 +319,45 @@ class TestomatioPipe {
         if (this.runUrl) {
           console.log(APP_PREFIX, '📊 Report Saved. Report URL:', chalk.magenta(this.runUrl));
         }
+        if (this.runPublicUrl) {
+          console.log(APP_PREFIX, '🌟 Public URL:', chalk.magenta(this.runPublicUrl));
+        }
       }
       if (this.runUrl && this.proceed) {
         const notFinishedMessage = chalk.yellow.bold('Run was not finished because of $TESTOMATIO_PROCEED');
         console.log(APP_PREFIX, `📊 ${notFinishedMessage}. Report URL: ${chalk.magenta(this.runUrl)}`);
         console.log(APP_PREFIX, `🛬 Run to finish it: TESTOMATIO_RUN=${this.runId} npx start-test-run --finish`);
       }
+      if (this.hasUnmatchedTests) {
+        console.log('');
+        // eslint-disable-next-line max-len
+        console.log(APP_PREFIX, chalk.yellow.bold('⚠️ Some reported tests were not found in Testomat.io project'));
+        // eslint-disable-next-line max-len
+        console.log(
+          APP_PREFIX,
+          `If you use Testomat.io as a reporter only, please re-run tests using ${chalk.bold('TESTOMATIO_CREATE=1')}`,
+        );
+        // eslint-disable-next-line max-len
+        console.log(
+          APP_PREFIX,
+          `But to keep your tests consistent it is recommended to ${chalk.bold('import tests first')}`,
+        );
+        console.log(APP_PREFIX, 'If tests were imported but still not matched, assign test IDs to your tests.');
+        console.log(APP_PREFIX, 'You can do that automatically via command line tools:');
+        console.log(APP_PREFIX, chalk.bold('npx check-tests ... --update-ids'), 'See: https://bit.ly/js-update-ids');
+        console.log(APP_PREFIX, 'or for Cucumber:');
+        // eslint-disable-next-line max-len
+        console.log(
+          APP_PREFIX,
+          chalk.bold('npx check-cucumber ... --update-ids'),
+          'See: https://bit.ly/bdd-update-ids',
+        );
+      }
     } catch (err) {
       console.log(APP_PREFIX, 'Error updating status, skipping...', err);
+      printCreateIssue(err);
     }
+    debug('Run finished');
   }
 
   toString() {
@@ -169,19 +365,27 @@ class TestomatioPipe {
   }
 }
 
-module.exports = TestomatioPipe;
-
+let registeredErrorHints = false;
+function printCreateIssue(err) {
+  if (registeredErrorHints) return;
+  registeredErrorHints = true;
+  process.on('exit', () => {
+    console.log();
+    console.log(APP_PREFIX, 'There was an error reporting to Testomat.io:');
+    console.log(
+      APP_PREFIX,
+      'If you think this is a bug please create an issue: https://github.com/testomatio/reporter/issues/new',
+    ); // eslint-disable-line max-len
+    console.log(APP_PREFIX, 'Provide this information:');
+    console.log('Error:', err.message || err.code);
+    if (!err.config) return;
 
-function setS3Credentials(artifacts) {  
-  if (!Object.keys(artifacts).length) return;
-
-  console.log(APP_PREFIX, 'S3 were credentials obtained from Testomat.io...');
-
-  if (artifacts.ACCESS_KEY_ID) process.env.S3_ACCESS_KEY_ID = artifacts.ACCESS_KEY_ID; 
-  if (artifacts.SECRET_ACCESS_KEY) process.env.S3_SECRET_ACCESS_KEY = artifacts.SECRET_ACCESS_KEY; 
-  if (artifacts.REGION) process.env.S3_REGION = artifacts.REGION; 
-  if (artifacts.BUCKET) process.env.S3_BUCKET = artifacts.BUCKET; 
-  if (artifacts.ENDPOINT) process.env.S3_ENDPOINT = artifacts.ENDPOINT; 
-  if (artifacts.presign) process.env.TESTOMATIO_PRIVATE_ARTIFACTS = '1'; 
-  resetConfig();
+    const time = new Date().toUTCString();
+    const { data, url, baseURL, method } = err?.config || {};
+    console.log('```js');
+    console.log({ data: data?.replace(/"(tstmt_[^"]+)"/g, 'tstmt_*'), url, baseURL, method, time });
+    console.log('```');
+  });
 }
+
+module.exports = TestomatioPipe;

package/lib/reporter-functions.js

@@ -0,0 +1,46 @@
+const { services } = require('./services');
+const { initPlaywrightForStorage } = require('./adapter/playwright');
+
+if (process.env.PLAYWRIGHT_TEST_BASE_URL) {
+  initPlaywrightForStorage();
+}
+
+/**
+ * Stores path to file as artifact and uploads it to the S3 storage
+ * @param {string | {path: string, type: string, name: string}} data - path to file or object with path, type and name
+ */
+function saveArtifact(data, context = null) {
+  if (!data) return;
+  services.artifacts.put(data, context);
+}
+
+/**
+ * Attach log message(s) to the test report
+ * @param  {...any} args
+ */
+function logMessage(...args) {
+  services.logger._templateLiteralLog(...args);
+}
+
+/**
+ * Similar to "log" function but marks message in report as a step
+ * @param {*} message
+ */
+function addStep(message) {
+  services.logger.step(message);
+}
+
+/**
+ * Add key-value pair(s) to the test report
+ * @param {*} keyValue
+ */
+function setKeyValue(keyValue) {
+  services.keyValues.put(keyValue);
+}
+
+module.exports = {
+  artifact: saveArtifact,
+  log: logMessage,
+  step: addStep,
+  keyValue: setKeyValue,
+};

package/lib/reporter.js

@@ -1,17 +1,19 @@
-const logger = require('./logger');
 const TestomatClient = require('./client');
 const TRConstants = require('./constants');
-const TRArtifacts = require('./_ArtifactStorageOld');
+const { services } = require('./services');
 
-const log = logger._log.bind(logger);
-const step = logger.step.bind(logger);
+const reporterFunctions = require('./reporter-functions');
 
 module.exports = {
-  logger,
-  log,
-  step,
+  // TODO: deprecate in future; use log or testomat.log
+  testomatioLogger: services.logger,
+
+  artifact: reporterFunctions.artifact,
+  log: reporterFunctions.log,
+  logger: services.logger,
+  meta: reporterFunctions.keyValue,
+  step: reporterFunctions.step,
+
   TestomatClient,
   TRConstants,
-  TRArtifacts,
-  addArtifact: TRArtifacts.artifact,
 };

package/lib/services/artifacts.js

@@ -0,0 +1,57 @@
+const debug = require('debug')('@testomatio/reporter:services-artifacts');
+const { dataStorage } = require('../data-storage');
+
+/**
+ * Artifact storage is supposed to store file paths
+ */
+class ArtifactStorage {
+  static #instance;
+
+  /**
+   * Singleton
+   * @returns {ArtifactStorage}
+   */
+  static getInstance() {
+    if (!this.#instance) {
+      this.#instance = new ArtifactStorage();
+    }
+    return this.#instance;
+  }
+
+  /**
+   * Stores path to file as artifact and uploads it to the S3 storage
+   * @param {string | {path: string, type: string, name: string}} data - path to file or object with path, type and name
+   * @param {*} context testId or test title
+   */
+  put(data, context = null) {
+    if (!data) return;
+    debug('Save artifact:', data);
+    dataStorage.putData('artifact', data, context);
+  }
+
+  /**
+   * Returns list of artifacts to upload
+   * @param {*} context testId or test context from test runner
+   * @returns {(string | {path: string, type: string, name: string})[]}
+   */
+  get(context) {
+    let artifacts = dataStorage.getData('artifact', context);
+    if (!artifacts || !artifacts.length) return [];
+
+    artifacts = artifacts.map(artifactData => {
+      // artifact could be an object ({type, path, name} props) or string (just path)
+      let artifact;
+      try {
+        artifact = JSON.parse(artifactData);
+      } catch (e) {
+        artifact = artifactData;
+      }
+      return artifact;
+    });
+    artifacts = artifacts.filter(artifact => !!artifact);
+    debug(`Artifacts for test ${context}:`, artifacts);
+    return artifacts.length ? artifacts : [];
+  }
+}
+
+module.exports.artifactStorage = ArtifactStorage.getInstance();

package/lib/services/index.js

@@ -0,0 +1,13 @@
+const { logger } = require('./logger');
+const { artifactStorage } = require('./artifacts');
+const { keyValueStorage } = require('./key-values');
+const { dataStorage } = require('../data-storage');
+
+module.exports.services = {
+  logger,
+  artifacts: artifactStorage,
+  keyValues: keyValueStorage,
+  setContext: context => {
+    dataStorage.setContext(context);
+  },
+};

package/lib/services/key-values.js

@@ -0,0 +1,58 @@
+const debug = require('debug')('@testomatio/reporter:services-key-value');
+const { dataStorage } = require('../data-storage');
+
+class KeyValueStorage {
+  static #instance;
+
+  /**
+   *
+   * @returns {KeyValueStorage}
+   */
+  static getInstance() {
+    if (!this.#instance) {
+      this.#instance = new KeyValueStorage();
+    }
+    return this.#instance;
+  }
+
+  /**
+   * Stores key-value pair and passes it to reporter
+   * @param {{key: string}} keyValue - key-value pair(s) as object
+   * @param {*} context - full test title
+   */
+  put(keyValue, context = null) {
+    if (!keyValue) return;
+    dataStorage.putData('keyvalue', keyValue, context);
+  }
+
+  #isKeyValueObject(smth) {
+    return smth && typeof smth === 'object' && !Array.isArray(smth) && smth !== null;
+  }
+
+  /**
+   * Returns key-values pairs for the test as object
+   * @param {*} context testId or test context from test runner
+   * @returns {{[key: string]: string} | {}} key-values pairs as object, e.g. {priority: 'high', browser: 'chrome'}
+   */
+  get(context = null) {
+    const keyValuesList = dataStorage.getData('keyvalue', context);
+    if (!keyValuesList || !keyValuesList?.length) return {};
+
+    const keyValues = {};
+    for (const keyValue of keyValuesList) {
+      if (this.#isKeyValueObject(keyValue)) {
+        Object.assign(keyValues, keyValue);
+      } else if (typeof keyValue === 'string') {
+        try {
+          Object.assign(keyValues, JSON.parse(keyValue));
+        } catch (e) {
+          debug(`Error parsing key-values for test ${context}`, keyValue);
+        }
+      }
+    }
+
+    return keyValues;
+  }
+}
+
+module.exports.keyValueStorage = KeyValueStorage.getInstance();