@testomatio/reporter 1.2.3-beta → 1.2.3-beta-batch-upload

package/lib/pipe/testomatio.js

@@ -1,14 +1,17 @@
 const debug = require('debug')('@testomatio/reporter:pipe:testomatio');
 const chalk = require('chalk');
+// Retry interceptor function
+const axiosRetry = require('axios-retry');
+// Default axios instance
 const axios = require('axios');
-const JsonCycle = require('json-cycle');
-const { APP_PREFIX, STATUS } = require('../constants');
-const { isValidUrl } = require('../util');
-const { resetConfig } = require('../fileUploader');
-
-const { TESTOMATIO_RUN } = process.env;
-if (TESTOMATIO_RUN) {
-  process.env.runId = TESTOMATIO_RUN;
+
+const { APP_PREFIX, STATUS, AXIOS_TIMEOUT, AXIOS_RETRY_TIMEOUT } = require('../constants');
+const { isValidUrl, foundedTestLog } = require('../utils/utils');
+const { parseFilterParams, generateFilterRequestParams, setS3Credentials } = require('../utils/pipe_utils');
+const config = require('../config');
+
+if (process.env.TESTOMATIO_RUN) {
+  process.env.runId = process.env.TESTOMATIO_RUN;
 }
 
 /**
@@ -18,26 +21,62 @@ if (TESTOMATIO_RUN) {
  * @implements {Pipe}
  */
 class TestomatioPipe {
+  batch = {
+    tests: [],
+    intervalFunction: null,
+    intervalTime: 5000, // how often tests are sent
+  };
+
   constructor(params, store) {
     this.isEnabled = false;
     this.url = params.testomatioUrl || process.env.TESTOMATIO_URL || 'https://app.testomat.io';
-    this.apiKey = params.apiKey || process.env.TESTOMATIO;
+    this.apiKey = params.apiKey || config.TESTOMATIO;
     debug('Testomatio Pipe: ', this.apiKey ? 'API KEY' : '*no api key*');
     if (!this.apiKey) {
       return;
     }
     debug('Testomatio Pipe: Enabled');
+    this.parallel = params.parallel;
     this.store = store || {};
     this.title = params.title || process.env.TESTOMATIO_TITLE;
     this.sharedRun = !!process.env.TESTOMATIO_SHARED_RUN;
     this.groupTitle = params.groupTitle || process.env.TESTOMATIO_RUNGROUP_TITLE;
     this.env = process.env.TESTOMATIO_ENV;
-    this.axios = axios.create();
+    this.label = process.env.TESTOMATIO_LABEL;
+    // Create a new instance of axios with a custom config
+    this.axios = axios.create({
+      baseURL: `${this.url.trim()}`,
+      timeout: AXIOS_TIMEOUT,
+    });
+    // Pass the axios instance to the retry function
+    axiosRetry(this.axios, {
+      retries: 3, // Number of retries (Defaults to 3)
+      shouldResetTimeout: true,
+      retryCondition: error => {
+        // Conditional check the error status code
+        switch (error.response.status) {
+          case 409:
+          case 429:
+          case 502:
+          case 503:
+            return true; // Retry request with response status code 409, 429, 502, 503
+          default:
+            return false; // Do not retry the others
+        }
+      },
+      retryDelay: retryCount => retryCount * AXIOS_RETRY_TIMEOUT, // sum = 15sec
+      onRetry: retryCount => {
+        debug(`Retry attempt #${retryCount} failed. Retrying again...`);
+      },
+    });
+
     this.isEnabled = true;
     // do not finish this run (for parallel testing)
     this.proceed = process.env.TESTOMATIO_PROCEED;
+    this.jiraId = process.env.TESTOMATIO_JIRA_ID;
     this.runId = params.runId || process.env.runId;
-    this.createNewTests = !!process.env.TESTOMATIO_CREATE;
+    this.createNewTests = params.createNewTests ?? !!process.env.TESTOMATIO_CREATE;
+    this.hasUnmatchedTests = false;
 
     if (!isValidUrl(this.url.trim())) {
       this.isEnabled = false;
@@ -45,91 +84,196 @@ class TestomatioPipe {
     }
   }
 
+  /**
+   * Asynchronously prepares and retrieves the Testomat.io test grepList based on the provided options.
+   * @param {Object} opts - The options for preparing the test grepList.
+   * @returns {Promise<string[]>} - An array containing the retrieved
+   * test grepList, or an empty array if no tests are found or the request is disabled.
+   * @throws {Error} - Throws an error if there was a problem while making the request.
+   */
+  async prepareRun(opts) {
+    if (!this.isEnabled) return [];
+
+    const { type, id } = parseFilterParams(opts);
+
+    try {
+      const q = generateFilterRequestParams({
+        type,
+        id,
+        apiKey: this.apiKey.trim(),
+      });
+
+      if (!q) {
+        return;
+      }
+
+      const resp = await this.axios.get('/api/test_grep', q);
+      const { data } = resp;
+
+      if (Array.isArray(data?.tests) && data?.tests?.length > 0) {
+        foundedTestLog(APP_PREFIX, data.tests);
+        return data.tests;
+      }
+
+      console.log(APP_PREFIX, `⛔  No tests found for your --filter --> ${type}=${id}`);
+    } catch (err) {
+      console.error(APP_PREFIX, `🚩 Error getting Testomat.io test grepList: ${err}`);
+    }
+  }
+
   /**
    * @returns Promise<void>
    */
   async createRun() {
+    debug('Creating run...');
     if (!this.isEnabled) return;
+    this.batch.intervalFunction = setInterval(this.#batchUpload, this.batch.intervalTime);
+
+    let buildUrl = process.env.BUILD_URL || process.env.CI_JOB_URL || process.env.CIRCLE_BUILD_URL;
+
+    // GitHub Actions Url
+    if (!buildUrl && process.env.GITHUB_RUN_ID) {
+      // eslint-disable-next-line max-len
+      buildUrl = `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`;
+    }
+
+    // Azure DevOps Url
+    if (!buildUrl && process.env.SYSTEM_TEAMFOUNDATIONCOLLECTIONURI) {
+      const collectionUri = process.env.SYSTEM_TEAMFOUNDATIONCOLLECTIONURI;
+      const project = process.env.SYSTEM_TEAMPROJECT;
+      const buildId = process.env.BUILD_BUILDID;
+      buildUrl = `${collectionUri}/${project}/_build/results?buildId=${buildId}`;
+    }
+
+    if (buildUrl && !buildUrl.startsWith('http')) buildUrl = undefined;
+
+    const accessEvent = process.env.TESTOMATIO_PUBLISH ? 'publish' : null;
 
     const runParams = Object.fromEntries(
       Object.entries({
+        ci_build_url: buildUrl,
+        parallel: this.parallel,
         api_key: this.apiKey.trim(),
         group_title: this.groupTitle,
+        access_event: accessEvent,
+        jira_id: this.jiraId,
         env: this.env,
         title: this.title,
+        label: this.label,
         shared_run: this.sharedRun,
-      }).filter(([, value]) => !!value)
+      }).filter(([, value]) => !!value),
     );
+    debug('Run params', JSON.stringify(runParams, null, 2));
 
     if (this.runId) {
-      const resp = await this.axios.put(`${this.url.trim()}/api/reporter/${this.runId}`, runParams);
+      debug(`Run with id ${this.runId} already created, updating...`);
+      const resp = await this.axios.put(`/api/reporter/${this.runId}`, runParams);
       if (resp.data.artifacts) setS3Credentials(resp.data.artifacts);
       return;
     }
 
     try {
-      const resp = await this.axios.post(`${this.url.trim()}/api/reporter`, runParams, {
+      const resp = await this.axios.post(`/api/reporter`, runParams, {
         maxContentLength: Infinity,
         maxBodyLength: Infinity,
       });
+
       this.runId = resp.data.uid;
       this.runUrl = `${this.url}/${resp.data.url.split('/').splice(3).join('/')}`;
+      this.runPublicUrl = resp.data.public_url;
+
       if (resp.data.artifacts) setS3Credentials(resp.data.artifacts);
+
       this.store.runUrl = this.runUrl;
+      this.store.runPublicUrl = this.runPublicUrl;
       this.store.runId = this.runId;
       console.log(APP_PREFIX, '📊 Report created. Report ID:', this.runId);
       process.env.runId = this.runId;
+      debug('Run created', this.runId);
     } catch (err) {
       console.error(
         APP_PREFIX,
-        'Error creating Testomat.io report, please check if your API key is valid. Skipping report',
+        'Error creating Testomat.io report, please check if your API key is valid. Skipping report | ',
+        err?.response?.statusText || err?.status || err.message,
       );
     }
+    debug('"createRun" function finished');
   }
 
   /**
-   * 
-   * @param testData data 
-   * @returns 
+   * Uploads tests as a batch (multiple tests at once). Intended to be used with a setInterval
+   */
+  #batchUpload = async () => {
+    if (!this.batch.tests.length) return;
+
+    // get tests from batch and clear batch
+    const testsToSend = this.batch.tests.splice(0);
+    debug('📨 Batch upload', testsToSend.length, 'tests');
+
+    return this.axios
+      .post(
+        `/api/reporter/${this.runId}/testrun`,
+        { api_key: this.apiKey, tests: testsToSend },
+        {
+          maxContentLength: Infinity,
+          maxBodyLength: Infinity,
+          headers: {
+            // Overwrite Axios's automatically set Content-Type
+            'Content-Type': 'application/json',
+          },
+        },
+      )
+      .catch(err => {
+        if (err.response) {
+          if (err.response.status >= 400) {
+            const responseData = err.response.data || { message: '' };
+            console.log(
+              APP_PREFIX,
+              chalk.yellow(`Warning: ${responseData.message} (${err.response.status})`),
+              // chalk.grey(data?.title || ''),
+            );
+            if (err.response.data.message.includes('could not be matched')) {
+              this.hasUnmatchedTests = true;
+            }
+            return;
+          }
+          console.log(
+            APP_PREFIX,
+            chalk.yellow(`Warning: (${err.response?.status})`),
+            `Report couldn't be processed: ${err?.response?.data?.message}`,
+          );
+        } else {
+          console.log(APP_PREFIX, "Report couldn't be processed", err);
+        }
+      });
+  };
+
+  /**
+   * Adds a test to the batch uploader
    */
   addTest(data) {
     if (!this.isEnabled) return;
     if (!this.runId) return;
     data.api_key = this.apiKey;
     data.create = this.createNewTests;
-    const json = JsonCycle.stringify(data);
-
-    return this.axios.post(`${this.url}/api/reporter/${this.runId}/testrun`, json, {
-      maxContentLength: Infinity,
-      maxBodyLength: Infinity,
-      headers: {
-        // Overwrite Axios's automatically set Content-Type
-        'Content-Type': 'application/json',
-      },
-    })
-    .catch((err) => {
-      if (err.response) {
-        if (err.response.status >= 400) {
-          const responseData = err.response.data || { message: '' };
-          console.log(
-            APP_PREFIX,
-            chalk.blue(this.title),
-            `Report couldn't be processed: (${err.response.status}) ${responseData.message}`,
-          );
-          return;
-        }
-        console.log(APP_PREFIX, chalk.blue(this.title), `Report couldn't be processed: ${err.response.data.message}`);
-      } else {
-        console.log(APP_PREFIX, chalk.blue(this.title), "Report couldn't be processed", err);
-      }
-    });
+    // const json = JsonCycle.stringify(data);
+    // debug('Adding test', JSON.stringify(data));
+
+    this.batch.tests.push(data);
+
+    // if test is added after run already finished
+    if (!this.batch.intervalFunction) this.#batchUpload();
   }
 
   /**
-   * @param {import('../../types').RunData} params 
-   * @returns 
+   * @param {import('../../types').RunData} params
+   * @returns
    */
   async finishRun(params) {
+    this.#batchUpload();
+    if (this.batch.intervalFunction) clearInterval(this.batch.intervalFunction);
+
+    debug('Finishing run...');
     if (!this.isEnabled) return;
 
     const { status, parallel } = params;
@@ -143,7 +287,7 @@ class TestomatioPipe {
 
     try {
       if (this.runId && !this.proceed) {
-        await this.axios.put(`${this.url}/api/reporter/${this.runId}`, {
+        await this.axios.put(`/api/reporter/${this.runId}`, {
           api_key: this.apiKey,
           status_event,
           tests: params.tests,
@@ -151,15 +295,44 @@ class TestomatioPipe {
         if (this.runUrl) {
           console.log(APP_PREFIX, '📊 Report Saved. Report URL:', chalk.magenta(this.runUrl));
         }
+        if (this.runPublicUrl) {
+          console.log(APP_PREFIX, '🌟 Public URL:', chalk.magenta(this.runPublicUrl));
+        }
       }
       if (this.runUrl && this.proceed) {
         const notFinishedMessage = chalk.yellow.bold('Run was not finished because of $TESTOMATIO_PROCEED');
         console.log(APP_PREFIX, `📊 ${notFinishedMessage}. Report URL: ${chalk.magenta(this.runUrl)}`);
         console.log(APP_PREFIX, `🛬 Run to finish it: TESTOMATIO_RUN=${this.runId} npx start-test-run --finish`);
       }
+      if (this.hasUnmatchedTests) {
+        console.log('');
+        // eslint-disable-next-line max-len
+        console.log(APP_PREFIX, chalk.yellow.bold('⚠️ Some reported tests were not found in Testomat.io project'));
+        // eslint-disable-next-line max-len
+        console.log(
+          APP_PREFIX,
+          `If you use Testomat.io as a reporter only, please re-run tests using ${chalk.bold('TESTOMATIO_CREATE=1')}`,
+        );
+        // eslint-disable-next-line max-len
+        console.log(
+          APP_PREFIX,
+          `But to keep your tests consistent it is recommended to ${chalk.bold('import tests first')}`,
+        );
+        console.log(APP_PREFIX, 'If tests were imported but still not matched, assign test IDs to your tests.');
+        console.log(APP_PREFIX, 'You can do that automatically via command line tools:');
+        console.log(APP_PREFIX, chalk.bold('npx check-tests ... --update-ids'), 'See: https://bit.ly/js-update-ids');
+        console.log(APP_PREFIX, 'or for Cucumber:');
+        // eslint-disable-next-line max-len
+        console.log(
+          APP_PREFIX,
+          chalk.bold('npx check-cucumber ... --update-ids'),
+          'See: https://bit.ly/bdd-update-ids',
+        );
+      }
     } catch (err) {
       console.log(APP_PREFIX, 'Error updating status, skipping...', err);
     }
+    debug('Run finished');
   }
 
   toString() {
@@ -168,18 +341,3 @@ class TestomatioPipe {
 }
 
 module.exports = TestomatioPipe;
-
-
-function setS3Credentials(artifacts) {  
-  if (!Object.keys(artifacts).length) return;
-
-  console.log(APP_PREFIX, 'S3 were credentials obtained from Testomat.io...');
-
-  if (artifacts.ACCESS_KEY_ID) process.env.S3_ACCESS_KEY_ID = artifacts.ACCESS_KEY_ID; 
-  if (artifacts.SECRET_ACCESS_KEY) process.env.S3_SECRET_ACCESS_KEY = artifacts.SECRET_ACCESS_KEY; 
-  if (artifacts.REGION) process.env.S3_REGION = artifacts.REGION; 
-  if (artifacts.BUCKET) process.env.S3_BUCKET = artifacts.BUCKET; 
-  if (artifacts.ENDPOINT) process.env.S3_ENDPOINT = artifacts.ENDPOINT; 
-  if (artifacts.presign) process.env.TESTOMATIO_PRIVATE_ARTIFACTS = '1'; 
-  resetConfig();
-}

package/lib/reporter-functions.js

@@ -0,0 +1,46 @@
+const { services } = require('./services');
+const { initPlaywrightForStorage } = require('./adapter/playwright');
+
+if (process.env.PLAYWRIGHT_TEST_BASE_URL) {
+  initPlaywrightForStorage();
+}
+
+/**
+ * Stores path to file as artifact and uploads it to the S3 storage
+ * @param {string | {path: string, type: string, name: string}} data - path to file or object with path, type and name
+ */
+function saveArtifact(data, context = null) {
+  if (!data) return;
+  services.artifacts.put(data, context);
+}
+
+/**
+ * Attach log message(s) to the test report
+ * @param  {...any} args
+ */
+function logMessage(...args) {
+  services.logger._templateLiteralLog(...args);
+}
+
+/**
+ * Similar to "log" function but marks message in report as a step
+ * @param {*} message
+ */
+function addStep(message) {
+  services.logger.step(message);
+}
+
+/**
+ * Add key-value pair(s) to the test report
+ * @param {*} keyValue
+ */
+function setKeyValue(keyValue) {
+  services.keyValues.put(keyValue);
+}
+
+module.exports = {
+  artifact: saveArtifact,
+  log: logMessage,
+  step: addStep,
+  keyValue: setKeyValue,
+};

package/lib/reporter.js

@@ -1,17 +1,19 @@
-const logger = require('./logger');
 const TestomatClient = require('./client');
 const TRConstants = require('./constants');
-const TRArtifacts = require('./_ArtifactStorageOld');
+const { services } = require('./services');
 
-const log = logger._log.bind(logger);
-const step = logger.step.bind(logger);
+const reporterFunctions = require('./reporter-functions');
 
 module.exports = {
-  logger,
-  log,
-  step,
+  // TODO: deprecate in future; use log or testomat.log
+  testomatioLogger: services.logger,
+
+  artifact: reporterFunctions.artifact,
+  log: reporterFunctions.log,
+  logger: services.logger,
+  meta: reporterFunctions.keyValue,
+  step: reporterFunctions.step,
+
   TestomatClient,
   TRConstants,
-  TRArtifacts,
-  addArtifact: TRArtifacts.artifact,
 };

package/lib/services/artifacts.js

@@ -0,0 +1,57 @@
+const debug = require('debug')('@testomatio/reporter:services-artifacts');
+const { dataStorage } = require('../data-storage');
+
+/**
+ * Artifact storage is supposed to store file paths
+ */
+class ArtifactStorage {
+  static #instance;
+
+  /**
+   * Singleton
+   * @returns {ArtifactStorage}
+   */
+  static getInstance() {
+    if (!this.#instance) {
+      this.#instance = new ArtifactStorage();
+    }
+    return this.#instance;
+  }
+
+  /**
+   * Stores path to file as artifact and uploads it to the S3 storage
+   * @param {string | {path: string, type: string, name: string}} data - path to file or object with path, type and name
+   * @param {*} context testId or test title
+   */
+  put(data, context = null) {
+    if (!data) return;
+    debug('Save artifact:', data);
+    dataStorage.putData('artifact', data, context);
+  }
+
+  /**
+   * Returns list of artifacts to upload
+   * @param {*} context testId or test context from test runner
+   * @returns {(string | {path: string, type: string, name: string})[]}
+   */
+  get(context) {
+    let artifacts = dataStorage.getData('artifact', context);
+    if (!artifacts || !artifacts.length) return [];
+
+    artifacts = artifacts.map(artifactData => {
+      // artifact could be an object ({type, path, name} props) or string (just path)
+      let artifact;
+      try {
+        artifact = JSON.parse(artifactData);
+      } catch (e) {
+        artifact = artifactData;
+      }
+      return artifact;
+    });
+    artifacts = artifacts.filter(artifact => !!artifact);
+    debug(`Artifacts for test ${context}:`, artifacts);
+    return artifacts.length ? artifacts : [];
+  }
+}
+
+module.exports.artifactStorage = ArtifactStorage.getInstance();

package/lib/services/index.js

@@ -0,0 +1,13 @@
+const { logger } = require('./logger');
+const { artifactStorage } = require('./artifacts');
+const { keyValueStorage } = require('./key-values');
+const { dataStorage } = require('../data-storage');
+
+module.exports.services = {
+  logger,
+  artifacts: artifactStorage,
+  keyValues: keyValueStorage,
+  setContext: context => {
+    dataStorage.setContext(context);
+  },
+};

package/lib/services/key-values.js

@@ -0,0 +1,58 @@
+const debug = require('debug')('@testomatio/reporter:services-key-value');
+const { dataStorage } = require('../data-storage');
+
+class KeyValueStorage {
+  static #instance;
+
+  /**
+   *
+   * @returns {KeyValueStorage}
+   */
+  static getInstance() {
+    if (!this.#instance) {
+      this.#instance = new KeyValueStorage();
+    }
+    return this.#instance;
+  }
+
+  /**
+   * Stores key-value pair and passes it to reporter
+   * @param {{key: string}} keyValue - key-value pair(s) as object
+   * @param {*} context - full test title
+   */
+  put(keyValue, context = null) {
+    if (!keyValue) return;
+    dataStorage.putData('keyvalue', keyValue, context);
+  }
+
+  #isKeyValueObject(smth) {
+    return smth && typeof smth === 'object' && !Array.isArray(smth) && smth !== null;
+  }
+
+  /**
+   * Returns key-values pairs for the test as object
+   * @param {*} context testId or test context from test runner
+   * @returns {{[key: string]: string} | {}} key-values pairs as object, e.g. {priority: 'high', browser: 'chrome'}
+   */
+  get(context = null) {
+    const keyValuesList = dataStorage.getData('keyvalue', context);
+    if (!keyValuesList || !keyValuesList?.length) return {};
+
+    const keyValues = {};
+    for (const keyValue of keyValuesList) {
+      if (this.#isKeyValueObject(keyValue)) {
+        Object.assign(keyValues, keyValue);
+      } else if (typeof keyValue === 'string') {
+        try {
+          Object.assign(keyValues, JSON.parse(keyValue));
+        } catch (e) {
+          debug(`Error parsing key-values for test ${context}`, keyValue);
+        }
+      }
+    }
+
+    return keyValues;
+  }
+}
+
+module.exports.keyValueStorage = KeyValueStorage.getInstance();