@testomatio/reporter 1.3.0-ignore-stack-for-passed.1 → 1.3.0

package/lib/bin/reportXml.js

@@ -44,10 +44,15 @@ program
 
     let timeoutTimer;
     if (opts.timelimit) {
-      timeoutTimer = setTimeout(() => {
-        console.log(`⚠️  Reached timeout of ${opts.timelimit}s. Exiting... (Exit code is 0 to not fail the pipeline)`);
-        process.exit(0);
-      }, parseInt(opts.timelimit, 10) * 1000);
+      timeoutTimer = setTimeout(
+        () => {
+          console.log(
+            `⚠️  Reached timeout of ${opts.timelimit}s. Exiting... (Exit code is 0 to not fail the pipeline)`,
+          );
+          process.exit(0);
+        },
+        parseInt(opts.timelimit, 10) * 1000,
+      );
     }
 
     try {

package/lib/constants.js

@@ -4,7 +4,6 @@ const path = require('path');
 
 const APP_PREFIX = chalk.gray('[TESTOMATIO]');
 const AXIOS_TIMEOUT = 20 * 1000; // sum = 20sec
-const AXIOS_RETRY_TIMEOUT = 5 * 1000; // sum = 5sec
 
 const TESTOMAT_TMP_STORAGE_DIR = path.join(os.tmpdir(), 'testomatio_tmp');
 
@@ -31,6 +30,19 @@ const HTML_REPORT = {
 
 const testomatLogoURL = 'https://avatars.githubusercontent.com/u/59105116?s=36&v=4';
 
+const REPORTER_REQUEST_RETRIES = {
+  retryTimeout: 5 * 1000, // sum = 5sec
+  retriesPerRequest: 2,
+  maxTotalRetries: Number(process.env.TESTOMATIO_MAX_REQUEST_FAILURES_COUNT) || 10,
+  withinTimeSeconds: Number(process.env.TESTOMATIO_MAX_REQUEST_RETRIES_WITHIN_TIME_SECONDS) || 60,
+};
+
+const RunStatus = {
+  Passed: 'passed',
+  Failed: 'failed',
+  Finished: 'finished',
+};
+
 module.exports = {
   APP_PREFIX,
   TESTOMAT_TMP_STORAGE_DIR,
@@ -38,6 +50,7 @@ module.exports = {
   STATUS,
   HTML_REPORT,
   AXIOS_TIMEOUT,
-  AXIOS_RETRY_TIMEOUT,
   testomatLogoURL,
+  REPORTER_REQUEST_RETRIES,
+  RunStatus,
 };

package/lib/fileUploader.js

@@ -20,6 +20,7 @@ const keys = [
   'S3_BUCKET',
   'S3_ACCESS_KEY_ID',
   'S3_SECRET_ACCESS_KEY',
+  'S3_SESSION_TOKEN',
   'TESTOMATIO_DISABLE_ARTIFACTS',
   'TESTOMATIO_PRIVATE_ARTIFACTS',
   'S3_FORCE_PATH_STYLE',
@@ -74,7 +75,8 @@ const _getFileExtBase64 = str => {
 };
 
 const _getS3Config = () => {
-  const { S3_REGION, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_FORCE_PATH_STYLE, S3_ENDPOINT } = getConfig();
+  const { S3_REGION, S3_SESSION_TOKEN, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_FORCE_PATH_STYLE, S3_ENDPOINT } =
+    getConfig();
 
   const cfg = {
     region: S3_REGION,
@@ -85,6 +87,10 @@ const _getS3Config = () => {
     },
   };
 
+  if (S3_SESSION_TOKEN) {
+    cfg.credentials.sessionToken = S3_SESSION_TOKEN;
+  }
+
   if (S3_ENDPOINT) {
     cfg.endpoint = S3_ENDPOINT;
   }

package/lib/pipe/gitlab.js

@@ -79,13 +79,13 @@ class GitLabPipe {
     let summary = `${this.hiddenCommentData}
     
   | [![Testomat.io Report](${testomatLogoURL})](https://testomat.io)  | ${statusEmoji(
-      runParams.status,
-    )} ${runParams.status.toUpperCase()} ${statusEmoji(runParams.status)} |
+    runParams.status,
+  )} ${runParams.status.toUpperCase()} ${statusEmoji(runParams.status)} |
   | --- | --- |
   | Tests | ✔️  **${this.tests.length}** tests run  |
   | Summary | ${statusEmoji('failed')} **${failedCount}** failed; ${statusEmoji(
-      'passed',
-    )} **${passedCount}** passed; **${statusEmoji('skipped')}** ${skippedCount} skipped |
+    'passed',
+  )} **${passedCount}** passed; **${statusEmoji('skipped')}** ${skippedCount} skipped |
   | Duration | 🕐  **${humanizeDuration(
     parseInt(
       this.tests.reduce((a, t) => a + (t.run_time || 0), 0),

package/lib/pipe/html.js

@@ -91,7 +91,7 @@ class HtmlPipe {
       // GENERATE HTML reports based on the results data
       this.buildReport({
         runParams,
-        // TODO: this.tests=[] in case of Mocha
+        // TODO: this.tests=[] in case of Mocha, need retest by Vitalii
         tests: this.tests,
         outputPath: this.htmlOutputPath,
         templatePath: this.templateHtmlPath,
@@ -123,30 +123,31 @@ class HtmlPipe {
     }
 
     tests.forEach(test => {
-      if (!test.message || test.message.trim() === '') {
+      if (!test.message?.trim()) {
         test.message = "This test has no 'message' code";
       }
 
-      if (!test.suite_title || test.suite_title.trim() === '') {
+      if (!test.suite_title?.trim()) {
         test.suite_title = 'Unknown suite';
       }
 
-      if (!test.title || test.title.trim() === '') {
+      if (!test.title?.trim()) {
         test.title = 'Unknown test title';
       }
 
-      if (!test.files || test.files.length === 0) {
+      if (!test.files?.length) {
         test.files = 'This test has no files';
       }
 
-      if (test.steps) {
-        if (!test.steps || test.steps.trim() === '') {
-          test.steps = "This test has no 'steps' code";
-        } else {
-          test.steps = removeAnsiColorCodes(test.steps);
-        }
+      if (!test.steps?.trim()) {
+        test.steps = "This test has no 'steps' code";
+      } else {
+        test.steps = removeAnsiColorCodes(test.steps);
       }
 
+      // TODO: future-proof: currently there is no need to display Artifacts and Metadata in HTML
+      test.artifacts = test.artifacts || [];
+      test.meta = test.meta || {};
       // TODO: u can added an additional test values to this checks in the future
     });
 
@@ -199,7 +200,8 @@ class HtmlPipe {
 
       return template(data);
     } catch (e) {
-      console.log('Unknown HTML report generation error: ', e);
+      console.log(chalk.red('❌ Oops! An unknown error occurred when generating an HTML report'));
+      console.log(chalk.red(e));
     }
   }
 
@@ -209,42 +211,77 @@ class HtmlPipe {
       (tests, status) => tests.filter(test => test.status.toLowerCase() === status.toLowerCase()).length,
     );
 
-    handlebars.registerHelper('json', tests => {
-      // TODO: please refactor the code below, add meaningful variable names, separate into functions
-      function replaceScriptTagsInArray(array) {
-        return array.map(obj => {
-          const keysToCheck = ['steps', 'stack', 'title', 'suite_title', 'message', 'code'];
-          const newObj = {};
-
-          for (const key in obj) {
-            if (Object.prototype.hasOwnProperty.call(obj, key)) {
-              if (key === 'example') {
-                newObj[key] = {};
-                for (const subKey in obj[key]) {
-                  if (Object.prototype.hasOwnProperty.call(obj[key], subKey)) {
-                    newObj[key][subKey] =
-                      typeof obj[key][subKey] === 'string'
-                        ? obj[key][subKey].replace(/<script>/g, '<$cript>').replace(/<\/script>/g, '</$cript>')
-                        : obj[key][subKey];
-                  }
-                }
-              } else if (keysToCheck.includes(key)) {
-                newObj[key] =
-                  typeof obj[key] === 'string'
-                    ? obj[key].replace(/<script>/g, '<$cript>').replace(/<\/script>/g, '</$cript>')
-                    : obj[key];
-              } else {
-                newObj[key] = obj[key];
-              }
+    handlebars.registerHelper(
+      'selectComponent',
+      () =>
+        new handlebars.SafeString(
+          `<select style="width: 70px;height: 38px;" class="form-select" aria-label="Tests counter on page">
+          <option value="0">10</option>
+          <option value="1">25</option>
+          <option value="2">50</option>
+        </select>`,
+        ),
+    );
+
+    handlebars.registerHelper(
+      'emptyDataComponent',
+      () =>
+        new handlebars.SafeString(
+          `<div class="noData m-0">
+          NO MATCHING TESTS
+        </div>`,
+        ),
+    );
+
+    handlebars.registerHelper('pageDispleyElements', tests => {
+      // We wrapp the lines to the HTML format we need
+      const totalTests = JSON.parse(
+        JSON.stringify(tests)
+          .replace(/<script>/g, '&lt;script&gt;')
+          .replace(/<\/script>/g, '&lt;/script&gt;'), // eslint-disable-line
+      );
+
+      const paginationOptions = {
+        0: 10,
+        1: 25,
+        2: 50,
+      };
+      const statuses = ['all', 'passed', 'failed', 'skipped'];
+      const pageItemGroups = {
+        all: {},
+        passed: {},
+        failed: {},
+        skipped: {},
+        totalTests,
+      };
+
+      function paginateItems(items, pageSize) {
+        const paginatedItems = [];
+        for (let i = 0; i < items.length; i += pageSize) {
+          paginatedItems.push(items.slice(i, i + pageSize));
+        }
+        return paginatedItems;
+      }
+
+      statuses.forEach(status => {
+        for (const option in paginationOptions) {
+          // eslint-disable-next-line no-prototype-builtins
+          if (paginationOptions.hasOwnProperty(option)) {
+            const pageSize = paginationOptions[option];
+            let filteredItems = totalTests;
+
+            if (status !== 'all') {
+              filteredItems = totalTests.filter(item => item.status === status);
             }
+
+            pageItemGroups[status][option] = paginateItems(filteredItems, pageSize);
           }
+        }
+      });
 
-          return newObj;
-        });
-      }
+      pageItemGroups.totalTests = totalTests;
 
-      // Remove ANSI escape codes
-      return JSON.stringify(replaceScriptTagsInArray(tests));
+      return JSON.stringify(pageItemGroups);
     });
   }
 
@@ -261,7 +298,7 @@ class HtmlPipe {
  */
 function testExecutionSumTime(tests) {
   const totalMilliseconds = tests.reduce((sum, test) => {
-    if (typeof test.run_time === 'number') {
+    if (typeof test.run_time === 'number' && !Number.isNaN(test.run_time)) {
       return sum + test.run_time;
     }
     return sum;

package/lib/pipe/testomatio.js

@@ -6,7 +6,7 @@ const axiosRetry = require('axios-retry');
 const axios = require('axios');
 const JsonCycle = require('json-cycle');
 
-const { APP_PREFIX, STATUS, AXIOS_TIMEOUT, AXIOS_RETRY_TIMEOUT } = require('../constants');
+const { APP_PREFIX, STATUS, AXIOS_TIMEOUT, REPORTER_REQUEST_RETRIES } = require('../constants');
 const { isValidUrl, foundedTestLog } = require('../utils/utils');
 const { parseFilterParams, generateFilterRequestParams, setS3Credentials } = require('../utils/pipe_utils');
 const config = require('../config');
@@ -23,6 +23,10 @@ if (process.env.TESTOMATIO_RUN) {
  */
 class TestomatioPipe {
   constructor(params, store) {
+    this.retriesTimestamps = [];
+    this.reportingCanceledDueToReqFailures = false;
+    this.notReportedTestsCount = 0;
+
     this.isEnabled = false;
     this.url = params.testomatioUrl || process.env.TESTOMATIO_URL || 'https://app.testomat.io';
     this.apiKey = params.apiKey || config.TESTOMATIO;
@@ -43,25 +47,30 @@ class TestomatioPipe {
       baseURL: `${this.url.trim()}`,
       timeout: AXIOS_TIMEOUT,
     });
+
     // Pass the axios instance to the retry function
     axiosRetry(this.axios, {
-      retries: 3, // Number of retries (Defaults to 3)
+      // do not use retries for unit tests
+      retries: REPORTER_REQUEST_RETRIES.retriesPerRequest, // Number of retries
       shouldResetTimeout: true,
       retryCondition: error => {
-        // Conditional check the error status code
-        switch (error.response.status) {
-          case 409:
-          case 429:
-          case 502:
-          case 503:
-            return true; // Retry request with response status code 409, 429, 502, 503
+        if (!error.response) return false;
+        switch (error.response?.status) {
+          case 400: // Bad request (probably wrong API key)
+          case 404: // Test not matched
+          case 429: // Rate limit exceeded
+          case 500: // Internal server error
+            return false;
           default:
-            return false; // Do not retry the others
+            break;
         }
+        return error.response?.status >= 401; // Retry on 401+ and 5xx
       },
-      retryDelay: retryCount => retryCount * AXIOS_RETRY_TIMEOUT, // sum = 15sec
-      onRetry: retryCount => {
-        debug(`Retry attempt #${retryCount} failed. Retrying again...`);
+      retryDelay: () => REPORTER_REQUEST_RETRIES.retryTimeout, // sum = 15sec
+      onRetry: async (retryCount, error) => {
+        this.retriesTimestamps.push(Date.now());
+
+        debug(`${error.message || `Request failed ${error.status}`}. Retry #${retryCount} ...`);
       },
     });
 
@@ -190,13 +199,45 @@ class TestomatioPipe {
         'Error creating Testomat.io report, please check if your API key is valid. Skipping report | ',
         err?.response?.statusText || err?.status || err.message,
       );
+      printCreateIssue(err);
     }
     debug('"createRun" function finished');
   }
 
+  /**
+   * Decides whether to skip test reporting in case of too many request failures
+   * @param {TestData} testData
+   * @returns {boolean}
+   */
+  #cancelTestReportingInCaseOfTooManyReqFailures(testData) {
+    if (this.reportingCanceledDueToReqFailures) return true;
+
+    const retriesCountWithinTime = this.retriesTimestamps.filter(
+      timestamp => Date.now() - timestamp < REPORTER_REQUEST_RETRIES.withinTimeSeconds * 1000,
+    ).length;
+    debug(`${retriesCountWithinTime} failed requests within ${REPORTER_REQUEST_RETRIES.withinTimeSeconds}s`);
+
+    if (retriesCountWithinTime > REPORTER_REQUEST_RETRIES.maxTotalRetries) {
+      const errorMessage = chalk.yellow(
+        `${retriesCountWithinTime} requests were failed within ${REPORTER_REQUEST_RETRIES.withinTimeSeconds}s,\
+ reporting for test "${testData.title}" to Testomat is skipped`,
+      );
+      console.warn(`${APP_PREFIX} ${errorMessage}`);
+
+      this.reportingCanceledDueToReqFailures = true;
+      this.notReportedTestsCount++;
+
+      return true;
+    }
+
+    return false;
+  }
+
   addTest(data) {
     if (!this.isEnabled) return;
     if (!this.runId) return;
+    if (this.#cancelTestReportingInCaseOfTooManyReqFailures(data)) return;
+
     data.api_key = this.apiKey;
     data.create = this.createNewTests;
 
@@ -227,7 +268,7 @@ class TestomatioPipe {
               chalk.yellow(`Warning: ${responseData.message} (${err.response.status})`),
               chalk.grey(data?.title || ''),
             );
-            if (err.response.data.message.includes('could not be matched')) {
+            if (err.response?.data?.message?.includes('could not be matched')) {
               this.hasUnmatchedTests = true;
             }
             return;
@@ -237,6 +278,7 @@ class TestomatioPipe {
             chalk.yellow(`Warning: ${data?.title || ''} (${err.response?.status})`),
             `Report couldn't be processed: ${err?.response?.data?.message}`,
           );
+          printCreateIssue(err);
         } else {
           console.log(APP_PREFIX, chalk.blue(data?.title || ''), "Report couldn't be processed", err);
         }
@@ -248,8 +290,15 @@ class TestomatioPipe {
    * @returns
    */
   async finishRun(params) {
-    debug('Finishing run...');
     if (!this.isEnabled) return;
+    debug('Finishing run...');
+
+    if (this.reportingCanceledDueToReqFailures) {
+      const errorMessage = chalk.red(
+        `⚠️ Due to request failures, ${this.notReportedTestsCount} test(s) were not reported to Testomat.io`,
+      );
+      console.warn(`${APP_PREFIX} ${errorMessage}`);
+    }
 
     const { status, parallel } = params;
 
@@ -306,6 +355,7 @@ class TestomatioPipe {
       }
     } catch (err) {
       console.log(APP_PREFIX, 'Error updating status, skipping...', err);
+      printCreateIssue(err);
     }
     debug('Run finished');
   }
@@ -315,4 +365,27 @@ class TestomatioPipe {
   }
 }
 
+let registeredErrorHints = false;
+function printCreateIssue(err) {
+  if (registeredErrorHints) return;
+  registeredErrorHints = true;
+  process.on('exit', () => {
+    console.log();
+    console.log(APP_PREFIX, 'There was an error reporting to Testomat.io:');
+    console.log(
+      APP_PREFIX,
+      'If you think this is a bug please create an issue: https://github.com/testomatio/reporter/issues/new',
+    ); // eslint-disable-line max-len
+    console.log(APP_PREFIX, 'Provide this information:');
+    console.log('Error:', err.message || err.code);
+    if (!err.config) return;
+
+    const time = new Date().toUTCString();
+    const { data, url, baseURL, method } = err?.config || {};
+    console.log('```js');
+    console.log({ data: data?.replace(/"(tstmt_[^"]+)"/g, 'tstmt_*'), url, baseURL, method, time });
+    console.log('```');
+  });
+}
+
 module.exports = TestomatioPipe;