@testomatio/reporter 1.2.2-beta-html-pagination-feature-v2 → 1.2.2-beta-cancel-reporting-after-multiple-fails

package/lib/constants.js

@@ -4,7 +4,6 @@ const path = require('path');
 
 const APP_PREFIX = chalk.gray('[TESTOMATIO]');
 const AXIOS_TIMEOUT = 20 * 1000; // sum = 20sec
-const AXIOS_RETRY_TIMEOUT = 5 * 1000; // sum = 5sec
 
 const TESTOMAT_TMP_STORAGE_DIR = path.join(os.tmpdir(), 'testomatio_tmp');
 
@@ -31,6 +30,19 @@ const HTML_REPORT = {
 
 const testomatLogoURL = 'https://avatars.githubusercontent.com/u/59105116?s=36&v=4';
 
+const REPORTER_REQUEST_RETRIES = {
+  retryTimeout: 5 * 1000, // sum = 5sec
+  retriesPerRequest: 2,
+  maxTotalRetries: Number(process.env.TESTOMATIO_MAX_REQUEST_FAILURES_COUNT) || 10,
+  withinTimeSeconds: Number(process.env.TESTOMATIO_MAX_REQUEST_RETRIES_WITHIN_TIME_SECONDS) || 60,
+};
+
+const RunStatus = {
+  Passed: 'passed',
+  Failed: 'failed',
+  Finished: 'finished',
+};
+
 module.exports = {
   APP_PREFIX,
   TESTOMAT_TMP_STORAGE_DIR,
@@ -38,6 +50,7 @@ module.exports = {
   STATUS,
   HTML_REPORT,
   AXIOS_TIMEOUT,
-  AXIOS_RETRY_TIMEOUT,
   testomatLogoURL,
+  REPORTER_REQUEST_RETRIES,
+  RunStatus,
 };

package/lib/pipe/html.js

@@ -91,7 +91,7 @@ class HtmlPipe {
       // GENERATE HTML reports based on the results data
       this.buildReport({
         runParams,
-        // TODO: this.tests=[] in case of Mocha, need retest by Vitalii
+        // TODO: this.tests=[] in case of Mocha
         tests: this.tests,
         outputPath: this.htmlOutputPath,
         templatePath: this.templateHtmlPath,
@@ -135,13 +135,11 @@ class HtmlPipe {
         test.title = 'Unknown test title';
       }
 
-      if ('files' in test) {
-        if (test.files.length === 0) {
-          test.files = 'This test has no files';
-        }
+      if (!test.files || test.files.length === 0) {
+        test.files = 'This test has no files';
       }
 
-      if ('steps' in test) {
+      if (test.steps) {
         if (!test.steps || test.steps.trim() === '') {
           test.steps = "This test has no 'steps' code";
         } else {
@@ -149,15 +147,6 @@ class HtmlPipe {
         }
       }
 
-      // TODO: future-proof: currently there is no need to display Artifacts and Metadata in HTML
-      if ('artifacts' in test) {
-        test.artifacts = [];
-      }
-
-      if ('meta' in test) {
-        test.meta = {};
-      }
-
       // TODO: u can added an additional test values to this checks in the future
     });
 
@@ -210,8 +199,7 @@ class HtmlPipe {
 
       return template(data);
     } catch (e) {
-      console.log(chalk.red('❌ Oops! An unknown error occurred when generating an HTML report'));
-      console.log(chalk.red(e));
+      console.log('Unknown HTML report generation error: ', e);
     }
   }
 
@@ -221,76 +209,42 @@ class HtmlPipe {
       (tests, status) => tests.filter(test => test.status.toLowerCase() === status.toLowerCase()).length,
     );
 
-    handlebars.registerHelper(
-      'selectComponent',
-      () =>
-        new handlebars.SafeString(
-          `<select style="width: 70px;height: 38px;" class="form-select" aria-label="Tests counter on page">
-          <option value="0">10</option>
-          <option value="1">25</option>
-          <option value="2">50</option>
-        </select>`,
-        ),
-    );
-
-    handlebars.registerHelper(
-      'emptyDataComponent',
-      () =>
-        new handlebars.SafeString(
-          `<div class="noData m-0">
-          NO MATCHING TESTS
-        </div>`,
-        ),
-    );
-
-    handlebars.registerHelper('pageDispleyElements', tests => {
-      // We wrapp the lines to the HTML format we need
-      const totalTests = JSON.parse(
-        JSON.stringify(tests)
-          .replace(/<script>/g, '&lt;script&gt;')
-          .replace(/<\/script>/g, '&lt;/script&gt;'), // eslint-disable-line
-      );
-
-      const paginationOptions = {
-        0: 10,
-        1: 25,
-        2: 50,
-      };
-      const statuses = ['all', 'passed', 'failed', 'skipped'];
-      const pageItemGroups = {
-        all: {},
-        passed: {},
-        failed: {},
-        skipped: {},
-        totalTests,
-      };
-
-      function paginateItems(items, pageSize) {
-        const paginatedItems = [];
-        for (let i = 0; i < items.length; i += pageSize) {
-          paginatedItems.push(items.slice(i, i + pageSize));
-        }
-        return paginatedItems;
-      }
-
-      statuses.forEach(status => {
-        for (const option in paginationOptions) {
-          if (Object.hasOwnProperty.call(paginationOptions, option)) {
-            const pageSize = paginationOptions[option];
-            let filteredItems = totalTests;
-
-            if (status !== 'all') {
-              filteredItems = totalTests.filter(item => item.status === status);
+    handlebars.registerHelper('json', tests => {
+      // TODO: please refactor the code below, add meaningful variable names, separate into functions
+      function replaceScriptTagsInArray(array) {
+        return array.map(obj => {
+          const keysToCheck = ['steps', 'stack', 'title', 'suite_title', 'message', 'code'];
+          const newObj = {};
+
+          for (const key in obj) {
+            if (Object.prototype.hasOwnProperty.call(obj, key)) {
+              if (key === 'example') {
+                newObj[key] = {};
+                for (const subKey in obj[key]) {
+                  if (Object.prototype.hasOwnProperty.call(obj[key], subKey)) {
+                    newObj[key][subKey] =
+                      typeof obj[key][subKey] === 'string'
+                        ? obj[key][subKey].replace(/<script>/g, '<$cript>').replace(/<\/script>/g, '</$cript>')
+                        : obj[key][subKey];
+                  }
+                }
+              } else if (keysToCheck.includes(key)) {
+                newObj[key] =
+                  typeof obj[key] === 'string'
+                    ? obj[key].replace(/<script>/g, '<$cript>').replace(/<\/script>/g, '</$cript>')
+                    : obj[key];
+              } else {
+                newObj[key] = obj[key];
+              }
             }
-
-            pageItemGroups[status][option] = paginateItems(filteredItems, pageSize);
           }
-        }
-      });
 
-      pageItemGroups.totalTests = totalTests;
+          return newObj;
+        });
+      }
 
-      return JSON.stringify(pageItemGroups);
+      // Remove ANSI escape codes
+      return JSON.stringify(replaceScriptTagsInArray(tests));
     });
   }
 
@@ -307,7 +261,7 @@ class HtmlPipe {
  */
 function testExecutionSumTime(tests) {
   const totalMilliseconds = tests.reduce((sum, test) => {
-    if (typeof test.run_time === 'number' && !Number.isNaN(test.run_time)) {
+    if (typeof test.run_time === 'number') {
       return sum + test.run_time;
     }
     return sum;

package/lib/pipe/testomatio.js

@@ -6,7 +6,7 @@ const axiosRetry = require('axios-retry');
 const axios = require('axios');
 const JsonCycle = require('json-cycle');
 
-const { APP_PREFIX, STATUS, AXIOS_TIMEOUT, AXIOS_RETRY_TIMEOUT } = require('../constants');
+const { APP_PREFIX, STATUS, AXIOS_TIMEOUT, REPORTER_REQUEST_RETRIES } = require('../constants');
 const { isValidUrl, foundedTestLog } = require('../utils/utils');
 const { parseFilterParams, generateFilterRequestParams, setS3Credentials } = require('../utils/pipe_utils');
 const config = require('../config');
@@ -23,6 +23,10 @@ if (process.env.TESTOMATIO_RUN) {
  */
 class TestomatioPipe {
   constructor(params, store) {
+    this.retriesTimestamps = [];
+    this.reportingCanceledDueToReqFailures = false;
+    this.notReportedTestsCount = 0;
+
     this.isEnabled = false;
     this.url = params.testomatioUrl || process.env.TESTOMATIO_URL || 'https://app.testomat.io';
     this.apiKey = params.apiKey || config.TESTOMATIO;
@@ -43,25 +47,29 @@ class TestomatioPipe {
       baseURL: `${this.url.trim()}`,
       timeout: AXIOS_TIMEOUT,
     });
+
     // Pass the axios instance to the retry function
     axiosRetry(this.axios, {
-      retries: 3, // Number of retries (Defaults to 3)
+      // do not use retries for unit tests
+      retries: REPORTER_REQUEST_RETRIES.retriesPerRequest, // Number of retries
       shouldResetTimeout: true,
       retryCondition: error => {
-        // Conditional check the error status code
-        switch (error.response.status) {
-          case 409:
-          case 429:
-          case 502:
-          case 503:
-            return true; // Retry request with response status code 409, 429, 502, 503
+        switch (error.response?.status) {
+          case 400: // Bad request (probably wrong API key)
+          case 404: // Test not matched
+          case 429: // Rate limit exceeded
+          case 500: // Internal server error
+            return false;
           default:
-            return false; // Do not retry the others
+            break;
         }
+        return error.response?.status >= 401; // Retry on 401+ and 5xx
       },
-      retryDelay: retryCount => retryCount * AXIOS_RETRY_TIMEOUT, // sum = 15sec
-      onRetry: retryCount => {
-        debug(`Retry attempt #${retryCount} failed. Retrying again...`);
+      retryDelay: () => REPORTER_REQUEST_RETRIES.retryTimeout, // sum = 15sec
+      onRetry: async (retryCount, error) => {
+        this.retriesTimestamps.push(Date.now());
+
+        debug(`${error.message || `Request failed ${error.status}`}. Retry #${retryCount} ...`);
       },
     });
 
@@ -194,9 +202,40 @@ class TestomatioPipe {
     debug('"createRun" function finished');
   }
 
+  /**
+   * Decides whether to skip test reporting in case of too many request failures
+   * @param {TestData} testData
+   * @returns {boolean}
+   */
+  #cancelTestReportingInCaseOfTooManyReqFailures(testData) {
+    if (this.reportingCanceledDueToReqFailures) return true;
+
+    const retriesCountWithinTime = this.retriesTimestamps.filter(
+      timestamp => Date.now() - timestamp < REPORTER_REQUEST_RETRIES.withinTimeSeconds * 1000,
+    ).length;
+    debug(`${retriesCountWithinTime} failed requests within ${REPORTER_REQUEST_RETRIES.withinTimeSeconds}s`);
+
+    if (retriesCountWithinTime > REPORTER_REQUEST_RETRIES.maxTotalRetries) {
+      const errorMessage = chalk.yellow(
+        `${retriesCountWithinTime} requests were failed within ${REPORTER_REQUEST_RETRIES.withinTimeSeconds}s,\
+ reporting for test "${testData.title}" to Testomat is skipped`,
+      );
+      console.warn(`${APP_PREFIX} ${errorMessage}`);
+
+      this.reportingCanceledDueToReqFailures = true;
+      this.notReportedTestsCount++;
+
+      return true;
+    }
+
+    return false;
+  }
+
   addTest(data) {
     if (!this.isEnabled) return;
     if (!this.runId) return;
+    if (this.#cancelTestReportingInCaseOfTooManyReqFailures(data)) return;
+
     data.api_key = this.apiKey;
     data.create = this.createNewTests;
     const json = JsonCycle.stringify(data);
@@ -221,7 +260,7 @@ class TestomatioPipe {
               chalk.yellow(`Warning: ${responseData.message} (${err.response.status})`),
               chalk.grey(data?.title || ''),
             );
-            if (err.response.data.message.includes('could not be matched')) {
+            if (err.response.data?.message?.includes('could not be matched')) {
               this.hasUnmatchedTests = true;
             }
             return;
@@ -242,8 +281,15 @@ class TestomatioPipe {
    * @returns
    */
   async finishRun(params) {
-    debug('Finishing run...');
     if (!this.isEnabled) return;
+    debug('Finishing run...');
+
+    if (this.reportingCanceledDueToReqFailures) {
+      const errorMessage = chalk.red(
+        `⚠️ Due to request failures, ${this.notReportedTestsCount} test(s) were not reported to Testomat.io`,
+      );
+      console.warn(`${APP_PREFIX} ${errorMessage}`);
+    }
 
     const { status, parallel } = params;
 

package/lib/template/template-draft.hbs

@@ -0,0 +1,249 @@
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <meta charset="utf-8" />
+        <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+        <link href="https://fonts.googleapis.com/css2?family=Fira+Mono:wght@400;500;700&display=swap" rel="stylesheet">
+        <link href="https://unpkg.com/tailwindcss@^2/dist/tailwind.min.css" rel="stylesheet">
+        <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/accordionjs@2.1.2/accordion.min.css">
+        <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css">
+        <title>Report {{runId}} - Testomat.io</title>
+        <link rel="stylesheet" href="./style.css">
+        {{!-- TODO: styling create as a new additional file instead of template => copy default one --}}
+        <style>
+            body {
+                font-family: "Fira Sans", sans-serif;
+            }
+        </style>
+    </head>
+
+    <body class="bg-gray-50 pt-6 px-4 lg:pt-4 lg:px-40 h-full;">
+        <div id="testomatio_report">                
+            <div class="flex items-center flex-col space-y-4 md:flex-row md:justify-between md:h-14 mb-6">
+                <div class="flex flex-col md:flex-row space-x-3 items-center">
+                    <h2 class="text-gray-900 leading-none text-3xl font-semibold mb-0">{{runId}}
+                    </h2>
+                    {{!-- <span class="text-sm text-gray-500">
+                        {{executionDate}}
+                    </span> --}}
+                </div>
+                <div class="flex space-x-3 items-center">                       
+                    <a class="inline-flex items-center border border-transparent leading-none text-white bg-indigo-500 hover:bg-indigo-700 rounded-md focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-indigo-500 cursor-pointer px-4 py-2 text-base leading-none" 
+                        style="min-height: 38px"
+                        href="{{runUrl}}" 
+                        target="_blank">
+                        Full Report
+                    </a>
+                </div>
+            </div>
+            <div class="bg-white border overflow-hidden rounded-lg divide-y divide-gray-200 mb-12">
+                <div class="flex flex-col lg:divide-x divide-gray-200 sm:flex-col md:flex-col lg:flex-row">
+                    <div class="w-full lg:w-1/2">
+                        <!-- Left chart section -->
+                        <div class="p-4">
+                            <h2 class="text-xl font-semibold mb-4">Testomatio Test Results</h2>
+                            <span class="text-sm text-gray-500 items-center">{{executionDate}}</span>
+                            <div class="hart-container" style="height: 200px;">
+                                <canvas id="testomatio_chart"></canvas>
+                                <div for="testomatio_chart"></div>
+                            </div>
+                        </div>
+                    </div>
+                    <div class="w-full lg:w-1/2">
+                        <!-- Right chart section -->
+                        <div class="p-4">
+                            <h2 class="text-xl font-semibold mb-4">Testomatio Run Info</h2>
+                            <dl>
+                                <div class="bg-gray-50 px-4 py-5 sm:grid sm:grid-cols-3 sm:gap-4 sm:px-6">
+                                    <dt class="text-sm font-medium text-gray-500">
+                                        RUN Status
+                                    </dt>
+                                    <dd class="mt-1 text-sm text-gray-900 sm:mt-0 sm:col-span-2">
+                                        <span class="inline-flex items-center justify-center px-2 py-0.5 text-xs font-bold leading-none bg-green-200 text-green-600 rounded-full">
+                                            {{status}}
+                                        </span>
+                                    </dd>
+                                    </div>
+                                    <div class="bg-gray-50 px-4 py-5 sm:grid sm:grid-cols-3 sm:gap-4 sm:px-6">
+                                    <dt class="text-sm font-medium text-gray-500">
+                                        Duration
+                                    </dt>
+                                    <dd class="mt-1 text-sm text-gray-900 sm:mt-0 sm:col-span-2">
+                                        <span>
+                                            <i class="fa fa-clock-o"></i>
+                                        </span>
+                                        {{executionTime}}
+                                    </dd>
+                                    </div>
+                                </div>
+                            </dl>
+                        </div>
+                    </div>
+                </div>
+            </div>
+            
+            <div class="px-4 py-5 lg:px-20 lg:py-5">
+                <!-- TODO: maybe add a search section by test NAME!! -->
+                <div class="px-4 py-5 lg:px-20 lg:py-5 border rounded-md mb-4">
+                    <div class="mb-2">
+                        <h3 class="text-2xl leading-10 font-medium text-gray-900">
+                            Search and Summary
+                        </h3>
+                    </div>
+                    <div class="flex items-center space-x-4">
+                        <div class="relative">
+                            <input type="text" id="search-input" placeholder="Search by test title..." class="border rounded-md py-1 px-2 w-48 focus:outline-none focus:ring focus:border-blue-300">
+
+                        </div>
+                        <div class="test-status">
+                            <div class="flex items-center">
+                                <span class="font-medium mr-2">All:</span>
+                                <span class="text-blue-600 font-semibold">{{tests.length}}</span>
+                                <span class="font-medium ml-4 mr-2">Passed:</span>
+                                <span class="text-green-600 font-semibold">{{getTestsByStatus tests "PASSED"}}</span>
+                                <span class="font-medium ml-4 mr-2">Failed:</span>
+                                <span class="text-red-600 font-semibold">{{getTestsByStatus tests "FAILED"}}</span>
+                                <span class="font-medium ml-4 mr-2">Skipped:</span>
+                                <span class="text-yellow-600 font-semibold">{{getTestsByStatus tests "SKIPPED"}}</span>
+                            </div>
+                        </div>
+                        
+                    </div>
+                </div>
+
+                <ul class="accordionjs bg-gray-50 rounded-md divide-y divide-gray-200 text-gray-900">
+                    <li class="pl-3 pr-4 py-3 text-sm cursor-pointer">
+                        <h4 class="text-lg flex items-center space-x-1">
+                            <span class="font-medium">Passed</span>
+                            <span class="inline-flex items-center justify-center px-2 py-0.5 text-xs font-bold leading-none bg-green-200 text-green-600 rounded-full">
+                            {{getTestsByStatus tests "PASSED"}}
+                            </span>
+                        </h4>
+                        <ul role="list" class="pl-2 divide-y divide-gray-200">
+                            {{#each tests}}
+                                <ul role="list" class="pl-2 divide-y divide-gray-200">                                
+                                    {{#eq status "passed"}}
+                                        <li class="py-3 flex justify-between items-center">
+                                            <div>
+                                                <div class="success"></div>
+                                                <a class="testrun hover:underline" data-toggle="test-{{test_id}}" data-target="test-code-{{test_id}}">
+                                                    {{title}} title
+                                                </a>                                     
+                                            </div>
+                                            <dd class="mt-1 text-sm text-gray-900 sm:mt-0 sm:col-span-2">
+                                                <span>
+                                                    <i class="fa fa-clock-o"></i>
+                                                </span>
+                                                {{#if run_time}}
+                                                    {{run_time}}ms
+                                                {{/if}}
+                                            </dd>
+                                        </li>
+                                        <li class="py-3 test-code-container test-code" id="test-code-{{test_id}}">
+                                            <pre>
+                                                <code>
+                                                    {{steps}}
+                                                </code>
+                                            </pre>
+                                        </li>
+                                    {{/eq}}
+                                </ul> 
+                            {{/each}}                               
+                        </ul>                            
+                    </li>
+
+                    <li class="pl-3 pr-4 py-3 text-sm cursor-pointer">
+                        <h4 class="text-lg flex items-center space-x-1">
+                            <span class="font-medium">Failed</span>
+                            <span class="inline-flex items-center justify-center px-2 py-0.5 text-xs font-bold leading-none bg-red-200 text-red-500 rounded-full">
+                            {{getTestsByStatus tests "FAILED"}}
+                            </span>
+                        </h4>
+                
+                        <ul role="list" class="pl-2 divide-y divide-gray-200">
+                            {{#each tests}}
+                                {{#eq status "failed"}}
+                                    <li class="py-3 flex justify-between items-center">
+                                            <div>
+                                                <div class="fail"></div>
+                                                <a class="testrun hover:underline" data-toggle="test-{{test_id}}" data-target="test-code-{{test_id}}">
+                                                    {{title}} title
+                                                </a>                                     
+                                            </div>
+                                            <dd class="mt-1 text-sm text-gray-900 sm:mt-0 sm:col-span-2">
+                                                <span>
+                                                    <i class="fa fa-clock-o"></i>
+                                                </span>
+                                                {{#if run_time}}
+                                                    {{run_time}}ms
+                                                {{/if}}
+                                            </dd>
+                                        </li>
+                                        <li class="py-3 test-code-container test-code" id="test-code-{{test_id}}">
+                                            <pre>
+                                                <code>
+                                                    {{steps}}
+                                                </code>
+                                            </pre>
+                                        </li>
+                                {{/eq}}
+                            {{/each}}
+                        </ul>
+                    </li>
+                </ul>
+            </div>
+        </div>
+        <div
+            class="fixed bg-white border-t inset-x-0 bottom-0 mx-auto py-1 px-3 sm:px-6 lg:px-8 flex items-center justify-between flex-wrap">
+            <p class="ml-3 text-gray-500 text-xs lg:truncate">
+                This is a public report generated by <a class="hover:underline text-gray-700 font-semibold"
+                    href="https://testomat.io" target="_blank">Testomat.io</a>. Everyone who has accesse to this link can
+                see this report.<br>
+            </p>
+        </div>
+
+    </body>
+    <!-- Jquery from a CDN -->
+    <script type="module" src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/4.3.2/chart.min.js"></script>
+    <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.2.1/jquery.min.js"></script>
+    <script src="https://cdn.jsdelivr.net/npm/accordionjs@2.1.2/accordion.min.js"></script>
+    <!-- Chart.js from a CDN -->
+    <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
+
+    <!-- Include Handlebars from a CDN -->
+    <script src="https://cdn.jsdelivr.net/npm/handlebars@latest/dist/handlebars.js"></script>
+    <!-- Handlebars tamplate actions -->
+    <script>
+        $(document).ready(function ($) {
+            $(".accordionjs").accordionjs();
+
+            $('#copy-url').on('click', () => {
+                navigator.clipboard.writeText(window.location.href);
+            })
+        });
+        //Open code section handler
+        document.querySelectorAll('.test-code').forEach(element => {
+            element.style.display = 'none';
+        });
+        const testrunLinks = document.querySelectorAll('.testrun');
+
+        testrunLinks.forEach(link => {
+            link.addEventListener('click', function () {
+                // get "data-toggle" & "data-target" from link env
+                const toggleId = this.getAttribute('data-toggle');
+                const targetId = this.getAttribute('data-target');
+                
+                // get "tests.code" elements and change view
+                const targetElement = document.querySelector(`#${targetId}`);
+                
+                // SET "tests.code" viewing after each click
+                if (targetElement.style.display === 'none') {
+                    targetElement.style.display = 'block';
+                } else {
+                    targetElement.style.display = 'none';
+                }
+            });
+        });
+    </script>
+    <script type="text/javascript" src="./chart-my.js"></script>
+</html>