@testomatio/reporter 1.1.0-beta-3 → 1.1.0-beta.label-assign

package/Changelog.md

@@ -1,4 +1,16 @@
 <!-- pending release updates -->
+# 1.1.0
+
+* Assign Run by label:
+
+```
+TESTOMATIO={API_KEY} TESTOMATIO_LABEL="release,module:checkout" <actual run command>
+```
+
+# 1.0.18
+
+* Fixed stack traces for CodeceptJS
+
 # 1.0.17
 
 Renamed `TESTOMATIO_STACK_FILTER` to `TESTOMATIO_STACK_IGNORE`

package/lib/adapter/codecept.js

@@ -147,9 +147,6 @@ function CodeceptReporter(config) {
     failedTests.push(id || title);
     let testId = parseTest(tags);
     const testObj = getTestAndMessage(title);
-    if (error && error.stack && test.steps && test.steps.length) {
-      error.stack = test.steps[test.steps.length - 1].line();
-    }
 
     const files = [];
     if (artifacts.screenshot) files.push({ path: artifacts.screenshot, type: 'image/png' });

package/lib/adapter/cypress-plugin/index.js

@@ -10,6 +10,7 @@ const testomatioReporter = on => {
   const client = new TestomatClient({ apiKey: process.env.TESTOMATIO });
 
   on('before:run', async (run) => {
+    // TODO: looks like client.env does not exist
     if (!client.env) {
       client.env = `${run.browser.displayName},${run.system.osName}`
     }
@@ -48,10 +49,16 @@ const testomatioReporter = on => {
         }
       }
 
-      const screenshots = results.screenshots
-        .filter(screenshot => screenshot.path.includes(title))
-        .filter(screenshot => screenshot.testAttemptIndex === lastAttemptIndex)
-        .map(screenshot => screenshot.path);
+      const screenshots = Array.isArray(results.screenshots)
+        ? results.screenshots
+            .filter(
+              screenshot =>
+                screenshot?.path &&
+                screenshot?.path.includes(title) &&
+                screenshot?.takenAt
+            )
+            .map(screenshot => screenshot.path)
+        : [];
 
       const files = [...videos, ...screenshots];
 

package/lib/bin/startTest.js

@@ -53,6 +53,12 @@ program
 
     let exitCode = 0;
 
+    if (!command.split) {
+      process.exitCode = 255;
+      console.log(APP_PREFIX, `No command provided. Use -c option to launch a test runner.`);
+      return;
+    }
+
     const client = new TestomatClient({ apiKey, title, parallel: true });
 
     if(filter) {
@@ -74,12 +80,6 @@ program
       }
     }
 
-    if (!command.split) {
-      process.exitCode = 255;
-      console.log(APP_PREFIX, `No command provided. Use -c option to launch a test runner.`);
-      return;
-    }
-
     const testCmds = command.split(' ');
     console.log(APP_PREFIX, `🚀 Running`, chalk.green(command));
 

package/lib/config.js

@@ -0,0 +1,24 @@
+// This file is used to read environment variables from .env file and process.env
+
+// ! uncommenting next line leads ro reading vars from .env file
+// require('dotenv').config();
+const debug = require('debug')('@testomatio/reporter:config');
+
+/* for possibility to use multiple env files (reading different paths)
+const dotenv = require('dotenv');
+const envFileVars = dotenv.config({ path: '.env' }).parsed; */
+
+// select only TESTOMATIO related variables (only to print them in debug)
+const testomatioEnvVars =
+  Object.keys(process.env)
+    .filter(key => key.startsWith('TESTOMATIO') || key.startsWith('S3_'))
+    .reduce((obj, key) => {
+      obj[key] = process.env[key];
+      return obj;
+    }, {}) || {};
+debug('TESTOMATIO variables:', testomatioEnvVars);
+
+// includes variables from .env file and process.env
+const config = process.env;
+
+module.exports = config;

package/lib/constants.js

@@ -20,10 +20,17 @@ const STATUS = {
   SKIPPED: 'skipped',
   FINISHED: 'finished',
 };
+// html pipe var
+const HTML_REPORT = {
+  FOLDER: "html-report",
+  REPORT_DEFAULT_NAME: "testomatio-report.html",
+  TEMPLATE_NAME: 'testomatio.hbs'
+};
 
 module.exports = {
   APP_PREFIX,
   TESTOMAT_TMP_STORAGE_DIR,
   CSV_HEADERS,
   STATUS,
+  HTML_REPORT
 }

package/lib/fileUploader.js

@@ -3,10 +3,16 @@ const { S3 } = require('@aws-sdk/client-s3');
 const { Upload } = require('@aws-sdk/lib-storage');
 
 const fs = require('fs');
+const util = require('util');
 const path = require('path');
+const promiseRetry = require('promise-retry');
+
+const readFile = util.promisify(fs.readFile);
+const stat = util.promisify(fs.stat);
 const chalk = require('chalk');
 const { randomUUID } = require('crypto');
 const memoize = require('lodash.memoize');
+
 const { APP_PREFIX } = require('./constants');
 
 const keys = [
@@ -93,51 +99,64 @@ const uploadUsingS3 = async (filePath, runId) => {
     Key = filePath.name;
   }
 
-  if (!fs.existsSync(filePath)) {
-    console.error(chalk.yellow(`Artifacts file ${filePath} does not exist. Skipping...`));
-    return;
-  }
-
   const {
-    TESTOMATIO_PRIVATE_ARTIFACTS, S3_BUCKET
+    TESTOMATIO_PRIVATE_ARTIFACTS,
+    S3_BUCKET
   } = getConfig();
 
-  debug('S3 config', getMaskedConfig());
-  debug('Uploading', filePath, 'to', S3_BUCKET);
-  
-const fileData = fs.readFileSync(filePath);
+  try {
+    debug('S3 config', getMaskedConfig());
+    debug('Started upload', filePath, 'to ', S3_BUCKET);
 
-  Key = `${runId}/${randomUUID()}-${Key || path.basename(filePath)}`;
-  const ACL = TESTOMATIO_PRIVATE_ARTIFACTS ? 'private' : 'public-read';
+    // Verification that the file was actually created: 20 attempts of 0.5 second => 10sec
+    const isFileExist = await checkFileExists(filePath, 20, 500);
 
-  const s3 = new S3(_getS3Config());
+    if (!isFileExist) {
+      console.error(chalk.yellow(`Artifacts file ${filePath} does not exist. Skipping...`));
+      return;
+    }
 
-  const params = {
-    Bucket: S3_BUCKET,
-    Key,
-    Body: fileData,
-    ContentType,
-    ACL,
-  };
+    debug('File: ', filePath, ' exists');
+    
+    const fileData = await readFile(filePath);
 
-  try {
+    Key = `${runId}/${randomUUID()}-${Key || path.basename(filePath)}`;
+
+    const ACL = TESTOMATIO_PRIVATE_ARTIFACTS ? 'private' : 'public-read';
+
+    if (!S3_BUCKET || !fileData) {
+      console.log(
+        APP_PREFIX,
+        chalk.bold.red(`Failed uploading '${Key}'. Please check S3 credentials`), getMaskedConfig());
+      return;
+    }
+
+    const s3 = new S3(_getS3Config());
+
+    const params = {
+      Bucket: S3_BUCKET,
+      Key,
+      Body: fileData,
+      ContentType,
+      ACL,
+    };
+  
     const out = new Upload({
       client: s3,
       params
     });
 
-    await out.done();
-    debug('Uploaded', out.singleUploadResult.Location)
-
-    return out.singleUploadResult.Location;
-  } catch (e) {
-    console.log(e);
-    console.log(APP_PREFIX, chalk.red(`Failed uploading '${Key}'. Please check S3 credentials`), getMaskedConfig());
-
+    return await getS3LocationLink(out);
+  } 
+  catch (e) {
+    debug('S3 file uploading error: ', e);
+   
     console.log(APP_PREFIX, `To ${chalk.bold('disable')} artifact uploads set: TESTOMATIO_DISABLE_ARTIFACTS=1`);
+
     if (!TESTOMATIO_PRIVATE_ARTIFACTS) {
       console.log(APP_PREFIX, `To enable ${chalk.bold('PRIVATE')} uploads set: TESTOMATIO_PRIVATE_ARTIFACTS=1`);
-    } else {
+    } 
+    else {
       console.log(
         APP_PREFIX,
         `To enable ${chalk.bold('PUBLIC')} uploads remove TESTOMATIO_PRIVATE_ARTIFACTS env variable`,
@@ -148,7 +167,6 @@ const fileData = fs.readFileSync(filePath);
 };
 
 const uploadUsingS3AsBuffer = async (buffer, fileName, runId) => {
-
   const {
     S3_REGION, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_ENDPOINT, TESTOMATIO_PRIVATE_ARTIFACTS, S3_BUCKET
   } = getConfig();
@@ -158,6 +176,18 @@ const uploadUsingS3AsBuffer = async (buffer, fileName, runId) => {
   const fileExtension = _getFileExtBase64(buffer.toString('base64'));
   const Key = `${runId}/${fileName}${fileExtension}`;
 
+  if (!S3_BUCKET || !buffer) {
+    console.log(APP_PREFIX, chalk.bold.red(`Failed uploading '${Key}'. Please check S3 credentials`), {
+      accessKeyId: S3_ACCESS_KEY_ID,
+      secretAccessKey: S3_SECRET_ACCESS_KEY ? '**** (hidden) ***' : '(empty)',
+      region: S3_REGION,
+      bucket: S3_BUCKET,
+      acl: ACL,
+      endpoint: S3_ENDPOINT,
+    });
+    return;
+  }
+
   const s3 = new S3(_getS3Config());
 
   try {
@@ -171,20 +201,14 @@ const uploadUsingS3AsBuffer = async (buffer, fileName, runId) => {
         ACL,
       }
     });
-    await out.done();
 
-    return out.singleUploadResult.Location;
-  } catch (e) {
-    console.log(APP_PREFIX, chalk.bold.red(`Failed uploading '${Key}'. Please check S3 credentials`), {
-      accessKeyId: S3_ACCESS_KEY_ID,
-      secretAccessKey: S3_SECRET_ACCESS_KEY ? '**** (hidden) ***' : '(empty)',
-      region: S3_REGION,
-      bucket: S3_BUCKET,
-      acl: ACL,
-      endpoint: S3_ENDPOINT,
-    });
+    return await getS3LocationLink(out);
+  } 
+  catch (e) {
+    debug('S3 buffer uploading error: ', e);
 
     console.log(APP_PREFIX, `To ${chalk.bold('disable')} artifact uploads set: TESTOMATIO_DISABLE_ARTIFACTS=1`);
+
     if (!TESTOMATIO_PRIVATE_ARTIFACTS) {
       console.log(APP_PREFIX, `To enable ${chalk.bold('PRIVATE')} uploads set: TESTOMATIO_PRIVATE_ARTIFACTS=1`);
     } else {
@@ -203,7 +227,9 @@ const uploadFileByPath = async (filePath, runId) => {
       return uploadUsingS3(filePath, runId);
     }
   } catch (e) {
-    console.error(chalk.red('Error occurred while uploading artifacts'), e);
+    debug(e);
+
+    console.error(chalk.red('Error occurred while uploading artifacts! '), e);
   }
 };
 
@@ -213,10 +239,61 @@ const uploadFileAsBuffer = async (buffer, fileName, runId) => {
       return uploadUsingS3AsBuffer(buffer, fileName, runId);
     }
   } catch (e) {
-    console.error(chalk.red('Error occurred while uploading artifacts'), e);
+    debug(e);
+
+    console.error(chalk.red('Error occurred while uploading artifacts! '), e);
   }
 };
 
+const checkFileExists = async (filePath, attempts = 5, intervalMs = 500) => {
+  const checkFile = async () => {
+    const fileStats = await stat(filePath);
+    if (fileStats.isFile()) {
+      return true;
+    }
+
+    throw new Error('File not found');
+  };
+
+  try {
+    await promiseRetry(
+      { 
+        retries: attempts,
+        minTimeout: intervalMs 
+      },
+      checkFile
+    );
+
+    return true;
+  } catch (err) {
+    console.error(
+      chalk.yellow(`File ${filePath} was not found or did not have time to be generated...`)
+    );
+
+    return false;
+  }
+};
+
+const getS3LocationLink = async (out) => {
+  const response = await out.done();
+
+  let s3Location = response?.Location;
+
+  debug('Uploaded response.Location', s3Location);
+
+  if (!s3Location) {
+    // TODO: out: a fallback case  - remove after deeper testing
+    s3Location = out?.singleUploadResult?.Location;
+    debug('Uploaded singleUploadResult.Location', s3Location);
+
+    if (!s3Location) {
+      throw new Error("Problems getting the S3 artifact's link. Please check S3 permissions!");
+    }
+  }
+
+  return s3Location;
+};
+
 module.exports = {
   uploadFileByPath: memoize(uploadFileByPath),
   uploadFileAsBuffer: memoize(uploadFileAsBuffer),

package/lib/pipe/gitlab.js

@@ -23,7 +23,7 @@ class GitLabPipe {
     this.store = store;
     this.tests = [];
     // GitLab PAT looks like glpat-nKGdja3jsG4850sGksh7
-    this.token = params.GITLAB_PAT || this.ENV.GITLAB_PAT;
+    this.token = params.GITLAB_PAT || process.env.GITLAB_PAT || this.ENV.GITLAB_PAT;
     this.hiddenCommentData = `<!--- testomat.io report ${process.env.CI_JOB_NAME || ''} -->`;
 
     debug(

package/lib/pipe/html.js

@@ -0,0 +1,316 @@
+const debug = require('debug')('@testomatio/reporter:pipe:html');
+const merge = require('lodash.merge');
+const fs = require('fs');
+const path = require('path');
+const chalk = require('chalk');
+const handlebars = require('handlebars');
+const fileUrl = require('file-url');
+
+const { fileSystem, isSameTest, ansiRegExp } = require('../utils/utils');
+const { HTML_REPORT } = require('../constants');
+
+class HtmlPipe {
+    constructor(params, store = {}) {
+        this.store = store || {};
+        this.title = params.title || process.env.TESTOMATIO_TITLE;
+        this.apiKey = params.apiKey || process.env.TESTOMATIO;
+        this.isHtml = process.env.TESTOMATIO_HTML_REPORT_SAVE;
+
+        debug('HTML Pipe: ', this.apiKey ? 'API KEY' : '*no api key provided*');
+
+        this.isEnabled = false;
+        this.htmlOutputPath = "";
+        this.fullHtmlOutputPath = "";
+        this.filenameMsg = "";
+        this.tests = [];
+
+        if (this.isHtml) {
+            this.isEnabled = true;
+            this.htmlReportDir = process.env.TESTOMATIO_HTML_REPORT_FOLDER || HTML_REPORT.FOLDER;
+
+            if (process.env.TESTOMATIO_HTML_FILENAME && process.env.TESTOMATIO_HTML_FILENAME.endsWith(".html")) {
+                this.htmlReportName = process.env.TESTOMATIO_HTML_FILENAME
+            }
+
+            if (process.env.TESTOMATIO_HTML_FILENAME && !process.env.TESTOMATIO_HTML_FILENAME.endsWith(".html")) {
+                this.htmlReportName = HTML_REPORT.REPORT_DEFAULT_NAME;
+                this.filenameMsg = "HTML filename must include the extension \".html\"." +
+                    ` The default report name "${this.htmlReportDir}/${HTML_REPORT.REPORT_DEFAULT_NAME}" is used!`;
+            }
+
+            if (!process.env.TESTOMATIO_HTML_FILENAME) {
+                this.htmlReportName = HTML_REPORT.REPORT_DEFAULT_NAME;
+            }
+
+            this.templateFolderPath = path.resolve(__dirname, '..', 'template');
+            this.templateHtmlPath = path.resolve(this.templateFolderPath, HTML_REPORT.TEMPLATE_NAME);
+            this.htmlOutputPath = path.join(this.htmlReportDir, this.htmlReportName);
+            // create a new folder for the HTML reports
+            fileSystem.createDir(this.htmlReportDir);
+
+            debug(
+                chalk.yellow('HTML Pipe:'),
+                `Save HTML report: ${this.isEnabled}`,
+                `HTML report folder: ${this.htmlReportDir}, report name: ${this.htmlReportName}`
+            );
+        }
+    }
+
+    async createRun() {
+        // empty
+    }
+
+    updateRun() {
+        // empty
+    }
+
+    /**
+     * Add test data to the result array for saving. As a result of this function, we get a result object to save.
+     * @param {Object} test - object which includes each test entry.
+     */
+    addTest(test) {
+        if (!this.isEnabled) return;
+
+        if (!test.steps || !test.status) return;
+
+        const index = this.tests.findIndex(t => isSameTest(t, test));
+        // update if they were already added
+        if (index >= 0) {
+            this.tests[index] = merge(this.tests[index], test);
+            return;
+        }
+
+        this.tests.push(test);
+    }
+
+    async finishRun(runParams) {
+        if (!this.isEnabled) return;
+
+        if (this.isHtml) {
+            // GENERATE HTML reports based on the results data
+            this.buildReport({
+                runParams,
+                tests: this.tests, 
+                outputPath: this.htmlOutputPath, 
+                templatePath: this.templateHtmlPath, 
+                warningMsg: this.filenameMsg
+            });
+        }
+    }
+    /**
+     * Generates an HTML report based on provided test data and a template.
+     * @param {object} opts - Test options used to generate the HTML report: 
+     * runParams, tests, outputPath, templatePath
+     * @returns {void} - This function does not return anything.
+     */
+
+    buildReport(opts) {
+        const { runParams, tests, outputPath, templatePath, warningMsg: msg } = opts;
+        
+        debug('HTML tests data:', tests);
+
+        if (!outputPath) {
+            console.log(chalk.yellow(`🚨 HTML export path is not set, ignoring...`));
+            return;
+        }
+
+        console.log(chalk.yellow(`⏳ The test results will be added to the HTML report. It will take some time...`));
+
+        if (msg) {
+            console.log(chalk.blue(msg));
+        }
+
+        tests.forEach(test => {
+
+            if (!test.message || test.message.trim() === "") {
+                test.message = "This test has no 'message' code";
+            }
+
+            if (!test.suite_title || test.suite_title.trim() === "") {
+                test.suite_title = "Unknown suite";
+            }
+
+            if (!test.title || test.title.trim() === "") {
+                test.title = "Unknown test title";
+            }
+
+            if (!test.files || test.files.length === 0) {
+                test.files = "This test has no files";
+            }
+
+            if (test.steps) {
+                if (!test.steps || test.steps.trim() === "") {
+                    test.steps = "This test has no 'steps' code";
+                }
+                else {
+                    test.steps = removeAnsiColorCodes(test.steps);
+                }
+            }
+            
+            // TODO: u can added an additional test values to this checks in the future
+        });
+
+        const data = {
+            runId: this.store.runId || "",
+            status: runParams.status || "No status info",
+            parallel: runParams.isParallel || "No parallel info",
+            runUrl: this.store.runUrl || "",
+            executionTime: testExecutionSumTime(tests),
+            executionDate: getCurrentDateTimeFormatted(),
+            tests
+        };
+        // generate output HTML based on the template
+        const html = this.#generateHTMLReport(data, templatePath);
+
+        if (!html) return;
+
+        fs.writeFileSync(outputPath, html, 'utf-8');
+        // Check if the file exists
+        if (fs.existsSync(outputPath)) {
+            // Get the absolute path of the file
+            const absolutePath = path.resolve(outputPath);
+            // Convert the file path to a file URL
+            const fileUrlPath = fileUrl(absolutePath, {resolve: true});
+
+            debug('HTML tests data:', fileUrlPath);
+
+            console.log(chalk.green(`📊 The HTML report was successfully generated. Full filepath: ${fileUrlPath}`));
+        } else {
+            console.log(chalk.red(`🚨 Failed to generate the HTML report.`));
+        }
+    }
+
+    /**
+     * Generates an HTML report based on provided test data and a template path.
+     * @param {any} data - Test data used to generate the HTML report.
+     * @param {string} [templatePath=""] - The path to the HTML template used for generating the report.
+     * @returns {string | void} - The generated HTML report as a string or void if templatePath is not provided.
+     */
+    #generateHTMLReport(data, templatePath = "") {
+        if (!templatePath) {
+            console.log(chalk.red(`🚨 HTML template not found. Report generation is impossible!`))
+            return;
+        }
+
+        const templateSource = fs.readFileSync(templatePath, 'utf8');
+        this.#loadReportHelpers();
+        try {
+            const template = handlebars.compile(templateSource);
+
+            return template(data);
+        }
+        catch (e) {
+            console.log('Unknown HTML report generation error: ', e);
+        }
+    }
+
+    #loadReportHelpers() {
+        handlebars.registerHelper('getTestsByStatus', (tests, status) => 
+            tests.filter(test => test.status.toLowerCase() === status.toLowerCase()).length
+        );
+
+        handlebars.registerHelper('json', (tests) => {
+            function replaceScriptTagsInArray(array) {
+                return array.map(obj => {
+                    const keysToCheck = ["steps", "stack", "title", "suite_title", "message", "code"];
+                    const newObj = {};
+            
+                    for (const key in obj) {
+                        if (Object.prototype.hasOwnProperty.call(obj, key)) {
+                            if (key === "example") {
+                                newObj[key] = {};
+                                for (const subKey in obj[key]) {
+                                    if (Object.prototype.hasOwnProperty.call(obj[key], subKey)) {
+                                        newObj[key][subKey] = typeof obj[key][subKey] === "string"
+                                            ? obj[key][subKey]
+                                                .replace(/<script>/g, "<$cript>")
+                                                .replace(/<\/script>/g, "</$cript>")
+                                            : obj[key][subKey];
+                                    }
+                                }
+                            } else if (keysToCheck.includes(key)) {
+                                newObj[key] = typeof obj[key] === "string"
+                                    ? obj[key].replace(/<script>/g, "<$cript>").replace(/<\/script>/g, "</$cript>")
+                                    : obj[key];
+                            } else {
+                                newObj[key] = obj[key];
+                            }
+                        }
+                    }
+            
+                    return newObj;
+                });
+            }
+
+            // Remove ANSI escape codes
+            return JSON.stringify(replaceScriptTagsInArray(tests));
+        });
+    }
+
+    toString() {
+        return 'HTML Reporter';
+    }
+}
+
+/**
+ * Calculates the total execution time for an array of tests.
+ * @param {Object[]} tests - An array of test objects.
+ * @param {number} tests[].run_time - The execution time of each test in milliseconds.
+ * @returns {string} - The total execution time in a formatted duration string.
+ */
+function testExecutionSumTime(tests) {
+    const totalMilliseconds = tests.reduce((sum, test) => {
+        if (typeof test.run_time === 'number') {
+            return sum + test.run_time;
+        }
+        return sum;
+    }, 0);
+
+    return formatDuration(totalMilliseconds);
+}
+
+/**
+ * Removes ANSI color codes and converts newline characters to HTML line breaks in a given string.
+ * @param {string} str - The input string containing ANSI color codes.
+ * @returns {string} - The updated string with removed ANSI color codes and replaced newline characters.
+ */
+function removeAnsiColorCodes(str) {
+    let updatedStr = str.replace(ansiRegExp(), "");        
+    updatedStr = updatedStr.replace(/\n/g, '<br>');
+
+    return updatedStr;
+}
+
+/**
+ * Formats duration in milliseconds into a human-readable string representation.
+ * @param {number} duration - The duration in milliseconds.
+ * @returns {string} - The formatted duration string (e.g., "2h 30m 15s 500ms").
+ */
+function formatDuration(duration) {
+    const milliseconds = duration % 1000;
+    duration = (duration - milliseconds) / 1000;
+    const seconds = duration % 60;
+    duration = (duration - seconds) / 60;
+    const minutes = duration % 60;
+    const hours = (duration - minutes) / 60;
+
+    return `${hours}h ${minutes}m ${seconds}s ${milliseconds}ms`;
+}
+
+/**
+ * Retrieves the current date and time in a formatted string.
+ * @returns {string} - The formatted date and time string (e.g., "(01/01/2023 12:00:00)").
+ */
+function getCurrentDateTimeFormatted() {
+    const currentDate = new Date();
+    const day = currentDate.getDate().toString().padStart(2, '0');
+    const month = (currentDate.getMonth() + 1).toString().padStart(2, '0');
+    const year = currentDate.getFullYear();
+    const hours = currentDate.getHours().toString().padStart(2, '0');
+    const minutes = currentDate.getMinutes().toString().padStart(2, '0');
+    const seconds = currentDate.getSeconds().toString().padStart(2, '0');
+
+    return `(${day}/${month}/${year} ${hours}:${minutes}:${seconds})`;
+}
+
+module.exports = HtmlPipe;