@testomatio/reporter 1.2.0-beta-3 → 1.2.0

package/lib/client.js

@@ -1,13 +1,17 @@
 const debug = require('debug')('@testomatio/reporter:client');
 const createCallsiteRecord = require('callsite-record');
 const { sep, join } = require('path');
-const { minimatch } = require('minimatch')
+const { minimatch } = require('minimatch');
 const fs = require('fs');
 const chalk = require('chalk');
 const { randomUUID } = require('crypto');
 const upload = require('./fileUploader');
 const { APP_PREFIX } = require('./constants');
 const pipesFactory = require('./pipe');
+const { glob } = require('glob');
+const path = require('path');
+
+let listOfTestFilesToExcludeFromReport = null;
 
 /**
  * @typedef {import('../types').TestData} TestData
@@ -27,6 +31,7 @@ class Client {
     this.pipes = pipesFactory(params, store);
     this.queue = Promise.resolve();
     this.totalUploaded = 0;
+    this.failedToUpload = 0;
     this.version = JSON.parse(fs.readFileSync(join(__dirname, '..', 'package.json')).toString()).version;
     this.executionList = Promise.resolve();
 
@@ -35,7 +40,7 @@ class Client {
 
   /**
    * Asynchronously prepares the execution list for running tests through various pipes.
-   * Each pipe in the client is checked for enablement, 
+   * Each pipe in the client is checked for enablement,
    * and if all pipes are disabled, the function returns a resolved Promise.
    * Otherwise, it executes the `prepareRun` method for each enabled pipe and collects the results.
    * The results are then filtered to remove any undefined values.
@@ -45,7 +50,7 @@ class Client {
    * @param {Object} params - The options for preparing the test execution list.
    * @param {string} params.pipe - Name of the executed pipe.
    * @param {string} params.pipeOptions - Filter option.
-   * @returns {Promise<any>} - A Promise that resolves to an 
+   * @returns {Promise<any>} - A Promise that resolves to an
    * array containing the prepared execution list,
    * or resolves to undefined if no valid results are found or if all pipes are disabled.
    */
@@ -62,22 +67,22 @@ class Client {
       if (!filterPipe.isEnabled) {
         // TODO:for the future for the another pipes
         console.warn(
-          APP_PREFIX, 
-          `At the moment processing is available only for the "testomatio" key. Example: "testomatio:tag-name=xxx"`
+          APP_PREFIX,
+          `At the moment processing is available only for the "testomatio" key. Example: "testomatio:tag-name=xxx"`,
         );
         return;
       }
 
-      const results = await Promise.all(this.pipes.map(async p => 
-        ({ pipe: p.toString(), result: await p.prepareRun(pipeOptions) })  
-      ));
-    
+      const results = await Promise.all(
+        this.pipes.map(async p => ({ pipe: p.toString(), result: await p.prepareRun(pipeOptions) })),
+      );
+
       const result = results.filter(p => p.pipe.includes('Testomatio'))[0]?.result;
 
       if (!result || result.length === 0) {
         return;
-      } 
-  
+      }
+
       debug('Execution tests list', result);
 
       return result;
@@ -96,9 +101,6 @@ class Client {
     // all pipes disabled, skipping
     if (!this.pipes?.filter(p => p.isEnabled).length) return Promise.resolve();
 
-    global.testomatioArtifacts = [];
-    if (!global.testomatioDataStore) global.testomatioDataStore = {};
-
     this.queue = this.queue
       .then(() => Promise.all(this.pipes.map(p => p.createRun())))
       .catch(err => console.log(APP_PREFIX, err))
@@ -112,14 +114,14 @@ class Client {
    *
    * @param {string|undefined} status
    * @param {TestData} [testData]
-   * @param {string[]} [storeArtifacts]
    * @returns {Promise<PipeResult[]>}
    */
-  async addTestRun(status, testData, storeArtifacts = []) {
-    debug('Adding test run for test', testData?.test_id || 'unknown test');
+  async addTestRun(status, testData) {
     // all pipes disabled, skipping
     if (!this.pipes?.filter(p => p.isEnabled).length) return [];
 
+    if (isTestShouldBeExculedFromReport(testData)) return [];
+
     if (!testData)
       testData = {
         title: 'Unknown test',
@@ -135,46 +137,31 @@ class Client {
       steps,
       code = null,
       title,
+      file,
       suite_title,
       suite_id,
       test_id,
+      manuallyAttachedArtifacts,
+      meta,
     } = testData;
     let { message = '' } = testData;
 
-    const uploadedFiles = [];
-
-    let stack = '';
-
+    let errorFormatted = '';
     if (error) {
-      stack = this.formatError(error) || '';
+      errorFormatted += this.formatError(error) || '';
       message = error?.message;
     }
-    if (steps) {
-      stack = this.formatSteps(stack, steps);
-    }
-
-    stack += testData.stack || '';
 
-    // in some cases (e.g. using cucumber) logger instance becomes empty object, if import at the top of the file
-    const logger = require('./logger');
-    const testLogs = logger.getLogs(test_id);
-    // debug(`Test logs for ${test_id}:\n`, testLogs);
-    if (stack) stack += '\n\n';
-    stack += testLogs ? `${chalk.bold('Logs:')}\n${testLogs}` : '';
+    // Attach logs
+    const fullLogs = this.formatLogs({ error: errorFormatted, steps, logs: testData.logs });
 
-    if (Array.isArray(storeArtifacts) && storeArtifacts.length) {
-      debug('CLIENT storeArtifact', storeArtifacts);
-      files.push(...storeArtifacts);
-    }
+    // add artifacts
+    if (manuallyAttachedArtifacts?.length) files.push(...manuallyAttachedArtifacts);
 
-    if (Array.isArray(global.testomatioArtifacts)) {
-      debug('CLIENT global[testomatioArtifacts]', global.testomatioArtifacts);
-      files.push(...global.testomatioArtifacts);
-      global.testomatioArtifacts = [];
-    }
+    const uploadedFiles = [];
 
-    for (const file of files) {
-      uploadedFiles.push(upload.uploadFileByPath(file, this.uuid));
+    for (const f of files) {
+      uploadedFiles.push(upload.uploadFileByPath(f, this.uuid));
     }
 
     for (const [idx, buffer] of filesBuffers.entries()) {
@@ -182,18 +169,22 @@ class Client {
       uploadedFiles.push(upload.uploadFileAsBuffer(buffer, fileName, this.uuid));
     }
 
-    const artifacts = await Promise.all(uploadedFiles);
+    const artifacts = (await Promise.all(uploadedFiles)).filter(n => !!n);
 
-    global.testomatioArtifacts = [];
+    if (artifacts.length < uploadedFiles.length) {
+      const failedUploading = uploadedFiles.length - artifacts.length;
+      this.failedToUpload += failedUploading;
+    }
 
-    this.totalUploaded += uploadedFiles.filter(n => n).length;
+    this.totalUploaded += artifacts.length;
 
     const data = {
       files,
       steps,
       status,
-      stack,
+      stack: fullLogs,
       example,
+      file,
       code,
       title,
       suite_title,
@@ -202,16 +193,19 @@ class Client {
       message,
       run_time: parseFloat(time),
       artifacts,
+      meta,
     };
 
+    // debug('Adding test run...', data);
+
     this.queue = this.queue.then(() =>
       Promise.all(
-        this.pipes.map(async p => {
+        this.pipes.map(async pipe => {
           try {
-            const result = await p.addTest(data);
-            return { pipe: p.toString(), result };
+            const result = await pipe.addTest(data);
+            return { pipe: pipe.toString(), result };
           } catch (err) {
-            console.log(APP_PREFIX, p.toString(), err);
+            console.log(APP_PREFIX, pipe.toString(), err);
           }
         }),
       ),
@@ -237,15 +231,27 @@ class Client {
     this.queue = this.queue
       .then(() => Promise.all(this.pipes.map(p => p.finishRun(runParams))))
       .then(() => {
-        debug('TOTAL uploaded files', this.totalUploaded);
+        debug('TOTAL artifacts', this.totalUploaded);
+        if (this.totalUploaded && !upload.isArtifactsEnabled())
+          debug(`${this.totalUploaded} artifacts are not uploaded, because artifacts uploading is not enabled`);
 
-        if (upload.isArtifactsEnabled() && this.totalUploaded > 0) {
+        if (this.totalUploaded && upload.isArtifactsEnabled()) {
           console.log(
             APP_PREFIX,
-            `🗄️  Total ${this.totalUploaded} artifacts ${
+            `🗄️ ${this.totalUploaded} artifacts ${
               process.env.TESTOMATIO_PRIVATE_ARTIFACTS ? 'privately' : chalk.bold('publicly')
-            } uploaded to S3 bucket  `,
+            } uploaded to S3 bucket`,
           );
+
+          if (this.failedToUpload > 0) {
+            console.log(
+              APP_PREFIX,
+              chalk.yellow(
+                `Some artifacts were not uploaded. ${this.failedToUpload} artifacts could not be uploaded.
+                Run tests with DEBUG="@testomatio/reporter:file-uploader" to see details"`,
+              ),
+            );
+          }
         }
       })
       .catch(err => console.log(APP_PREFIX, err));
@@ -253,15 +259,27 @@ class Client {
     return this.queue;
   }
 
-  formatSteps(stack, steps) {
-    return stack ? `${steps}\n\n${chalk.bold.red('################[ Failure ]################')}\n${stack}` : steps;
+  /**
+   * Returns the formatted stack including the stack trace, steps, and logs.
+   * @returns {string}
+   */
+  formatLogs({ error, steps, logs }) {
+    error = error?.trim();
+    steps = steps?.trim();
+    logs = logs?.trim();
+
+    let testLogs = '';
+    if (steps) testLogs += `${chalk.bold.blue('################[ Steps ]################')}\n${steps}\n\n`;
+    if (logs) testLogs += `${chalk.bold.gray('################[ Logs ]################')}\n${logs}\n\n`;
+    if (error) testLogs += `${chalk.bold.red('################[ Failure ]################')}\n${error}`;
+    return testLogs;
   }
 
   formatError(error, message) {
     if (!message) message = error.message;
     if (error.inspect) message = error.inspect() || '';
 
-    let stack = `\n${chalk.bold(message)}\n`;
+    let stack = `${message}\n`;
 
     // diffs for mocha, cypress, codeceptjs style
     if (error.actual && error.expected) {
@@ -278,11 +296,11 @@ class Client {
       const record = createCallsiteRecord({
         forError: error,
         isCallsiteFrame: frame => {
-          if (customFilter && minimatch(frame.getFileName(), customFilter)) return false;          
+          if (customFilter && minimatch(frame.getFileName(), customFilter)) return false;
           if (hasFrame) return false;
           if (isNotInternalFrame(frame)) hasFrame = true;
           return hasFrame;
-        }
+        },
       });
       if (record && !record.filename.startsWith('http')) {
         stack += record.renderSync({ stackFilter: isNotInternalFrame });
@@ -295,9 +313,48 @@ class Client {
 }
 
 function isNotInternalFrame(frame) {
-  return frame.getFileName().includes(sep) &&
+  return (
+    frame.getFileName() &&
+    frame.getFileName().includes(sep) &&
     !frame.getFileName().includes('node_modules') &&
     !frame.getFileName().includes('internal')
+  );
+}
+
+/**
+ *
+ * @param {TestData} testData
+ * @returns boolean
+ */
+function isTestShouldBeExculedFromReport(testData) {
+  // const fileName = path.basename(test.location?.file || '');
+  const globExcludeFilesPattern = process.env.TESTOMATIO_EXCLUDE_FILES_FROM_REPORT_GLOB_PATTERN;
+  if (!globExcludeFilesPattern) return false;
+
+  if (!testData.file) {
+    debug('No "file" property found for test ', testData.title);
+    return false;
+  }
+
+  const excludeParretnsList = globExcludeFilesPattern.split(';');
+
+  // as scanning files is time consuming operation, just save the result in variable to avoid multiple scans
+  if (!listOfTestFilesToExcludeFromReport) {
+    // list of files with relative paths
+    listOfTestFilesToExcludeFromReport = glob.sync(excludeParretnsList, { ignore: '**/node_modules/**' });
+    debug('Tests from next files will not be reported:', listOfTestFilesToExcludeFromReport);
+  }
+
+  const testFileRelativePath = path.relative(process.cwd(), testData.file);
+
+  // no files found matching the exclusion pattern
+  if (!listOfTestFilesToExcludeFromReport.length) return false;
+
+  if (listOfTestFilesToExcludeFromReport.includes(testFileRelativePath)) {
+    debug(`Excluding test '${testData.title}' <${testFileRelativePath}> from reporting`);
+    return true;
+  }
+  return false;
 }
 
 module.exports = Client;

package/lib/config.js

@@ -0,0 +1,34 @@
+// This file is used to read environment variables from .env file
+
+// require('dotenv').config({ path: process.env.TESTOMATIO_ENV_FILE_PATH });
+
+const debug = require('debug')('@testomatio/reporter:config');
+
+/* for possibility to use multiple env files (reading different paths)
+const dotenv = require('dotenv');
+const envFileVars = dotenv.config({ path: '.env' }).parsed; */
+
+if (process.env.TESTOMATIO_API_KEY) {
+  process.env.TESTOMATIO = process.env.TESTOMATIO_API_KEY;
+}
+if (process.env.TESTOMATIO_TOKEN) {
+  process.env.TESTOMATIO = process.env.TESTOMATIO_TOKEN;
+}
+
+if (process.env.TESTOMATIO === 'undefined')
+  console.error('TESTOMATIO is "undefined". Something went wrong. Contact dev team.');
+
+// select only TESTOMATIO related variables (only to print them in debug)
+const testomatioEnvVars =
+  Object.keys(process.env)
+    .filter(key => key.startsWith('TESTOMATIO') || key.startsWith('S3_'))
+    .reduce((obj, key) => {
+      obj[key] = process.env[key];
+      return obj;
+    }, {}) || {};
+debug('TESTOMATIO variables:', testomatioEnvVars);
+
+// includes variables from .env file and process.env
+const config = process.env;
+
+module.exports = config;

package/lib/constants.js

@@ -1,11 +1,12 @@
 const chalk = require('chalk');
+const os = require('os');
+const path = require('path');
 
 const APP_PREFIX = chalk.gray('[TESTOMATIO]');
-const TESTOMAT_ARTIFACT_SUFFIX = "testomatio_artifact_";
+const AXIOS_TIMEOUT = 20 * 1000; // sum = 20sec
+const AXIOS_RETRY_TIMEOUT = 5 * 1000; // sum = 5sec
 
-const TESTOMAT_TMP_STORAGE = {
-  mainDir: "testomatio_tmp",
-}
+const TESTOMAT_TMP_STORAGE_DIR = path.join(os.tmpdir(), 'testomatio_tmp');
 
 const CSV_HEADERS = [
   { id: 'suite_title', title: 'Suite_title' },
@@ -30,9 +31,10 @@ const HTML_REPORT = {
 
 module.exports = {
   APP_PREFIX,
-  TESTOMAT_ARTIFACT_SUFFIX,
-  TESTOMAT_TMP_STORAGE,
+  TESTOMAT_TMP_STORAGE_DIR,
   CSV_HEADERS,
   STATUS,
-  HTML_REPORT
+  HTML_REPORT,
+  AXIOS_TIMEOUT,
+  AXIOS_RETRY_TIMEOUT
 }

package/lib/data-storage.js

@@ -0,0 +1,203 @@
+const debug = require('debug')('@testomatio/reporter:storage');
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const { join } = require('path');
+const { TESTOMAT_TMP_STORAGE_DIR } = require('./constants');
+const { fileSystem, testRunnerHelper } = require('./utils/utils');
+const crypto = require('crypto');
+
+class DataStorage {
+  static #instance;
+
+  context;
+
+  /**
+   *
+   * @returns {DataStorage}
+   */
+  static getInstance() {
+    if (!this.#instance) {
+      this.#instance = new DataStorage();
+    }
+    return this.#instance;
+  }
+
+  setContext(context) {
+    this.context = context;
+  }
+
+  /**
+   * Creates data storage instance as singleton
+   * Stores data to global variable or to file depending on what is applicable for current test runner (adapter)
+   * Recommend to use composition while using this class (instead of inheritance).
+   * ! Also the class which will use data storage should be singleton (to avoid data loss).
+   */
+  constructor() {
+    // some frameworks use global variable to store data, some use file storage
+    this.isFileStorage = true;
+  }
+
+  /**
+   * Puts any data to storage (file or global variable).
+   * If file: stores data as text, if global variable – stores as array of data.
+   * @param {'log' | 'artifact' | 'keyvalue'} dataType
+   * @param {*} data anything you want to store (string, object, array, etc)
+   * @param {*} context could be testId or any context (test name, suite name, including their IDs etc)
+   * suite name + test name is used by default
+   * @returns
+   */
+  putData(dataType, data, context = null) {
+    if (!dataType || !data) return;
+
+    context = context || this.context || testRunnerHelper.getNameOfCurrentlyRunningTest();
+    if (!context) {
+      debug(`No context provided for "${dataType}" data:`, data);
+      return;
+    }
+    const contextHash = stringToMD5Hash(context);
+
+    if (this.isFileStorage) {
+      const dataDirPath = path.join(TESTOMAT_TMP_STORAGE_DIR, dataType);
+      fileSystem.createDir(dataDirPath);
+      this.#putDataToFile(dataType, data, contextHash);
+    } else {
+      this.#putDataToGlobalVar(dataType, data, contextHash);
+    }
+  }
+
+  /**
+   * Returns data, stored for specific test/context (or data which was stored without test id specified).
+   * This method will get data from global variable and/or from from file (previosly saved with put method).
+   *
+   * @param {'log' | 'artifact' | 'keyvalue'} dataType
+   * @param {string} context
+   * @returns {any []} array of data (any type), null (if no data found for context) or string (if data type is log)
+   */
+  getData(dataType, context) {
+    // TODO: think if it could be useful
+    // context = context || this.context || testRunnerHelper.getNameOfCurrentlyRunningTest();
+
+    if (!context) {
+      debug(`Trying to get "${dataType}" data without context`);
+      return null;
+    }
+
+    const contextHash = stringToMD5Hash(context);
+
+    let testDataFromFile = [];
+    let testDataFromGlobalVar = [];
+
+    if (global?.testomatioDataStore) {
+      testDataFromGlobalVar = this.#getDataFromGlobalVar(dataType, contextHash);
+      if (testDataFromGlobalVar) {
+        if (testDataFromGlobalVar.length) return testDataFromGlobalVar;
+      }
+      // don't return nothing if no data in global variable
+    }
+
+    testDataFromFile = this.#getDataFromFile(dataType, contextHash);
+
+    if (testDataFromFile.length) {
+      return testDataFromFile;
+    }
+    debug(`No "${dataType}" data for context "${contextHash}" in both file and global variable`);
+
+    // in case no data found for context
+    return null;
+  }
+
+  /**
+   * @param {'log' | 'artifact' | 'keyvalue'} dataType
+   * @param {string} context
+   * @returns aray of data (any type)
+   */
+  #getDataFromGlobalVar(dataType, context) {
+    try {
+      if (global?.testomatioDataStore[dataType]) {
+        const testData = global.testomatioDataStore[dataType][context];
+        if (testData) debug(`"${dataType}" data for constext "${context}":`, testData.join(', '));
+        return testData || [];
+      }
+      // debug(`No ${this.dataType} data for context ${context} in <global> storage`);
+      return [];
+    } catch (e) {
+      // there could be no data, ignore
+    }
+  }
+
+  /**
+   * @param {'log' | 'artifact' | 'keyvalue'} dataType
+   * @param {*} context
+   * @returns array of data (any type)
+   */
+  #getDataFromFile(dataType, context) {
+    const dataDirPath = path.join(TESTOMAT_TMP_STORAGE_DIR, dataType);
+    try {
+      const filepath = join(dataDirPath, `${dataType}_${context}`);
+      if (fs.existsSync(filepath)) {
+        const testDataAsText = fs.readFileSync(filepath, 'utf-8');
+        if (testDataAsText) debug(`"${dataType}" data for context "${context}":`, testDataAsText);
+        const testDataArr = testDataAsText?.split(os.EOL) || [];
+        return testDataArr;
+      }
+      // debug(`No ${this.dataType} data for ${context} in <file> storage`);
+      return [];
+    } catch (e) {
+      // there could be no data, ignore
+    }
+    return [];
+  }
+
+  /**
+   * Puts data to global variable. Unlike the file storage, stores data in array (file storage just append as string).
+   * @param {'log' | 'artifact' | 'keyvalue'} dataType
+   * @param {*} data
+   * @param {*} context
+   */
+  #putDataToGlobalVar(dataType, data, context) {
+    debug('Saving data to global variable for ', context, ':', data);
+    if (!global.testomatioDataStore) global.testomatioDataStore = {};
+    if (!global.testomatioDataStore?.[dataType]) global.testomatioDataStore[dataType] = {};
+
+    if (!global.testomatioDataStore?.[dataType][context]) global.testomatioDataStore[dataType][context] = [];
+    global.testomatioDataStore[dataType][context].push(data);
+  }
+
+  /**
+   * Puts data to file. Unlike the global variable storage, stores data as string
+   * @param {'log' | 'artifact' | 'keyvalue'} dataType
+   * @param {*} data
+   * @param {string} context
+   * @returns
+   */
+  #putDataToFile(dataType, data, context) {
+    const dataDirPath = path.join(TESTOMAT_TMP_STORAGE_DIR, dataType);
+    if (typeof data !== 'string') data = JSON.stringify(data);
+    const filename = `${dataType}_${context}`;
+    const filepath = join(dataDirPath, filename);
+    if (!fs.existsSync(dataDirPath)) fileSystem.createDir(dataDirPath);
+    debug(`Saving data to file for context "${context}" to ${filepath}. Data: ${JSON.stringify(data)}`);
+
+    // append new line if file already exists (in this case its definitely includes some data)
+    if (fs.existsSync(filepath)) {
+      fs.appendFileSync(filepath, os.EOL + data, 'utf-8');
+    } else {
+      fs.writeFileSync(filepath, data, 'utf-8');
+    }
+  }
+}
+
+function stringToMD5Hash(str) {
+  const md5 = crypto.createHash('md5');
+  md5.update(str);
+  const hash = md5.digest('hex');
+
+  return hash;
+}
+
+module.exports.dataStorage = DataStorage.getInstance();
+module.exports.stringToMD5Hash = stringToMD5Hash;
+
+// TODO: consider using fs promises instead of writeSync/appendFileSync to
+// prevent blocking and improve performance (probably queue usage will be required)