@testomatio/reporter 1.2.1-beta → 1.2.1-beta.codecept-id

package/lib/client.js

@@ -1,13 +1,17 @@
 const debug = require('debug')('@testomatio/reporter:client');
 const createCallsiteRecord = require('callsite-record');
 const { sep, join } = require('path');
+const { minimatch } = require('minimatch');
 const fs = require('fs');
 const chalk = require('chalk');
 const { randomUUID } = require('crypto');
 const upload = require('./fileUploader');
 const { APP_PREFIX } = require('./constants');
 const pipesFactory = require('./pipe');
-const logger = require('./logger');
+const { glob } = require('glob');
+const path = require('path');
+
+let listOfTestFilesToExcludeFromReport = null;
 
 /**
  * @typedef {import('../types').TestData} TestData
@@ -22,39 +26,86 @@ class Client {
    * @param {*} params
    */
   constructor(params = {}) {
-    this.parallel = params.parallel;
     const store = {};
     this.uuid = randomUUID();
     this.pipes = pipesFactory(params, store);
     this.queue = Promise.resolve();
     this.totalUploaded = 0;
+    this.failedToUpload = 0;
     this.version = JSON.parse(fs.readFileSync(join(__dirname, '..', 'package.json')).toString()).version;
+    this.executionList = Promise.resolve();
+
     console.log(APP_PREFIX, `Testomatio Reporter v${this.version}`);
   }
 
+  /**
+   * Asynchronously prepares the execution list for running tests through various pipes.
+   * Each pipe in the client is checked for enablement,
+   * and if all pipes are disabled, the function returns a resolved Promise.
+   * Otherwise, it executes the `prepareRun` method for each enabled pipe and collects the results.
+   * The results are then filtered to remove any undefined values.
+   * If no valid results are found, the function returns undefined.
+   * Otherwise, it returns the first non-empty array from the filtered results.
+   *
+   * @param {Object} params - The options for preparing the test execution list.
+   * @param {string} params.pipe - Name of the executed pipe.
+   * @param {string} params.pipeOptions - Filter option.
+   * @returns {Promise<any>} - A Promise that resolves to an
+   * array containing the prepared execution list,
+   * or resolves to undefined if no valid results are found or if all pipes are disabled.
+   */
+  async prepareRun(params) {
+    const { pipe, pipeOptions } = params;
+    // all pipes disabled, skipping
+    if (!this.pipes.some(p => p.isEnabled)) {
+      return Promise.resolve();
+    }
+
+    try {
+      const filterPipe = this.pipes.find(p => p.constructor.name.toLowerCase() === `${pipe.toLowerCase()}pipe`);
+
+      if (!filterPipe.isEnabled) {
+        // TODO:for the future for the another pipes
+        console.warn(
+          APP_PREFIX,
+          `At the moment processing is available only for the "testomatio" key. Example: "testomatio:tag-name=xxx"`,
+        );
+        return;
+      }
+
+      const results = await Promise.all(
+        this.pipes.map(async p => ({ pipe: p.toString(), result: await p.prepareRun(pipeOptions) })),
+      );
+
+      const result = results.filter(p => p.pipe.includes('Testomatio'))[0]?.result;
+
+      if (!result || result.length === 0) {
+        return;
+      }
+
+      debug('Execution tests list', result);
+
+      return result;
+    } catch (err) {
+      console.error(APP_PREFIX, err);
+    }
+  }
+
   /**
    * Used to create a new Test run
    *
    * @returns {Promise<any>} - resolves to Run id which should be used to update / add test
    */
   createRun() {
+    debug('Creating run...');
     // all pipes disabled, skipping
-    if (!this.pipes.filter(p => p.isEnabled).length) return Promise.resolve();
-
-    const runParams = {
-      title: this.title,
-      parallel: this.parallel,
-      env: this.env,
-    };
-
-    global.testomatioArtifacts = [];
-    // TODO: create global storage
+    if (!this.pipes?.filter(p => p.isEnabled).length) return Promise.resolve();
 
     this.queue = this.queue
-      .then(() => Promise.all(this.pipes.map(p => p.createRun(runParams))))
+      .then(() => Promise.all(this.pipes.map(p => p.createRun())))
       .catch(err => console.log(APP_PREFIX, err))
       .then(() => undefined); // fixes return type
-    debug('Run', this.queue);
+    // debug('Run', this.queue);
     return this.queue;
   }
 
@@ -63,12 +114,13 @@ class Client {
    *
    * @param {string|undefined} status
    * @param {TestData} [testData]
-   * @param {string[]} [storeArtifacts]
    * @returns {Promise<PipeResult[]>}
    */
-  async addTestRun(status, testData, storeArtifacts = []) {
+  async addTestRun(status, testData) {
     // all pipes disabled, skipping
-    if (!this.pipes.filter(p => p.isEnabled).length) return;
+    if (!this.pipes?.filter(p => p.isEnabled).length) return [];
+
+    if (isTestShouldBeExculedFromReport(testData)) return [];
 
     if (!testData)
       testData = {
@@ -85,42 +137,31 @@ class Client {
       steps,
       code = null,
       title,
+      file,
       suite_title,
       suite_id,
       test_id,
+      manuallyAttachedArtifacts,
+      meta,
     } = testData;
     let { message = '' } = testData;
 
-    const uploadedFiles = [];
-
-    let stack = '';
-
+    let errorFormatted = '';
     if (error) {
-      stack = this.formatError(error) || '';
+      errorFormatted += this.formatError(error) || '';
       message = error?.message;
     }
-    if (steps) {
-      stack = this.formatSteps(stack, steps);
-    }
 
-    const testLogs = logger.getLogs(test_id);
-    debug(`Test logs for ${test_id}:\n`, testLogs);
-    if (stack) stack += '\n\n';
-    stack += testLogs;
+    // Attach logs
+    const fullLogs = this.formatLogs({ error: errorFormatted, steps, logs: testData.logs });
 
-    if (Array.isArray(storeArtifacts) && storeArtifacts.length) {
-      debug('CLIENT storeArtifact', storeArtifacts);
-      files.push(...storeArtifacts);
-    }
+    // add artifacts
+    if (manuallyAttachedArtifacts?.length) files.push(...manuallyAttachedArtifacts);
 
-    if (Array.isArray(global.testomatioArtifacts)) {
-      debug('CLIENT global[testomatioArtifacts]', global.testomatioArtifacts);
-      files.push(...global.testomatioArtifacts);
-      global.testomatioArtifacts = [];
-    }
+    const uploadedFiles = [];
 
-    for (const file of files) {
-      uploadedFiles.push(upload.uploadFileByPath(file, this.uuid));
+    for (const f of files) {
+      uploadedFiles.push(upload.uploadFileByPath(f, this.uuid));
     }
 
     for (const [idx, buffer] of filesBuffers.entries()) {
@@ -128,18 +169,22 @@ class Client {
       uploadedFiles.push(upload.uploadFileAsBuffer(buffer, fileName, this.uuid));
     }
 
-    const artifacts = await Promise.all(uploadedFiles);
+    const artifacts = (await Promise.all(uploadedFiles)).filter(n => !!n);
 
-    global.testomatioArtifacts = [];
+    if (artifacts.length < uploadedFiles.length) {
+      const failedUploading = uploadedFiles.length - artifacts.length;
+      this.failedToUpload += failedUploading;
+    }
 
-    this.totalUploaded += uploadedFiles.filter(n => n).length;
+    this.totalUploaded += artifacts.length;
 
     const data = {
       files,
       steps,
       status,
-      stack,
+      stack: fullLogs,
       example,
+      file,
       code,
       title,
       suite_title,
@@ -148,19 +193,24 @@ class Client {
       message,
       run_time: parseFloat(time),
       artifacts,
+      meta,
     };
 
-    this.queue = this.queue
-      .then(() => Promise.all(this.pipes.map(async p => {
-        try {
-          const result = await p.addTest(data);
-          return { pipe: p.toString(), result };
-        } catch (err) {
-          console.log(APP_PREFIX, p.toString(), err);
-        }
-      }
-    )));
-    
+    // debug('Adding test run...', data);
+
+    this.queue = this.queue.then(() =>
+      Promise.all(
+        this.pipes.map(async pipe => {
+          try {
+            const result = await pipe.addTest(data);
+            return { pipe: pipe.toString(), result };
+          } catch (err) {
+            console.log(APP_PREFIX, pipe.toString(), err);
+          }
+        }),
+      ),
+    );
+
     return this.queue;
   }
 
@@ -172,23 +222,36 @@ class Client {
    * @returns {Promise<any>} - A Promise that resolves when finishes the run.
    */
   updateRunStatus(status, isParallel = false) {
+    debug('Updating run status...');
     // all pipes disabled, skipping
-    if (!this.pipes.filter(p => p.isEnabled).length) return Promise.resolve();
+    if (!this.pipes?.filter(p => p.isEnabled).length) return Promise.resolve();
 
     const runParams = { status, parallel: isParallel };
 
     this.queue = this.queue
       .then(() => Promise.all(this.pipes.map(p => p.finishRun(runParams))))
       .then(() => {
-        debug('TOTAL uploaded files', this.totalUploaded);
+        debug('TOTAL artifacts', this.totalUploaded);
+        if (this.totalUploaded && !upload.isArtifactsEnabled())
+          debug(`${this.totalUploaded} artifacts are not uploaded, because artifacts uploading is not enabled`);
 
-        if (upload.isArtifactsEnabled() && this.totalUploaded > 0) {
+        if (this.totalUploaded && upload.isArtifactsEnabled()) {
           console.log(
             APP_PREFIX,
-            `🗄️  Total ${this.totalUploaded} artifacts ${
+            `🗄️ ${this.totalUploaded} artifacts ${
               process.env.TESTOMATIO_PRIVATE_ARTIFACTS ? 'privately' : chalk.bold('publicly')
-            } uploaded to S3 bucket  `,
+            } uploaded to S3 bucket`,
           );
+
+          if (this.failedToUpload > 0) {
+            console.log(
+              APP_PREFIX,
+              chalk.yellow(
+                `Some artifacts were not uploaded. ${this.failedToUpload} artifacts could not be uploaded.
+                Run tests with DEBUG="@testomatio/reporter:file-uploader" to see details"`,
+              ),
+            );
+          }
         }
       })
       .catch(err => console.log(APP_PREFIX, err));
@@ -196,15 +259,27 @@ class Client {
     return this.queue;
   }
 
-  formatSteps(stack, steps) {
-    return stack ? `${steps}\n\n${chalk.bold.red('################[ Failure ]################')}\n${stack}` : steps;
+  /**
+   * Returns the formatted stack including the stack trace, steps, and logs.
+   * @returns {string}
+   */
+  formatLogs({ error, steps, logs }) {
+    error = error?.trim();
+    steps = steps?.trim();
+    logs = logs?.trim();
+
+    let testLogs = '';
+    if (steps) testLogs += `${chalk.bold.blue('################[ Steps ]################')}\n${steps}\n\n`;
+    if (logs) testLogs += `${chalk.bold.gray('################[ Logs ]################')}\n${logs}\n\n`;
+    if (error) testLogs += `${chalk.bold.red('################[ Failure ]################')}\n${error}`;
+    return testLogs;
   }
 
   formatError(error, message) {
     if (!message) message = error.message;
     if (error.inspect) message = error.inspect() || '';
 
-    let stack = `\n${chalk.bold(message)}\n`;
+    let stack = `${message}\n`;
 
     // diffs for mocha, cypress, codeceptjs style
     if (error.actual && error.expected) {
@@ -214,17 +289,21 @@ class Client {
       stack += '\n\n';
     }
 
+    const customFilter = process.env.TESTOMATIO_STACK_IGNORE;
+
     try {
+      let hasFrame = false;
       const record = createCallsiteRecord({
         forError: error,
+        isCallsiteFrame: frame => {
+          if (customFilter && minimatch(frame.getFileName(), customFilter)) return false;
+          if (hasFrame) return false;
+          if (isNotInternalFrame(frame)) hasFrame = true;
+          return hasFrame;
+        },
       });
       if (record && !record.filename.startsWith('http')) {
-        stack += record.renderSync({
-          stackFilter: frame =>
-            frame.getFileName().indexOf(sep) > -1 &&
-            frame.getFileName().indexOf('node_modules') < 0 &&
-            frame.getFileName().indexOf('internal') < 0,
-        });
+        stack += record.renderSync({ stackFilter: isNotInternalFrame });
       }
       return stack;
     } catch (e) {
@@ -233,4 +312,49 @@ class Client {
   }
 }
 
+function isNotInternalFrame(frame) {
+  return (
+    frame.getFileName() &&
+    frame.getFileName().includes(sep) &&
+    !frame.getFileName().includes('node_modules') &&
+    !frame.getFileName().includes('internal')
+  );
+}
+
+/**
+ *
+ * @param {TestData} testData
+ * @returns boolean
+ */
+function isTestShouldBeExculedFromReport(testData) {
+  // const fileName = path.basename(test.location?.file || '');
+  const globExcludeFilesPattern = process.env.TESTOMATIO_EXCLUDE_FILES_FROM_REPORT_GLOB_PATTERN;
+  if (!globExcludeFilesPattern) return false;
+
+  if (!testData.file) {
+    debug('No "file" property found for test ', testData.title);
+    return false;
+  }
+
+  const excludeParretnsList = globExcludeFilesPattern.split(';');
+
+  // as scanning files is time consuming operation, just save the result in variable to avoid multiple scans
+  if (!listOfTestFilesToExcludeFromReport) {
+    // list of files with relative paths
+    listOfTestFilesToExcludeFromReport = glob.sync(excludeParretnsList, { ignore: '**/node_modules/**' });
+    debug('Tests from next files will not be reported:', listOfTestFilesToExcludeFromReport);
+  }
+
+  const testFileRelativePath = path.relative(process.cwd(), testData.file);
+
+  // no files found matching the exclusion pattern
+  if (!listOfTestFilesToExcludeFromReport.length) return false;
+
+  if (listOfTestFilesToExcludeFromReport.includes(testFileRelativePath)) {
+    debug(`Excluding test '${testData.title}' <${testFileRelativePath}> from reporting`);
+    return true;
+  }
+  return false;
+}
+
 module.exports = Client;

package/lib/config.js

@@ -0,0 +1,34 @@
+// This file is used to read environment variables from .env file
+
+// require('dotenv').config({ path: process.env.TESTOMATIO_ENV_FILE_PATH });
+
+const debug = require('debug')('@testomatio/reporter:config');
+
+/* for possibility to use multiple env files (reading different paths)
+const dotenv = require('dotenv');
+const envFileVars = dotenv.config({ path: '.env' }).parsed; */
+
+if (process.env.TESTOMATIO_API_KEY) {
+  process.env.TESTOMATIO = process.env.TESTOMATIO_API_KEY;
+}
+if (process.env.TESTOMATIO_TOKEN) {
+  process.env.TESTOMATIO = process.env.TESTOMATIO_TOKEN;
+}
+
+if (process.env.TESTOMATIO === 'undefined')
+  console.error('TESTOMATIO is "undefined". Something went wrong. Contact dev team.');
+
+// select only TESTOMATIO related variables (only to print them in debug)
+const testomatioEnvVars =
+  Object.keys(process.env)
+    .filter(key => key.startsWith('TESTOMATIO') || key.startsWith('S3_'))
+    .reduce((obj, key) => {
+      obj[key] = process.env[key];
+      return obj;
+    }, {}) || {};
+debug('TESTOMATIO variables:', testomatioEnvVars);
+
+// includes variables from .env file and process.env
+const config = process.env;
+
+module.exports = config;

package/lib/constants.js

@@ -1,11 +1,12 @@
 const chalk = require('chalk');
+const os = require('os');
+const path = require('path');
 
 const APP_PREFIX = chalk.gray('[TESTOMATIO]');
-const TESTOMAT_ARTIFACT_SUFFIX = "testomatio_artifact_";
+const AXIOS_TIMEOUT = 20 * 1000; // sum = 20sec
+const AXIOS_RETRY_TIMEOUT = 5 * 1000; // sum = 5sec
 
-const TESTOMAT_TMP_STORAGE = {
-  mainDir: "testomatio_tmp",
-}
+const TESTOMAT_TMP_STORAGE_DIR = path.join(os.tmpdir(), 'testomatio_tmp');
 
 const CSV_HEADERS = [
   { id: 'suite_title', title: 'Suite_title' },
@@ -21,11 +22,22 @@ const STATUS = {
   SKIPPED: 'skipped',
   FINISHED: 'finished',
 };
+// html pipe var
+const HTML_REPORT = {
+  FOLDER: 'html-report',
+  REPORT_DEFAULT_NAME: 'testomatio-report.html',
+  TEMPLATE_NAME: 'testomatio.hbs',
+};
+
+const testomatLogoURL = 'https://avatars.githubusercontent.com/u/59105116?s=36&v=4';
 
 module.exports = {
   APP_PREFIX,
-  TESTOMAT_ARTIFACT_SUFFIX,
-  TESTOMAT_TMP_STORAGE,
+  TESTOMAT_TMP_STORAGE_DIR,
   CSV_HEADERS,
   STATUS,
-}
+  HTML_REPORT,
+  AXIOS_TIMEOUT,
+  AXIOS_RETRY_TIMEOUT,
+  testomatLogoURL,
+};

package/lib/data-storage.js

@@ -0,0 +1,203 @@
+const debug = require('debug')('@testomatio/reporter:storage');
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const { join } = require('path');
+const { TESTOMAT_TMP_STORAGE_DIR } = require('./constants');
+const { fileSystem, testRunnerHelper } = require('./utils/utils');
+const crypto = require('crypto');
+
+class DataStorage {
+  static #instance;
+
+  context;
+
+  /**
+   *
+   * @returns {DataStorage}
+   */
+  static getInstance() {
+    if (!this.#instance) {
+      this.#instance = new DataStorage();
+    }
+    return this.#instance;
+  }
+
+  setContext(context) {
+    this.context = context;
+  }
+
+  /**
+   * Creates data storage instance as singleton
+   * Stores data to global variable or to file depending on what is applicable for current test runner (adapter)
+   * Recommend to use composition while using this class (instead of inheritance).
+   * ! Also the class which will use data storage should be singleton (to avoid data loss).
+   */
+  constructor() {
+    // some frameworks use global variable to store data, some use file storage
+    this.isFileStorage = true;
+  }
+
+  /**
+   * Puts any data to storage (file or global variable).
+   * If file: stores data as text, if global variable – stores as array of data.
+   * @param {'log' | 'artifact' | 'keyvalue'} dataType
+   * @param {*} data anything you want to store (string, object, array, etc)
+   * @param {*} context could be testId or any context (test name, suite name, including their IDs etc)
+   * suite name + test name is used by default
+   * @returns
+   */
+  putData(dataType, data, context = null) {
+    if (!dataType || !data) return;
+
+    context = context || this.context || testRunnerHelper.getNameOfCurrentlyRunningTest();
+    if (!context) {
+      debug(`No context provided for "${dataType}" data:`, data);
+      return;
+    }
+    const contextHash = stringToMD5Hash(context);
+
+    if (this.isFileStorage) {
+      const dataDirPath = path.join(TESTOMAT_TMP_STORAGE_DIR, dataType);
+      fileSystem.createDir(dataDirPath);
+      this.#putDataToFile(dataType, data, contextHash);
+    } else {
+      this.#putDataToGlobalVar(dataType, data, contextHash);
+    }
+  }
+
+  /**
+   * Returns data, stored for specific test/context (or data which was stored without test id specified).
+   * This method will get data from global variable and/or from from file (previosly saved with put method).
+   *
+   * @param {'log' | 'artifact' | 'keyvalue'} dataType
+   * @param {string} context
+   * @returns {any []} array of data (any type), null (if no data found for context) or string (if data type is log)
+   */
+  getData(dataType, context) {
+    // TODO: think if it could be useful
+    // context = context || this.context || testRunnerHelper.getNameOfCurrentlyRunningTest();
+
+    if (!context) {
+      debug(`Trying to get "${dataType}" data without context`);
+      return null;
+    }
+
+    const contextHash = stringToMD5Hash(context);
+
+    let testDataFromFile = [];
+    let testDataFromGlobalVar = [];
+
+    if (global?.testomatioDataStore) {
+      testDataFromGlobalVar = this.#getDataFromGlobalVar(dataType, contextHash);
+      if (testDataFromGlobalVar) {
+        if (testDataFromGlobalVar.length) return testDataFromGlobalVar;
+      }
+      // don't return nothing if no data in global variable
+    }
+
+    testDataFromFile = this.#getDataFromFile(dataType, contextHash);
+
+    if (testDataFromFile.length) {
+      return testDataFromFile;
+    }
+    debug(`No "${dataType}" data for context "${contextHash}" in both file and global variable`);
+
+    // in case no data found for context
+    return null;
+  }
+
+  /**
+   * @param {'log' | 'artifact' | 'keyvalue'} dataType
+   * @param {string} context
+   * @returns aray of data (any type)
+   */
+  #getDataFromGlobalVar(dataType, context) {
+    try {
+      if (global?.testomatioDataStore[dataType]) {
+        const testData = global.testomatioDataStore[dataType][context];
+        if (testData) debug(`"${dataType}" data for constext "${context}":`, testData.join(', '));
+        return testData || [];
+      }
+      // debug(`No ${this.dataType} data for context ${context} in <global> storage`);
+      return [];
+    } catch (e) {
+      // there could be no data, ignore
+    }
+  }
+
+  /**
+   * @param {'log' | 'artifact' | 'keyvalue'} dataType
+   * @param {*} context
+   * @returns array of data (any type)
+   */
+  #getDataFromFile(dataType, context) {
+    const dataDirPath = path.join(TESTOMAT_TMP_STORAGE_DIR, dataType);
+    try {
+      const filepath = join(dataDirPath, `${dataType}_${context}`);
+      if (fs.existsSync(filepath)) {
+        const testDataAsText = fs.readFileSync(filepath, 'utf-8');
+        if (testDataAsText) debug(`"${dataType}" data for context "${context}":`, testDataAsText);
+        const testDataArr = testDataAsText?.split(os.EOL) || [];
+        return testDataArr;
+      }
+      // debug(`No ${this.dataType} data for ${context} in <file> storage`);
+      return [];
+    } catch (e) {
+      // there could be no data, ignore
+    }
+    return [];
+  }
+
+  /**
+   * Puts data to global variable. Unlike the file storage, stores data in array (file storage just append as string).
+   * @param {'log' | 'artifact' | 'keyvalue'} dataType
+   * @param {*} data
+   * @param {*} context
+   */
+  #putDataToGlobalVar(dataType, data, context) {
+    debug('Saving data to global variable for ', context, ':', data);
+    if (!global.testomatioDataStore) global.testomatioDataStore = {};
+    if (!global.testomatioDataStore?.[dataType]) global.testomatioDataStore[dataType] = {};
+
+    if (!global.testomatioDataStore?.[dataType][context]) global.testomatioDataStore[dataType][context] = [];
+    global.testomatioDataStore[dataType][context].push(data);
+  }
+
+  /**
+   * Puts data to file. Unlike the global variable storage, stores data as string
+   * @param {'log' | 'artifact' | 'keyvalue'} dataType
+   * @param {*} data
+   * @param {string} context
+   * @returns
+   */
+  #putDataToFile(dataType, data, context) {
+    const dataDirPath = path.join(TESTOMAT_TMP_STORAGE_DIR, dataType);
+    if (typeof data !== 'string') data = JSON.stringify(data);
+    const filename = `${dataType}_${context}`;
+    const filepath = join(dataDirPath, filename);
+    if (!fs.existsSync(dataDirPath)) fileSystem.createDir(dataDirPath);
+    debug(`Saving data to file for context "${context}" to ${filepath}. Data: ${JSON.stringify(data)}`);
+
+    // append new line if file already exists (in this case its definitely includes some data)
+    if (fs.existsSync(filepath)) {
+      fs.appendFileSync(filepath, os.EOL + data, 'utf-8');
+    } else {
+      fs.writeFileSync(filepath, data, 'utf-8');
+    }
+  }
+}
+
+function stringToMD5Hash(str) {
+  const md5 = crypto.createHash('md5');
+  md5.update(str);
+  const hash = md5.digest('hex');
+
+  return hash;
+}
+
+module.exports.dataStorage = DataStorage.getInstance();
+module.exports.stringToMD5Hash = stringToMD5Hash;
+
+// TODO: consider using fs promises instead of writeSync/appendFileSync to
+// prevent blocking and improve performance (probably queue usage will be required)