@testomatio/reporter 1.0.0-beta.4 → 1.0.1-6.1

package/lib/dataStorage.js

@@ -0,0 +1,232 @@
+const debug = require('debug')('@testomatio/reporter:storage');
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const { join } = require('path');
+const JestReporter = require('./adapter/jest');
+const { TESTOMAT_TMP_STORAGE } = require('./constants');
+const { fileSystem } = require('./utils/utils');
+const getTestIdFromTestTitle = require('./utils/utils').parseTest;
+
+class DataStorage {
+  /**
+   * Creates data storage instance for specific data type.
+   * Stores data to global variable or to file depending on what is applicable for current test runner
+   * (running environment).
+   * @param {*} dataType storage type (like 'log', 'artifact'); could be any string, used only to define file path
+   */
+  constructor(dataType) {
+    this.dataType = dataType || 'data';
+    this.isFileStorage = true;
+    this.#refreshStorageType();
+
+    this.dataDirPath = path.join(TESTOMAT_TMP_STORAGE.mainDir, this.dataType);
+    fileSystem.createDir(this.dataDirPath);
+  }
+
+  /**
+   * Try to define the running environment. Not 100% reliable and used as additional check
+   * @returns jest | mocha | ...
+   */
+  getRunningEnviroment() {
+    // jest
+    if (process.env.JEST_WORKER_ID) return 'jest';
+
+    if (global.codeceptjs) return 'codeceptjs';
+
+    // 'cucumber:current', 'cucumber:legacy'
+    if (global.testomatioRunningEnvironment) return global.testomatioRunningEnvironment;
+
+    if (process.env.PLAYWRIGHT_TEST_BASE_URL || process.env.TEST_WORKER_INDEX) return 'playwright';
+
+    // mocha - can't detect
+    return null;
+  }
+
+  /**
+   * Puts any data to storage (file or global variable)
+   * @param {*} data anything you want to store
+   * @param {*} context could be testId or any context (test, suite, etc) which is used to define testId
+   * @returns
+   */
+  putData(data, context = null) {
+    this.#refreshStorageType();
+
+    let testId = this.#tryToRetrieveTestId(context) || null;
+
+    if (this.runningEnvironment === 'codeceptjs') {
+      this.isFileStorage = false;
+      testId = testId ?? global.testomatioDataStore?.currentlyRunningTestId;
+    }
+
+    if (this.runningEnvironment === 'jest') {
+      testId = testId ?? JestReporter.getIdOfCurrentlyRunningTest();
+    }
+
+    // logs in playwright are gathered by pw framework itself
+    if (this.runningEnvironment === 'playwright' && this.dataType === 'log') return;
+
+    // get id from global store
+    if (global.testomatioDataStore && global.testomatioDataStore.currentlyRunningTestId)
+      testId = testId ?? global.testomatioDataStore?.currentlyRunningTestId;
+
+    if (!testId && global?.currentlyRunningTestTitle)
+      testId = this.#tryToRetrieveTestId(global.currentlyRunningTestTitle);
+
+    if (!testId) {
+      debug(`No test id provided for ${this.dataType} data: ${data}`);
+      return;
+    }
+
+    if (this.isFileStorage) {
+      this.#putDataToFile(data, testId);
+    } else {
+      this.#putDataToGlobalVar(data, testId);
+    }
+  }
+
+  /**
+   * Returns data, stored for specific testId (or data which was stored without test id specified).
+   * This method will get data from global variable. But if it is not available, it will try to get data from file.
+   * Thus, good approach is to remove file storage folder before each test run (and after, for sure).
+   *
+   * Defining the execution environment is not guaranteed! Is used only as additional check.
+   * @param {*} context could be testId or any context (test, suite, etc) which is used to define testId
+   * @returns
+   */
+  getData(context) {
+    this.#refreshStorageType();
+
+    let testId = null;
+    if (typeof context === 'string') {
+      testId = this.#tryToRetrieveTestId(context) || context;
+    } else {
+      // TODO: derive testId from context
+      // testId = context...
+    }
+
+    if (!testId) {
+      debug(`Cannot get test id from passed context:`, context);
+      return null;
+    }
+
+    let testData = '';
+
+    if (global?.testomatioDataStore) {
+      testData = this.#getDataFromGlobalVar(testId);
+      // these frameworks use global variable storage
+      if (testData && ['cucumber', 'codeceptjs'].includes(this.runningEnvironment)) return testData;
+    }
+
+    /* condition is removed for mocha
+    mocha has created a global storage, but just in some cases, generally it is not available
+    */
+    // if (this.isFileStorage || !testData) {
+    //   testData = this.#getDataFromFile(testId);
+    //   if (testData) return testData;
+    // }
+
+    const testDataFromFile = this.#getDataFromFile(testId);
+    testData += testDataFromFile || '';
+
+    debug(`No ${this.dataType} data for test id ${testId} in both file and global variable`);
+    return testData || '';
+  }
+
+  /**
+   * This method is named as "try" because it does not guarantee that test id could be retrieved.
+   * Context could be anything (test, suite, string, etc) which is used to define testId.
+   * Or it could represent any other entity (which does not contain test id).
+   * @param {*} context
+   */
+  #tryToRetrieveTestId(context) {
+    if (!context) return null;
+    this.#refreshStorageType();
+
+    if (this.runningEnvironment === 'playwright' || context?.title) {
+      // context is testInfo
+      const testId = getTestIdFromTestTitle(context.title);
+      if (testId) return testId;
+    }
+
+    if (typeof context === 'string') {
+      const testId = getTestIdFromTestTitle(context);
+      if (testId) return testId;
+    }
+
+    return null;
+  }
+
+  /**
+   * Refreshes storage type (file or global variable) depending on current environment
+   * This method should be run on each attempt to put or get data from/to storage
+   * Because storage instance is created before the test runner is started. And storage type could be changed
+   */
+  #refreshStorageType() {
+    this.runningEnvironment = this.getRunningEnviroment();
+
+    // some test frameworks do not persist global variables, thus file storage is used for them (by default)
+    if (['playwright', 'codeceptjs'].includes(this.runningEnvironment)) this.isFileStorage = false;
+  }
+
+  #getDataFromGlobalVar(testId) {
+    try {
+      if (global?.testomatioDataStore[this.dataType]) {
+        const testData = global.testomatioDataStore[this.dataType][testId];
+        debug(`Data for test id ${testId}:\n${testData}`);
+        return testData || '';
+      }
+      debug(`No ${this.dataType} data for test id ${testId} in <global> storage`);
+      return '';
+    } catch (e) {
+      // there could be no data, ignore
+    }
+  }
+
+  #getDataFromFile(testId) {
+    try {
+      const filepath = join(this.dataDirPath, `${this.dataType}_${testId}`);
+      if (fs.existsSync(filepath)) {
+        const testData = fs.readFileSync(filepath, 'utf-8');
+        debug(`Data for test id ${testId}:\n${testData}`);
+        return testData || '';
+      }
+      debug(`No ${this.dataType} data for test id ${testId} in <file> storage`);
+      return '';
+    } catch (e) {
+      // there could be no data, ignore
+    }
+    return '';
+  }
+
+  #putDataToGlobalVar(data, testId) {
+    debug('Saving data to global variable for test', testId, ':\n', data, '\n');
+    if (!global.testomatioDataStore) global.testomatioDataStore = {};
+    if (!global.testomatioDataStore?.[this.dataType]) global.testomatioDataStore[this.dataType] = {};
+    global.testomatioDataStore?.[this.dataType][testId] // eslint-disable-line no-unused-expressions
+      ? (global.testomatioDataStore[this.dataType][testId] += `\n${data}`)
+      : (global.testomatioDataStore[this.dataType][testId] = data);
+  }
+
+  #putDataToFile(data, testId) {
+    if (typeof data !== 'string') data = JSON.stringify(data);
+    const filename = `${this.dataType}_${testId}`;
+    const filepath = join(this.dataDirPath, filename);
+    if (!fs.existsSync(this.dataDirPath)) fileSystem.createDir(this.dataDirPath);
+    debug('Saving data to file for test', testId, 'to', filepath, ':\n', data, '\n');
+
+    // TODO: handle multiple invocations of JSON.stringify.
+    // UPD: not actual because decided not to use it - it created extra wrapping quotes "" and // (escaped slashes)
+    fs.appendFileSync(filepath, data + os.EOL, 'utf-8');
+  }
+}
+
+module.exports.DataStorage = DataStorage;
+
+// TODO: consider using fs promises instead of writeSync/appendFileSync to
+// prevent blocking and improve performance (probably queue usage will be required)
+
+// TODO: rewrite client - everything regarding storing artifacts
+// TODO: try to define adapter inside client
+// TODO: use .env
+// TODO: ability to intercept multiple loggers (upd: no need)

package/lib/junit-adapter/java.js

@@ -10,24 +10,42 @@ class JavaAdapter extends Adapter {
 
   formatTest(t) {
     const fileParts = t.suite_title.split('.')
-    const example = t.title.match(/\[(.*)\]/)?.[1];
-    if (example) t.example = { "#": example }
 
     t.file = namespaceToFileName(t.suite_title);
     t.title = t.title.split('(')[0];
+
+    // detect params
+    const paramMatches = t.title.match(/\[(.*?)\]/g);
+
+    if (paramMatches) {
+      const params = paramMatches.map((_match, index) => `param${index + 1}`);
+      if (params.length === 1) params[0] = 'param';
+      let paramIndex = 0;
+    
+      t.title = t.title.replace(/: \[(.*?)\]/g, () => {
+        if (params.length < 2) return `\${param}`
+        const paramName = params[paramIndex] || `param${paramIndex + 1}`;
+        paramIndex++;
+        return `\${${paramName}}`;
+      });
+      const example = {};
+      paramMatches.forEach((match, index) => { example[params[index]] = match.replace(/[[\]]/g, '') });
+      t.example = example;
+    }
+
     t.suite_title = fileParts[fileParts.length - 1].replace(/\$/g, ' | ')
     return t;
   }
 
-  formatStack(t) {
-    const stack = super.formatStack(t);
+  // formatStack(t) {
+  //   const stack = super.formatStack(t);
 
-    const file = t.suite_title.split('.');
+  //   const file = t.suite_title.split('.');
 
-    const fileLine = `at .*${file[file.length - 1]}\.java:(\\d+)` // eslint-disable-line no-useless-escape
-    const regexp = new RegExp(fileLine,"g")
-    return stack.replace(regexp, `${this.opts.javaTests}${path.sep}${namespaceToFileName(t.suite_title)}:$1:`);
-  }
+  //   const fileLine = `at .*${file[file.length - 1]}\.java:(\\d+)` // eslint-disable-line no-useless-escape
+  //   const regexp = new RegExp(fileLine,"g")
+  //   return stack.replace(regexp, `${this.opts.javaTests}${path.sep}${namespaceToFileName(t.suite_title)}:$1:`);
+  // }
 }
 
 function namespaceToFileName(fileName) {

package/lib/logger.js

@@ -0,0 +1,319 @@
+const chalk = require('chalk');
+const debug = require('debug')('@testomatio/reporter:logger');
+const { DataStorage } = require('./dataStorage');
+
+const LOG_METHODS = ['assert', 'debug', 'error', 'info', 'log', 'trace', 'warn'];
+const LEVELS = {
+  ALL: { severity: 1, color: '' },
+  VERBOSE: { severity: 3, color: 'grey' },
+  TRACE: { severity: 5, color: 'grey' },
+  DEBUG: { severity: 7, color: 'cyan' },
+  INFO: { severity: 9, color: 'black' },
+  LOG: { severity: 11, color: 'black' },
+  WARN: { severity: 13, color: 'yellow' },
+  ERROR: { severity: 15, color: 'red' },
+};
+
+// ! DON'T use console.log, console.warn, etc in this file, because it will lead to infinite loop
+// use debug() instead
+
+/**
+ * Logger allows to intercept logs from any logger (console.log, tracer, pino, etc)
+ * and save in the testomatio reporter.
+ * Supports different syntaxes to satisfy any user preferences.
+ */
+class Logger {
+  // set default logger to be used in log, warn, error, etc methods
+  #originalUserLogger = { ...console };
+
+  #dataStorage = new DataStorage('log');
+
+  logLevel = process?.env?.LOG_LEVEL?.toUpperCase() || 'ALL';
+
+  constructor() {
+    // intercept console by default
+    this.intercept(console);
+
+    // singleton
+    if (!Logger.instance) {
+      Logger.instance = this;
+    }
+
+    // add beforeEach hook for mocha. it does not override existing hook, just add new one
+    try {
+      // @ts-ignore
+      if (!beforeEach) return;
+      // @ts-ignore-next-line
+      beforeEach(function () {
+        if (this.currentTest?.__mocha_id__) {
+          global.testTitle = this.currentTest.fullTitle();
+        }
+      });
+    } catch (e) {
+      // ignore
+    }
+  }
+
+  /**
+   * Allows you to define a step inside a test. Step name is attached to the report and
+   * helps to understand the test flow.
+   * @param {*} strings
+   * @param  {...any} values
+   */
+  step(strings, ...values) {
+    // get testId for mocha
+    const context = global.testTitle ?? null;
+
+    let logs = '';
+    for (let i = 0; i < strings.length; i++) {
+      logs += strings[i];
+      if (i < values.length) {
+        logs += values[i];
+      }
+    }
+    logs = chalk.blue(`> ${logs}`);
+    this.#dataStorage.putData(logs, context);
+  }
+
+  /**
+   *
+   * @param {*} context testId or test context from test runner
+   * @returns
+   */
+  getLogs(context) {
+    const logs = this.#dataStorage.getData(context);
+    return logs || '';
+  }
+
+  #stringifyLogs(...args) {
+    const logs = [];
+    // stringify everything except strings
+    for (const arg of args) {
+      // ignore empty strings
+      if (arg === '') continue;
+      if (typeof arg === 'string') {
+        logs.push(arg);
+      } else if (Array.isArray(arg)) {
+        logs.push(arg.join(' '));
+      } else {
+        try {
+          // eslint-disable-next-line no-unused-expressions
+          this.prettyObjects ? logs.push(JSON.stringify(arg, null, 2)) : logs.push(JSON.stringify(arg));
+        } catch (e) {
+          debug('Error while stringify object', e);
+          logs.push(arg);
+        }
+      }
+    }
+    return logs.join(' ');
+  }
+
+  /**
+   * Tagget template literal. Allows to use different syntaxes:
+   * 1. Tagget template: log`text ${someVar}`
+   * 2. Standard: log(`text ${someVar}`)
+   * 3. Standard with multiple arguments: log('text', someVar)
+   */
+  templateLiteralLog(strings, ...args) {
+    if (Array.isArray(strings)) strings = strings.filter(item => item !== '').map(item => item.trim());
+    if (Array.isArray(args)) args = args.filter(item => item !== '');
+    // entity which is used to define testId
+    let context = null;
+
+    // get testId for mocha
+    context = global.testTitle ?? null;
+
+    let logs;
+    // this block means tagged template is used (syntax like $`text ${someVar}`)
+    if (Array.isArray(strings) && strings.length === args.length + 1) {
+      logs = strings.reduce(
+        (result, current, index) =>
+          result +
+          current +
+          // strings are splitted by args when use tagged template, thus we add arg after each string
+          // it looks like: `string1 arg1 string2 arg2 string3`
+          (args[index] !== undefined // eslint-disable-line no-nested-ternary
+            ? typeof args[index] === 'string'
+              ? args[index] // add arg as it is
+              : this.#stringifyLogs(args[index]) // stringify arg
+            : ''),
+        // initial accumulator value
+        '',
+      );
+    } else {
+      // this block means arguments syntax is used (syntax like $('text', someVar))
+      // in this case strings represents just a first argument
+      logs = this.#stringifyLogs(strings, ...args);
+    }
+    this.#originalUserLogger.log(logs);
+    this.#dataStorage.putData(logs, context);
+  }
+
+  /**
+   * This function is a wrapper for each logging methods (log, warn, error etc) (not to repeat the same code)
+   * @param {*} argsArray
+   * @param {*} level
+   * @returns
+   */
+  #logWrapper(argsArray, level) {
+    if (!argsArray.length) return;
+
+    // entity which is used to define testId
+    let context = null;
+
+    // get context for mocha
+    context = global.testTitle ?? null;
+
+    const severity = LEVELS[level].severity;
+    if (severity < LEVELS[this.logLevel]?.severity) return;
+
+    const logs = this.#stringifyLogs(...argsArray);
+    const colorizedLogs = chalk[LEVELS[level].color](logs);
+    this.#dataStorage.putData(colorizedLogs, context);
+    try {
+      // level.toLowerCase() represents method name (log, warn, error, etc)
+      this.#originalUserLogger[level.toLowerCase()](colorizedLogs);
+    } catch (e) {
+      // method could be unexisting, ignore error
+    }
+  }
+
+  assert(...args) {
+    this.#logWrapper(args, 'ERROR');
+  }
+
+  debug(...args) {
+    this.#logWrapper(args, 'DEBUG');
+  }
+
+  error(...args) {
+    this.#logWrapper(args, 'ERROR');
+  }
+
+  info(...args) {
+    this.#logWrapper(args, 'INFO');
+  }
+
+  log(...args) {
+    this.#logWrapper(args, 'LOG');
+  }
+
+  trace(...args) {
+    this.#logWrapper(args, 'TRACE');
+  }
+
+  warn(...args) {
+    this.#logWrapper(args, 'WARN');
+  }
+
+  /**
+   * Intercepts user logger messages.
+   * When call this method, Logger start to control the user logger
+   * @param {*} userLogger
+   */
+  intercept(userLogger) {
+    /* prevent multiple console interceptions (cause of infinite loop)
+    actual only for "console", because its used as default output and is intercepted by default */
+    const isUserLoggerConsole = userLogger.toString?.().toLowerCase() === '[object console]';
+    if (isUserLoggerConsole && process.env.TESTOMATIO_LOGGER_CONSOLE_INTERCEPTED) {
+      debug(`Try to intercept console, but it is already intercepted`);
+      return;
+    }
+
+    process.env.TESTOMATIO_LOGGER_CONSOLE_INTERCEPTED = 'true';
+    debug(isUserLoggerConsole ? 'console intercepted' : 'User logger intercepted');
+
+    // override user logger (any, e.g. console) methods to intercept log messages
+    for (const method of LOG_METHODS) {
+      /*
+      its better to create method even if it does not exist in user logger;
+      on method invocation, we will store the data anyway and catch block will prevent potential errors
+      while trying to output the message to terminal
+      */
+      // if (!this._loggerToIntercept[method]) continue;
+      userLogger[method] = (...args) => this[method](...args);
+    }
+
+    /* Playwright
+    Playwright intercepts console messages by default. Thus when we intercept any logger and provide output by console,
+    messages are intercepted by Playwright.
+    */
+
+    /*
+    Initial idea was to intercept any logger (tracer, pino, etc),
+    intercept message and provide output by the same logger.
+    But reality brings some problems: the same messages are intercepted multiple times
+    (because of multiple loggers are created at the same terminal process).
+    Also its difficult to understand (actually did not find the way to do it) if logger was already intercepted or not.
+    Thus, decided to intercept only console by default and provide output by default console.
+    It means, if user uses his own logger, its messages will be intercepted,
+    but the output will be provided by console.
+    TODO: try to implement the providing output to terminal by user logger
+    */
+  }
+
+  /**
+   * Allows to configure logger. Make sure you do it before the logger usage in your code.
+   *
+   * @param {Object} [config={}] - The configuration object.
+   * @param {string} [config.logLevel] - The desired log level. Valid values are 'DEBUG', 'INFO', 'WARN', and 'ERROR'.
+   * @param {boolean} [config.prettyObjects] - Specifies whether to enable pretty printing of objects.
+   * @returns {void}
+   */
+  configure(config = {}) {
+    if (!config) return;
+    if (config.prettyObjects === false || config.prettyObjects === true) this.prettyObjects = config.prettyObjects;
+    if (config.logLevel) this.logLevel = config.logLevel.toUpperCase();
+  }
+}
+
+Logger.instance = null;
+
+// module.exports.Logger = Logger;
+const logger = new Logger();
+
+module.exports = logger;
+
+// TODO: parse passed arguments as {level: 'str', message: 'str'} because some loggers use such syntax;
+// upd: did not face such loggers, but still could be useful
+
+/* Cypress
+There is no listener like "after:test" in cypress, only "after:spec" is available.
+Thus, cannot separate logs even when I gather them (because I don't know when the test is done, just know about suite).
+Also there is no easy way to access the message from cy.log() function.
+(Using testomatio logger – logger.log() is not convenient because Cypress chains its commands,
+thus such command will interrupt the chain.)
+
+(Could not implement intercepting of cy.log('message'));
+I found the only ability to get any logs using .task('log', 'message') (this is custom, not default cypress command)
+and then intercept it with:
+  on('task', {
+    log (message) {
+      console.log(message)
+      return null
+    }
+  })
+
+but:
+1) it does not solve problem with getting current running testId;
+2) leads to warning "Warning: Multiple attempts to register the following task(s):".)
+
+My way to get test id:
+add cypress command to save test title to file)):
+  Cypress.Commands.add('writeTestTitleToFile', () => {
+    const testTitle = cy.state('runnable').title;
+    cy.writeFile('testomatio_test_title', testTitle);
+  });
+
+Finally, in the test it will look like:
+    cy
+      .writeTestTitleToFile() // <<<
+      .task('log', 'This is a log message from the test') // <<<
+
+      .get('element)
+      .type('text')
+      .click()
+
+
+Parallelization in Cypress is only available if using Cypress Dashboard??
+*/

package/lib/pipe/csv.js

@@ -4,7 +4,7 @@ const fs = require('fs');
 const csvWriter = require('csv-writer');
 const chalk = require('chalk');
 const merge = require('lodash.merge');
-const { isSameTest, getCurrentDateTime } = require('../util');
+const { isSameTest, getCurrentDateTime } = require('../utils/utils');
 const { CSV_HEADERS } = require('../constants');
 
 /**
@@ -40,6 +40,9 @@ class CsvPipe {
     }
   }
 
+  // TODO: to using SET opts as argument => prepareRun(opts)
+  async prepareRun() {}
+  
   async createRun() {
     // empty
   }