@testomatio/reporter 1.0.0 → 1.0.1-6.1

package/lib/bin/startTest.js

@@ -13,9 +13,11 @@ program
   .option('--launch', 'Start a new run and return its ID')
   .option('--finish', 'Finish Run by its ID')
   .option("--env-file <envfile>", "Load environment variables from env file")
-  .action(opts => {
+  .option("--filter <filter>", "Additional execution filter")
+  .action(async (opts) => {
+    const { launch, finish, filter } = opts;
+    let { command } = opts;
 
-    const { command, launch, finish } = opts;
     if (opts.envFile) require('dotenv').config(opts.envFile); // eslint-disable-line
 
     const apiKey = process.env['INPUT_TESTOMATIO-KEY'] || process.env.TESTOMATIO;
@@ -51,6 +53,27 @@ program
 
     let exitCode = 0;
 
+    const client = new TestomatClient({ apiKey, title, parallel: true });
+
+    if(filter) {
+      const [pipe, ...optsArray] = filter.split(":");
+      const pipeOptions = optsArray.join(":");
+
+      try { 
+        const tests = await client.prepareRun({pipe, pipeOptions});
+
+        if(!tests || tests.length === 0) {
+          return;
+        }
+
+        const grep = ` --grep (${tests.join('|')})`;
+        command += grep;
+      }
+      catch(err) {
+        console.log(APP_PREFIX, err);
+      }
+    }
+
     if (!command.split) {
       process.exitCode = 255;
       console.log(APP_PREFIX, `No command provided. Use -c option to launch a test runner.`);
@@ -73,8 +96,6 @@ program
       return;
     }
 
-    const client = new TestomatClient({ apiKey, title, parallel: true });
-
     client.createRun().then(() => {
       const cmd = spawn(testCmds[0], testCmds.slice(1), { stdio: 'inherit' });
 

package/lib/client.js

@@ -1,6 +1,7 @@
 const debug = require('debug')('@testomatio/reporter:client');
 const createCallsiteRecord = require('callsite-record');
 const { sep, join } = require('path');
+const { minimatch } = require('minimatch')
 const fs = require('fs');
 const chalk = require('chalk');
 const { randomUUID } = require('crypto');
@@ -11,7 +12,7 @@ const pipesFactory = require('./pipe');
 /**
  * @typedef {import('../types').TestData} TestData
  * @typedef {import('../types').RunStatus} RunStatus
- * @typedef {import('../types').PipeResult} PipeResult  
+ * @typedef {import('../types').PipeResult} PipeResult
  */
 
 class Client {
@@ -21,57 +22,109 @@ class Client {
    * @param {*} params
    */
   constructor(params = {}) {
-    this.parallel = params.parallel;
     const store = {};
     this.uuid = randomUUID();
     this.pipes = pipesFactory(params, store);
     this.queue = Promise.resolve();
     this.totalUploaded = 0;
     this.version = JSON.parse(fs.readFileSync(join(__dirname, '..', 'package.json')).toString()).version;
+    this.executionList = Promise.resolve();
+
     console.log(APP_PREFIX, `Testomatio Reporter v${this.version}`);
   }
 
+  /**
+   * Asynchronously prepares the execution list for running tests through various pipes.
+   * Each pipe in the client is checked for enablement, 
+   * and if all pipes are disabled, the function returns a resolved Promise.
+   * Otherwise, it executes the `prepareRun` method for each enabled pipe and collects the results.
+   * The results are then filtered to remove any undefined values.
+   * If no valid results are found, the function returns undefined.
+   * Otherwise, it returns the first non-empty array from the filtered results.
+   *
+   * @param {Object} params - The options for preparing the test execution list.
+   * @param {string} params.pipe - Name of the executed pipe.
+   * @param {string} params.pipeOptions - Filter option.
+   * @returns {Promise<any>} - A Promise that resolves to an 
+   * array containing the prepared execution list,
+   * or resolves to undefined if no valid results are found or if all pipes are disabled.
+   */
+  async prepareRun(params) {
+    const { pipe, pipeOptions } = params;
+    // all pipes disabled, skipping
+    if (!this.pipes.some(p => p.isEnabled)) {
+      return Promise.resolve();
+    }
+
+    try {
+      const filterPipe = this.pipes.find(p => p.constructor.name.toLowerCase() === `${pipe.toLowerCase()}pipe`);
+
+      if (!filterPipe.isEnabled) {
+        // TODO:for the future for the another pipes
+        console.warn(
+          APP_PREFIX, 
+          `At the moment processing is available only for the "testomatio" key. Example: "testomatio:tag-name=xxx"`
+        );
+        return;
+      }
+
+      const results = await Promise.all(this.pipes.map(async p => 
+        ({ pipe: p.toString(), result: await p.prepareRun(pipeOptions) })  
+      ));
+    
+      const result = results.filter(p => p.pipe.includes('Testomatio'))[0]?.result;
+
+      if (!result || result.length === 0) {
+        return;
+      } 
+  
+      debug('Execution tests list', result);
+
+      return result;
+    } catch (err) {
+      console.error(APP_PREFIX, err);
+    }
+  }
+
   /**
    * Used to create a new Test run
    *
    * @returns {Promise<any>} - resolves to Run id which should be used to update / add test
    */
   createRun() {
+    debug('Creating run...');
     // all pipes disabled, skipping
-    if (!this.pipes.filter(p => p.isEnabled).length) return Promise.resolve();
-
-    const runParams = {
-      title: this.title,
-      parallel: this.parallel,
-      env: this.env,
-    };
+    if (!this.pipes?.filter(p => p.isEnabled).length) return Promise.resolve();
 
     global.testomatioArtifacts = [];
+    if (!global.testomatioDataStore) global.testomatioDataStore = {};
 
     this.queue = this.queue
-      .then(() => Promise.all(this.pipes.map(p => p.createRun(runParams))))
+      .then(() => Promise.all(this.pipes.map(p => p.createRun())))
       .catch(err => console.log(APP_PREFIX, err))
       .then(() => undefined); // fixes return type
-    debug('Run', this.queue);
+    // debug('Run', this.queue);
     return this.queue;
   }
 
   /**
    * Updates test status and its data
-   * 
+   *
    * @param {string|undefined} status
    * @param {TestData} [testData]
    * @param {string[]} [storeArtifacts]
    * @returns {Promise<PipeResult[]>}
    */
   async addTestRun(status, testData, storeArtifacts = []) {
+    debug('Adding test run for test', testData?.test_id || 'unknown test');
     // all pipes disabled, skipping
-    if (!this.pipes.filter(p => p.isEnabled).length) return;
+    if (!this.pipes?.filter(p => p.isEnabled).length) return [];
 
-    if (!testData) testData = {
-      title: 'Unknown test',
-      suite_title: 'Unknown suite',
-    }
+    if (!testData)
+      testData = {
+        title: 'Unknown test',
+        suite_title: 'Unknown suite',
+      };
 
     const {
       error = null,
@@ -100,13 +153,22 @@ class Client {
       stack = this.formatSteps(stack, steps);
     }
 
+    stack += testData.stack || '';
+
+    // in some cases (e.g. using cucumber) logger instance becomes empty object, if import at the top of the file
+    const logger = require('./logger');
+    const testLogs = logger.getLogs(test_id);
+    // debug(`Test logs for ${test_id}:\n`, testLogs);
+    if (stack) stack += '\n\n';
+    stack += testLogs ? `${chalk.bold('Logs:')}\n${testLogs}` : '';
+
     if (Array.isArray(storeArtifacts) && storeArtifacts.length) {
-      debug("CLIENT storeArtifact", storeArtifacts);
+      debug('CLIENT storeArtifact', storeArtifacts);
       files.push(...storeArtifacts);
     }
 
     if (Array.isArray(global.testomatioArtifacts)) {
-      debug("CLIENT global[testomatioArtifacts]", global.testomatioArtifacts);
+      debug('CLIENT global[testomatioArtifacts]', global.testomatioArtifacts);
       files.push(...global.testomatioArtifacts);
       global.testomatioArtifacts = [];
     }
@@ -142,17 +204,19 @@ class Client {
       artifacts,
     };
 
-    this.queue = this.queue
-      .then(() => Promise.all(this.pipes.map(async p => {
-        try {
-          const result = await p.addTest(data);
-          return { pipe: p.toString(), result };
-        } catch (err) {
-          console.log(APP_PREFIX, p.toString(), err);
-        }
-      }
-    )));
-    
+    this.queue = this.queue.then(() =>
+      Promise.all(
+        this.pipes.map(async p => {
+          try {
+            const result = await p.addTest(data);
+            return { pipe: p.toString(), result };
+          } catch (err) {
+            console.log(APP_PREFIX, p.toString(), err);
+          }
+        }),
+      ),
+    );
+
     return this.queue;
   }
 
@@ -164,16 +228,17 @@ class Client {
    * @returns {Promise<any>} - A Promise that resolves when finishes the run.
    */
   updateRunStatus(status, isParallel = false) {
+    debug('Updating run status...');
     // all pipes disabled, skipping
-    if (!this.pipes.filter(p => p.isEnabled).length) return Promise.resolve();
+    if (!this.pipes?.filter(p => p.isEnabled).length) return Promise.resolve();
 
     const runParams = { status, parallel: isParallel };
 
     this.queue = this.queue
       .then(() => Promise.all(this.pipes.map(p => p.finishRun(runParams))))
       .then(() => {
-        debug("TOTAL uploaded files", this.totalUploaded);
-        
+        debug('TOTAL uploaded files', this.totalUploaded);
+
         if (upload.isArtifactsEnabled() && this.totalUploaded > 0) {
           console.log(
             APP_PREFIX,
@@ -206,17 +271,21 @@ class Client {
       stack += '\n\n';
     }
 
+    const customFilter = process.env.TESTOMATIO_STACK_IGNORE;
+
     try {
+      let hasFrame = false;
       const record = createCallsiteRecord({
         forError: error,
+        isCallsiteFrame: frame => {
+          if (customFilter && minimatch(frame.getFileName(), customFilter)) return false;          
+          if (hasFrame) return false;
+          if (isNotInternalFrame(frame)) hasFrame = true;
+          return hasFrame;
+        }
       });
       if (record && !record.filename.startsWith('http')) {
-        stack += record.renderSync({
-          stackFilter: frame =>
-            frame.getFileName().indexOf(sep) > -1 &&
-            frame.getFileName().indexOf('node_modules') < 0 &&
-            frame.getFileName().indexOf('internal') < 0,
-        });
+        stack += record.renderSync({ stackFilter: isNotInternalFrame });
       }
       return stack;
     } catch (e) {
@@ -225,4 +294,10 @@ class Client {
   }
 }
 
+function isNotInternalFrame(frame) {
+  return frame.getFileName().includes(sep) &&
+    !frame.getFileName().includes('node_modules') &&
+    !frame.getFileName().includes('internal')
+}
+
 module.exports = Client;

package/lib/constants.js

@@ -3,6 +3,10 @@ const chalk = require('chalk');
 const APP_PREFIX = chalk.gray('[TESTOMATIO]');
 const TESTOMAT_ARTIFACT_SUFFIX = "testomatio_artifact_";
 
+const TESTOMAT_TMP_STORAGE = {
+  mainDir: "testomatio_tmp",
+}
+
 const CSV_HEADERS = [
   { id: 'suite_title', title: 'Suite_title' },
   { id: 'title', title: 'Title' },
@@ -21,6 +25,7 @@ const STATUS = {
 module.exports = {
   APP_PREFIX,
   TESTOMAT_ARTIFACT_SUFFIX,
+  TESTOMAT_TMP_STORAGE,
   CSV_HEADERS,
   STATUS,
 }

package/lib/dataStorage.js

@@ -0,0 +1,232 @@
+const debug = require('debug')('@testomatio/reporter:storage');
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const { join } = require('path');
+const JestReporter = require('./adapter/jest');
+const { TESTOMAT_TMP_STORAGE } = require('./constants');
+const { fileSystem } = require('./utils/utils');
+const getTestIdFromTestTitle = require('./utils/utils').parseTest;
+
+class DataStorage {
+  /**
+   * Creates data storage instance for specific data type.
+   * Stores data to global variable or to file depending on what is applicable for current test runner
+   * (running environment).
+   * @param {*} dataType storage type (like 'log', 'artifact'); could be any string, used only to define file path
+   */
+  constructor(dataType) {
+    this.dataType = dataType || 'data';
+    this.isFileStorage = true;
+    this.#refreshStorageType();
+
+    this.dataDirPath = path.join(TESTOMAT_TMP_STORAGE.mainDir, this.dataType);
+    fileSystem.createDir(this.dataDirPath);
+  }
+
+  /**
+   * Try to define the running environment. Not 100% reliable and used as additional check
+   * @returns jest | mocha | ...
+   */
+  getRunningEnviroment() {
+    // jest
+    if (process.env.JEST_WORKER_ID) return 'jest';
+
+    if (global.codeceptjs) return 'codeceptjs';
+
+    // 'cucumber:current', 'cucumber:legacy'
+    if (global.testomatioRunningEnvironment) return global.testomatioRunningEnvironment;
+
+    if (process.env.PLAYWRIGHT_TEST_BASE_URL || process.env.TEST_WORKER_INDEX) return 'playwright';
+
+    // mocha - can't detect
+    return null;
+  }
+
+  /**
+   * Puts any data to storage (file or global variable)
+   * @param {*} data anything you want to store
+   * @param {*} context could be testId or any context (test, suite, etc) which is used to define testId
+   * @returns
+   */
+  putData(data, context = null) {
+    this.#refreshStorageType();
+
+    let testId = this.#tryToRetrieveTestId(context) || null;
+
+    if (this.runningEnvironment === 'codeceptjs') {
+      this.isFileStorage = false;
+      testId = testId ?? global.testomatioDataStore?.currentlyRunningTestId;
+    }
+
+    if (this.runningEnvironment === 'jest') {
+      testId = testId ?? JestReporter.getIdOfCurrentlyRunningTest();
+    }
+
+    // logs in playwright are gathered by pw framework itself
+    if (this.runningEnvironment === 'playwright' && this.dataType === 'log') return;
+
+    // get id from global store
+    if (global.testomatioDataStore && global.testomatioDataStore.currentlyRunningTestId)
+      testId = testId ?? global.testomatioDataStore?.currentlyRunningTestId;
+
+    if (!testId && global?.currentlyRunningTestTitle)
+      testId = this.#tryToRetrieveTestId(global.currentlyRunningTestTitle);
+
+    if (!testId) {
+      debug(`No test id provided for ${this.dataType} data: ${data}`);
+      return;
+    }
+
+    if (this.isFileStorage) {
+      this.#putDataToFile(data, testId);
+    } else {
+      this.#putDataToGlobalVar(data, testId);
+    }
+  }
+
+  /**
+   * Returns data, stored for specific testId (or data which was stored without test id specified).
+   * This method will get data from global variable. But if it is not available, it will try to get data from file.
+   * Thus, good approach is to remove file storage folder before each test run (and after, for sure).
+   *
+   * Defining the execution environment is not guaranteed! Is used only as additional check.
+   * @param {*} context could be testId or any context (test, suite, etc) which is used to define testId
+   * @returns
+   */
+  getData(context) {
+    this.#refreshStorageType();
+
+    let testId = null;
+    if (typeof context === 'string') {
+      testId = this.#tryToRetrieveTestId(context) || context;
+    } else {
+      // TODO: derive testId from context
+      // testId = context...
+    }
+
+    if (!testId) {
+      debug(`Cannot get test id from passed context:`, context);
+      return null;
+    }
+
+    let testData = '';
+
+    if (global?.testomatioDataStore) {
+      testData = this.#getDataFromGlobalVar(testId);
+      // these frameworks use global variable storage
+      if (testData && ['cucumber', 'codeceptjs'].includes(this.runningEnvironment)) return testData;
+    }
+
+    /* condition is removed for mocha
+    mocha has created a global storage, but just in some cases, generally it is not available
+    */
+    // if (this.isFileStorage || !testData) {
+    //   testData = this.#getDataFromFile(testId);
+    //   if (testData) return testData;
+    // }
+
+    const testDataFromFile = this.#getDataFromFile(testId);
+    testData += testDataFromFile || '';
+
+    debug(`No ${this.dataType} data for test id ${testId} in both file and global variable`);
+    return testData || '';
+  }
+
+  /**
+   * This method is named as "try" because it does not guarantee that test id could be retrieved.
+   * Context could be anything (test, suite, string, etc) which is used to define testId.
+   * Or it could represent any other entity (which does not contain test id).
+   * @param {*} context
+   */
+  #tryToRetrieveTestId(context) {
+    if (!context) return null;
+    this.#refreshStorageType();
+
+    if (this.runningEnvironment === 'playwright' || context?.title) {
+      // context is testInfo
+      const testId = getTestIdFromTestTitle(context.title);
+      if (testId) return testId;
+    }
+
+    if (typeof context === 'string') {
+      const testId = getTestIdFromTestTitle(context);
+      if (testId) return testId;
+    }
+
+    return null;
+  }
+
+  /**
+   * Refreshes storage type (file or global variable) depending on current environment
+   * This method should be run on each attempt to put or get data from/to storage
+   * Because storage instance is created before the test runner is started. And storage type could be changed
+   */
+  #refreshStorageType() {
+    this.runningEnvironment = this.getRunningEnviroment();
+
+    // some test frameworks do not persist global variables, thus file storage is used for them (by default)
+    if (['playwright', 'codeceptjs'].includes(this.runningEnvironment)) this.isFileStorage = false;
+  }
+
+  #getDataFromGlobalVar(testId) {
+    try {
+      if (global?.testomatioDataStore[this.dataType]) {
+        const testData = global.testomatioDataStore[this.dataType][testId];
+        debug(`Data for test id ${testId}:\n${testData}`);
+        return testData || '';
+      }
+      debug(`No ${this.dataType} data for test id ${testId} in <global> storage`);
+      return '';
+    } catch (e) {
+      // there could be no data, ignore
+    }
+  }
+
+  #getDataFromFile(testId) {
+    try {
+      const filepath = join(this.dataDirPath, `${this.dataType}_${testId}`);
+      if (fs.existsSync(filepath)) {
+        const testData = fs.readFileSync(filepath, 'utf-8');
+        debug(`Data for test id ${testId}:\n${testData}`);
+        return testData || '';
+      }
+      debug(`No ${this.dataType} data for test id ${testId} in <file> storage`);
+      return '';
+    } catch (e) {
+      // there could be no data, ignore
+    }
+    return '';
+  }
+
+  #putDataToGlobalVar(data, testId) {
+    debug('Saving data to global variable for test', testId, ':\n', data, '\n');
+    if (!global.testomatioDataStore) global.testomatioDataStore = {};
+    if (!global.testomatioDataStore?.[this.dataType]) global.testomatioDataStore[this.dataType] = {};
+    global.testomatioDataStore?.[this.dataType][testId] // eslint-disable-line no-unused-expressions
+      ? (global.testomatioDataStore[this.dataType][testId] += `\n${data}`)
+      : (global.testomatioDataStore[this.dataType][testId] = data);
+  }
+
+  #putDataToFile(data, testId) {
+    if (typeof data !== 'string') data = JSON.stringify(data);
+    const filename = `${this.dataType}_${testId}`;
+    const filepath = join(this.dataDirPath, filename);
+    if (!fs.existsSync(this.dataDirPath)) fileSystem.createDir(this.dataDirPath);
+    debug('Saving data to file for test', testId, 'to', filepath, ':\n', data, '\n');
+
+    // TODO: handle multiple invocations of JSON.stringify.
+    // UPD: not actual because decided not to use it - it created extra wrapping quotes "" and // (escaped slashes)
+    fs.appendFileSync(filepath, data + os.EOL, 'utf-8');
+  }
+}
+
+module.exports.DataStorage = DataStorage;
+
+// TODO: consider using fs promises instead of writeSync/appendFileSync to
+// prevent blocking and improve performance (probably queue usage will be required)
+
+// TODO: rewrite client - everything regarding storing artifacts
+// TODO: try to define adapter inside client
+// TODO: use .env
+// TODO: ability to intercept multiple loggers (upd: no need)

package/lib/junit-adapter/java.js

@@ -10,24 +10,42 @@ class JavaAdapter extends Adapter {
 
   formatTest(t) {
     const fileParts = t.suite_title.split('.')
-    const example = t.title.match(/\[(.*)\]/)?.[1];
-    if (example) t.example = { "#": example }
 
     t.file = namespaceToFileName(t.suite_title);
     t.title = t.title.split('(')[0];
+
+    // detect params
+    const paramMatches = t.title.match(/\[(.*?)\]/g);
+
+    if (paramMatches) {
+      const params = paramMatches.map((_match, index) => `param${index + 1}`);
+      if (params.length === 1) params[0] = 'param';
+      let paramIndex = 0;
+    
+      t.title = t.title.replace(/: \[(.*?)\]/g, () => {
+        if (params.length < 2) return `\${param}`
+        const paramName = params[paramIndex] || `param${paramIndex + 1}`;
+        paramIndex++;
+        return `\${${paramName}}`;
+      });
+      const example = {};
+      paramMatches.forEach((match, index) => { example[params[index]] = match.replace(/[[\]]/g, '') });
+      t.example = example;
+    }
+
     t.suite_title = fileParts[fileParts.length - 1].replace(/\$/g, ' | ')
     return t;
   }
 
-  formatStack(t) {
-    const stack = super.formatStack(t);
+  // formatStack(t) {
+  //   const stack = super.formatStack(t);
 
-    const file = t.suite_title.split('.');
+  //   const file = t.suite_title.split('.');
 
-    const fileLine = `at .*${file[file.length - 1]}\.java:(\\d+)` // eslint-disable-line no-useless-escape
-    const regexp = new RegExp(fileLine,"g")
-    return stack.replace(regexp, `${this.opts.javaTests}${path.sep}${namespaceToFileName(t.suite_title)}:$1:`);
-  }
+  //   const fileLine = `at .*${file[file.length - 1]}\.java:(\\d+)` // eslint-disable-line no-useless-escape
+  //   const regexp = new RegExp(fileLine,"g")
+  //   return stack.replace(regexp, `${this.opts.javaTests}${path.sep}${namespaceToFileName(t.suite_title)}:$1:`);
+  // }
 }
 
 function namespaceToFileName(fileName) {