@testomatio/reporter 1.0.0-beta.4 → 1.0.1-6.1

package/lib/adapter/jest.js

@@ -1,6 +1,6 @@
 const TestomatClient = require('../client');
-const { STATUS } = require('../constants');
-const { parseTest, ansiRegExp } = require('../util');
+const { STATUS, TESTOMAT_TMP_STORAGE } = require('../constants');
+const { parseTest, ansiRegExp, fileSystem } = require('../utils/utils');
 
 class JestReporter {
   constructor(globalConfig, options) {
@@ -11,6 +11,22 @@ class JestReporter {
     this.client.createRun();
   }
 
+  static getIdOfCurrentlyRunningTest() {
+    if (!process.env.JEST_WORKER_ID) return null;
+    try {
+      // @ts-expect-error "expect" could only be defined inside Jest environement (forbidden to import it outside)
+      // eslint-disable-next-line no-undef
+      if (expect && expect?.getState()?.currentTestName) return parseTest(expect?.getState()?.currentTestName); 
+    } catch (e) {
+      return null; 
+    }
+  }
+
+  onRunStart() {
+    // clear tmp dir
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE.mainDir);
+  }
+
   onTestResult(test, testResult) {
     if (!this.client) return;
 
@@ -26,12 +42,21 @@ class JestReporter {
         steps = failureMessages[0];
       }
       const testId = parseTest(title);
+      let suite_title;
+      // this is test without a suite
+      if (result.fullName === result.title) {
+        suite_title = testResult.testFilePath.split('/').pop();
+      } else {
+        suite_title = result.fullName.replace(result.title, '');
+      }
+
       const deducedStatus = status === 'pending' ? 'skipped' : status;
       // In jest if test is not matched with test name pattern it is considered as skipped.
       // So adding a check if it is skipped for real or because of test pattern
       if (!this._globalConfig.testNamePattern || deducedStatus !== 'skipped') {
         this.client.addTestRun(deducedStatus, {
           test_id: testId,
+          suite_title,
           error,
           steps,
           title,
@@ -47,6 +72,9 @@ class JestReporter {
     const { numFailedTests } = results;
     const status = numFailedTests === 0 ? STATUS.PASSED : STATUS.FAILED;
     this.client.updateRunStatus(status);
+
+    // clear tmp dir
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE.mainDir);
   }
 }
 

package/lib/adapter/mocha.js

@@ -3,55 +3,54 @@ const Mocha = require('mocha');
 const debug = require('debug')('@testomatio/reporter:adapter:mocha');
 const chalk = require('chalk');
 const TestomatClient = require('../client');
-const { STATUS } = require('../constants');
-const { parseTest, specificTestInfo } = require('../util');
-const ArtifactStorage = require('../ArtifactStorage');
+const { STATUS, TESTOMAT_TMP_STORAGE } = require('../constants');
+const { parseTest, specificTestInfo, fileSystem } = require('../utils/utils');
+const ArtifactStorage = require('../_ArtifactStorageOld');
 
-const { EVENT_RUN_BEGIN, EVENT_RUN_END, EVENT_TEST_FAIL, EVENT_TEST_PASS, EVENT_TEST_PENDING } = Mocha.Runner.constants;
+const { EVENT_RUN_BEGIN, EVENT_RUN_END, EVENT_TEST_FAIL, EVENT_TEST_PASS, EVENT_TEST_PENDING } =
+  Mocha.Runner.constants;
 
 function MochaReporter(runner, opts) {
   Mocha.reporters.Base.call(this, runner);
-  let passes = 0; let failures = 0; let skipped = 0;
+  let passes = 0;
+  let failures = 0;
+  let skipped = 0;
   let artifactStore;
 
   const apiKey = opts?.reporterOptions?.apiKey || process.env.TESTOMATIO;
 
-  if (!apiKey) {
-    debug('TESTOMATIO key is empty, ignoring reports');
-    return;
-  }
   const client = new TestomatClient({ apiKey });
 
   runner.on(EVENT_RUN_BEGIN, () => {
     client.createRun();
 
-    const params = { 
-      toFile: true
+    const params = {
+      toFile: true,
     };
-    
-    artifactStore = runner._workerReporter !== undefined
-      ? new ArtifactStorage(params)
-      : new ArtifactStorage();
+
+    artifactStore = runner._workerReporter !== undefined ? new ArtifactStorage(params) : new ArtifactStorage();
+
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE.mainDir);
   });
 
-  runner.on(EVENT_TEST_PASS, async(test) => {
+  runner.on(EVENT_TEST_PASS, async test => {
     passes += 1;
     console.log(chalk.bold.green('✔'), test.fullTitle());
     const testId = parseTest(test.title);
 
-    const specificTest =  specificTestInfo(test);
+    const specificTest = specificTestInfo(test);
     const content = await artifactStore.artifactByTestName(specificTest);
 
     debug(`test=${specificTest} content = `, content);
 
     client.addTestRun(
-      STATUS.PASSED, 
+      STATUS.PASSED,
       {
         test_id: testId,
         title: test.title,
         time: test.duration,
       },
-      content
+      content,
     );
   });
 
@@ -66,24 +65,25 @@ function MochaReporter(runner, opts) {
     });
   });
 
-  runner.on(EVENT_TEST_FAIL, async(test, err) => {
+  runner.on(EVENT_TEST_FAIL, async (test, err) => {
     failures += 1;
     console.log(chalk.bold.red('✖'), test.fullTitle(), chalk.gray(err.message));
     const testId = parseTest(test.title);
 
-    const specificTest =  specificTestInfo(test);
+    const specificTest = specificTestInfo(test);
     const content = await artifactStore.artifactByTestName(specificTest);
 
     debug(`fail test=${specificTest} content = `, content);
 
     client.addTestRun(
-      STATUS.FAILED, {
+      STATUS.FAILED,
+      {
         error: err,
         test_id: testId,
         title: test.title,
         time: test.duration,
-      }, 
-      content
+      },
+      content,
     );
   });
 

package/lib/adapter/playwright.js

@@ -3,14 +3,14 @@ const crypto = require('crypto');
 const os = require('os');
 const path = require('path');
 const fs = require('fs');
-const { APP_PREFIX, STATUS: Status } = require('../constants');
+const { APP_PREFIX, STATUS: Status, TESTOMAT_TMP_STORAGE } = require('../constants');
 const TestomatioClient = require('../client');
 const { isArtifactsEnabled } = require('../fileUploader');
-const { parseTest } = require('../util');
+const { parseTest, fileSystem } = require('../utils/utils');
 
 const reportTestPromises = [];
 
-class TestomatioReporter {
+class PlaywrightReporter {
   constructor(config = {}) {
     this.client = new TestomatioClient({ apiKey: config?.apiKey });
 
@@ -18,11 +18,17 @@ class TestomatioReporter {
   }
 
   onBegin(_config, suite) {
+    // clean data storage
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE.mainDir);
     if (!this.client) return;
     this.suite = suite;
     this.client.createRun();
   }
 
+  // onTestBegin(test) {
+  //   const testId = parseTest(test.title);
+  // }
+
   onTestEnd(test, result) {
     if (!this.client) return;
 
@@ -37,7 +43,9 @@ class TestomatioReporter {
     for (const step of result.steps) {
       appendStep(step, steps);
     }
-    
+
+    const logs = `\n\n${chalk.bold('Logs:')}\n${chalk.red(result.stderr.join(''))}\n${result.stdout.join('')}`;
+
     const reportTestPromise = this.client.addTestRun(checkStatus(result.status), {
       error,
       test_id: testId,
@@ -45,13 +53,14 @@ class TestomatioReporter {
       title,
       steps: steps.join('\n'),
       time: duration,
+      stack: logs,
     }).then(pipes => {
-      testId = pipes.filter(p => p.pipe.includes('Testomatio'))[0]?.result?.data?.test_id;
+      testId = pipes?.filter(p => p.pipe.includes('Testomatio'))[0]?.result?.data?.test_id;
 
       this.uploads.push({
         testId, title, suite_title, files: result.attachments.filter((a) => a.body || a.path)
-      })
-    });;
+      });
+    });
 
     reportTestPromises.push(reportTestPromise);
   }
@@ -67,10 +76,9 @@ class TestomatioReporter {
       const promises = [];
 
       for (const upload of this.uploads) {
-
         const { title, testId, suite_title } = upload;
 
-        const files = upload.files.map((attachment) => {
+        const files = upload.files.map(attachment => {
           if (attachment.body) {
             const fileName = tmpFile();
             fs.writeFileSync(fileName, attachment.body);
@@ -78,7 +86,6 @@ class TestomatioReporter {
           return { path: attachment.path, title, type: attachment.contentType };
         });
 
-
         promises.push(
           this.client.addTestRun(undefined, {
             test_id: testId,
@@ -124,4 +131,4 @@ function tmpFile(prefix = 'tmp.') {
   return path.join(tmpdir, prefix + crypto.randomBytes(16).toString('hex'));
 }
 
-module.exports = TestomatioReporter;
+module.exports = PlaywrightReporter;

package/lib/adapter/webdriver.js

@@ -1,7 +1,7 @@
 // eslint-disable-next-line global-require, import/no-extraneous-dependencies
 const WDIOReporter = require('@wdio/reporter').default;
 const TestomatClient = require('../client');
-const { parseTest } = require('../util');
+const { parseTest } = require('../utils/utils');
 
 class WebdriverReporter extends WDIOReporter {
   constructor(options) {

package/lib/artifactStorage.js

@@ -0,0 +1,25 @@
+// const debug = require('debug')('@testomatio/reporter:logger');
+const { DataStorage } = require('./dataStorage');
+
+class ArtifactStorage {
+  constructor() {
+    this.dataStorage = new DataStorage('artifact');
+
+    // singleton
+    if (!ArtifactStorage.instance) {
+      ArtifactStorage.instance = this;
+    }
+  }
+
+  save(data, context = null) {
+    this.dataStorage.putData(data, context);
+  }
+
+  get(context) {
+    this.dataStorage.getData(context);
+  }
+}
+
+ArtifactStorage.instance = null;
+
+module.exports = new ArtifactStorage();

package/lib/bin/reportXml.js

@@ -23,6 +23,7 @@ program
     }
     let { javaTests, lang } = opts;
     if (opts.envFile) {
+      console.log(APP_PREFIX, 'Loading env file:', opts.envFile);
       debug('Loading env file: %s', opts.envFile) 
       require('dotenv').config({ path: opts.envFile }); // eslint-disable-line
     }

package/lib/bin/startTest.js

@@ -13,9 +13,11 @@ program
   .option('--launch', 'Start a new run and return its ID')
   .option('--finish', 'Finish Run by its ID')
   .option("--env-file <envfile>", "Load environment variables from env file")
-  .action(opts => {
+  .option("--filter <filter>", "Additional execution filter")
+  .action(async (opts) => {
+    const { launch, finish, filter } = opts;
+    let { command } = opts;
 
-    const { command, launch, finish } = opts;
     if (opts.envFile) require('dotenv').config(opts.envFile); // eslint-disable-line
 
     const apiKey = process.env['INPUT_TESTOMATIO-KEY'] || process.env.TESTOMATIO;
@@ -51,6 +53,27 @@ program
 
     let exitCode = 0;
 
+    const client = new TestomatClient({ apiKey, title, parallel: true });
+
+    if(filter) {
+      const [pipe, ...optsArray] = filter.split(":");
+      const pipeOptions = optsArray.join(":");
+
+      try { 
+        const tests = await client.prepareRun({pipe, pipeOptions});
+
+        if(!tests || tests.length === 0) {
+          return;
+        }
+
+        const grep = ` --grep (${tests.join('|')})`;
+        command += grep;
+      }
+      catch(err) {
+        console.log(APP_PREFIX, err);
+      }
+    }
+
     if (!command.split) {
       process.exitCode = 255;
       console.log(APP_PREFIX, `No command provided. Use -c option to launch a test runner.`);
@@ -73,8 +96,6 @@ program
       return;
     }
 
-    const client = new TestomatClient({ apiKey, title, parallel: true });
-
     client.createRun().then(() => {
       const cmd = spawn(testCmds[0], testCmds.slice(1), { stdio: 'inherit' });
 

package/lib/client.js

@@ -1,6 +1,7 @@
 const debug = require('debug')('@testomatio/reporter:client');
 const createCallsiteRecord = require('callsite-record');
 const { sep, join } = require('path');
+const { minimatch } = require('minimatch')
 const fs = require('fs');
 const chalk = require('chalk');
 const { randomUUID } = require('crypto');
@@ -11,6 +12,7 @@ const pipesFactory = require('./pipe');
 /**
  * @typedef {import('../types').TestData} TestData
  * @typedef {import('../types').RunStatus} RunStatus
+ * @typedef {import('../types').PipeResult} PipeResult
  */
 
 class Client {
@@ -20,57 +22,109 @@ class Client {
    * @param {*} params
    */
   constructor(params = {}) {
-    this.parallel = params.parallel;
     const store = {};
     this.uuid = randomUUID();
     this.pipes = pipesFactory(params, store);
     this.queue = Promise.resolve();
     this.totalUploaded = 0;
     this.version = JSON.parse(fs.readFileSync(join(__dirname, '..', 'package.json')).toString()).version;
+    this.executionList = Promise.resolve();
+
     console.log(APP_PREFIX, `Testomatio Reporter v${this.version}`);
   }
 
+  /**
+   * Asynchronously prepares the execution list for running tests through various pipes.
+   * Each pipe in the client is checked for enablement, 
+   * and if all pipes are disabled, the function returns a resolved Promise.
+   * Otherwise, it executes the `prepareRun` method for each enabled pipe and collects the results.
+   * The results are then filtered to remove any undefined values.
+   * If no valid results are found, the function returns undefined.
+   * Otherwise, it returns the first non-empty array from the filtered results.
+   *
+   * @param {Object} params - The options for preparing the test execution list.
+   * @param {string} params.pipe - Name of the executed pipe.
+   * @param {string} params.pipeOptions - Filter option.
+   * @returns {Promise<any>} - A Promise that resolves to an 
+   * array containing the prepared execution list,
+   * or resolves to undefined if no valid results are found or if all pipes are disabled.
+   */
+  async prepareRun(params) {
+    const { pipe, pipeOptions } = params;
+    // all pipes disabled, skipping
+    if (!this.pipes.some(p => p.isEnabled)) {
+      return Promise.resolve();
+    }
+
+    try {
+      const filterPipe = this.pipes.find(p => p.constructor.name.toLowerCase() === `${pipe.toLowerCase()}pipe`);
+
+      if (!filterPipe.isEnabled) {
+        // TODO:for the future for the another pipes
+        console.warn(
+          APP_PREFIX, 
+          `At the moment processing is available only for the "testomatio" key. Example: "testomatio:tag-name=xxx"`
+        );
+        return;
+      }
+
+      const results = await Promise.all(this.pipes.map(async p => 
+        ({ pipe: p.toString(), result: await p.prepareRun(pipeOptions) })  
+      ));
+    
+      const result = results.filter(p => p.pipe.includes('Testomatio'))[0]?.result;
+
+      if (!result || result.length === 0) {
+        return;
+      } 
+  
+      debug('Execution tests list', result);
+
+      return result;
+    } catch (err) {
+      console.error(APP_PREFIX, err);
+    }
+  }
+
   /**
    * Used to create a new Test run
    *
-   * @returns {Promise<void>} - resolves to Run id which should be used to update / add test
+   * @returns {Promise<any>} - resolves to Run id which should be used to update / add test
    */
   createRun() {
+    debug('Creating run...');
     // all pipes disabled, skipping
-    if (!this.pipes.filter(p => p.isEnabled).length) return Promise.resolve();
-
-    const runParams = {
-      title: this.title,
-      parallel: this.parallel,
-      env: this.env,
-    };
+    if (!this.pipes?.filter(p => p.isEnabled).length) return Promise.resolve();
 
     global.testomatioArtifacts = [];
+    if (!global.testomatioDataStore) global.testomatioDataStore = {};
 
     this.queue = this.queue
-      .then(() => Promise.all(this.pipes.map(p => p.createRun(runParams))))
+      .then(() => Promise.all(this.pipes.map(p => p.createRun())))
       .catch(err => console.log(APP_PREFIX, err))
       .then(() => undefined); // fixes return type
-    debug('Run', this.queue);
+    // debug('Run', this.queue);
     return this.queue;
   }
 
   /**
    * Updates test status and its data
-   * 
+   *
    * @param {string|undefined} status
    * @param {TestData} [testData]
    * @param {string[]} [storeArtifacts]
-   * @returns {Promise<void>}
+   * @returns {Promise<PipeResult[]>}
    */
   async addTestRun(status, testData, storeArtifacts = []) {
+    debug('Adding test run for test', testData?.test_id || 'unknown test');
     // all pipes disabled, skipping
-    if (!this.pipes.filter(p => p.isEnabled).length) return;
+    if (!this.pipes?.filter(p => p.isEnabled).length) return [];
 
-    if (!testData) testData = {
-      title: 'Unknown test',
-      suite_title: 'Unknown suite',
-    }
+    if (!testData)
+      testData = {
+        title: 'Unknown test',
+        suite_title: 'Unknown suite',
+      };
 
     const {
       error = null,
@@ -99,13 +153,22 @@ class Client {
       stack = this.formatSteps(stack, steps);
     }
 
+    stack += testData.stack || '';
+
+    // in some cases (e.g. using cucumber) logger instance becomes empty object, if import at the top of the file
+    const logger = require('./logger');
+    const testLogs = logger.getLogs(test_id);
+    // debug(`Test logs for ${test_id}:\n`, testLogs);
+    if (stack) stack += '\n\n';
+    stack += testLogs ? `${chalk.bold('Logs:')}\n${testLogs}` : '';
+
     if (Array.isArray(storeArtifacts) && storeArtifacts.length) {
-      debug("CLIENT storeArtifact", storeArtifacts);
+      debug('CLIENT storeArtifact', storeArtifacts);
       files.push(...storeArtifacts);
     }
 
     if (Array.isArray(global.testomatioArtifacts)) {
-      debug("CLIENT global[testomatioArtifacts]", global.testomatioArtifacts);
+      debug('CLIENT global[testomatioArtifacts]', global.testomatioArtifacts);
       files.push(...global.testomatioArtifacts);
       global.testomatioArtifacts = [];
     }
@@ -141,17 +204,19 @@ class Client {
       artifacts,
     };
 
-    this.queue = this.queue
-      .then(() => Promise.all(this.pipes.map(async p => {
-        try {
-          const result = await p.addTest(data);
-          return { pipe: p.toString(), result };
-        } catch (err) {
-          console.log(APP_PREFIX, pipe.toString(), err);
-        }
-      }
-    )));
-    
+    this.queue = this.queue.then(() =>
+      Promise.all(
+        this.pipes.map(async p => {
+          try {
+            const result = await p.addTest(data);
+            return { pipe: p.toString(), result };
+          } catch (err) {
+            console.log(APP_PREFIX, p.toString(), err);
+          }
+        }),
+      ),
+    );
+
     return this.queue;
   }
 
@@ -160,19 +225,20 @@ class Client {
    * Updates the status of the current test run and finishes the run.
    * @param {RunStatus} status - The status of the current test run. Must be one of "passed", "failed", or "finished"
    * @param {boolean} [isParallel] - Whether the current test run was executed in parallel with other tests.
-   * @returns {Promise<void>} - A Promise that resolves when finishes the run.
+   * @returns {Promise<any>} - A Promise that resolves when finishes the run.
    */
   updateRunStatus(status, isParallel = false) {
+    debug('Updating run status...');
     // all pipes disabled, skipping
-    if (!this.pipes.filter(p => p.isEnabled).length) return Promise.resolve();
+    if (!this.pipes?.filter(p => p.isEnabled).length) return Promise.resolve();
 
     const runParams = { status, parallel: isParallel };
 
     this.queue = this.queue
       .then(() => Promise.all(this.pipes.map(p => p.finishRun(runParams))))
       .then(() => {
-        debug("TOTAL uploaded files", this.totalUploaded);
-        
+        debug('TOTAL uploaded files', this.totalUploaded);
+
         if (upload.isArtifactsEnabled() && this.totalUploaded > 0) {
           console.log(
             APP_PREFIX,
@@ -205,17 +271,21 @@ class Client {
       stack += '\n\n';
     }
 
+    const customFilter = process.env.TESTOMATIO_STACK_IGNORE;
+
     try {
+      let hasFrame = false;
       const record = createCallsiteRecord({
         forError: error,
+        isCallsiteFrame: frame => {
+          if (customFilter && minimatch(frame.getFileName(), customFilter)) return false;          
+          if (hasFrame) return false;
+          if (isNotInternalFrame(frame)) hasFrame = true;
+          return hasFrame;
+        }
       });
       if (record && !record.filename.startsWith('http')) {
-        stack += record.renderSync({
-          stackFilter: frame =>
-            frame.getFileName().indexOf(sep) > -1 &&
-            frame.getFileName().indexOf('node_modules') < 0 &&
-            frame.getFileName().indexOf('internal') < 0,
-        });
+        stack += record.renderSync({ stackFilter: isNotInternalFrame });
       }
       return stack;
     } catch (e) {
@@ -224,4 +294,10 @@ class Client {
   }
 }
 
+function isNotInternalFrame(frame) {
+  return frame.getFileName().includes(sep) &&
+    !frame.getFileName().includes('node_modules') &&
+    !frame.getFileName().includes('internal')
+}
+
 module.exports = Client;

package/lib/constants.js

@@ -3,6 +3,10 @@ const chalk = require('chalk');
 const APP_PREFIX = chalk.gray('[TESTOMATIO]');
 const TESTOMAT_ARTIFACT_SUFFIX = "testomatio_artifact_";
 
+const TESTOMAT_TMP_STORAGE = {
+  mainDir: "testomatio_tmp",
+}
+
 const CSV_HEADERS = [
   { id: 'suite_title', title: 'Suite_title' },
   { id: 'title', title: 'Title' },
@@ -21,6 +25,7 @@ const STATUS = {
 module.exports = {
   APP_PREFIX,
   TESTOMAT_ARTIFACT_SUFFIX,
+  TESTOMAT_TMP_STORAGE,
   CSV_HEADERS,
   STATUS,
 }