@testomatio/reporter 1.1.0-beta → 1.1.0-beta-2

package/lib/adapter/mocha.js

@@ -1,58 +1,40 @@
 // eslint-disable-next-line global-require, import/no-extraneous-dependencies
 const Mocha = require('mocha');
-const debug = require('debug')('@testomatio/reporter:adapter:mocha');
 const chalk = require('chalk');
 const TestomatClient = require('../client');
-const { STATUS } = require('../constants');
-const { parseTest, specificTestInfo } = require('../util');
-const ArtifactStorage = require('../_ArtifactStorageOld');
+const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
+const { parseTest, fileSystem } = require('../utils/utils');
 
-const { EVENT_RUN_BEGIN, EVENT_RUN_END, EVENT_TEST_FAIL, EVENT_TEST_PASS, EVENT_TEST_PENDING } = Mocha.Runner.constants;
+const { EVENT_RUN_BEGIN, EVENT_RUN_END, EVENT_TEST_FAIL, EVENT_TEST_PASS, EVENT_TEST_PENDING } =
+  Mocha.Runner.constants;
 
 function MochaReporter(runner, opts) {
   Mocha.reporters.Base.call(this, runner);
-  let passes = 0; let failures = 0; let skipped = 0;
-  let artifactStore;
+  let passes = 0;
+  let failures = 0;
+  let skipped = 0;
+  // let artifactStore;
 
   const apiKey = opts?.reporterOptions?.apiKey || process.env.TESTOMATIO;
 
-  if (!apiKey) {
-    debug('TESTOMATIO key is empty, ignoring reports');
-    return;
-  }
   const client = new TestomatClient({ apiKey });
 
   runner.on(EVENT_RUN_BEGIN, () => {
     client.createRun();
 
-    const params = { 
-      toFile: true
-    };
-    
-    artifactStore = runner._workerReporter !== undefined
-      ? new ArtifactStorage(params)
-      : new ArtifactStorage();
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
   });
 
-  runner.on(EVENT_TEST_PASS, async(test) => {
+  runner.on(EVENT_TEST_PASS, async test => {
     passes += 1;
     console.log(chalk.bold.green('✔'), test.fullTitle());
     const testId = parseTest(test.title);
 
-    const specificTest =  specificTestInfo(test);
-    const content = await artifactStore.artifactByTestName(specificTest);
-
-    debug(`test=${specificTest} content = `, content);
-
-    client.addTestRun(
-      STATUS.PASSED, 
-      {
-        test_id: testId,
-        title: test.title,
-        time: test.duration,
-      },
-      content
-    );
+    client.addTestRun(STATUS.PASSED, {
+      test_id: testId,
+      title: test.title,
+      time: test.duration,
+    });
   });
 
   runner.on(EVENT_TEST_PENDING, test => {
@@ -66,33 +48,23 @@ function MochaReporter(runner, opts) {
     });
   });
 
-  runner.on(EVENT_TEST_FAIL, async(test, err) => {
+  runner.on(EVENT_TEST_FAIL, async (test, err) => {
     failures += 1;
     console.log(chalk.bold.red('✖'), test.fullTitle(), chalk.gray(err.message));
     const testId = parseTest(test.title);
 
-    const specificTest =  specificTestInfo(test);
-    const content = await artifactStore.artifactByTestName(specificTest);
-
-    debug(`fail test=${specificTest} content = `, content);
-
-    client.addTestRun(
-      STATUS.FAILED, {
-        error: err,
-        test_id: testId,
-        title: test.title,
-        time: test.duration,
-      }, 
-      content
-    );
+    client.addTestRun(STATUS.FAILED, {
+      error: err,
+      test_id: testId,
+      title: test.title,
+      time: test.duration,
+    });
   });
 
   runner.on(EVENT_RUN_END, () => {
     const status = failures === 0 ? STATUS.PASSED : STATUS.FAILED;
     console.log(chalk.bold(status), `${passes} passed, ${failures} failed, ${skipped} skipped`);
     client.updateRunStatus(status);
-
-    artifactStore.cleanup();
   });
 }
 

package/lib/adapter/playwright.js

@@ -3,14 +3,14 @@ const crypto = require('crypto');
 const os = require('os');
 const path = require('path');
 const fs = require('fs');
-const { APP_PREFIX, STATUS: Status } = require('../constants');
+const { APP_PREFIX, STATUS: Status, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
 const TestomatioClient = require('../client');
 const { isArtifactsEnabled } = require('../fileUploader');
-const { parseTest } = require('../util');
+const { parseTest, fileSystem } = require('../utils/utils');
 
 const reportTestPromises = [];
 
-class TestomatioReporter {
+class PlaywrightReporter {
   constructor(config = {}) {
     this.client = new TestomatioClient({ apiKey: config?.apiKey });
 
@@ -18,11 +18,19 @@ class TestomatioReporter {
   }
 
   onBegin(_config, suite) {
+    // clean data storage
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
     if (!this.client) return;
     this.suite = suite;
     this.client.createRun();
   }
 
+  /*
+  onTestBegin(test) {
+    // does not work; value is not to the storage context
+    global.testTitle = test.title;
+  } */
+
   onTestEnd(test, result) {
     if (!this.client) return;
 
@@ -37,21 +45,34 @@ class TestomatioReporter {
     for (const step of result.steps) {
       appendStep(step, steps);
     }
-    
-    const reportTestPromise = this.client.addTestRun(checkStatus(result.status), {
-      error,
-      test_id: testId,
-      suite_title,
-      title,
-      steps: steps.join('\n'),
-      time: duration,
-    }).then(pipes => {
-      testId = pipes.filter(p => p.pipe.includes('Testomatio'))[0]?.result?.data?.test_id;
-
-      this.uploads.push({
-        testId, title, suite_title, files: result.attachments.filter((a) => a.body || a.path)
+
+    let logs = '';
+    if (result.stderr.length || result.stdout.length) {
+      logs = `\n\n${chalk.bold('Logs:')}\n${chalk.red(result.stderr.join(''))}\n${result.stdout.join('')}`;
+    }
+
+    const reportTestPromise = this.client
+      .addTestRun(checkStatus(result.status), {
+        error,
+        test_id: testId,
+        suite_title,
+        title,
+        steps: steps.join('\n'),
+        time: duration,
+        stack: logs,
       })
-    });;
+      .then(pipes => {
+        testId = pipes?.filter(p => p.pipe.includes('Testomatio'))[0]?.result?.data?.test_id;
+
+        this.uploads.push({
+          testId,
+          title,
+          suite_title,
+          files: result.attachments.filter(a => a.body || a.path),
+        });
+        // remove empty uploads
+        this.uploads = this.uploads.filter(upload => upload.files.length);
+      });
 
     reportTestPromises.push(reportTestPromise);
   }
@@ -67,10 +88,9 @@ class TestomatioReporter {
       const promises = [];
 
       for (const upload of this.uploads) {
-
         const { title, testId, suite_title } = upload;
 
-        const files = upload.files.map((attachment) => {
+        const files = upload.files.map(attachment => {
           if (attachment.body) {
             const fileName = tmpFile();
             fs.writeFileSync(fileName, attachment.body);
@@ -78,7 +98,6 @@ class TestomatioReporter {
           return { path: attachment.path, title, type: attachment.contentType };
         });
 
-
         promises.push(
           this.client.addTestRun(undefined, {
             test_id: testId,
@@ -124,4 +143,4 @@ function tmpFile(prefix = 'tmp.') {
   return path.join(tmpdir, prefix + crypto.randomBytes(16).toString('hex'));
 }
 
-module.exports = TestomatioReporter;
+module.exports = PlaywrightReporter;

package/lib/adapter/webdriver.js

@@ -1,7 +1,7 @@
 // eslint-disable-next-line global-require, import/no-extraneous-dependencies
 const WDIOReporter = require('@wdio/reporter').default;
 const TestomatClient = require('../client');
-const { parseTest } = require('../util');
+const { parseTest } = require('../utils/utils');
 
 class WebdriverReporter extends WDIOReporter {
   constructor(options) {

package/lib/bin/reportXml.js

@@ -23,6 +23,7 @@ program
     }
     let { javaTests, lang } = opts;
     if (opts.envFile) {
+      console.log(APP_PREFIX, 'Loading env file:', opts.envFile);
       debug('Loading env file: %s', opts.envFile) 
       require('dotenv').config({ path: opts.envFile }); // eslint-disable-line
     }

package/lib/bin/startTest.js

@@ -13,9 +13,11 @@ program
   .option('--launch', 'Start a new run and return its ID')
   .option('--finish', 'Finish Run by its ID')
   .option("--env-file <envfile>", "Load environment variables from env file")
-  .action(opts => {
+  .option("--filter <filter>", "Additional execution filter")
+  .action(async (opts) => {
+    const { launch, finish, filter } = opts;
+    let { command } = opts;
 
-    const { command, launch, finish } = opts;
     if (opts.envFile) require('dotenv').config(opts.envFile); // eslint-disable-line
 
     const apiKey = process.env['INPUT_TESTOMATIO-KEY'] || process.env.TESTOMATIO;
@@ -51,6 +53,27 @@ program
 
     let exitCode = 0;
 
+    const client = new TestomatClient({ apiKey, title, parallel: true });
+
+    if(filter) {
+      const [pipe, ...optsArray] = filter.split(":");
+      const pipeOptions = optsArray.join(":");
+
+      try { 
+        const tests = await client.prepareRun({pipe, pipeOptions});
+
+        if(!tests || tests.length === 0) {
+          return;
+        }
+
+        const grep = ` --grep (${tests.join('|')})`;
+        command += grep;
+      }
+      catch(err) {
+        console.log(APP_PREFIX, err);
+      }
+    }
+
     if (!command.split) {
       process.exitCode = 255;
       console.log(APP_PREFIX, `No command provided. Use -c option to launch a test runner.`);
@@ -73,8 +96,6 @@ program
       return;
     }
 
-    const client = new TestomatClient({ apiKey, title, parallel: true });
-
     client.createRun().then(() => {
       const cmd = spawn(testCmds[0], testCmds.slice(1), { stdio: 'inherit' });
 

package/lib/client.js

@@ -1,18 +1,19 @@
 const debug = require('debug')('@testomatio/reporter:client');
 const createCallsiteRecord = require('callsite-record');
 const { sep, join } = require('path');
+const { minimatch } = require('minimatch')
 const fs = require('fs');
 const chalk = require('chalk');
 const { randomUUID } = require('crypto');
 const upload = require('./fileUploader');
 const { APP_PREFIX } = require('./constants');
 const pipesFactory = require('./pipe');
-const logger = require('./logger');
+const artifactStorage = require('./storages/artifact-storage');
 
 /**
  * @typedef {import('../types').TestData} TestData
  * @typedef {import('../types').RunStatus} RunStatus
- * @typedef {import('../types').PipeResult} PipeResult  
+ * @typedef {import('../types').PipeResult} PipeResult
  */
 
 class Client {
@@ -22,39 +23,85 @@ class Client {
    * @param {*} params
    */
   constructor(params = {}) {
-    this.parallel = params.parallel;
     const store = {};
     this.uuid = randomUUID();
     this.pipes = pipesFactory(params, store);
     this.queue = Promise.resolve();
     this.totalUploaded = 0;
     this.version = JSON.parse(fs.readFileSync(join(__dirname, '..', 'package.json')).toString()).version;
+    this.executionList = Promise.resolve();
+
     console.log(APP_PREFIX, `Testomatio Reporter v${this.version}`);
   }
 
+  /**
+   * Asynchronously prepares the execution list for running tests through various pipes.
+   * Each pipe in the client is checked for enablement, 
+   * and if all pipes are disabled, the function returns a resolved Promise.
+   * Otherwise, it executes the `prepareRun` method for each enabled pipe and collects the results.
+   * The results are then filtered to remove any undefined values.
+   * If no valid results are found, the function returns undefined.
+   * Otherwise, it returns the first non-empty array from the filtered results.
+   *
+   * @param {Object} params - The options for preparing the test execution list.
+   * @param {string} params.pipe - Name of the executed pipe.
+   * @param {string} params.pipeOptions - Filter option.
+   * @returns {Promise<any>} - A Promise that resolves to an 
+   * array containing the prepared execution list,
+   * or resolves to undefined if no valid results are found or if all pipes are disabled.
+   */
+  async prepareRun(params) {
+    const { pipe, pipeOptions } = params;
+    // all pipes disabled, skipping
+    if (!this.pipes.some(p => p.isEnabled)) {
+      return Promise.resolve();
+    }
+
+    try {
+      const filterPipe = this.pipes.find(p => p.constructor.name.toLowerCase() === `${pipe.toLowerCase()}pipe`);
+
+      if (!filterPipe.isEnabled) {
+        // TODO:for the future for the another pipes
+        console.warn(
+          APP_PREFIX, 
+          `At the moment processing is available only for the "testomatio" key. Example: "testomatio:tag-name=xxx"`
+        );
+        return;
+      }
+
+      const results = await Promise.all(this.pipes.map(async p => 
+        ({ pipe: p.toString(), result: await p.prepareRun(pipeOptions) })  
+      ));
+    
+      const result = results.filter(p => p.pipe.includes('Testomatio'))[0]?.result;
+
+      if (!result || result.length === 0) {
+        return;
+      } 
+  
+      debug('Execution tests list', result);
+
+      return result;
+    } catch (err) {
+      console.error(APP_PREFIX, err);
+    }
+  }
+
   /**
    * Used to create a new Test run
    *
    * @returns {Promise<any>} - resolves to Run id which should be used to update / add test
    */
   createRun() {
+    debug('Creating run...');
     // all pipes disabled, skipping
-    if (!this.pipes.filter(p => p.isEnabled).length) return Promise.resolve();
-
-    const runParams = {
-      title: this.title,
-      parallel: this.parallel,
-      env: this.env,
-    };
-
-    global.testomatioArtifacts = [];
-    // TODO: create global storage
+    if (!this.pipes?.filter(p => p.isEnabled).length) return Promise.resolve();
 
     this.queue = this.queue
-      .then(() => Promise.all(this.pipes.map(p => p.createRun(runParams))))
+      .then(() => Promise.all(this.pipes.map(p => p.createRun())))
       .catch(err => console.log(APP_PREFIX, err))
       .then(() => undefined); // fixes return type
-    debug('Run', this.queue);
+    // debug('Run', this.queue);
     return this.queue;
   }
 
@@ -63,12 +110,11 @@ class Client {
    *
    * @param {string|undefined} status
    * @param {TestData} [testData]
-   * @param {string[]} [storeArtifacts]
    * @returns {Promise<PipeResult[]>}
    */
-  async addTestRun(status, testData, storeArtifacts = []) {
+  async addTestRun(status, testData) {
     // all pipes disabled, skipping
-    if (!this.pipes.filter(p => p.isEnabled).length) return;
+    if (!this.pipes?.filter(p => p.isEnabled).length) return [];
 
     if (!testData)
       testData = {
@@ -103,20 +149,23 @@ class Client {
       stack = this.formatSteps(stack, steps);
     }
 
+    stack += testData.stack || '';
+
+    // ATTACH LOGS from storage
+    // in some cases (e.g. using cucumber) logger instance becomes empty object, if import at the top of the file
+    const logger = require('./storages/logger');
     const testLogs = logger.getLogs(test_id);
-    debug(`Test logs for ${test_id}:\n`, testLogs);
-    stack += testLogs;
+    // debug(`Test logs for ${test_id}:\n`, testLogs);
+    if (stack) stack += '\n\n';
+    stack += testLogs ? `${chalk.bold('Logs:')}\n${testLogs}` : '';
 
-    if (Array.isArray(storeArtifacts) && storeArtifacts.length) {
-      debug('CLIENT storeArtifact', storeArtifacts);
-      files.push(...storeArtifacts);
-    }
+    // GET ARTIFACTS from storage
+    const artifactFiles = artifactStorage.get(test_id);
+    if (artifactFiles) files.push(...artifactFiles);
 
-    if (Array.isArray(global.testomatioArtifacts)) {
-      debug('CLIENT global[testomatioArtifacts]', global.testomatioArtifacts);
-      files.push(...global.testomatioArtifacts);
-      global.testomatioArtifacts = [];
-    }
+    // GET KEY-VALUEs from storage
+    const keyValueStorage = require('./storages/key-value-storage');
+    const keyValues = keyValueStorage.get(test_id);
 
     for (const file of files) {
       uploadedFiles.push(upload.uploadFileByPath(file, this.uuid));
@@ -129,7 +178,7 @@ class Client {
 
     const artifacts = await Promise.all(uploadedFiles);
 
-    global.testomatioArtifacts = [];
+    // global.testomatioArtifacts = [];
 
     this.totalUploaded += uploadedFiles.filter(n => n).length;
 
@@ -147,19 +196,22 @@ class Client {
       message,
       run_time: parseFloat(time),
       artifacts,
+      meta: keyValues,
     };
 
-    this.queue = this.queue
-      .then(() => Promise.all(this.pipes.map(async p => {
-        try {
-          const result = await p.addTest(data);
-          return { pipe: p.toString(), result };
-        } catch (err) {
-          console.log(APP_PREFIX, p.toString(), err);
-        }
-      }
-    )));
-    
+    this.queue = this.queue.then(() =>
+      Promise.all(
+        this.pipes.map(async p => {
+          try {
+            const result = await p.addTest(data);
+            return { pipe: p.toString(), result };
+          } catch (err) {
+            console.log(APP_PREFIX, p.toString(), err);
+          }
+        }),
+      ),
+    );
+
     return this.queue;
   }
 
@@ -171,22 +223,25 @@ class Client {
    * @returns {Promise<any>} - A Promise that resolves when finishes the run.
    */
   updateRunStatus(status, isParallel = false) {
+    debug('Updating run status...');
     // all pipes disabled, skipping
-    if (!this.pipes.filter(p => p.isEnabled).length) return Promise.resolve();
+    if (!this.pipes?.filter(p => p.isEnabled).length) return Promise.resolve();
 
     const runParams = { status, parallel: isParallel };
 
     this.queue = this.queue
       .then(() => Promise.all(this.pipes.map(p => p.finishRun(runParams))))
       .then(() => {
-        debug('TOTAL uploaded files', this.totalUploaded);
+        debug('TOTAL artifacts', this.totalUploaded);
+        if (this.totalUploaded && !upload.isArtifactsEnabled())
+          debug(`${this.totalUploaded} artifacts are not uploaded, because artifacts uploading is not enabled`);
 
-        if (upload.isArtifactsEnabled() && this.totalUploaded > 0) {
+        if (this.totalUploaded && upload.isArtifactsEnabled()) {
           console.log(
             APP_PREFIX,
-            `🗄️  Total ${this.totalUploaded} artifacts ${
+            `🗄️ ${this.totalUploaded} artifacts ${
               process.env.TESTOMATIO_PRIVATE_ARTIFACTS ? 'privately' : chalk.bold('publicly')
-            } uploaded to S3 bucket  `,
+            } uploaded to S3 bucket`,
           );
         }
       })
@@ -213,17 +268,21 @@ class Client {
       stack += '\n\n';
     }
 
+    const customFilter = process.env.TESTOMATIO_STACK_IGNORE;
+
     try {
+      let hasFrame = false;
       const record = createCallsiteRecord({
         forError: error,
+        isCallsiteFrame: frame => {
+          if (customFilter && minimatch(frame.getFileName(), customFilter)) return false;          
+          if (hasFrame) return false;
+          if (isNotInternalFrame(frame)) hasFrame = true;
+          return hasFrame;
+        }
       });
       if (record && !record.filename.startsWith('http')) {
-        stack += record.renderSync({
-          stackFilter: frame =>
-            frame.getFileName().indexOf(sep) > -1 &&
-            frame.getFileName().indexOf('node_modules') < 0 &&
-            frame.getFileName().indexOf('internal') < 0,
-        });
+        stack += record.renderSync({ stackFilter: isNotInternalFrame });
       }
       return stack;
     } catch (e) {
@@ -232,4 +291,10 @@ class Client {
   }
 }
 
+function isNotInternalFrame(frame) {
+  return frame.getFileName().includes(sep) &&
+    !frame.getFileName().includes('node_modules') &&
+    !frame.getFileName().includes('internal')
+}
+
 module.exports = Client;

package/lib/constants.js

@@ -1,11 +1,10 @@
 const chalk = require('chalk');
+const os = require('os');
+const path = require('path');
 
 const APP_PREFIX = chalk.gray('[TESTOMATIO]');
-const TESTOMAT_ARTIFACT_SUFFIX = "testomatio_artifact_";
 
-const TESTOMAT_TMP_STORAGE = {
-  mainDir: "testomatio_tmp",
-}
+const TESTOMAT_TMP_STORAGE_DIR = path.join(os.tmpdir(), 'testomatio_tmp');
 
 const CSV_HEADERS = [
   { id: 'suite_title', title: 'Suite_title' },
@@ -24,8 +23,7 @@ const STATUS = {
 
 module.exports = {
   APP_PREFIX,
-  TESTOMAT_ARTIFACT_SUFFIX,
-  TESTOMAT_TMP_STORAGE,
+  TESTOMAT_TMP_STORAGE_DIR,
   CSV_HEADERS,
   STATUS,
 }

package/lib/junit-adapter/java.js

@@ -10,24 +10,42 @@ class JavaAdapter extends Adapter {
 
   formatTest(t) {
     const fileParts = t.suite_title.split('.')
-    const example = t.title.match(/\[(.*)\]/)?.[1];
-    if (example) t.example = { "#": example }
 
     t.file = namespaceToFileName(t.suite_title);
     t.title = t.title.split('(')[0];
+
+    // detect params
+    const paramMatches = t.title.match(/\[(.*?)\]/g);
+
+    if (paramMatches) {
+      const params = paramMatches.map((_match, index) => `param${index + 1}`);
+      if (params.length === 1) params[0] = 'param';
+      let paramIndex = 0;
+    
+      t.title = t.title.replace(/: \[(.*?)\]/g, () => {
+        if (params.length < 2) return `\${param}`
+        const paramName = params[paramIndex] || `param${paramIndex + 1}`;
+        paramIndex++;
+        return `\${${paramName}}`;
+      });
+      const example = {};
+      paramMatches.forEach((match, index) => { example[params[index]] = match.replace(/[[\]]/g, '') });
+      t.example = example;
+    }
+
     t.suite_title = fileParts[fileParts.length - 1].replace(/\$/g, ' | ')
     return t;
   }
 
-  formatStack(t) {
-    const stack = super.formatStack(t);
+  // formatStack(t) {
+  //   const stack = super.formatStack(t);
 
-    const file = t.suite_title.split('.');
+  //   const file = t.suite_title.split('.');
 
-    const fileLine = `at .*${file[file.length - 1]}\.java:(\\d+)` // eslint-disable-line no-useless-escape
-    const regexp = new RegExp(fileLine,"g")
-    return stack.replace(regexp, `${this.opts.javaTests}${path.sep}${namespaceToFileName(t.suite_title)}:$1:`);
-  }
+  //   const fileLine = `at .*${file[file.length - 1]}\.java:(\\d+)` // eslint-disable-line no-useless-escape
+  //   const regexp = new RegExp(fileLine,"g")
+  //   return stack.replace(regexp, `${this.opts.javaTests}${path.sep}${namespaceToFileName(t.suite_title)}:$1:`);
+  // }
 }
 
 function namespaceToFileName(fileName) {

package/lib/pipe/csv.js

@@ -4,7 +4,7 @@ const fs = require('fs');
 const csvWriter = require('csv-writer');
 const chalk = require('chalk');
 const merge = require('lodash.merge');
-const { isSameTest, getCurrentDateTime } = require('../util');
+const { isSameTest, getCurrentDateTime } = require('../utils/utils');
 const { CSV_HEADERS } = require('../constants');
 
 /**
@@ -40,6 +40,9 @@ class CsvPipe {
     }
   }
 
+  // TODO: to using SET opts as argument => prepareRun(opts)
+  async prepareRun() {}
+  
   async createRun() {
     // empty
   }