@testomatio/reporter 1.3.1-beta → 1.3.1-beta.1-exclude-skipped-tests

package/lib/adapter/webdriver.js

@@ -1,7 +1,7 @@
 // eslint-disable-next-line global-require, import/no-extraneous-dependencies
 const WDIOReporter = require('@wdio/reporter').default;
 const TestomatClient = require('../client');
-const { parseTest } = require('../util');
+const { getTestomatIdFromTestTitle } = require('../utils/utils');
 
 class WebdriverReporter extends WDIOReporter {
   constructor(options) {
@@ -31,12 +31,24 @@ class WebdriverReporter extends WDIOReporter {
     this._addTestPromises.push(this.addTest(test));
   }
 
+  // wdio-cucumber does not trigger onTestEnd hook, thus, using this one
+  /**
+   *
+   * @param {} scerario
+   * @returns
+   */
+  onSuiteEnd(scerario) {
+    if (scerario.type === 'scenario') {
+      this._addTestPromises.push(this.addBddScenario(scerario));
+    }
+  }
+
   async addTest(test) {
     if (!this.client) return;
 
     const { title, _duration: duration, state, error, output } = test;
 
-    const testId = parseTest(title);
+    const testId = getTestomatIdFromTestTitle(title);
 
     const screenshotEndpoint = '/session/:sessionId/screenshot';
     const screenshotsBuffers = output
@@ -51,6 +63,39 @@ class WebdriverReporter extends WDIOReporter {
       filesBuffers: screenshotsBuffers,
     });
   }
+
+  /**
+   * @param {import('../../types').WebdriverIOScenario} scenario
+   */
+  addBddScenario(scenario) {
+    if (!this.client) return;
+
+    const { title, _duration: duration } = scenario;
+
+    const testId = getTestomatIdFromTestTitle(title || scenario.tags.map(tag => tag.name).join(' '));
+
+    let scenarioState = scenario.tests.every(test => test.state === 'passed') ? 'passed' : 'failed';
+    if (scenario.tests.every(test => test.state === 'skipped')) {
+      scenarioState = 'skipped';
+    }
+    const errors = scenario.tests
+      .filter(test => test.state === 'failed')
+      .map(test => test.error?.stack)
+      .filter(Boolean);
+    const error = errors.join('\n');
+
+    const tags = scenario.tags.map(tag => tag.name);
+
+    return this.client.addTestRun(scenarioState, {
+      error: error ? Error(error) : null,
+      title,
+      test_id: testId,
+      time: duration,
+      tags,
+      file: scenario.file,
+      // filesBuffers: screenshotsBuffers,
+    });
+  }
 }
 
 module.exports = WebdriverReporter;

package/lib/bin/reportXml.js

@@ -1,29 +1,30 @@
 #!/usr/bin/env node
-const program = require("commander");
-const chalk = require("chalk");
+const program = require('commander');
+const chalk = require('chalk');
 const glob = require('glob');
 const debug = require('debug')('@testomatio/reporter:xml-cli');
 const { APP_PREFIX } = require('../constants');
-const XmlReader = require("../xmlReader");
+const XmlReader = require('../xmlReader');
 
 const { version } = require('../../package.json');
 
 console.log(chalk.cyan.bold(` 🤩 Testomat.io XML Reporter v${version}`));
 
 program
-  .arguments("<pattern>")
-  .option("-d, --dir <dir>", "Project directory")
-  .option("--java-tests [java-path]", "Load Java tests from path, by default: src/test/java")
-  .option("--lang <lang>", "Language used (python, ruby, java)")
-  .option("--timelimit <time>", "default time limit in seconds to kill a stuck process")
-  .option("--env-file <envfile>", "Load environment variables from env file")
+  .arguments('<pattern>')
+  .option('-d, --dir <dir>', 'Project directory')
+  .option('--java-tests [java-path]', 'Load Java tests from path, by default: src/test/java')
+  .option('--lang <lang>', 'Language used (python, ruby, java)')
+  .option('--timelimit <time>', 'default time limit in seconds to kill a stuck process')
+  .option('--env-file <envfile>', 'Load environment variables from env file')
   .action(async (pattern, opts) => {
     if (!pattern.endsWith('.xml')) {
       pattern += '.xml';
     }
     let { javaTests, lang } = opts;
     if (opts.envFile) {
-      debug('Loading env file: %s', opts.envFile) 
+      console.log(APP_PREFIX, 'Loading env file:', opts.envFile);
+      debug('Loading env file: %s', opts.envFile);
       require('dotenv').config({ path: opts.envFile }); // eslint-disable-line
     }
     if (javaTests === true) javaTests = 'src/test/java';
@@ -37,16 +38,21 @@ program
     }
 
     for (const file of files) {
-      console.log(APP_PREFIX,`Parsed ${file}`);
+      console.log(APP_PREFIX, `Parsed ${file}`);
       runReader.parse(file);
     }
 
     let timeoutTimer;
     if (opts.timelimit) {
-      timeoutTimer = setTimeout(() => {
-        console.log(`⚠️  Reached timeout of ${opts.timelimit}s. Exiting... (Exit code is 0 to not fail the pipeline)`);
-        process.exit(0);
-      }, parseInt(opts.timelimit, 10) * 1000)
+      timeoutTimer = setTimeout(
+        () => {
+          console.log(
+            `⚠️  Reached timeout of ${opts.timelimit}s. Exiting... (Exit code is 0 to not fail the pipeline)`,
+          );
+          process.exit(0);
+        },
+        parseInt(opts.timelimit, 10) * 1000,
+      );
     }
 
     try {
@@ -56,7 +62,7 @@ program
       console.log(APP_PREFIX, 'Error updating status, skipping...', err);
     }
 
-    if (timeoutTimer) clearTimeout(timeoutTimer)
+    if (timeoutTimer) clearTimeout(timeoutTimer);
   });
 
 if (process.argv.length < 3) {

package/lib/bin/startTest.js

@@ -5,6 +5,7 @@ const chalk = require('chalk');
 const TestomatClient = require('../client');
 const { APP_PREFIX, STATUS } = require('../constants');
 const { version } = require('../../package.json');
+const config = require('../config');
 
 console.log(chalk.cyan.bold(` 🤩 Testomat.io Reporter v${version}`));
 
@@ -12,13 +13,15 @@ program
   .option('-c, --command <cmd>', 'Test runner command')
   .option('--launch', 'Start a new run and return its ID')
   .option('--finish', 'Finish Run by its ID')
-  .option("--env-file <envfile>", "Load environment variables from env file")
-  .action(opts => {
+  .option('--env-file <envfile>', 'Load environment variables from env file')
+  .option('--filter <filter>', 'Additional execution filter')
+  .action(async opts => {
+    const { launch, finish, filter } = opts;
+    let { command } = opts;
 
-    const { command, launch, finish } = opts;
     if (opts.envFile) require('dotenv').config(opts.envFile); // eslint-disable-line
 
-    const apiKey = process.env['INPUT_TESTOMATIO-KEY'] || process.env.TESTOMATIO;
+    const apiKey = process.env['INPUT_TESTOMATIO-KEY'] || config.TESTOMATIO;
     const title = process.env.TESTOMATIO_TITLE;
 
     if (launch) {
@@ -57,6 +60,26 @@ program
       return;
     }
 
+    const client = new TestomatClient({ apiKey, title, parallel: true });
+
+    if (filter) {
+      const [pipe, ...optsArray] = filter.split(':');
+      const pipeOptions = optsArray.join(':');
+
+      try {
+        const tests = await client.prepareRun({ pipe, pipeOptions });
+
+        if (!tests || tests.length === 0) {
+          return;
+        }
+
+        const grep = ` --grep (${tests.join('|')})`;
+        command += grep;
+      } catch (err) {
+        console.log(APP_PREFIX, err);
+      }
+    }
+
     const testCmds = command.split(' ');
     console.log(APP_PREFIX, `🚀 Running`, chalk.green(command));
 
@@ -73,8 +96,6 @@ program
       return;
     }
 
-    const client = new TestomatClient({ apiKey, title, parallel: true });
-
     client.createRun().then(() => {
       const cmd = spawn(testCmds[0], testCmds.slice(1), { stdio: 'inherit' });
 

package/lib/client.js

@@ -1,12 +1,17 @@
 const debug = require('debug')('@testomatio/reporter:client');
 const createCallsiteRecord = require('callsite-record');
 const { sep, join } = require('path');
+const { minimatch } = require('minimatch');
 const fs = require('fs');
 const chalk = require('chalk');
 const { randomUUID } = require('crypto');
 const upload = require('./fileUploader');
-const { APP_PREFIX } = require('./constants');
+const { APP_PREFIX, STATUS } = require('./constants');
 const pipesFactory = require('./pipe');
+const { glob } = require('glob');
+const path = require('path');
+
+let listOfTestFilesToExcludeFromReport = null;
 
 /**
  * @typedef {import('../types').TestData} TestData
@@ -21,36 +26,83 @@ class Client {
    * @param {*} params
    */
   constructor(params = {}) {
-    this.parallel = params.parallel;
     const store = {};
     this.uuid = randomUUID();
     this.pipes = pipesFactory(params, store);
     this.queue = Promise.resolve();
     this.totalUploaded = 0;
+    this.failedToUpload = 0;
     this.version = JSON.parse(fs.readFileSync(join(__dirname, '..', 'package.json')).toString()).version;
+    this.executionList = Promise.resolve();
+
     console.log(APP_PREFIX, `Testomatio Reporter v${this.version}`);
   }
 
+  /**
+   * Asynchronously prepares the execution list for running tests through various pipes.
+   * Each pipe in the client is checked for enablement,
+   * and if all pipes are disabled, the function returns a resolved Promise.
+   * Otherwise, it executes the `prepareRun` method for each enabled pipe and collects the results.
+   * The results are then filtered to remove any undefined values.
+   * If no valid results are found, the function returns undefined.
+   * Otherwise, it returns the first non-empty array from the filtered results.
+   *
+   * @param {Object} params - The options for preparing the test execution list.
+   * @param {string} params.pipe - Name of the executed pipe.
+   * @param {string} params.pipeOptions - Filter option.
+   * @returns {Promise<any>} - A Promise that resolves to an
+   * array containing the prepared execution list,
+   * or resolves to undefined if no valid results are found or if all pipes are disabled.
+   */
+  async prepareRun(params) {
+    const { pipe, pipeOptions } = params;
+    // all pipes disabled, skipping
+    if (!this.pipes.some(p => p.isEnabled)) {
+      return Promise.resolve();
+    }
+
+    try {
+      const filterPipe = this.pipes.find(p => p.constructor.name.toLowerCase() === `${pipe.toLowerCase()}pipe`);
+
+      if (!filterPipe.isEnabled) {
+        // TODO:for the future for the another pipes
+        console.warn(
+          APP_PREFIX,
+          `At the moment processing is available only for the "testomatio" key. Example: "testomatio:tag-name=xxx"`,
+        );
+        return;
+      }
+
+      const results = await Promise.all(
+        this.pipes.map(async p => ({ pipe: p.toString(), result: await p.prepareRun(pipeOptions) })),
+      );
+
+      const result = results.filter(p => p.pipe.includes('Testomatio'))[0]?.result;
+
+      if (!result || result.length === 0) {
+        return;
+      }
+
+      debug('Execution tests list', result);
+
+      return result;
+    } catch (err) {
+      console.error(APP_PREFIX, err);
+    }
+  }
+
   /**
    * Used to create a new Test run
    *
    * @returns {Promise<any>} - resolves to Run id which should be used to update / add test
    */
   createRun() {
+    debug('Creating run...');
     // all pipes disabled, skipping
     if (!this.pipes?.filter(p => p.isEnabled).length) return Promise.resolve();
 
-    const runParams = {
-      title: this.title,
-      parallel: this.parallel,
-      env: this.env,
-    };
-
-    global.testomatioArtifacts = [];
-    if (!global.testomatioDataStore) global.testomatioDataStore = {};
-
     this.queue = this.queue
-      .then(() => Promise.all(this.pipes.map(p => p.createRun(runParams))))
+      .then(() => Promise.all(this.pipes.map(p => p.createRun())))
       .catch(err => console.log(APP_PREFIX, err))
       .then(() => undefined); // fixes return type
     // debug('Run', this.queue);
@@ -62,13 +114,18 @@ class Client {
    *
    * @param {string|undefined} status
    * @param {TestData} [testData]
-   * @param {string[]} [storeArtifacts]
    * @returns {Promise<PipeResult[]>}
    */
-  async addTestRun(status, testData, storeArtifacts = []) {
+  async addTestRun(status, testData) {
     // all pipes disabled, skipping
     if (!this.pipes?.filter(p => p.isEnabled).length) return [];
 
+    if (isTestShouldBeExculedFromReport(testData)) return [];
+
+    if (status === STATUS.SKIPPED && process.env.TESTOMATIO_IGNORE_SKIPPED) {
+      return; // do not log skipped tests
+    } 
+
     if (!testData)
       testData = {
         title: 'Unknown test',
@@ -84,46 +141,31 @@ class Client {
       steps,
       code = null,
       title,
+      file,
       suite_title,
       suite_id,
       test_id,
+      manuallyAttachedArtifacts,
+      meta,
     } = testData;
     let { message = '' } = testData;
 
-    const uploadedFiles = [];
-
-    let stack = '';
-
+    let errorFormatted = '';
     if (error) {
-      stack = this.formatError(error) || '';
+      errorFormatted += this.formatError(error) || '';
       message = error?.message;
     }
-    if (steps) {
-      stack += this.formatSteps(stack, steps);
-    }
-
-    stack += testData.stack || '';
 
-    // in some cases (e.g. using cucumber) logger instance becomes empty object, if import at the top of the file
-    const logger = require('./logger');
-    const testLogs = logger.getLogs(test_id);
-    // debug(`Test logs for ${test_id}:\n`, testLogs);
-    if (stack) stack += '\n\n';
-    stack += testLogs ? `${chalk.bold('Logs:')}\n${testLogs}` : '';
+    // Attach logs
+    const fullLogs = this.formatLogs({ error: errorFormatted, steps, logs: testData.logs });
 
-    if (Array.isArray(storeArtifacts) && storeArtifacts.length) {
-      debug('CLIENT storeArtifact', storeArtifacts);
-      files.push(...storeArtifacts);
-    }
+    // add artifacts
+    if (manuallyAttachedArtifacts?.length) files.push(...manuallyAttachedArtifacts);
 
-    if (Array.isArray(global.testomatioArtifacts)) {
-      debug('CLIENT global[testomatioArtifacts]', global.testomatioArtifacts);
-      files.push(...global.testomatioArtifacts);
-      global.testomatioArtifacts = [];
-    }
+    const uploadedFiles = [];
 
-    for (const file of files) {
-      uploadedFiles.push(upload.uploadFileByPath(file, this.uuid));
+    for (const f of files) {
+      uploadedFiles.push(upload.uploadFileByPath(f, this.uuid));
     }
 
     for (const [idx, buffer] of filesBuffers.entries()) {
@@ -131,18 +173,22 @@ class Client {
       uploadedFiles.push(upload.uploadFileAsBuffer(buffer, fileName, this.uuid));
     }
 
-    const artifacts = await Promise.all(uploadedFiles);
+    const artifacts = (await Promise.all(uploadedFiles)).filter(n => !!n);
 
-    global.testomatioArtifacts = [];
+    if (artifacts.length < uploadedFiles.length) {
+      const failedUploading = uploadedFiles.length - artifacts.length;
+      this.failedToUpload += failedUploading;
+    }
 
-    this.totalUploaded += uploadedFiles.filter(n => n).length;
+    this.totalUploaded += artifacts.length;
 
     const data = {
       files,
       steps,
       status,
-      stack,
+      stack: fullLogs,
       example,
+      file,
       code,
       title,
       suite_title,
@@ -151,16 +197,19 @@ class Client {
       message,
       run_time: parseFloat(time),
       artifacts,
+      meta,
     };
 
+    // debug('Adding test run...', data);
+
     this.queue = this.queue.then(() =>
       Promise.all(
-        this.pipes.map(async p => {
+        this.pipes.map(async pipe => {
           try {
-            const result = await p.addTest(data);
-            return { pipe: p.toString(), result };
+            const result = await pipe.addTest(data);
+            return { pipe: pipe.toString(), result };
           } catch (err) {
-            console.log(APP_PREFIX, p.toString(), err);
+            console.log(APP_PREFIX, pipe.toString(), err);
           }
         }),
       ),
@@ -177,6 +226,7 @@ class Client {
    * @returns {Promise<any>} - A Promise that resolves when finishes the run.
    */
   updateRunStatus(status, isParallel = false) {
+    debug('Updating run status...');
     // all pipes disabled, skipping
     if (!this.pipes?.filter(p => p.isEnabled).length) return Promise.resolve();
 
@@ -185,15 +235,27 @@ class Client {
     this.queue = this.queue
       .then(() => Promise.all(this.pipes.map(p => p.finishRun(runParams))))
       .then(() => {
-        debug('TOTAL uploaded files', this.totalUploaded);
+        debug('TOTAL artifacts', this.totalUploaded);
+        if (this.totalUploaded && !upload.isArtifactsEnabled())
+          debug(`${this.totalUploaded} artifacts are not uploaded, because artifacts uploading is not enabled`);
 
-        if (upload.isArtifactsEnabled() && this.totalUploaded > 0) {
+        if (this.totalUploaded && upload.isArtifactsEnabled()) {
           console.log(
             APP_PREFIX,
-            `🗄️  Total ${this.totalUploaded} artifacts ${
+            `🗄️ ${this.totalUploaded} artifacts ${
               process.env.TESTOMATIO_PRIVATE_ARTIFACTS ? 'privately' : chalk.bold('publicly')
-            } uploaded to S3 bucket  `,
+            } uploaded to S3 bucket`,
           );
+
+          if (this.failedToUpload > 0) {
+            console.log(
+              APP_PREFIX,
+              chalk.yellow(
+                `Some artifacts were not uploaded. ${this.failedToUpload} artifacts could not be uploaded.
+                Run tests with DEBUG="@testomatio/reporter:file-uploader" to see details"`,
+              ),
+            );
+          }
         }
       })
       .catch(err => console.log(APP_PREFIX, err));
@@ -201,15 +263,27 @@ class Client {
     return this.queue;
   }
 
-  formatSteps(stack, steps) {
-    return stack ? `${steps}\n\n${chalk.bold.red('################[ Failure ]################')}\n${stack}` : steps;
+  /**
+   * Returns the formatted stack including the stack trace, steps, and logs.
+   * @returns {string}
+   */
+  formatLogs({ error, steps, logs }) {
+    error = error?.trim();
+    steps = steps?.trim();
+    logs = logs?.trim();
+
+    let testLogs = '';
+    if (steps) testLogs += `${chalk.bold.blue('################[ Steps ]################')}\n${steps}\n\n`;
+    if (logs) testLogs += `${chalk.bold.gray('################[ Logs ]################')}\n${logs}\n\n`;
+    if (error) testLogs += `${chalk.bold.red('################[ Failure ]################')}\n${error}`;
+    return testLogs;
   }
 
   formatError(error, message) {
     if (!message) message = error.message;
     if (error.inspect) message = error.inspect() || '';
 
-    let stack = `\n${chalk.bold(message)}\n`;
+    let stack = `${message}\n`;
 
     // diffs for mocha, cypress, codeceptjs style
     if (error.actual && error.expected) {
@@ -219,17 +293,21 @@ class Client {
       stack += '\n\n';
     }
 
+    const customFilter = process.env.TESTOMATIO_STACK_IGNORE;
+
     try {
+      let hasFrame = false;
       const record = createCallsiteRecord({
         forError: error,
+        isCallsiteFrame: frame => {
+          if (customFilter && minimatch(frame.getFileName(), customFilter)) return false;
+          if (hasFrame) return false;
+          if (isNotInternalFrame(frame)) hasFrame = true;
+          return hasFrame;
+        },
       });
       if (record && !record.filename.startsWith('http')) {
-        stack += record.renderSync({
-          stackFilter: frame =>
-            frame.getFileName().indexOf(sep) > -1 &&
-            frame.getFileName().indexOf('node_modules') < 0 &&
-            frame.getFileName().indexOf('internal') < 0,
-        });
+        stack += record.renderSync({ stackFilter: isNotInternalFrame });
       }
       return stack;
     } catch (e) {
@@ -238,4 +316,49 @@ class Client {
   }
 }
 
+function isNotInternalFrame(frame) {
+  return (
+    frame.getFileName() &&
+    frame.getFileName().includes(sep) &&
+    !frame.getFileName().includes('node_modules') &&
+    !frame.getFileName().includes('internal')
+  );
+}
+
+/**
+ *
+ * @param {TestData} testData
+ * @returns boolean
+ */
+function isTestShouldBeExculedFromReport(testData) {
+  // const fileName = path.basename(test.location?.file || '');
+  const globExcludeFilesPattern = process.env.TESTOMATIO_EXCLUDE_FILES_FROM_REPORT_GLOB_PATTERN;
+  if (!globExcludeFilesPattern) return false;
+
+  if (!testData.file) {
+    debug('No "file" property found for test ', testData.title);
+    return false;
+  }
+
+  const excludeParretnsList = globExcludeFilesPattern.split(';');
+
+  // as scanning files is time consuming operation, just save the result in variable to avoid multiple scans
+  if (!listOfTestFilesToExcludeFromReport) {
+    // list of files with relative paths
+    listOfTestFilesToExcludeFromReport = glob.sync(excludeParretnsList, { ignore: '**/node_modules/**' });
+    debug('Tests from next files will not be reported:', listOfTestFilesToExcludeFromReport);
+  }
+
+  const testFileRelativePath = path.relative(process.cwd(), testData.file);
+
+  // no files found matching the exclusion pattern
+  if (!listOfTestFilesToExcludeFromReport.length) return false;
+
+  if (listOfTestFilesToExcludeFromReport.includes(testFileRelativePath)) {
+    debug(`Excluding test '${testData.title}' <${testFileRelativePath}> from reporting`);
+    return true;
+  }
+  return false;
+}
+
 module.exports = Client;

package/lib/config.js

@@ -0,0 +1,34 @@
+// This file is used to read environment variables from .env file
+
+// require('dotenv').config({ path: process.env.TESTOMATIO_ENV_FILE_PATH });
+
+const debug = require('debug')('@testomatio/reporter:config');
+
+/* for possibility to use multiple env files (reading different paths)
+const dotenv = require('dotenv');
+const envFileVars = dotenv.config({ path: '.env' }).parsed; */
+
+if (process.env.TESTOMATIO_API_KEY) {
+  process.env.TESTOMATIO = process.env.TESTOMATIO_API_KEY;
+}
+if (process.env.TESTOMATIO_TOKEN) {
+  process.env.TESTOMATIO = process.env.TESTOMATIO_TOKEN;
+}
+
+if (process.env.TESTOMATIO === 'undefined')
+  console.error('TESTOMATIO is "undefined". Something went wrong. Contact dev team.');
+
+// select only TESTOMATIO related variables (only to print them in debug)
+const testomatioEnvVars =
+  Object.keys(process.env)
+    .filter(key => key.startsWith('TESTOMATIO') || key.startsWith('S3_'))
+    .reduce((obj, key) => {
+      obj[key] = process.env[key];
+      return obj;
+    }, {}) || {};
+debug('TESTOMATIO variables:', testomatioEnvVars);
+
+// includes variables from .env file and process.env
+const config = process.env;
+
+module.exports = config;