@testomatio/reporter 1.4.1-beta-1 → 1.4.1

package/lib/adapter/jest.js

@@ -1,6 +1,10 @@
+const chalk = require('chalk');
 const TestomatClient = require('../client');
 const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
-const { parseTest, ansiRegExp, fileSystem } = require('../utils/utils');
+const { getTestomatIdFromTestTitle, ansiRegExp, fileSystem } = require('../utils/utils');
+const { services } = require('../services');
+const debug = require('debug')('@testomatio/reporter:adapter-jest');
+const path = require('path');
 
 class JestReporter {
   constructor(globalConfig, options) {
@@ -16,6 +20,19 @@ class JestReporter {
     fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
   }
 
+  // start of test file (including beforeAll)
+  onTestStart(testFile) {
+    debug('Start running test file:', testFile.path);
+    services.setContext(testFile.path);
+  }
+
+  // start of the test (including beforeEach)
+  onTestCaseStart(test, testCase) {
+    debug('Start running test:', testCase.fullName);
+    services.setContext(testCase.fullName);
+  }
+
+  // end of test file! (there is also onTestCaseResult listener)
   onTestResult(test, testResult) {
     if (!this.client) return;
 
@@ -30,14 +47,16 @@ class JestReporter {
         error = new Error(errorMessage);
         steps = failureMessages[0];
       }
-      const testId = parseTest(title);
-      let suite_title;
-      // this is test without a suite
-      if (result.fullName === result.title) {
-        suite_title = testResult.testFilePath.split('/').pop();
-      } else {
-        suite_title = result.fullName.replace(result.title, '');
-      }
+      const testId = getTestomatIdFromTestTitle(title);
+
+      // suite titles from most outer to most inner, separated by space
+      let fullSuiteTitle = testResult.ancestorTitles?.join(' ');
+      // if no suite titles, use file name
+      if (!fullSuiteTitle && testResult.testFilePath) fullSuiteTitle = path.basename(testResult.testFilePath);
+
+      const logs = getTestLogs(result);
+      const artifacts = services.artifacts.get(result.fullName);
+      const keyValues = services.keyValues.get(result.fullName);
 
       const deducedStatus = status === 'pending' ? 'skipped' : status;
       // In jest if test is not matched with test name pattern it is considered as skipped.
@@ -45,11 +64,14 @@ class JestReporter {
       if (!this._globalConfig.testNamePattern || deducedStatus !== 'skipped') {
         this.client.addTestRun(deducedStatus, {
           test_id: testId,
-          suite_title,
+          suite_title: fullSuiteTitle,
           error,
           steps,
           title,
           time: duration,
+          logs,
+          manuallyAttachedArtifacts: artifacts,
+          meta: keyValues,
         });
       }
     }
@@ -64,4 +86,20 @@ class JestReporter {
   }
 }
 
+function getTestLogs(testResult) {
+  const suiteLogsArr = services.logger.getLogs(testResult.testFilePath);
+  const suiteLogs = suiteLogsArr ? suiteLogsArr.join('\n').trim() : '';
+  const testLogsArr = services.logger.getLogs(testResult.fullName);
+  const testLogs = testLogsArr ? testLogsArr.join('\n').trim() : '';
+
+  let logs = '';
+  if (suiteLogs) {
+    logs += `${chalk.bold('\t--- Suite ---')}\n${suiteLogs}`;
+  }
+  if (testLogs) {
+    logs += `\n${chalk.bold('\t--- Test ---')}\n${testLogs}`;
+  }
+  return logs;
+}
+
 module.exports = JestReporter;

package/lib/adapter/mocha.js

@@ -3,10 +3,21 @@ const Mocha = require('mocha');
 const chalk = require('chalk');
 const TestomatClient = require('../client');
 const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
-const { parseTest, fileSystem } = require('../utils/utils');
+const { getTestomatIdFromTestTitle, fileSystem } = require('../utils/utils');
+const config = require('../config');
+const { services } = require('../services');
 
-const { EVENT_RUN_BEGIN, EVENT_RUN_END, EVENT_TEST_FAIL, EVENT_TEST_PASS, EVENT_TEST_PENDING } =
-  Mocha.Runner.constants;
+const {
+  EVENT_RUN_BEGIN,
+  EVENT_RUN_END,
+  EVENT_TEST_FAIL,
+  EVENT_TEST_PASS,
+  EVENT_TEST_PENDING,
+  EVENT_SUITE_BEGIN,
+  EVENT_SUITE_END,
+  EVENT_TEST_BEGIN,
+  EVENT_TEST_END,
+} = Mocha.Runner.constants;
 
 function MochaReporter(runner, opts) {
   Mocha.reporters.Base.call(this, runner);
@@ -15,7 +26,7 @@ function MochaReporter(runner, opts) {
   let skipped = 0;
   // let artifactStore;
 
-  const apiKey = opts?.reporterOptions?.apiKey || process.env.TESTOMATIO;
+  const apiKey = opts?.reporterOptions?.apiKey || config.TESTOMATIO;
 
   const client = new TestomatClient({ apiKey });
 
@@ -25,24 +36,54 @@ function MochaReporter(runner, opts) {
     fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
   });
 
+  runner.on(EVENT_SUITE_BEGIN, async suite => {
+    services.setContext(suite.fullTitle());
+  });
+
+  runner.on(EVENT_SUITE_END, async () => {
+    services.setContext(null);
+  });
+
+  runner.on(EVENT_TEST_BEGIN, async test => {
+    services.setContext(test.fullTitle());
+  });
+
+  runner.on(EVENT_TEST_END, async () => {
+    services.setContext(null);
+  });
+
   runner.on(EVENT_TEST_PASS, async test => {
     passes += 1;
+
     console.log(chalk.bold.green('✔'), test.fullTitle());
-    const testId = parseTest(test.title);
+    const testId = getTestomatIdFromTestTitle(test.title);
+
+    const logs = getTestLogs(test);
+    const artifacts = services.artifacts.get(test.fullTitle());
+    const keyValues = services.keyValues.get(test.fullTitle());
 
     client.addTestRun(STATUS.PASSED, {
       test_id: testId,
-      title: test.title,
+      suite_title: getSuiteTitle(test),
+      title: getTestName(test),
+      code: test.body.toString(),
+      file: getFile(test),
       time: test.duration,
+      logs,
+      manuallyAttachedArtifacts: artifacts,
+      meta: keyValues,
     });
   });
 
   runner.on(EVENT_TEST_PENDING, test => {
     skipped += 1;
     console.log('skip: %s', test.fullTitle());
-    const testId = parseTest(test.title);
+    const testId = getTestomatIdFromTestTitle(test.title);
     client.addTestRun(STATUS.SKIPPED, {
-      title: test.title,
+      title: getTestName(test),
+      suite_title: getSuiteTitle(test),
+      code: test.body.toString(),
+      file: getFile(test),
       test_id: testId,
       time: test.duration,
     });
@@ -51,13 +92,19 @@ function MochaReporter(runner, opts) {
   runner.on(EVENT_TEST_FAIL, async (test, err) => {
     failures += 1;
     console.log(chalk.bold.red('✖'), test.fullTitle(), chalk.gray(err.message));
-    const testId = parseTest(test.title);
+    const testId = getTestomatIdFromTestTitle(test.title);
+
+    const logs = getTestLogs(test);
 
     client.addTestRun(STATUS.FAILED, {
       error: err,
+      suite_title: getSuiteTitle(test),
+      file: getFile(test),
       test_id: testId,
-      title: test.title,
+      title: getTestName(test),
+      code: test.body.toString(),
       time: test.duration,
+      logs,
     });
   });
 
@@ -68,7 +115,40 @@ function MochaReporter(runner, opts) {
   });
 }
 
+function getTestLogs(test) {
+  const suiteLogsArr = services.logger.getLogs(test.parent.fullTitle());
+  const suiteLogs = suiteLogsArr ? suiteLogsArr.join('\n').trim() : '';
+  const testLogsArr = services.logger.getLogs(test.fullTitle());
+  const testLogs = testLogsArr ? testLogsArr.join('\n').trim() : '';
+
+  let logs = '';
+  if (suiteLogs) {
+    logs += `${chalk.bold('\t--- BeforeSuite ---')}\n${suiteLogs}`;
+  }
+  if (testLogs) {
+    logs += `\n${chalk.bold('\t--- Test ---')}\n${testLogs}`;
+  }
+  return logs;
+}
+
 // To have this reporter "extend" a built-in reporter uncomment the following line:
 Mocha.utils.inherits(MochaReporter, Mocha.reporters.Spec);
 
 module.exports = MochaReporter;
+
+function getSuiteTitle(test, pathArr = []) {
+  if (test.parent.parent) getSuiteTitle(test.parent, pathArr);
+
+  pathArr.push(test.parent.title);
+
+  return pathArr.filter(t => !!t)[0];
+}
+
+function getFile(test) {
+  return test.parent.file?.replace(process.cwd(), '');
+}
+
+function getTestName(test) {
+  if (process.env.TESTOMATIO_CREATE === 'fulltitle') return test.fullTitle();
+  return test.title;
+}

package/lib/adapter/playwright.js

@@ -6,7 +6,10 @@ const fs = require('fs');
 const { APP_PREFIX, STATUS: Status, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
 const TestomatioClient = require('../client');
 const { isArtifactsEnabled } = require('../fileUploader');
-const { parseTest, fileSystem } = require('../utils/utils');
+const { getTestomatIdFromTestTitle, fileSystem } = require('../utils/utils');
+// const debug = require('debug')('@testomatio/reporter:adapter:playwright');
+const { services } = require('../services');
+const { dataStorage } = require('../data-storage');
 
 const reportTestPromises = [];
 
@@ -17,39 +20,43 @@ class PlaywrightReporter {
     this.uploads = [];
   }
 
-  onBegin(_config, suite) {
+  onBegin(config, suite) {
     // clean data storage
     fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
     if (!this.client) return;
     this.suite = suite;
+    this.config = config;
     this.client.createRun();
   }
 
-  /*
-  onTestBegin(test) {
-    // does not work; value is not to the storage context
-    global.testTitle = test.title;
-  } */
+  onTestBegin(testInfo) {
+    const fullTestTitle = getTestContextName(testInfo);
+    dataStorage.setContext(fullTestTitle);
+  }
 
   onTestEnd(test, result) {
     if (!this.client) return;
 
-    let testId = parseTest(test.title);
-
     const { title } = test;
+
+    let testId = getTestomatIdFromTestTitle(`${title} ${test.tags?.join(' ')}`);
+
     const { error, duration } = result;
 
-    const suite_title = test.parent ? test.parent.title : null;
+    const suite_title = test.parent ? test.parent?.title : path.basename(test?.location?.file);
 
     const steps = [];
     for (const step of result.steps) {
       appendStep(step, steps);
     }
 
+    const fullTestTitle = getTestContextName(test);
     let logs = '';
     if (result.stderr.length || result.stdout.length) {
       logs = `\n\n${chalk.bold('Logs:')}\n${chalk.red(result.stderr.join(''))}\n${result.stdout.join('')}`;
     }
+    const manuallyAttachedArtifacts = services.artifacts.get(fullTestTitle);
+    const keyValues = services.keyValues.get(fullTestTitle);
 
     const reportTestPromise = this.client
       .addTestRun(checkStatus(result.status), {
@@ -59,7 +66,10 @@ class PlaywrightReporter {
         title,
         steps: steps.join('\n'),
         time: duration,
-        stack: logs,
+        logs,
+        manuallyAttachedArtifacts,
+        meta: keyValues,
+        file: test.location?.file,
       })
       .then(pipes => {
         testId = pipes?.filter(p => p.pipe.includes('Testomatio'))[0]?.result?.data?.test_id;
@@ -69,6 +79,7 @@ class PlaywrightReporter {
           title,
           suite_title,
           files: result.attachments.filter(a => a.body || a.path),
+          file: test.location?.file,
         });
         // remove empty uploads
         this.uploads = this.uploads.filter(upload => upload.files.length);
@@ -77,6 +88,22 @@ class PlaywrightReporter {
     reportTestPromises.push(reportTestPromise);
   }
 
+  #getArtifactPath(artifact) {
+    if (artifact.path) {
+      if (path.isAbsolute(artifact.path)) return artifact.path;
+
+      return path.join(this.config.outputDir || this.config.projects[0].outputDir, artifact.path);
+    }
+
+    if (artifact.body) {
+      const fileName = tmpFile();
+      fs.writeFileSync(fileName, artifact.body);
+      return fileName;
+    }
+
+    return null;
+  }
+
   async onEnd(result) {
     if (!this.client) return;
 
@@ -90,13 +117,11 @@ class PlaywrightReporter {
       for (const upload of this.uploads) {
         const { title, testId, suite_title } = upload;
 
-        const files = upload.files.map(attachment => {
-          if (attachment.body) {
-            const fileName = tmpFile();
-            fs.writeFileSync(fileName, attachment.body);
-          }
-          return { path: attachment.path, title, type: attachment.contentType };
-        });
+        const files = upload.files.map(attachment => ({
+          path: this.#getArtifactPath(attachment),
+          title,
+          type: attachment.contentType,
+        }));
 
         promises.push(
           this.client.addTestRun(undefined, {
@@ -104,6 +129,7 @@ class PlaywrightReporter {
             title,
             suite_title,
             files,
+            file: upload.file,
           }),
         );
       }
@@ -143,4 +169,28 @@ function tmpFile(prefix = 'tmp.') {
   return path.join(tmpdir, prefix + crypto.randomBytes(16).toString('hex'));
 }
 
+/**
+ * Returns filename + test title
+ * @param {*} test - testInfo object from Playwright
+ * @returns
+ */
+function getTestContextName(test) {
+  return `${test._requireFile || ''}_${test.title}`;
+}
+
+function initPlaywrightForStorage() {
+  try {
+    // @ts-ignore-next-line
+    // eslint-disable-next-line import/no-unresolved
+    const { test } = require('@playwright/test');
+    // eslint-disable-next-line no-empty-pattern
+    test.beforeEach(async ({}, testInfo) => {
+      global.testomatioTestTitle = `${testInfo.file || ''}_${testInfo.title}`;
+    });
+  } catch (e) {
+    // ignore
+  }
+}
+
 module.exports = PlaywrightReporter;
+module.exports.initPlaywrightForStorage = initPlaywrightForStorage;

package/lib/adapter/webdriver.js

@@ -1,7 +1,7 @@
 // eslint-disable-next-line global-require, import/no-extraneous-dependencies
 const WDIOReporter = require('@wdio/reporter').default;
 const TestomatClient = require('../client');
-const { parseTest } = require('../utils/utils');
+const { getTestomatIdFromTestTitle } = require('../utils/utils');
 
 class WebdriverReporter extends WDIOReporter {
   constructor(options) {
@@ -31,12 +31,24 @@ class WebdriverReporter extends WDIOReporter {
     this._addTestPromises.push(this.addTest(test));
   }
 
+  // wdio-cucumber does not trigger onTestEnd hook, thus, using this one
+  /**
+   *
+   * @param {} scerario
+   * @returns
+   */
+  onSuiteEnd(scerario) {
+    if (scerario.type === 'scenario') {
+      this._addTestPromises.push(this.addBddScenario(scerario));
+    }
+  }
+
   async addTest(test) {
     if (!this.client) return;
 
     const { title, _duration: duration, state, error, output } = test;
 
-    const testId = parseTest(title);
+    const testId = getTestomatIdFromTestTitle(title);
 
     const screenshotEndpoint = '/session/:sessionId/screenshot';
     const screenshotsBuffers = output
@@ -51,6 +63,39 @@ class WebdriverReporter extends WDIOReporter {
       filesBuffers: screenshotsBuffers,
     });
   }
+
+  /**
+   * @param {import('../../types').WebdriverIOScenario} scenario
+   */
+  addBddScenario(scenario) {
+    if (!this.client) return;
+
+    const { title, _duration: duration } = scenario;
+
+    const testId = getTestomatIdFromTestTitle(title || scenario.tags.map(tag => tag.name).join(' '));
+
+    let scenarioState = scenario.tests.every(test => test.state === 'passed') ? 'passed' : 'failed';
+    if (scenario.tests.every(test => test.state === 'skipped')) {
+      scenarioState = 'skipped';
+    }
+    const errors = scenario.tests
+      .filter(test => test.state === 'failed')
+      .map(test => test.error?.stack)
+      .filter(Boolean);
+    const error = errors.join('\n');
+
+    const tags = scenario.tags.map(tag => tag.name);
+
+    return this.client.addTestRun(scenarioState, {
+      error: error ? Error(error) : null,
+      title,
+      test_id: testId,
+      time: duration,
+      tags,
+      file: scenario.file,
+      // filesBuffers: screenshotsBuffers,
+    });
+  }
 }
 
 module.exports = WebdriverReporter;

package/lib/bin/reportXml.js

@@ -1,22 +1,22 @@
 #!/usr/bin/env node
-const program = require("commander");
-const chalk = require("chalk");
+const program = require('commander');
+const chalk = require('chalk');
 const glob = require('glob');
 const debug = require('debug')('@testomatio/reporter:xml-cli');
 const { APP_PREFIX } = require('../constants');
-const XmlReader = require("../xmlReader");
+const XmlReader = require('../xmlReader');
 
 const { version } = require('../../package.json');
 
 console.log(chalk.cyan.bold(` 🤩 Testomat.io XML Reporter v${version}`));
 
 program
-  .arguments("<pattern>")
-  .option("-d, --dir <dir>", "Project directory")
-  .option("--java-tests [java-path]", "Load Java tests from path, by default: src/test/java")
-  .option("--lang <lang>", "Language used (python, ruby, java)")
-  .option("--timelimit <time>", "default time limit in seconds to kill a stuck process")
-  .option("--env-file <envfile>", "Load environment variables from env file")
+  .arguments('<pattern>')
+  .option('-d, --dir <dir>', 'Project directory')
+  .option('--java-tests [java-path]', 'Load Java tests from path, by default: src/test/java')
+  .option('--lang <lang>', 'Language used (python, ruby, java)')
+  .option('--timelimit <time>', 'default time limit in seconds to kill a stuck process')
+  .option('--env-file <envfile>', 'Load environment variables from env file')
   .action(async (pattern, opts) => {
     if (!pattern.endsWith('.xml')) {
       pattern += '.xml';
@@ -24,7 +24,7 @@ program
     let { javaTests, lang } = opts;
     if (opts.envFile) {
       console.log(APP_PREFIX, 'Loading env file:', opts.envFile);
-      debug('Loading env file: %s', opts.envFile) 
+      debug('Loading env file: %s', opts.envFile);
       require('dotenv').config({ path: opts.envFile }); // eslint-disable-line
     }
     if (javaTests === true) javaTests = 'src/test/java';
@@ -38,16 +38,21 @@ program
     }
 
     for (const file of files) {
-      console.log(APP_PREFIX,`Parsed ${file}`);
+      console.log(APP_PREFIX, `Parsed ${file}`);
       runReader.parse(file);
     }
 
     let timeoutTimer;
     if (opts.timelimit) {
-      timeoutTimer = setTimeout(() => {
-        console.log(`⚠️  Reached timeout of ${opts.timelimit}s. Exiting... (Exit code is 0 to not fail the pipeline)`);
-        process.exit(0);
-      }, parseInt(opts.timelimit, 10) * 1000)
+      timeoutTimer = setTimeout(
+        () => {
+          console.log(
+            `⚠️  Reached timeout of ${opts.timelimit}s. Exiting... (Exit code is 0 to not fail the pipeline)`,
+          );
+          process.exit(0);
+        },
+        parseInt(opts.timelimit, 10) * 1000,
+      );
     }
 
     try {
@@ -57,7 +62,7 @@ program
       console.log(APP_PREFIX, 'Error updating status, skipping...', err);
     }
 
-    if (timeoutTimer) clearTimeout(timeoutTimer)
+    if (timeoutTimer) clearTimeout(timeoutTimer);
   });
 
 if (process.argv.length < 3) {

package/lib/bin/startTest.js

@@ -5,6 +5,7 @@ const chalk = require('chalk');
 const TestomatClient = require('../client');
 const { APP_PREFIX, STATUS } = require('../constants');
 const { version } = require('../../package.json');
+const config = require('../config');
 
 console.log(chalk.cyan.bold(` 🤩 Testomat.io Reporter v${version}`));
 
@@ -12,15 +13,15 @@ program
   .option('-c, --command <cmd>', 'Test runner command')
   .option('--launch', 'Start a new run and return its ID')
   .option('--finish', 'Finish Run by its ID')
-  .option("--env-file <envfile>", "Load environment variables from env file")
-  .option("--filter <filter>", "Additional execution filter")
-  .action(async (opts) => {
+  .option('--env-file <envfile>', 'Load environment variables from env file')
+  .option('--filter <filter>', 'Additional execution filter')
+  .action(async opts => {
     const { launch, finish, filter } = opts;
     let { command } = opts;
 
     if (opts.envFile) require('dotenv').config(opts.envFile); // eslint-disable-line
 
-    const apiKey = process.env['INPUT_TESTOMATIO-KEY'] || process.env.TESTOMATIO;
+    const apiKey = process.env['INPUT_TESTOMATIO-KEY'] || config.TESTOMATIO;
     const title = process.env.TESTOMATIO_TITLE;
 
     if (launch) {
@@ -61,21 +62,20 @@ program
 
     const client = new TestomatClient({ apiKey, title, parallel: true });
 
-    if(filter) {
-      const [pipe, ...optsArray] = filter.split(":");
-      const pipeOptions = optsArray.join(":");
+    if (filter) {
+      const [pipe, ...optsArray] = filter.split(':');
+      const pipeOptions = optsArray.join(':');
 
-      try { 
-        const tests = await client.prepareRun({pipe, pipeOptions});
+      try {
+        const tests = await client.prepareRun({ pipe, pipeOptions });
 
-        if(!tests || tests.length === 0) {
+        if (!tests || tests.length === 0) {
           return;
         }
 
         const grep = ` --grep (${tests.join('|')})`;
         command += grep;
-      }
-      catch(err) {
+      } catch (err) {
         console.log(APP_PREFIX, err);
       }
     }