@testomatio/reporter 1.3.5-beta → 1.4.0-beta-wdio-bdd

package/lib/adapter/cypress-plugin/index.js

@@ -1,17 +1,19 @@
 const { STATUS } = require('../../constants');
-const { parseTest, parseSuite } = require('../../util');
+const { parseTest, parseSuite } = require('../../utils/utils');
 const TestomatClient = require('../../client');
+const config = require('../../config');
 
 const testomatioReporter = on => {
-  if (!process.env.TESTOMATIO) {
+  if (!config.TESTOMATIO) {
     console.log('TESTOMATIO key is empty, ignoring reports');
-    return
+    return;
   }
-  const client = new TestomatClient({ apiKey: process.env.TESTOMATIO });
+  const client = new TestomatClient({ apiKey: config.TESTOMATIO });
 
-  on('before:run', async (run) => {
+  on('before:run', async run => {
+    // TODO: looks like client.env does not exist
     if (!client.env) {
-      client.env = `${run.browser.displayName},${run.system.osName}`
+      client.env = `${run.browser.displayName},${run.system.osName}`;
     }
     await client.createRun();
   });
@@ -25,8 +27,9 @@ const testomatioReporter = on => {
       const lastAttemptIndex = test.attempts.length - 1;
       const latestAttempt = test.attempts[lastAttemptIndex];
 
-      const time = latestAttempt.duration;
-      const error = latestAttempt.error;
+      // latestAttempt.duration && latestAttempt.error were available in adapters version up to 13 JFYI
+      const time = latestAttempt.duration || latestAttempt.wallClockDuration || test.duration;
+      let error = latestAttempt.error;
 
       let title = test.title.pop();
       const examples = title.match(/\(example (#\d+)\)/);
@@ -39,35 +42,59 @@ const testomatioReporter = on => {
       const testId = parseTest(title);
       const suiteId = parseSuite(suiteTitle);
 
-      if (error) {
-        error.inspect = function() { // eslint-disable-line func-names
-          if (this && this.codeFrame) {
-            return this.codeFrame.frame;
-          }
-          return '';
-        }
+      if (!error && test.displayError) {
+        error = { message: test.displayError };
+        error.inspect = function () {
+          // eslint-disable-line func-names
+          return this.message;
+        };
       }
 
-      const screenshots = results.screenshots
-        .filter(screenshot => screenshot.path.includes(title))
-        .filter(screenshot => screenshot.testAttemptIndex === lastAttemptIndex)
-        .map(screenshot => screenshot.path);
+      const formattedError = error
+        ? {
+            message: error.message,
+            inspect:
+              error.inspect ||
+              function () {
+                return this.message;
+              },
+          }
+        : '';
+
+      const screenshots = Array.isArray(results.screenshots)
+        ? results.screenshots
+            .filter(screenshot => screenshot?.path && screenshot?.path.includes(title) && screenshot?.takenAt)
+            .map(screenshot => screenshot.path)
+        : [];
 
       const files = [...videos, ...screenshots];
 
       let state;
       switch (test.state) {
-        case 'passed': state = STATUS.PASSED; break;
-        case 'failed': state = STATUS.FAILED; break;
+        case 'passed':
+          state = STATUS.PASSED;
+          break;
+        case 'failed':
+          state = STATUS.FAILED;
+          break;
         case 'skipped':
-        case 'pending': 
+        case 'pending':
         default:
           state = STATUS.SKIPPED;
       }
 
-      addSpecTestsPromises.push(client.addTestRun(state, {
-        title, time, example, error, files, suite_title: suiteTitle, test_id: testId, suite_id: suiteId
-      }));
+      addSpecTestsPromises.push(
+        client.addTestRun(state, {
+          title,
+          time,
+          example,
+          error: formattedError,
+          files,
+          suite_title: suiteTitle,
+          test_id: testId,
+          suite_id: suiteId,
+        }),
+      );
     }
 
     await Promise.all(addSpecTestsPromises);

package/lib/adapter/jasmine.js

@@ -1,5 +1,5 @@
 const TestomatClient = require('../client');
-const { parseTest, ansiRegExp } = require('../util');
+const { parseTest, ansiRegExp } = require('../utils/utils');
 const { STATUS } = require('../constants');
 
 class JasmineReporter {

package/lib/adapter/jest.js

@@ -1,6 +1,10 @@
+const chalk = require('chalk');
 const TestomatClient = require('../client');
-const { STATUS, TESTOMAT_TMP_STORAGE } = require('../constants');
-const { parseTest, ansiRegExp, fileSystem } = require('../util');
+const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
+const { parseTest, ansiRegExp, fileSystem } = require('../utils/utils');
+const { services } = require('../services');
+const debug = require('debug')('@testomatio/reporter:adapter-jest');
+const path = require('path');
 
 class JestReporter {
   constructor(globalConfig, options) {
@@ -11,22 +15,24 @@ class JestReporter {
     this.client.createRun();
   }
 
-  static getIdOfCurrentlyRunningTest() {
-    if (!process.env.JEST_WORKER_ID) return null;
-    try {
-      // @ts-expect-error "expect" could only be defined inside Jest environement (forbidden to import it outside)
-      // eslint-disable-next-line no-undef
-      if (expect && expect?.getState()?.currentTestName) return parseTest(expect?.getState()?.currentTestName); 
-    } catch (e) {
-      return null; 
-    }
-  }
-
   onRunStart() {
     // clear tmp dir
-    fileSystem.clearDir(TESTOMAT_TMP_STORAGE.mainDir);
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
+  }
+
+  // start of test file (including beforeAll)
+  onTestStart(testFile) {
+    debug('Start running test file:', testFile.path);
+    services.setContext(testFile.path);
+  }
+
+  // start of the test (including beforeEach)
+  onTestCaseStart(test, testCase) {
+    debug('Start running test:', testCase.fullName);
+    services.setContext(testCase.fullName);
   }
 
+  // end of test file! (there is also onTestCaseResult listener)
   onTestResult(test, testResult) {
     if (!this.client) return;
 
@@ -42,16 +48,30 @@ class JestReporter {
         steps = failureMessages[0];
       }
       const testId = parseTest(title);
+
+      // suite titles from most outer to most inner, separated by space
+      let fullSuiteTitle = testResult.ancestorTitles?.join(' ');
+      // if no suite titles, use file name
+      if (!fullSuiteTitle && testResult.testFilePath) fullSuiteTitle = path.basename(testResult.testFilePath);
+
+      const logs = getTestLogs(result);
+      const artifacts = services.artifacts.get(result.fullName);
+      const keyValues = services.keyValues.get(result.fullName);
+
       const deducedStatus = status === 'pending' ? 'skipped' : status;
       // In jest if test is not matched with test name pattern it is considered as skipped.
       // So adding a check if it is skipped for real or because of test pattern
       if (!this._globalConfig.testNamePattern || deducedStatus !== 'skipped') {
         this.client.addTestRun(deducedStatus, {
           test_id: testId,
+          suite_title: fullSuiteTitle,
           error,
           steps,
           title,
           time: duration,
+          logs,
+          manuallyAttachedArtifacts: artifacts,
+          meta: keyValues,
         });
       }
     }
@@ -63,10 +83,23 @@ class JestReporter {
     const { numFailedTests } = results;
     const status = numFailedTests === 0 ? STATUS.PASSED : STATUS.FAILED;
     this.client.updateRunStatus(status);
+  }
+}
 
-    // clear tmp dir
-    fileSystem.clearDir(TESTOMAT_TMP_STORAGE.mainDir);
+function getTestLogs(testResult) {
+  const suiteLogsArr = services.logger.getLogs(testResult.testFilePath);
+  const suiteLogs = suiteLogsArr ? suiteLogsArr.join('\n').trim() : '';
+  const testLogsArr = services.logger.getLogs(testResult.fullName);
+  const testLogs = testLogsArr ? testLogsArr.join('\n').trim() : '';
+
+  let logs = '';
+  if (suiteLogs) {
+    logs += `${chalk.bold('\t--- Suite ---')}\n${suiteLogs}`;
+  }
+  if (testLogs) {
+    logs += `\n${chalk.bold('\t--- Test ---')}\n${testLogs}`;
   }
+  return logs;
 }
 
 module.exports = JestReporter;

package/lib/adapter/mocha.js

@@ -1,58 +1,78 @@
 // eslint-disable-next-line global-require, import/no-extraneous-dependencies
 const Mocha = require('mocha');
-const debug = require('debug')('@testomatio/reporter:adapter:mocha');
 const chalk = require('chalk');
 const TestomatClient = require('../client');
-const { STATUS } = require('../constants');
-const { parseTest, specificTestInfo } = require('../util');
-const ArtifactStorage = require('../_ArtifactStorageOld');
-
-const { EVENT_RUN_BEGIN, EVENT_RUN_END, EVENT_TEST_FAIL, EVENT_TEST_PASS, EVENT_TEST_PENDING } = Mocha.Runner.constants;
+const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
+const { parseTest, fileSystem } = require('../utils/utils');
+const config = require('../config');
+const { services } = require('../services');
+
+const {
+  EVENT_RUN_BEGIN,
+  EVENT_RUN_END,
+  EVENT_TEST_FAIL,
+  EVENT_TEST_PASS,
+  EVENT_TEST_PENDING,
+  EVENT_SUITE_BEGIN,
+  EVENT_SUITE_END,
+  EVENT_TEST_BEGIN,
+  EVENT_TEST_END,
+} = Mocha.Runner.constants;
 
 function MochaReporter(runner, opts) {
   Mocha.reporters.Base.call(this, runner);
-  let passes = 0; let failures = 0; let skipped = 0;
-  let artifactStore;
+  let passes = 0;
+  let failures = 0;
+  let skipped = 0;
+  // let artifactStore;
 
-  const apiKey = opts?.reporterOptions?.apiKey || process.env.TESTOMATIO;
+  const apiKey = opts?.reporterOptions?.apiKey || config.TESTOMATIO;
 
-  if (!apiKey) {
-    debug('TESTOMATIO key is empty, ignoring reports');
-    return;
-  }
   const client = new TestomatClient({ apiKey });
 
   runner.on(EVENT_RUN_BEGIN, () => {
     client.createRun();
 
-    const params = { 
-      toFile: true
-    };
-    
-    artifactStore = runner._workerReporter !== undefined
-      ? new ArtifactStorage(params)
-      : new ArtifactStorage();
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
   });
 
-  runner.on(EVENT_TEST_PASS, async(test) => {
+  runner.on(EVENT_SUITE_BEGIN, async suite => {
+    services.setContext(suite.fullTitle());
+  });
+
+  runner.on(EVENT_SUITE_END, async () => {
+    services.setContext(null);
+  });
+
+  runner.on(EVENT_TEST_BEGIN, async test => {
+    services.setContext(test.fullTitle());
+  });
+
+  runner.on(EVENT_TEST_END, async () => {
+    services.setContext(null);
+  });
+
+  runner.on(EVENT_TEST_PASS, async test => {
     passes += 1;
+
     console.log(chalk.bold.green('✔'), test.fullTitle());
     const testId = parseTest(test.title);
 
-    const specificTest =  specificTestInfo(test);
-    const content = await artifactStore.artifactByTestName(specificTest);
+    const logs = getTestLogs(test);
+    const artifacts = services.artifacts.get(test.fullTitle());
+    const keyValues = services.keyValues.get(test.fullTitle());
 
-    debug(`test=${specificTest} content = `, content);
-
-    client.addTestRun(
-      STATUS.PASSED, 
-      {
-        test_id: testId,
-        title: test.title,
-        time: test.duration,
-      },
-      content
-    );
+    client.addTestRun(STATUS.PASSED, {
+      test_id: testId,
+      suite_title: getSuiteTitle(test),
+      title: getTestName(test),
+      code: test.body.toString(),
+      file: getFile(test),
+      time: test.duration,
+      logs,
+      manuallyAttachedArtifacts: artifacts,
+      meta: keyValues,
+    });
   });
 
   runner.on(EVENT_TEST_PENDING, test => {
@@ -60,43 +80,75 @@ function MochaReporter(runner, opts) {
     console.log('skip: %s', test.fullTitle());
     const testId = parseTest(test.title);
     client.addTestRun(STATUS.SKIPPED, {
-      title: test.title,
+      title: getTestName(test),
+      suite_title: getSuiteTitle(test),
+      code: test.body.toString(),
+      file: getFile(test),
       test_id: testId,
       time: test.duration,
     });
   });
 
-  runner.on(EVENT_TEST_FAIL, async(test, err) => {
+  runner.on(EVENT_TEST_FAIL, async (test, err) => {
     failures += 1;
     console.log(chalk.bold.red('✖'), test.fullTitle(), chalk.gray(err.message));
     const testId = parseTest(test.title);
 
-    const specificTest =  specificTestInfo(test);
-    const content = await artifactStore.artifactByTestName(specificTest);
+    const logs = getTestLogs(test);
 
-    debug(`fail test=${specificTest} content = `, content);
-
-    client.addTestRun(
-      STATUS.FAILED, {
-        error: err,
-        test_id: testId,
-        title: test.title,
-        time: test.duration,
-      }, 
-      content
-    );
+    client.addTestRun(STATUS.FAILED, {
+      error: err,
+      suite_title: getSuiteTitle(test),
+      file: getFile(test),
+      test_id: testId,
+      title: getTestName(test),
+      code: test.body.toString(),
+      time: test.duration,
+      logs,
+    });
   });
 
   runner.on(EVENT_RUN_END, () => {
     const status = failures === 0 ? STATUS.PASSED : STATUS.FAILED;
     console.log(chalk.bold(status), `${passes} passed, ${failures} failed, ${skipped} skipped`);
     client.updateRunStatus(status);
-
-    artifactStore.cleanup();
   });
 }
 
+function getTestLogs(test) {
+  const suiteLogsArr = services.logger.getLogs(test.parent.fullTitle());
+  const suiteLogs = suiteLogsArr ? suiteLogsArr.join('\n').trim() : '';
+  const testLogsArr = services.logger.getLogs(test.fullTitle());
+  const testLogs = testLogsArr ? testLogsArr.join('\n').trim() : '';
+
+  let logs = '';
+  if (suiteLogs) {
+    logs += `${chalk.bold('\t--- BeforeSuite ---')}\n${suiteLogs}`;
+  }
+  if (testLogs) {
+    logs += `\n${chalk.bold('\t--- Test ---')}\n${testLogs}`;
+  }
+  return logs;
+}
+
 // To have this reporter "extend" a built-in reporter uncomment the following line:
 Mocha.utils.inherits(MochaReporter, Mocha.reporters.Spec);
 
 module.exports = MochaReporter;
+
+function getSuiteTitle(test, pathArr = []) {
+  if (test.parent.parent) getSuiteTitle(test.parent, pathArr);
+
+  pathArr.push(test.parent.title);
+
+  return pathArr.filter(t => !!t)[0];
+}
+
+function getFile(test) {
+  return test.parent.file?.replace(process.cwd(), '');
+}
+
+function getTestName(test) {
+  if (process.env.TESTOMATIO_CREATE === 'fulltitle') return test.fullTitle();
+  return test.title;
+}

package/lib/adapter/playwright.js

@@ -3,10 +3,13 @@ const crypto = require('crypto');
 const os = require('os');
 const path = require('path');
 const fs = require('fs');
-const { APP_PREFIX, STATUS: Status, TESTOMAT_TMP_STORAGE } = require('../constants');
+const { APP_PREFIX, STATUS: Status, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
 const TestomatioClient = require('../client');
 const { isArtifactsEnabled } = require('../fileUploader');
-const { parseTest, fileSystem } = require('../util');
+const { parseTest, fileSystem } = require('../utils/utils');
+// const debug = require('debug')('@testomatio/reporter:adapter:playwright');
+const { services } = require('../services');
+const { dataStorage } = require('../data-storage');
 
 const reportTestPromises = [];
 
@@ -17,54 +20,90 @@ class PlaywrightReporter {
     this.uploads = [];
   }
 
-  onBegin(_config, suite) {
+  onBegin(config, suite) {
     // clean data storage
-    fileSystem.clearDir(TESTOMAT_TMP_STORAGE.mainDir);
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
     if (!this.client) return;
     this.suite = suite;
+    this.config = config;
     this.client.createRun();
   }
 
-  // onTestBegin(test) {
-  //   const testId = parseTest(test.title);
-  // }
+  onTestBegin(testInfo) {
+    const fullTestTitle = getTestContextName(testInfo);
+    dataStorage.setContext(fullTestTitle);
+  }
 
   onTestEnd(test, result) {
     if (!this.client) return;
 
-    let testId = parseTest(test.title);
-
     const { title } = test;
+
+    let testId = parseTest(title);
+
     const { error, duration } = result;
 
-    const suite_title = test.parent ? test.parent.title : null;
+    const suite_title = test.parent ? test.parent?.title : path.basename(test?.location?.file);
 
     const steps = [];
     for (const step of result.steps) {
       appendStep(step, steps);
     }
 
-    const logs = `\n\n${chalk.bold('Logs:')}\n${chalk.red(result.stderr.join(''))}\n${result.stdout.join('')}`;
-
-    const reportTestPromise = this.client.addTestRun(checkStatus(result.status), {
-      error,
-      test_id: testId,
-      suite_title,
-      title,
-      steps: steps.join('\n'),
-      time: duration,
-      stack: logs,
-    }).then(pipes => {
-      testId = pipes?.filter(p => p.pipe.includes('Testomatio'))[0]?.result?.data?.test_id;
-
-      this.uploads.push({
-        testId, title, suite_title, files: result.attachments.filter((a) => a.body || a.path)
+    const fullTestTitle = getTestContextName(test);
+    let logs = '';
+    if (result.stderr.length || result.stdout.length) {
+      logs = `\n\n${chalk.bold('Logs:')}\n${chalk.red(result.stderr.join(''))}\n${result.stdout.join('')}`;
+    }
+    const manuallyAttachedArtifacts = services.artifacts.get(fullTestTitle);
+    const keyValues = services.keyValues.get(fullTestTitle);
+
+    const reportTestPromise = this.client
+      .addTestRun(checkStatus(result.status), {
+        error,
+        test_id: testId,
+        suite_title,
+        title,
+        steps: steps.join('\n'),
+        time: duration,
+        logs,
+        manuallyAttachedArtifacts,
+        meta: keyValues,
+        file: test.location?.file,
+      })
+      .then(pipes => {
+        testId = pipes?.filter(p => p.pipe.includes('Testomatio'))[0]?.result?.data?.test_id;
+
+        this.uploads.push({
+          testId,
+          title,
+          suite_title,
+          files: result.attachments.filter(a => a.body || a.path),
+          file: test.location?.file,
+        });
+        // remove empty uploads
+        this.uploads = this.uploads.filter(upload => upload.files.length);
       });
-    });
 
     reportTestPromises.push(reportTestPromise);
   }
 
+  #getArtifactPath(artifact) {
+    if (artifact.path) {
+      if (path.isAbsolute(artifact.path)) return artifact.path;
+
+      return path.join(this.config.outputDir || this.config.projects[0].outputDir, artifact.path);
+    }
+
+    if (artifact.body) {
+      const fileName = tmpFile();
+      fs.writeFileSync(fileName, artifact.body);
+      return fileName;
+    }
+
+    return null;
+  }
+
   async onEnd(result) {
     if (!this.client) return;
 
@@ -78,13 +117,11 @@ class PlaywrightReporter {
       for (const upload of this.uploads) {
         const { title, testId, suite_title } = upload;
 
-        const files = upload.files.map(attachment => {
-          if (attachment.body) {
-            const fileName = tmpFile();
-            fs.writeFileSync(fileName, attachment.body);
-          }
-          return { path: attachment.path, title, type: attachment.contentType };
-        });
+        const files = upload.files.map(attachment => ({
+          path: this.#getArtifactPath(attachment),
+          title,
+          type: attachment.contentType,
+        }));
 
         promises.push(
           this.client.addTestRun(undefined, {
@@ -92,6 +129,7 @@ class PlaywrightReporter {
             title,
             suite_title,
             files,
+            file: upload.file,
           }),
         );
       }
@@ -131,4 +169,28 @@ function tmpFile(prefix = 'tmp.') {
   return path.join(tmpdir, prefix + crypto.randomBytes(16).toString('hex'));
 }
 
+/**
+ * Returns filename + test title
+ * @param {*} test - testInfo object from Playwright
+ * @returns
+ */
+function getTestContextName(test) {
+  return `${test._requireFile || ''}_${test.title}`;
+}
+
+function initPlaywrightForStorage() {
+  try {
+    // @ts-ignore-next-line
+    // eslint-disable-next-line import/no-unresolved
+    const { test } = require('@playwright/test');
+    // eslint-disable-next-line no-empty-pattern
+    test.beforeEach(async ({}, testInfo) => {
+      global.testomatioTestTitle = `${testInfo.file || ''}_${testInfo.title}`;
+    });
+  } catch (e) {
+    // ignore
+  }
+}
+
 module.exports = PlaywrightReporter;
+module.exports.initPlaywrightForStorage = initPlaywrightForStorage;

package/lib/adapter/webdriver.js

@@ -1,7 +1,7 @@
 // eslint-disable-next-line global-require, import/no-extraneous-dependencies
 const WDIOReporter = require('@wdio/reporter').default;
 const TestomatClient = require('../client');
-const { parseTest } = require('../util');
+const { parseTest } = require('../utils/utils');
 
 class WebdriverReporter extends WDIOReporter {
   constructor(options) {
@@ -31,6 +31,18 @@ class WebdriverReporter extends WDIOReporter {
     this._addTestPromises.push(this.addTest(test));
   }
 
+  // wdio-cucumber does not trigger onTestEnd hook, thus, using this one
+  /**
+   *
+   * @param {} scerario
+   * @returns
+   */
+  onSuiteEnd(scerario) {
+    if (scerario.type === 'scenario') {
+      this._addTestPromises.push(this.addBddScenario(scerario));
+    }
+  }
+
   async addTest(test) {
     if (!this.client) return;
 
@@ -51,6 +63,39 @@ class WebdriverReporter extends WDIOReporter {
       filesBuffers: screenshotsBuffers,
     });
   }
+
+  /**
+   * @param {import('../../types').WebdriverIOScenario} scenario
+   */
+  addBddScenario(scenario) {
+    if (!this.client) return;
+
+    const { title, _duration: duration } = scenario;
+
+    const testId = parseTest(title) || parseTest(scenario.tags.map(tag => tag.name).join(' '));
+
+    let scenarioState = scenario.tests.every(test => test.state === 'passed') ? 'passed' : 'failed';
+    if (scenario.tests.every(test => test.state === 'skipped')) {
+      scenarioState = 'skipped';
+    }
+    const errors = scenario.tests
+      .filter(test => test.state === 'failed')
+      .map(test => test.error?.stack)
+      .filter(Boolean);
+    const error = errors.join('\n');
+
+    const tags = scenario.tags.map(tag => tag.name);
+
+    return this.client.addTestRun(scenarioState, {
+      error: error ? Error(error) : null,
+      title,
+      test_id: testId,
+      time: duration,
+      tags,
+      file: scenario.file,
+      // filesBuffers: screenshotsBuffers,
+    });
+  }
 }
 
 module.exports = WebdriverReporter;