@testomatio/reporter 1.2.0-beta-4 → 1.2.0

package/README.md

@@ -65,23 +65,16 @@ yarn add @testomatio/reporter --dev
 
 ### 1️⃣ Attach Reporter to the Test Runner
 
-* #### [Playwright](./docs/frameworks.md#playwright)
-* #### [CodeceptJS](./docs/frameworks.md#CodeceptJS)
-* #### [Cypress](./docs/frameworks.md#Cypress)
-* #### [Jest](./docs/frameworks.md#Jest)
-* #### [Mocha](./docs/frameworks.md#Mocha)
-* #### [WebDriverIO](./docs/frameworks.md#WebDriverIO)
-* #### [TestCafe](./docs/frameworks.md#TestCafe)
-* #### [Detox](./docs/frameworks.md#Detox)
-* #### [Codeception](https://github.com/testomatio/php-reporter)
-* #### [Newman (Postman)](./docs/frameworks.md#Newman)
-* #### [JUnit](./docs/junit.md#junit)
-* #### [NUnit](./docs/junit.md#nunit)
-* #### [PyTest](./docs/junit.md#pytest)
-* #### [PHPUnit](./docs/junit.md#phpunit)
-* #### [Protractor](./docs/frameworks.md#protractor)
-
-or any [other via JUnit](./docs/junit.md) report....
+| | | |
+|--|--|--|
+| [Playwright](./docs/frameworks.md#playwright) | [CodeceptJS](./docs/frameworks.md#CodeceptJS) | [Cypress](./docs/frameworks.md#Cypress) |
+| [Jest](./docs/frameworks.md#Jest) | [Mocha](./docs/frameworks.md#Mocha) | [WebDriverIO](./docs/frameworks.md#WebDriverIO) |
+| [TestCafe](./docs/frameworks.md#TestCafe) | [Detox](./docs/frameworks.md#Detox) | [Codeception](https://github.com/testomatio/php-reporter) |
+| [Newman (Postman)](./docs/frameworks.md#Newman) | [JUnit](./docs/junit.md#junit) | [NUnit](./docs/junit.md#nunit) |
+| [PyTest](./docs/junit.md#pytest) | [PHPUnit](./docs/junit.md#phpunit) | [Protractor](./docs/frameworks.md#protractor) |
+
+
+or **any [other via JUnit](./docs/junit.md)** report....
 
 ### 2️⃣ Configure Reports
 
@@ -104,7 +97,21 @@ GitHub report published as a comment to Pull Request:
 
 Intercept your logger messages or log anything with our [Logger](./docs/logger.md) (_work in progress_).
 
-### 5️⃣ Add to CI Pipeline
+### 5️⃣ TESTOMATIO HTML Report
+
+Testomatio now features a custom reporting tool that allows you to generate a standalone HTML/CSS report for easy visualization of your test runs. This mode provides a quick way to gain a clear and visually appealing overview of your test execution.
+
+With our reporter, you can:
+
+- Easily analyze the results of your test runs in a convenient HTML format.
+- Visualize data on successful and failed tests, including statistics and error details.
+- Quickly share reports with your team members or stakeholders.
+
+![HTML report](./docs/images/html_reporter_example.gif)
+
+Learn more about generating HTML reports [here](./docs/pipes/html.md)
+
+### 6️⃣ Add to CI Pipeline
 
 After you tested reporter locally add it to your CI pipeline.
 
@@ -125,6 +132,7 @@ Bring this reporter on CI and never lose test results again!
   * [GitHub](./docs/pipes/github.md)
   * [Gitlab](./docs/pipes/gitlab.md)
   * [CSV](./docs/pipes/csv.md)
+  * [HTML report](./docs/pipes/html.md)
 * 📓 [JUnit](./docs/junit.md)
 * 🗄️ [Artifacts](./docs/artifacts.md)
 * 🔂 [Workflows](./docs/workflows.md)
@@ -153,3 +161,4 @@ To print all reporter logs of a specific pipe:
 ```
 DEBUG=@testomatio/reporter:pipe:github
 ```
+

package/lib/adapter/codecept.js

@@ -1,9 +1,10 @@
 const debug = require('debug')('@testomatio/reporter:adapter:codeceptjs');
 const chalk = require('chalk');
 const TestomatClient = require('../client');
-const { STATUS, APP_PREFIX, TESTOMAT_TMP_STORAGE } = require('../constants');
+const { STATUS, APP_PREFIX, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
 const upload = require('../fileUploader');
 const { parseTest: getIdFromTestTitle, fileSystem } = require('../utils/utils');
+const { services } = require('../services');
 
 if (!global.codeceptjs) {
   // eslint-disable-next-line global-require, import/no-extraneous-dependencies
@@ -51,21 +52,56 @@ function CodeceptReporter(config) {
 
   recorder.startUnlessRunning();
 
-  global.testomatioRunningEnvironment = 'codeceptjs';
-
   // Listening to events
   event.dispatcher.on(event.all.before, () => {
     // clear tmp dir
-    fileSystem.clearDir(TESTOMAT_TMP_STORAGE.mainDir);
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
 
-    recorder.add('Creating new run', () => client.createRun());
+    // recorder.add('Creating new run', () => );
+    client.createRun();
     videos = [];
     traces = [];
 
     if (!global.testomatioDataStore) global.testomatioDataStore = {};
   });
 
-  event.dispatcher.on(event.test.before, () => {
+  let hookSteps = [];
+  let suiteHookRunning = false;
+
+  event.dispatcher.on(event.suite.before, suite => {
+    suiteHookRunning = true;
+    hookSteps = [];
+    global.testomatioDataStore.steps = [];
+
+    services.setContext(suite.fullTitle());
+  });
+
+  event.dispatcher.on(event.suite.after, () => {
+    services.setContext(null);
+  });
+
+  event.dispatcher.on(event.hook.started, () => {
+    // global.testomatioDataStore.steps = [];
+  });
+
+  event.dispatcher.on(event.hook.passed, () => {
+    if (suiteHookRunning) {
+      hookSteps.push(...global.testomatioDataStore.steps);
+      services.setContext(null);
+    }
+  });
+
+  event.dispatcher.on(event.hook.failed, () => {
+    if (suiteHookRunning) {
+      hookSteps.push(...global.testomatioDataStore.steps);
+      services.setContext(null);
+    }
+  });
+
+  event.dispatcher.on(event.test.before, test => {
+    suiteHookRunning = false;
+    global.testomatioDataStore.steps = [];
+
     recorder.add(() => {
       currentMetaStep = [];
       // output.reset();
@@ -76,11 +112,16 @@ function CodeceptReporter(config) {
     if (!global.testomatioDataStore) global.testomatioDataStore = {};
     // reset steps
     global.testomatioDataStore.steps = [];
+
+    services.setContext(test.fullTitle());
   });
 
   event.dispatcher.on(event.test.started, test => {
+    services.setContext(test.fullTitle());
+
     testTimeMap[test.id] = Date.now();
     if (global.testomatioDataStore) global.testomatioDataStore.currentlyRunningTestId = getIdFromTestTitle(test.title);
+    // start logging
   });
 
   event.dispatcher.on(event.all.result, async () => {
@@ -104,6 +145,12 @@ function CodeceptReporter(config) {
     }
     const testId = parseTest(tags);
     const testObj = getTestAndMessage(title);
+
+    const logs = getTestLogs(test);
+    const manuallyAttachedArtifacts = services.artifacts.get(test.fullTitle());
+    const keyValues = services.keyValues.get(test.fullTitle());
+    services.setContext(null);
+
     client.addTestRun(STATUS.PASSED, {
       ...stripExampleFromTitle(title),
       suite_title: test.parent && test.parent.title,
@@ -111,6 +158,9 @@ function CodeceptReporter(config) {
       time: getDuration(test),
       steps: global.testomatioDataStore.steps.join('\n') || null,
       test_id: testId,
+      logs,
+      manuallyAttachedArtifacts,
+      meta: keyValues,
     });
     // output.stop();
   });
@@ -147,14 +197,16 @@ function CodeceptReporter(config) {
     failedTests.push(id || title);
     let testId = parseTest(tags);
     const testObj = getTestAndMessage(title);
-    if (error && error.stack && test.steps && test.steps.length) {
-      error.stack = test.steps[test.steps.length - 1].line();
-    }
 
     const files = [];
     if (artifacts.screenshot) files.push({ path: artifacts.screenshot, type: 'image/png' });
     // todo: video must be uploaded later....
 
+    const logs = getTestLogs(test);
+    const manuallyAttachedArtifacts = services.artifacts.get(test.fullTitle());
+    const keyValues = services.keyValues.get(test.fullTitle());
+    services.setContext(null);
+
     const reportTestPromise = client
       .addTestRun(STATUS.FAILED, {
         ...stripExampleFromTitle(title),
@@ -165,6 +217,9 @@ function CodeceptReporter(config) {
         time: getDuration(test),
         files,
         steps: global.testomatioDataStore?.steps?.join('\n') || null,
+        logs,
+        manuallyAttachedArtifacts,
+        meta: keyValues,
       })
       .then(pipes => {
         testId = pipes.filter(p => p.pipe.includes('Testomatio'))[0]?.result?.data?.test_id;
@@ -233,7 +288,7 @@ function CodeceptReporter(config) {
     currentMetaStep = metaSteps;
     stepShift = 2 * shift;
 
-    const durationMs = +new Date() - (+stepStart);
+    const durationMs = +new Date() - +stepStart;
     let duration = '';
     if (durationMs) {
       duration = repeat(1) + chalk.grey(`(${durationMs}ms)`);
@@ -258,7 +313,7 @@ async function uploadAttachments(client, attachments, messagePrefix, attachmentT
   if (attachments.length > 0) {
     console.log(APP_PREFIX, `Attachments: ${messagePrefix} ${attachments.length} ${attachmentType}/-s ...`);
 
-    const promises = attachments.map(async (attachment) => {
+    const promises = attachments.map(async attachment => {
       const { testId, title, path, type } = attachment;
       const file = { path, type, title };
       return client.addTestRun(undefined, {
@@ -306,4 +361,21 @@ function repeat(num) {
   return ''.padStart(num, ' ');
 }
 
+// TODO: think about moving to some common utils
+function getTestLogs(test) {
+  const suiteLogsArr = services.logger.getLogs(test.parent.fullTitle());
+  const suiteLogs = suiteLogsArr ? suiteLogsArr.join('\n').trim() : '';
+  const testLogsArr = services.logger.getLogs(test.fullTitle());
+  const testLogs = testLogsArr ? testLogsArr.join('\n').trim() : '';
+
+  let logs = '';
+  if (suiteLogs) {
+    logs += `${chalk.bold('\t--- BeforeSuite ---')}\n${suiteLogs}`;
+  }
+  if (testLogs) {
+    logs += `\n${chalk.bold('\t--- Test ---')}\n${testLogs}`;
+  }
+  return logs;
+}
+
 module.exports = CodeceptReporter;

package/lib/adapter/cucumber/current.js

@@ -2,10 +2,11 @@
 const { Formatter, formatterHelpers } = require('@cucumber/cucumber');
 const chalk = require('chalk');
 const fs = require('fs');
-const { STATUS, TESTOMAT_TMP_STORAGE } = require('../../constants');
+const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../../constants');
 const TestomatClient = require('../../client');
-const logger = require('../../logger');
 const { parseTest, fileSystem } = require('../../utils/utils');
+const config = require('../../config');
+const { services } = require('../../services');
 
 const { GherkinDocumentParser, PickleParser } = formatterHelpers;
 const { getGherkinScenarioLocationMap, getGherkinStepMap } = GherkinDocumentParser;
@@ -29,15 +30,13 @@ class CucumberReporter extends Formatter {
     this.failures = [];
     this.cases = [];
 
-    this.client = new TestomatClient({ apiKey: options.apiKey || process.env.TESTOMATIO });
+    this.client = new TestomatClient({ apiKey: options.apiKey || config.TESTOMATIO });
     this.status = STATUS.PASSED;
-
-    global.testomatioRunningEnvironment = 'cucumber:current';
   }
 
   parseEnvelope(envelope) {
     if (envelope.testRunStarted) {
-      fileSystem.clearDir(TESTOMAT_TMP_STORAGE.mainDir);
+      fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
     }
     if (envelope.testCaseStarted && this.client) {
       this.client.createRun();
@@ -49,9 +48,10 @@ class CucumberReporter extends Formatter {
 
   onTestCaseStarted(testCaseStarted) {
     const testCaseAttempt = this.eventDataCollector.getTestCaseAttempt(testCaseStarted.id);
-    const testId = getTestId(testCaseAttempt.pickle);
     if (!global.testomatioDataStore) global.testomatioDataStore = {};
-    global.testomatioDataStore.currentlyRunningTestId = testId;
+
+    const testTitle = testCaseAttempt.pickle.name + testCaseAttempt.pickle.uri;
+    services.setContext(testTitle);
   }
 
   onTestCaseFinished(testCaseFinished) {
@@ -109,7 +109,10 @@ class CucumberReporter extends Formatter {
 
     if (!this.client) return;
 
-    const logs = logger.getLogs(testId);
+    const testTitle = testCaseAttempt.pickle.name + testCaseAttempt.pickle.uri;
+    const logs = services.logger.getLogs(testTitle).join('\n');
+    const artifacts = services.artifacts.get(testTitle);
+    const keyValues = services.keyValues.get(testTitle);
 
     this.client.addTestRun(status, {
       // error: testCaseAttempt.worstTestStepResult.message,
@@ -119,11 +122,15 @@ class CucumberReporter extends Formatter {
         .join('\n')
         .trim(),
       example: { ...example },
-      stack: logs,
+      logs,
+      manuallyAttachedArtifacts: artifacts,
+      meta: keyValues,
       title: scenario,
       test_id: testId,
       time,
     });
+
+    services.setContext(null);
   }
 
   onTestRunFinished(envelope) {

package/lib/adapter/cucumber/legacy.js

@@ -2,8 +2,9 @@
 const { Formatter } = require('cucumber');
 const chalk = require('chalk');
 const { parseTest, fileSystem } = require('../../utils/utils');
-const { STATUS, TESTOMAT_TMP_STORAGE } = require('../../constants');
+const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../../constants');
 const TestomatClient = require('../../client');
+const config = require('../../config');
 
 const createTestomatFormatter = apiKey => {
   if (!apiKey || apiKey === '') {
@@ -97,7 +98,7 @@ const createTestomatFormatter = apiKey => {
 
       options.eventBroadcaster.on('gherkin-document', addDocument);
       options.eventBroadcaster.on('test-run-started', () => {
-        fileSystem.clearDir(TESTOMAT_TMP_STORAGE.mainDir);
+        fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
         this?.client?.createRun();
       });
       options.eventBroadcaster.on('test-case-finished', this.onTestCaseFinished.bind(this));
@@ -151,4 +152,4 @@ const createTestomatFormatter = apiKey => {
   };
 };
 
-module.exports = createTestomatFormatter(process.env.TESTOMATIO);
+module.exports = createTestomatFormatter(config.TESTOMATIO);

package/lib/adapter/cypress-plugin/index.js

@@ -1,17 +1,19 @@
 const { STATUS } = require('../../constants');
 const { parseTest, parseSuite } = require('../../utils/utils');
 const TestomatClient = require('../../client');
+const config = require('../../config');
 
 const testomatioReporter = on => {
-  if (!process.env.TESTOMATIO) {
+  if (!config.TESTOMATIO) {
     console.log('TESTOMATIO key is empty, ignoring reports');
-    return
+    return;
   }
-  const client = new TestomatClient({ apiKey: process.env.TESTOMATIO });
+  const client = new TestomatClient({ apiKey: config.TESTOMATIO });
 
-  on('before:run', async (run) => {
+  on('before:run', async run => {
+    // TODO: looks like client.env does not exist
     if (!client.env) {
-      client.env = `${run.browser.displayName},${run.system.osName}`
+      client.env = `${run.browser.displayName},${run.system.osName}`;
     }
     await client.createRun();
   });
@@ -25,8 +27,9 @@ const testomatioReporter = on => {
       const lastAttemptIndex = test.attempts.length - 1;
       const latestAttempt = test.attempts[lastAttemptIndex];
 
-      const time = latestAttempt.duration;
-      const error = latestAttempt.error;
+      // latestAttempt.duration && latestAttempt.error were available in adapters version up to 13 JFYI
+      const time = latestAttempt.duration || latestAttempt.wallClockDuration || test.duration;
+      let error = latestAttempt.error;
 
       let title = test.title.pop();
       const examples = title.match(/\(example (#\d+)\)/);
@@ -39,35 +42,59 @@ const testomatioReporter = on => {
       const testId = parseTest(title);
       const suiteId = parseSuite(suiteTitle);
 
-      if (error) {
-        error.inspect = function() { // eslint-disable-line func-names
-          if (this && this.codeFrame) {
-            return this.codeFrame.frame;
-          }
-          return '';
-        }
+      if (!error && test.displayError) {
+        error = { message: test.displayError };
+        error.inspect = function () {
+          // eslint-disable-line func-names
+          return this.message;
+        };
       }
 
-      const screenshots = results.screenshots
-        .filter(screenshot => screenshot.path.includes(title))
-        .filter(screenshot => screenshot.testAttemptIndex === lastAttemptIndex)
-        .map(screenshot => screenshot.path);
+      const formattedError = error
+        ? {
+            message: error.message,
+            inspect:
+              error.inspect ||
+              function () {
+                return this.message;
+              },
+          }
+        : '';
+
+      const screenshots = Array.isArray(results.screenshots)
+        ? results.screenshots
+            .filter(screenshot => screenshot?.path && screenshot?.path.includes(title) && screenshot?.takenAt)
+            .map(screenshot => screenshot.path)
+        : [];
 
       const files = [...videos, ...screenshots];
 
       let state;
       switch (test.state) {
-        case 'passed': state = STATUS.PASSED; break;
-        case 'failed': state = STATUS.FAILED; break;
+        case 'passed':
+          state = STATUS.PASSED;
+          break;
+        case 'failed':
+          state = STATUS.FAILED;
+          break;
         case 'skipped':
-        case 'pending': 
+        case 'pending':
         default:
           state = STATUS.SKIPPED;
       }
 
-      addSpecTestsPromises.push(client.addTestRun(state, {
-        title, time, example, error, files, suite_title: suiteTitle, test_id: testId, suite_id: suiteId
-      }));
+      addSpecTestsPromises.push(
+        client.addTestRun(state, {
+          title,
+          time,
+          example,
+          error: formattedError,
+          files,
+          suite_title: suiteTitle,
+          test_id: testId,
+          suite_id: suiteId,
+        }),
+      );
     }
 
     await Promise.all(addSpecTestsPromises);

package/lib/adapter/jest.js

@@ -1,6 +1,10 @@
+const chalk = require('chalk');
 const TestomatClient = require('../client');
-const { STATUS, TESTOMAT_TMP_STORAGE } = require('../constants');
+const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
 const { parseTest, ansiRegExp, fileSystem } = require('../utils/utils');
+const { services } = require('../services');
+const debug = require('debug')('@testomatio/reporter:adapter-jest');
+const path = require('path');
 
 class JestReporter {
   constructor(globalConfig, options) {
@@ -11,22 +15,24 @@ class JestReporter {
     this.client.createRun();
   }
 
-  static getIdOfCurrentlyRunningTest() {
-    if (!process.env.JEST_WORKER_ID) return null;
-    try {
-      // @ts-expect-error "expect" could only be defined inside Jest environement (forbidden to import it outside)
-      // eslint-disable-next-line no-undef
-      if (expect && expect?.getState()?.currentTestName) return parseTest(expect?.getState()?.currentTestName); 
-    } catch (e) {
-      return null; 
-    }
-  }
-
   onRunStart() {
     // clear tmp dir
-    fileSystem.clearDir(TESTOMAT_TMP_STORAGE.mainDir);
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
+  }
+
+  // start of test file (including beforeAll)
+  onTestStart(testFile) {
+    debug('Start running test file:', testFile.path);
+    services.setContext(testFile.path);
+  }
+
+  // start of the test (including beforeEach)
+  onTestCaseStart(test, testCase) {
+    debug('Start running test:', testCase.fullName);
+    services.setContext(testCase.fullName);
   }
 
+  // end of test file! (there is also onTestCaseResult listener)
   onTestResult(test, testResult) {
     if (!this.client) return;
 
@@ -42,13 +48,15 @@ class JestReporter {
         steps = failureMessages[0];
       }
       const testId = parseTest(title);
-      let suite_title;
-      // this is test without a suite
-      if (result.fullName === result.title) {
-        suite_title = testResult.testFilePath.split('/').pop();
-      } else {
-        suite_title = result.fullName.replace(result.title, '');
-      }
+      
+      // suite titles from most outer to most inner, separated by space
+      let fullSuiteTitle = testResult.ancestorTitles?.join(' ');
+      // if no suite titles, use file name
+      if (!fullSuiteTitle && testResult.testFilePath) fullSuiteTitle = path.basename(testResult.testFilePath);
+
+      const logs = getTestLogs(result);
+      const artifacts = services.artifacts.get(result.fullName);
+      const keyValues = services.keyValues.get(result.fullName);
 
       const deducedStatus = status === 'pending' ? 'skipped' : status;
       // In jest if test is not matched with test name pattern it is considered as skipped.
@@ -56,11 +64,14 @@ class JestReporter {
       if (!this._globalConfig.testNamePattern || deducedStatus !== 'skipped') {
         this.client.addTestRun(deducedStatus, {
           test_id: testId,
-          suite_title,
+          suite_title: fullSuiteTitle,
           error,
           steps,
           title,
           time: duration,
+          logs,
+          manuallyAttachedArtifacts: artifacts,
+          meta: keyValues,
         });
       }
     }
@@ -72,10 +83,23 @@ class JestReporter {
     const { numFailedTests } = results;
     const status = numFailedTests === 0 ? STATUS.PASSED : STATUS.FAILED;
     this.client.updateRunStatus(status);
+  }
+}
 
-    // clear tmp dir
-    fileSystem.clearDir(TESTOMAT_TMP_STORAGE.mainDir);
+function getTestLogs(testResult) {
+  const suiteLogsArr = services.logger.getLogs(testResult.testFilePath);
+  const suiteLogs = suiteLogsArr ? suiteLogsArr.join('\n').trim() : '';
+  const testLogsArr = services.logger.getLogs(testResult.fullName);
+  const testLogs = testLogsArr ? testLogsArr.join('\n').trim() : '';
+
+  let logs = '';
+  if (suiteLogs) {
+    logs += `${chalk.bold('\t--- Suite ---')}\n${suiteLogs}`;
+  }
+  if (testLogs) {
+    logs += `\n${chalk.bold('\t--- Test ---')}\n${testLogs}`;
   }
+  return logs;
 }
 
 module.exports = JestReporter;