@testomatio/reporter 1.2.1-beta → 1.2.1-beta.codecept-id

package/lib/adapter/cucumber/current.js

@@ -1,35 +1,59 @@
-// eslint-disable-next-line global-require, import/no-extraneous-dependencies
-const { Formatter, formatterHelpers } = require("@cucumber/cucumber");
+// eslint-disable-next-line global-require, import/no-extraneous-dependencies, import/no-unresolved
+const { Formatter, formatterHelpers } = require('@cucumber/cucumber');
 const chalk = require('chalk');
 const fs = require('fs');
-const { STATUS } = require('../../constants');
+const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../../constants');
 const TestomatClient = require('../../client');
+const { parseTest, fileSystem } = require('../../utils/utils');
+const config = require('../../config');
+const { services } = require('../../services');
+
+const { GherkinDocumentParser, PickleParser } = formatterHelpers;
+const { getGherkinScenarioLocationMap, getGherkinStepMap } = GherkinDocumentParser;
+const { getPickleStepMap } = PickleParser;
+
+function getTestId(scenario) {
+  if (scenario) {
+    for (const tag of scenario.tags) {
+      const testId = parseTest(tag.name);
+      if (testId) return testId;
+    }
+  }
 
-const { GherkinDocumentParser, PickleParser } = formatterHelpers
-const {
-  getGherkinScenarioLocationMap,
-  getGherkinStepMap,
-} = GherkinDocumentParser
-const { getPickleStepMap } = PickleParser
+  return null;
+}
 
 class CucumberReporter extends Formatter {
   constructor(options) {
     super(options);
-    options.eventBroadcaster.on('envelope', this.parseEnvelope.bind(this))
+    options.eventBroadcaster.on('envelope', this.parseEnvelope.bind(this));
     this.failures = [];
     this.cases = [];
 
-    this.client = new TestomatClient();
+    this.client = new TestomatClient({ apiKey: options.apiKey || config.TESTOMATIO });
     this.status = STATUS.PASSED;
-
   }
 
   parseEnvelope(envelope) {
-    if (envelope.testCaseStarted && this.client) this.client.createRun()
+    if (envelope.testRunStarted) {
+      fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
+    }
+    if (envelope.testCaseStarted && this.client) {
+      this.client.createRun();
+      this.onTestCaseStarted(envelope.testCaseStarted);
+    }
     if (envelope.testCaseFinished) this.onTestCaseFinished(envelope.testCaseFinished);
     if (envelope.testRunFinished) this.onTestRunFinished(envelope);
   }
 
+  onTestCaseStarted(testCaseStarted) {
+    const testCaseAttempt = this.eventDataCollector.getTestCaseAttempt(testCaseStarted.id);
+    if (!global.testomatioDataStore) global.testomatioDataStore = {};
+
+    const testTitle = testCaseAttempt.pickle.name + testCaseAttempt.pickle.uri;
+    services.setContext(testTitle);
+  }
+
   onTestCaseFinished(testCaseFinished) {
     const testCaseAttempt = this.eventDataCollector.getTestCaseAttempt(testCaseFinished.testCaseStartedId);
 
@@ -45,10 +69,10 @@ class CucumberReporter extends Formatter {
       if (testCaseAttempt.worstTestStepResult.status === 'SKIPPED') {
         status = STATUS.SKIPPED;
       }
-      if (testCaseAttempt.worstTestStepResult.status === 'UNDEFINED') {
-        status = STATUS.SKIPPED;
-        message = 'Undefined steps. Implement missing steps and rerun this scenario';
-      }
+      // if (testCaseAttempt.worstTestStepResult.status === 'UNDEFINED') {
+      //   status = STATUS.SKIPPED;
+      //   message = 'Undefined steps. Implement missing steps and rerun this scenario';
+      // }
       if (testCaseAttempt.worstTestStepResult.status === 'FAILED') {
         message = testCaseAttempt?.worstTestStepResult?.message;
         status = STATUS.FAILED;
@@ -64,10 +88,12 @@ class CucumberReporter extends Formatter {
     }
 
     const scenario = testCaseAttempt.pickle.name;
-    const testId = testCaseAttempt.pickle.id;
+    // this may broke something (it is supposed to work, but I am sure it did not)
+    // const testId = testCaseAttempt.pickle.id;
+    const testId = getTestId(testCaseAttempt.pickle);
 
     let exampleString = '';
-    if (example) exampleString = ` ${example.join(' | ')}`
+    if (example) exampleString = ` ${example.join(' | ')}`;
     let cliMessage = `${chalk.bold(scenario)}${exampleString}: ${chalk[color].bold(status.toUpperCase())} `;
 
     if (message) cliMessage += chalk.gray(message.split('\n')[0]);
@@ -79,19 +105,32 @@ class CucumberReporter extends Formatter {
 
     const time = Object.values(testCaseAttempt.stepResults)
       .map(t => t.duration)
-      .reduce((sum, duration) => sum + (duration.seconds * 1000)  + (duration.nanos / 1000000), 0)
+      .reduce((sum, duration) => sum + duration.seconds * 1000 + duration.nanos / 1000000, 0);
 
     if (!this.client) return;
 
+    const testTitle = testCaseAttempt.pickle.name + testCaseAttempt.pickle.uri;
+    const logs = services.logger.getLogs(testTitle).join('\n');
+    const artifacts = services.artifacts.get(testTitle);
+    const keyValues = services.keyValues.get(testTitle);
+
     this.client.addTestRun(status, {
       // error: testCaseAttempt.worstTestStepResult.message,
       message,
-      steps: getSteps(testCaseAttempt).map(s => s.toString()).join('\n').trim(),
-      example: { ...example},
+      steps: getSteps(testCaseAttempt)
+        .map(s => s.toString())
+        .join('\n')
+        .trim(),
+      example: { ...example },
+      logs,
+      manuallyAttachedArtifacts: artifacts,
+      meta: keyValues,
       title: scenario,
       test_id: testId,
       time,
     });
+
+    services.setContext(null);
   }
 
   onTestRunFinished(envelope) {
@@ -99,21 +138,20 @@ class CucumberReporter extends Formatter {
       console.log(chalk.bold('\nSUMMARY:\n\n'));
 
       this.failures.forEach((tc, i) => {
-        let message = `  ${i+1}) ${tc.pickle.name}\n`;
+        let message = `  ${i + 1}) ${tc.pickle.name}\n`;
 
         const steps = getSteps(tc);
 
         steps.forEach(s => {
-          message += `     ${s.toString()}\n`
-        })
+          message += `     ${s.toString()}\n`;
+        });
 
         console.log(message);
         if (tc?.worstTestStepResult?.message) {
           console.log(chalk.red(tc?.worstTestStepResult?.message));
         }
         console.log();
-
-      })
+      });
     }
 
     const { testRunFinished } = envelope;
@@ -125,46 +163,48 @@ class CucumberReporter extends Formatter {
 
     if (!this.client) return;
 
-    this.client.updateRunStatus(testRunFinished.success ? STATUS.PASSED : STATUS.FAILED)
+    this.client.updateRunStatus(testRunFinished.success ? STATUS.PASSED : STATUS.FAILED);
   }
 }
 
 function getSteps(tc) {
   const stepIds = Object.keys(tc.stepResults);
-  const pickleSteps = getPickleStepMap(tc.pickle)
-  return stepIds.map(stepId => {
-    const ts = tc.testCase.testSteps.find(t => t.id === stepId);
-    if (!ts) return;
-    if (!ts.pickleStepId) return;
-    const result = tc.stepResults[stepId];
-    const pickleStep = pickleSteps[ts.pickleStepId];
-    const sourceStepId = pickleStep.astNodeIds[0];
-    const step = {
-      text: pickleStep.text,
-      duration: result.duration,
-      status: result.status,
-    }
-    const color = getStatusColor(result.status);
-    if (sourceStepId && getGherkinStepMap(tc.gherkinDocument)[sourceStepId]) {
-      step.keyword = getGherkinStepMap(tc.gherkinDocument)[sourceStepId].keyword;
-    }
-    step.toString = function toString() {
-      const duration = step.duration.seconds * 1000 + (step.duration.nanos / 1000000)
-      const durationString = ` ${  chalk.gray(`(${Number(duration).toFixed(2)}ms)`)}`;
-      const stepString = `${chalk.bold(this.keyword)}${this.text}`.trim();
-      if (color === 'red') return chalk.red(stepString) + durationString;
-      if (color === 'yellow') return chalk.yellow(stepString) + durationString;
-      return stepString + durationString;
-    }
-    return step;
-  }).filter(s => !!s)
+  const pickleSteps = getPickleStepMap(tc.pickle);
+  return stepIds
+    .map(stepId => {
+      const ts = tc.testCase.testSteps.find(t => t.id === stepId);
+      if (!ts) return;
+      if (!ts.pickleStepId) return;
+      const result = tc.stepResults[stepId];
+      const pickleStep = pickleSteps[ts.pickleStepId];
+      const sourceStepId = pickleStep.astNodeIds[0];
+      const step = {
+        text: pickleStep.text,
+        duration: result.duration,
+        status: result.status,
+      };
+      const color = getStatusColor(result.status);
+      if (sourceStepId && getGherkinStepMap(tc.gherkinDocument)[sourceStepId]) {
+        step.keyword = getGherkinStepMap(tc.gherkinDocument)[sourceStepId].keyword;
+      }
+      step.toString = function toString() {
+        const duration = step.duration.seconds * 1000 + step.duration.nanos / 1000000;
+        const durationString = ` ${chalk.gray(`(${Number(duration).toFixed(2)}ms)`)}`;
+        const stepString = `${chalk.bold(this.keyword)}${this.text}`.trim();
+        if (color === 'red') return chalk.red(stepString) + durationString;
+        if (color === 'yellow') return chalk.yellow(stepString) + durationString;
+        return stepString + durationString;
+      };
+      return step;
+    })
+    .filter(s => !!s);
 }
 
 function getStatusColor(status) {
   if (status === 'UNDEFINED') return 'yellow';
   if (status === 'SKIPPED') return 'yellow';
   if (status === 'FAILED') return 'red';
-  return 'green'
+  return 'green';
 }
 
 function getExample(testCaseAttempt) {
@@ -173,11 +213,14 @@ function getExample(testCaseAttempt) {
   if (!nodesMap[exampleNodeId]) return;
   const featureDoc = fs.readFileSync(testCaseAttempt.gherkinDocument.uri).toString();
   const { line } = nodesMap[exampleNodeId];
-  const example = featureDoc.split('\n')[line-1];
+  const example = featureDoc.split('\n')[line - 1];
   if (example) {
-    return example.trim().split('|').filter(r => !!r).map(r => r.trim());
-  };
-
+    return example
+      .trim()
+      .split('|')
+      .filter(r => !!r)
+      .map(r => r.trim());
+  }
 }
 
 module.exports = CucumberReporter;

package/lib/adapter/cucumber/legacy.js

@@ -1,9 +1,10 @@
 // eslint-disable-next-line global-require, import/no-extraneous-dependencies, import/no-unresolved
 const { Formatter } = require('cucumber');
 const chalk = require('chalk');
-const { parseTest } = require('../../util');
-const { STATUS } = require('../../constants');
+const { parseTest, fileSystem } = require('../../utils/utils');
+const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../../constants');
 const TestomatClient = require('../../client');
+const config = require('../../config');
 
 const createTestomatFormatter = apiKey => {
   if (!apiKey || apiKey === '') {
@@ -96,9 +97,23 @@ const createTestomatFormatter = apiKey => {
       this.status = STATUS.PASSED.toString();
 
       options.eventBroadcaster.on('gherkin-document', addDocument);
-      options.eventBroadcaster.on('test-run-started', () => this?.client?.createRun());
+      options.eventBroadcaster.on('test-run-started', () => {
+        fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
+        this?.client?.createRun();
+      });
       options.eventBroadcaster.on('test-case-finished', this.onTestCaseFinished.bind(this));
+      options.eventBroadcaster.on('test-case-started', this.onTestCaseStarted.bind(this));
       options.eventBroadcaster.on('test-run-finished', () => this?.client?.updateRunStatus(this.status));
+
+      global.testomatioRunningEnvironment = 'cucumber:legacy';
+    }
+
+    onTestCaseStarted(event) {
+      const scenario = getScenario(event.sourceLocation);
+      const testId = getTestId(scenario);
+
+      if (!global.testomatioDataStore) global.testomatioDataStore = {};
+      global.testomatioDataStore.currentlyRunningTestId = testId;
     }
 
     onTestCaseFinished(event) {
@@ -111,15 +126,15 @@ const createTestomatFormatter = apiKey => {
 
       if (!scenario.name) return;
 
-      let message = '';
-      let cliMessage = `- ${scenario.name}: ${chalk.bold(status.toUpperCase())}`;
+      const message = '';
+      const cliMessage = `- ${scenario.name}: ${chalk.bold(status.toUpperCase())}`;
 
-      if (event.result.status === 'undefined') {
-        cliMessage += chalk.yellow(
-          ' (undefined steps. Run Cucumber without this formatter and implement missing steps)',
-        );
-        message = 'Undefined steps. Implement missing steps and rerun this scenario';
-      }
+      // if (event.result.status === 'undefined') {
+      //   cliMessage += chalk.yellow(
+      //     ' (undefined steps. Run Cucumber without this formatter and implement missing steps)',
+      //   );
+      //   message = 'Undefined steps. Implement missing steps and rerun this scenario';
+      // }
       console.log(cliMessage);
       if (status !== STATUS.PASSED && status !== STATUS.SKIPPED) {
         this.status = STATUS.FAILED;
@@ -137,4 +152,4 @@ const createTestomatFormatter = apiKey => {
   };
 };
 
-module.exports = createTestomatFormatter(process.env.TESTOMATIO);
+module.exports = createTestomatFormatter(config.TESTOMATIO);

package/lib/adapter/cucumber.js

@@ -1,6 +1,6 @@
 try {
   // eslint-disable-next-line import/no-unresolved
-  require.resolve("@cucumber/cucumber");
+  require.resolve('@cucumber/cucumber');
   // eslint-disable-next-line global-require, import/no-extraneous-dependencies
   module.exports = require('./cucumber/current');
 } catch (_e) {
@@ -11,6 +11,6 @@ try {
     module.exports = require('./cucumber/legacy');
   } catch (_err) {
     console.error('Cucumber packages: "@cucumber/cucumber" or "cucumber" were not detected. Report won\'t be sent');
-    module.exports = {}
+    module.exports = {};
   }
 }

package/lib/adapter/cypress-plugin/index.js

@@ -1,17 +1,19 @@
 const { STATUS } = require('../../constants');
-const { parseTest, parseSuite } = require('../../util');
+const { parseTest, parseSuite } = require('../../utils/utils');
 const TestomatClient = require('../../client');
+const config = require('../../config');
 
 const testomatioReporter = on => {
-  if (!process.env.TESTOMATIO) {
+  if (!config.TESTOMATIO) {
     console.log('TESTOMATIO key is empty, ignoring reports');
-    return
+    return;
   }
-  const client = new TestomatClient({ apiKey: process.env.TESTOMATIO });
+  const client = new TestomatClient({ apiKey: config.TESTOMATIO });
 
-  on('before:run', async (run) => {
+  on('before:run', async run => {
+    // TODO: looks like client.env does not exist
     if (!client.env) {
-      client.env = `${run.browser.displayName},${run.system.osName}`
+      client.env = `${run.browser.displayName},${run.system.osName}`;
     }
     await client.createRun();
   });
@@ -25,8 +27,9 @@ const testomatioReporter = on => {
       const lastAttemptIndex = test.attempts.length - 1;
       const latestAttempt = test.attempts[lastAttemptIndex];
 
-      const time = latestAttempt.duration;
-      const error = latestAttempt.error;
+      // latestAttempt.duration && latestAttempt.error were available in adapters version up to 13 JFYI
+      const time = latestAttempt.duration || latestAttempt.wallClockDuration || test.duration;
+      let error = latestAttempt.error;
 
       let title = test.title.pop();
       const examples = title.match(/\(example (#\d+)\)/);
@@ -39,35 +42,59 @@ const testomatioReporter = on => {
       const testId = parseTest(title);
       const suiteId = parseSuite(suiteTitle);
 
-      if (error) {
-        error.inspect = function() { // eslint-disable-line func-names
-          if (this && this.codeFrame) {
-            return this.codeFrame.frame;
-          }
-          return '';
-        }
+      if (!error && test.displayError) {
+        error = { message: test.displayError };
+        error.inspect = function () {
+          // eslint-disable-line func-names
+          return this.message;
+        };
       }
 
-      const screenshots = results.screenshots
-        .filter(screenshot => screenshot.path.includes(title))
-        .filter(screenshot => screenshot.testAttemptIndex === lastAttemptIndex)
-        .map(screenshot => screenshot.path);
+      const formattedError = error
+        ? {
+            message: error.message,
+            inspect:
+              error.inspect ||
+              function () {
+                return this.message;
+              },
+          }
+        : '';
+
+      const screenshots = Array.isArray(results.screenshots)
+        ? results.screenshots
+            .filter(screenshot => screenshot?.path && screenshot?.path.includes(title) && screenshot?.takenAt)
+            .map(screenshot => screenshot.path)
+        : [];
 
       const files = [...videos, ...screenshots];
 
       let state;
       switch (test.state) {
-        case 'passed': state = STATUS.PASSED; break;
-        case 'failed': state = STATUS.FAILED; break;
+        case 'passed':
+          state = STATUS.PASSED;
+          break;
+        case 'failed':
+          state = STATUS.FAILED;
+          break;
         case 'skipped':
-        case 'pending': 
+        case 'pending':
         default:
           state = STATUS.SKIPPED;
       }
 
-      addSpecTestsPromises.push(client.addTestRun(state, {
-        title, time, example, error, files, suite_title: suiteTitle, test_id: testId, suite_id: suiteId
-      }));
+      addSpecTestsPromises.push(
+        client.addTestRun(state, {
+          title,
+          time,
+          example,
+          error: formattedError,
+          files,
+          suite_title: suiteTitle,
+          test_id: testId,
+          suite_id: suiteId,
+        }),
+      );
     }
 
     await Promise.all(addSpecTestsPromises);

package/lib/adapter/jasmine.js

@@ -1,5 +1,5 @@
 const TestomatClient = require('../client');
-const { parseTest, ansiRegExp } = require('../util');
+const { parseTest, ansiRegExp } = require('../utils/utils');
 const { STATUS } = require('../constants');
 
 class JasmineReporter {

package/lib/adapter/jest.js

@@ -1,6 +1,10 @@
+const chalk = require('chalk');
 const TestomatClient = require('../client');
-const { STATUS, TESTOMAT_TMP_STORAGE } = require('../constants');
-const { parseTest, ansiRegExp, fileSystem } = require('../util');
+const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
+const { parseTest, ansiRegExp, fileSystem } = require('../utils/utils');
+const { services } = require('../services');
+const debug = require('debug')('@testomatio/reporter:adapter-jest');
+const path = require('path');
 
 class JestReporter {
   constructor(globalConfig, options) {
@@ -11,17 +15,24 @@ class JestReporter {
     this.client.createRun();
   }
 
-  static getIdOfCurrentlyRunningTest() {
-    if (!process.env.JEST_WORKER_ID) return null;
-    // @ts-expect-error "expect" could only be defined inside Jest environement (forbidden to import it outside)
-    return parseTest(expect?.getState()?.currentTestName) || null; // eslint-disable-line
-  }
-
   onRunStart() {
     // clear tmp dir
-    fileSystem.clearDir(TESTOMAT_TMP_STORAGE.mainDir);
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
+  }
+
+  // start of test file (including beforeAll)
+  onTestStart(testFile) {
+    debug('Start running test file:', testFile.path);
+    services.setContext(testFile.path);
   }
 
+  // start of the test (including beforeEach)
+  onTestCaseStart(test, testCase) {
+    debug('Start running test:', testCase.fullName);
+    services.setContext(testCase.fullName);
+  }
+
+  // end of test file! (there is also onTestCaseResult listener)
   onTestResult(test, testResult) {
     if (!this.client) return;
 
@@ -37,16 +48,30 @@ class JestReporter {
         steps = failureMessages[0];
       }
       const testId = parseTest(title);
+
+      // suite titles from most outer to most inner, separated by space
+      let fullSuiteTitle = testResult.ancestorTitles?.join(' ');
+      // if no suite titles, use file name
+      if (!fullSuiteTitle && testResult.testFilePath) fullSuiteTitle = path.basename(testResult.testFilePath);
+
+      const logs = getTestLogs(result);
+      const artifacts = services.artifacts.get(result.fullName);
+      const keyValues = services.keyValues.get(result.fullName);
+
       const deducedStatus = status === 'pending' ? 'skipped' : status;
       // In jest if test is not matched with test name pattern it is considered as skipped.
       // So adding a check if it is skipped for real or because of test pattern
       if (!this._globalConfig.testNamePattern || deducedStatus !== 'skipped') {
         this.client.addTestRun(deducedStatus, {
           test_id: testId,
+          suite_title: fullSuiteTitle,
           error,
           steps,
           title,
           time: duration,
+          logs,
+          manuallyAttachedArtifacts: artifacts,
+          meta: keyValues,
         });
       }
     }
@@ -58,10 +83,23 @@ class JestReporter {
     const { numFailedTests } = results;
     const status = numFailedTests === 0 ? STATUS.PASSED : STATUS.FAILED;
     this.client.updateRunStatus(status);
+  }
+}
 
-    // clear tmp dir
-    fileSystem.clearDir(TESTOMAT_TMP_STORAGE.mainDir);
+function getTestLogs(testResult) {
+  const suiteLogsArr = services.logger.getLogs(testResult.testFilePath);
+  const suiteLogs = suiteLogsArr ? suiteLogsArr.join('\n').trim() : '';
+  const testLogsArr = services.logger.getLogs(testResult.fullName);
+  const testLogs = testLogsArr ? testLogsArr.join('\n').trim() : '';
+
+  let logs = '';
+  if (suiteLogs) {
+    logs += `${chalk.bold('\t--- Suite ---')}\n${suiteLogs}`;
+  }
+  if (testLogs) {
+    logs += `\n${chalk.bold('\t--- Test ---')}\n${testLogs}`;
   }
+  return logs;
 }
 
 module.exports = JestReporter;