@testomatio/reporter 1.1.0-beta → 1.1.0-beta-3

package/lib/adapter/cucumber/current.js

@@ -1,35 +1,59 @@
-// eslint-disable-next-line global-require, import/no-extraneous-dependencies
-const { Formatter, formatterHelpers } = require("@cucumber/cucumber");
+// eslint-disable-next-line global-require, import/no-extraneous-dependencies, import/no-unresolved
+const { Formatter, formatterHelpers } = require('@cucumber/cucumber');
 const chalk = require('chalk');
 const fs = require('fs');
-const { STATUS } = require('../../constants');
+const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../../constants');
 const TestomatClient = require('../../client');
+const logger = require('../../storages/logger');
+const { parseTest, fileSystem } = require('../../utils/utils');
+
+const { GherkinDocumentParser, PickleParser } = formatterHelpers;
+const { getGherkinScenarioLocationMap, getGherkinStepMap } = GherkinDocumentParser;
+const { getPickleStepMap } = PickleParser;
+
+function getTestId(scenario) {
+  if (scenario) {
+    for (const tag of scenario.tags) {
+      const testId = parseTest(tag.name);
+      if (testId) return testId;
+    }
+  }
 
-const { GherkinDocumentParser, PickleParser } = formatterHelpers
-const {
-  getGherkinScenarioLocationMap,
-  getGherkinStepMap,
-} = GherkinDocumentParser
-const { getPickleStepMap } = PickleParser
+  return null;
+}
 
 class CucumberReporter extends Formatter {
   constructor(options) {
     super(options);
-    options.eventBroadcaster.on('envelope', this.parseEnvelope.bind(this))
+    options.eventBroadcaster.on('envelope', this.parseEnvelope.bind(this));
     this.failures = [];
     this.cases = [];
 
-    this.client = new TestomatClient();
+    this.client = new TestomatClient({ apiKey: options.apiKey || process.env.TESTOMATIO });
     this.status = STATUS.PASSED;
 
+    global.testomatioRunningEnvironment = 'cucumber:current';
   }
 
   parseEnvelope(envelope) {
-    if (envelope.testCaseStarted && this.client) this.client.createRun()
+    if (envelope.testRunStarted) {
+      fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
+    }
+    if (envelope.testCaseStarted && this.client) {
+      this.client.createRun();
+      this.onTestCaseStarted(envelope.testCaseStarted);
+    }
     if (envelope.testCaseFinished) this.onTestCaseFinished(envelope.testCaseFinished);
     if (envelope.testRunFinished) this.onTestRunFinished(envelope);
   }
 
+  onTestCaseStarted(testCaseStarted) {
+    const testCaseAttempt = this.eventDataCollector.getTestCaseAttempt(testCaseStarted.id);
+    const testId = getTestId(testCaseAttempt.pickle);
+    if (!global.testomatioDataStore) global.testomatioDataStore = {};
+    global.testomatioDataStore.currentlyRunningTestId = testId;
+  }
+
   onTestCaseFinished(testCaseFinished) {
     const testCaseAttempt = this.eventDataCollector.getTestCaseAttempt(testCaseFinished.testCaseStartedId);
 
@@ -45,10 +69,10 @@ class CucumberReporter extends Formatter {
       if (testCaseAttempt.worstTestStepResult.status === 'SKIPPED') {
         status = STATUS.SKIPPED;
       }
-      if (testCaseAttempt.worstTestStepResult.status === 'UNDEFINED') {
-        status = STATUS.SKIPPED;
-        message = 'Undefined steps. Implement missing steps and rerun this scenario';
-      }
+      // if (testCaseAttempt.worstTestStepResult.status === 'UNDEFINED') {
+      //   status = STATUS.SKIPPED;
+      //   message = 'Undefined steps. Implement missing steps and rerun this scenario';
+      // }
       if (testCaseAttempt.worstTestStepResult.status === 'FAILED') {
         message = testCaseAttempt?.worstTestStepResult?.message;
         status = STATUS.FAILED;
@@ -64,10 +88,12 @@ class CucumberReporter extends Formatter {
     }
 
     const scenario = testCaseAttempt.pickle.name;
-    const testId = testCaseAttempt.pickle.id;
+    // this may broke something (it is supposed to work, but I am sure it did not)
+    // const testId = testCaseAttempt.pickle.id;
+    const testId = getTestId(testCaseAttempt.pickle);
 
     let exampleString = '';
-    if (example) exampleString = ` ${example.join(' | ')}`
+    if (example) exampleString = ` ${example.join(' | ')}`;
     let cliMessage = `${chalk.bold(scenario)}${exampleString}: ${chalk[color].bold(status.toUpperCase())} `;
 
     if (message) cliMessage += chalk.gray(message.split('\n')[0]);
@@ -79,15 +105,21 @@ class CucumberReporter extends Formatter {
 
     const time = Object.values(testCaseAttempt.stepResults)
       .map(t => t.duration)
-      .reduce((sum, duration) => sum + (duration.seconds * 1000)  + (duration.nanos / 1000000), 0)
+      .reduce((sum, duration) => sum + duration.seconds * 1000 + duration.nanos / 1000000, 0);
 
     if (!this.client) return;
 
+    const logs = logger.getLogs(testId);
+
     this.client.addTestRun(status, {
       // error: testCaseAttempt.worstTestStepResult.message,
       message,
-      steps: getSteps(testCaseAttempt).map(s => s.toString()).join('\n').trim(),
-      example: { ...example},
+      steps: getSteps(testCaseAttempt)
+        .map(s => s.toString())
+        .join('\n')
+        .trim(),
+      example: { ...example },
+      stack: logs,
       title: scenario,
       test_id: testId,
       time,
@@ -99,21 +131,20 @@ class CucumberReporter extends Formatter {
       console.log(chalk.bold('\nSUMMARY:\n\n'));
 
       this.failures.forEach((tc, i) => {
-        let message = `  ${i+1}) ${tc.pickle.name}\n`;
+        let message = `  ${i + 1}) ${tc.pickle.name}\n`;
 
         const steps = getSteps(tc);
 
         steps.forEach(s => {
-          message += `     ${s.toString()}\n`
-        })
+          message += `     ${s.toString()}\n`;
+        });
 
         console.log(message);
         if (tc?.worstTestStepResult?.message) {
           console.log(chalk.red(tc?.worstTestStepResult?.message));
         }
         console.log();
-
-      })
+      });
     }
 
     const { testRunFinished } = envelope;
@@ -125,46 +156,48 @@ class CucumberReporter extends Formatter {
 
     if (!this.client) return;
 
-    this.client.updateRunStatus(testRunFinished.success ? STATUS.PASSED : STATUS.FAILED)
+    this.client.updateRunStatus(testRunFinished.success ? STATUS.PASSED : STATUS.FAILED);
   }
 }
 
 function getSteps(tc) {
   const stepIds = Object.keys(tc.stepResults);
-  const pickleSteps = getPickleStepMap(tc.pickle)
-  return stepIds.map(stepId => {
-    const ts = tc.testCase.testSteps.find(t => t.id === stepId);
-    if (!ts) return;
-    if (!ts.pickleStepId) return;
-    const result = tc.stepResults[stepId];
-    const pickleStep = pickleSteps[ts.pickleStepId];
-    const sourceStepId = pickleStep.astNodeIds[0];
-    const step = {
-      text: pickleStep.text,
-      duration: result.duration,
-      status: result.status,
-    }
-    const color = getStatusColor(result.status);
-    if (sourceStepId && getGherkinStepMap(tc.gherkinDocument)[sourceStepId]) {
-      step.keyword = getGherkinStepMap(tc.gherkinDocument)[sourceStepId].keyword;
-    }
-    step.toString = function toString() {
-      const duration = step.duration.seconds * 1000 + (step.duration.nanos / 1000000)
-      const durationString = ` ${  chalk.gray(`(${Number(duration).toFixed(2)}ms)`)}`;
-      const stepString = `${chalk.bold(this.keyword)}${this.text}`.trim();
-      if (color === 'red') return chalk.red(stepString) + durationString;
-      if (color === 'yellow') return chalk.yellow(stepString) + durationString;
-      return stepString + durationString;
-    }
-    return step;
-  }).filter(s => !!s)
+  const pickleSteps = getPickleStepMap(tc.pickle);
+  return stepIds
+    .map(stepId => {
+      const ts = tc.testCase.testSteps.find(t => t.id === stepId);
+      if (!ts) return;
+      if (!ts.pickleStepId) return;
+      const result = tc.stepResults[stepId];
+      const pickleStep = pickleSteps[ts.pickleStepId];
+      const sourceStepId = pickleStep.astNodeIds[0];
+      const step = {
+        text: pickleStep.text,
+        duration: result.duration,
+        status: result.status,
+      };
+      const color = getStatusColor(result.status);
+      if (sourceStepId && getGherkinStepMap(tc.gherkinDocument)[sourceStepId]) {
+        step.keyword = getGherkinStepMap(tc.gherkinDocument)[sourceStepId].keyword;
+      }
+      step.toString = function toString() {
+        const duration = step.duration.seconds * 1000 + step.duration.nanos / 1000000;
+        const durationString = ` ${chalk.gray(`(${Number(duration).toFixed(2)}ms)`)}`;
+        const stepString = `${chalk.bold(this.keyword)}${this.text}`.trim();
+        if (color === 'red') return chalk.red(stepString) + durationString;
+        if (color === 'yellow') return chalk.yellow(stepString) + durationString;
+        return stepString + durationString;
+      };
+      return step;
+    })
+    .filter(s => !!s);
 }
 
 function getStatusColor(status) {
   if (status === 'UNDEFINED') return 'yellow';
   if (status === 'SKIPPED') return 'yellow';
   if (status === 'FAILED') return 'red';
-  return 'green'
+  return 'green';
 }
 
 function getExample(testCaseAttempt) {
@@ -173,11 +206,14 @@ function getExample(testCaseAttempt) {
   if (!nodesMap[exampleNodeId]) return;
   const featureDoc = fs.readFileSync(testCaseAttempt.gherkinDocument.uri).toString();
   const { line } = nodesMap[exampleNodeId];
-  const example = featureDoc.split('\n')[line-1];
+  const example = featureDoc.split('\n')[line - 1];
   if (example) {
-    return example.trim().split('|').filter(r => !!r).map(r => r.trim());
-  };
-
+    return example
+      .trim()
+      .split('|')
+      .filter(r => !!r)
+      .map(r => r.trim());
+  }
 }
 
 module.exports = CucumberReporter;

package/lib/adapter/cucumber/legacy.js

@@ -1,8 +1,8 @@
 // eslint-disable-next-line global-require, import/no-extraneous-dependencies, import/no-unresolved
 const { Formatter } = require('cucumber');
 const chalk = require('chalk');
-const { parseTest } = require('../../util');
-const { STATUS } = require('../../constants');
+const { parseTest, fileSystem } = require('../../utils/utils');
+const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../../constants');
 const TestomatClient = require('../../client');
 
 const createTestomatFormatter = apiKey => {
@@ -96,9 +96,23 @@ const createTestomatFormatter = apiKey => {
       this.status = STATUS.PASSED.toString();
 
       options.eventBroadcaster.on('gherkin-document', addDocument);
-      options.eventBroadcaster.on('test-run-started', () => this?.client?.createRun());
+      options.eventBroadcaster.on('test-run-started', () => {
+        fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
+        this?.client?.createRun();
+      });
       options.eventBroadcaster.on('test-case-finished', this.onTestCaseFinished.bind(this));
+      options.eventBroadcaster.on('test-case-started', this.onTestCaseStarted.bind(this));
       options.eventBroadcaster.on('test-run-finished', () => this?.client?.updateRunStatus(this.status));
+
+      global.testomatioRunningEnvironment = 'cucumber:legacy';
+    }
+
+    onTestCaseStarted(event) {
+      const scenario = getScenario(event.sourceLocation);
+      const testId = getTestId(scenario);
+      
+      if (!global.testomatioDataStore) global.testomatioDataStore = {};
+      global.testomatioDataStore.currentlyRunningTestId = testId;
     }
 
     onTestCaseFinished(event) {
@@ -111,15 +125,15 @@ const createTestomatFormatter = apiKey => {
 
       if (!scenario.name) return;
 
-      let message = '';
-      let cliMessage = `- ${scenario.name}: ${chalk.bold(status.toUpperCase())}`;
+      const message = '';
+      const cliMessage = `- ${scenario.name}: ${chalk.bold(status.toUpperCase())}`;
 
-      if (event.result.status === 'undefined') {
-        cliMessage += chalk.yellow(
-          ' (undefined steps. Run Cucumber without this formatter and implement missing steps)',
-        );
-        message = 'Undefined steps. Implement missing steps and rerun this scenario';
-      }
+      // if (event.result.status === 'undefined') {
+      //   cliMessage += chalk.yellow(
+      //     ' (undefined steps. Run Cucumber without this formatter and implement missing steps)',
+      //   );
+      //   message = 'Undefined steps. Implement missing steps and rerun this scenario';
+      // }
       console.log(cliMessage);
       if (status !== STATUS.PASSED && status !== STATUS.SKIPPED) {
         this.status = STATUS.FAILED;

package/lib/adapter/cypress-plugin/index.js

@@ -1,5 +1,5 @@
 const { STATUS } = require('../../constants');
-const { parseTest, parseSuite } = require('../../util');
+const { parseTest, parseSuite } = require('../../utils/utils');
 const TestomatClient = require('../../client');
 
 const testomatioReporter = on => {

package/lib/adapter/jasmine.js

@@ -1,5 +1,5 @@
 const TestomatClient = require('../client');
-const { parseTest, ansiRegExp } = require('../util');
+const { parseTest, ansiRegExp } = require('../utils/utils');
 const { STATUS } = require('../constants');
 
 class JasmineReporter {

package/lib/adapter/jest.js

@@ -1,6 +1,6 @@
 const TestomatClient = require('../client');
-const { STATUS, TESTOMAT_TMP_STORAGE } = require('../constants');
-const { parseTest, ansiRegExp, fileSystem } = require('../util');
+const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
+const { parseTest, ansiRegExp, fileSystem } = require('../utils/utils');
 
 class JestReporter {
   constructor(globalConfig, options) {
@@ -11,14 +11,9 @@ class JestReporter {
     this.client.createRun();
   }
 
-  static getIdOfCurrentlyRunningTest() {
-    // @ts-expect-error "expect" could only be defined inside Jest environement (forbidden to import it outside)
-    return parseTest(expect?.getState()?.currentTestName) || null; // eslint-disable-line
-  }
-
   onRunStart() {
     // clear tmp dir
-    fileSystem.clearDir(TESTOMAT_TMP_STORAGE.mainDir);
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
   }
 
   onTestResult(test, testResult) {
@@ -36,12 +31,21 @@ class JestReporter {
         steps = failureMessages[0];
       }
       const testId = parseTest(title);
+      let suite_title;
+      // this is test without a suite
+      if (result.fullName === result.title) {
+        suite_title = testResult.testFilePath.split('/').pop();
+      } else {
+        suite_title = result.fullName.replace(result.title, '');
+      }
+
       const deducedStatus = status === 'pending' ? 'skipped' : status;
       // In jest if test is not matched with test name pattern it is considered as skipped.
       // So adding a check if it is skipped for real or because of test pattern
       if (!this._globalConfig.testNamePattern || deducedStatus !== 'skipped') {
         this.client.addTestRun(deducedStatus, {
           test_id: testId,
+          suite_title,
           error,
           steps,
           title,
@@ -57,9 +61,6 @@ class JestReporter {
     const { numFailedTests } = results;
     const status = numFailedTests === 0 ? STATUS.PASSED : STATUS.FAILED;
     this.client.updateRunStatus(status);
-
-    // clear tmp dir
-    fileSystem.clearDir(TESTOMAT_TMP_STORAGE.mainDir);
   }
 }
 

package/lib/adapter/mocha.js

@@ -1,58 +1,40 @@
 // eslint-disable-next-line global-require, import/no-extraneous-dependencies
 const Mocha = require('mocha');
-const debug = require('debug')('@testomatio/reporter:adapter:mocha');
 const chalk = require('chalk');
 const TestomatClient = require('../client');
-const { STATUS } = require('../constants');
-const { parseTest, specificTestInfo } = require('../util');
-const ArtifactStorage = require('../_ArtifactStorageOld');
+const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
+const { parseTest, fileSystem } = require('../utils/utils');
 
-const { EVENT_RUN_BEGIN, EVENT_RUN_END, EVENT_TEST_FAIL, EVENT_TEST_PASS, EVENT_TEST_PENDING } = Mocha.Runner.constants;
+const { EVENT_RUN_BEGIN, EVENT_RUN_END, EVENT_TEST_FAIL, EVENT_TEST_PASS, EVENT_TEST_PENDING } =
+  Mocha.Runner.constants;
 
 function MochaReporter(runner, opts) {
   Mocha.reporters.Base.call(this, runner);
-  let passes = 0; let failures = 0; let skipped = 0;
-  let artifactStore;
+  let passes = 0;
+  let failures = 0;
+  let skipped = 0;
+  // let artifactStore;
 
   const apiKey = opts?.reporterOptions?.apiKey || process.env.TESTOMATIO;
 
-  if (!apiKey) {
-    debug('TESTOMATIO key is empty, ignoring reports');
-    return;
-  }
   const client = new TestomatClient({ apiKey });
 
   runner.on(EVENT_RUN_BEGIN, () => {
     client.createRun();
 
-    const params = { 
-      toFile: true
-    };
-    
-    artifactStore = runner._workerReporter !== undefined
-      ? new ArtifactStorage(params)
-      : new ArtifactStorage();
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
   });
 
-  runner.on(EVENT_TEST_PASS, async(test) => {
+  runner.on(EVENT_TEST_PASS, async test => {
     passes += 1;
     console.log(chalk.bold.green('✔'), test.fullTitle());
     const testId = parseTest(test.title);
 
-    const specificTest =  specificTestInfo(test);
-    const content = await artifactStore.artifactByTestName(specificTest);
-
-    debug(`test=${specificTest} content = `, content);
-
-    client.addTestRun(
-      STATUS.PASSED, 
-      {
-        test_id: testId,
-        title: test.title,
-        time: test.duration,
-      },
-      content
-    );
+    client.addTestRun(STATUS.PASSED, {
+      test_id: testId,
+      title: test.title,
+      time: test.duration,
+    });
   });
 
   runner.on(EVENT_TEST_PENDING, test => {
@@ -66,33 +48,23 @@ function MochaReporter(runner, opts) {
     });
   });
 
-  runner.on(EVENT_TEST_FAIL, async(test, err) => {
+  runner.on(EVENT_TEST_FAIL, async (test, err) => {
     failures += 1;
     console.log(chalk.bold.red('✖'), test.fullTitle(), chalk.gray(err.message));
     const testId = parseTest(test.title);
 
-    const specificTest =  specificTestInfo(test);
-    const content = await artifactStore.artifactByTestName(specificTest);
-
-    debug(`fail test=${specificTest} content = `, content);
-
-    client.addTestRun(
-      STATUS.FAILED, {
-        error: err,
-        test_id: testId,
-        title: test.title,
-        time: test.duration,
-      }, 
-      content
-    );
+    client.addTestRun(STATUS.FAILED, {
+      error: err,
+      test_id: testId,
+      title: test.title,
+      time: test.duration,
+    });
   });
 
   runner.on(EVENT_RUN_END, () => {
     const status = failures === 0 ? STATUS.PASSED : STATUS.FAILED;
     console.log(chalk.bold(status), `${passes} passed, ${failures} failed, ${skipped} skipped`);
     client.updateRunStatus(status);
-
-    artifactStore.cleanup();
   });
 }
 

package/lib/adapter/playwright.js

@@ -3,14 +3,14 @@ const crypto = require('crypto');
 const os = require('os');
 const path = require('path');
 const fs = require('fs');
-const { APP_PREFIX, STATUS: Status } = require('../constants');
+const { APP_PREFIX, STATUS: Status, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
 const TestomatioClient = require('../client');
 const { isArtifactsEnabled } = require('../fileUploader');
-const { parseTest } = require('../util');
+const { parseTest, fileSystem } = require('../utils/utils');
 
 const reportTestPromises = [];
 
-class TestomatioReporter {
+class PlaywrightReporter {
   constructor(config = {}) {
     this.client = new TestomatioClient({ apiKey: config?.apiKey });
 
@@ -18,11 +18,19 @@ class TestomatioReporter {
   }
 
   onBegin(_config, suite) {
+    // clean data storage
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
     if (!this.client) return;
     this.suite = suite;
     this.client.createRun();
   }
 
+  /*
+  onTestBegin(test) {
+    // does not work; value is not to the storage context
+    global.testTitle = test.title;
+  } */
+
   onTestEnd(test, result) {
     if (!this.client) return;
 
@@ -37,21 +45,34 @@ class TestomatioReporter {
     for (const step of result.steps) {
       appendStep(step, steps);
     }
-    
-    const reportTestPromise = this.client.addTestRun(checkStatus(result.status), {
-      error,
-      test_id: testId,
-      suite_title,
-      title,
-      steps: steps.join('\n'),
-      time: duration,
-    }).then(pipes => {
-      testId = pipes.filter(p => p.pipe.includes('Testomatio'))[0]?.result?.data?.test_id;
-
-      this.uploads.push({
-        testId, title, suite_title, files: result.attachments.filter((a) => a.body || a.path)
+
+    let logs = '';
+    if (result.stderr.length || result.stdout.length) {
+      logs = `\n\n${chalk.bold('Logs:')}\n${chalk.red(result.stderr.join(''))}\n${result.stdout.join('')}`;
+    }
+
+    const reportTestPromise = this.client
+      .addTestRun(checkStatus(result.status), {
+        error,
+        test_id: testId,
+        suite_title,
+        title,
+        steps: steps.join('\n'),
+        time: duration,
+        stack: logs,
       })
-    });;
+      .then(pipes => {
+        testId = pipes?.filter(p => p.pipe.includes('Testomatio'))[0]?.result?.data?.test_id;
+
+        this.uploads.push({
+          testId,
+          title,
+          suite_title,
+          files: result.attachments.filter(a => a.body || a.path),
+        });
+        // remove empty uploads
+        this.uploads = this.uploads.filter(upload => upload.files.length);
+      });
 
     reportTestPromises.push(reportTestPromise);
   }
@@ -67,10 +88,9 @@ class TestomatioReporter {
       const promises = [];
 
       for (const upload of this.uploads) {
-
         const { title, testId, suite_title } = upload;
 
-        const files = upload.files.map((attachment) => {
+        const files = upload.files.map(attachment => {
           if (attachment.body) {
             const fileName = tmpFile();
             fs.writeFileSync(fileName, attachment.body);
@@ -78,7 +98,6 @@ class TestomatioReporter {
           return { path: attachment.path, title, type: attachment.contentType };
         });
 
-
         promises.push(
           this.client.addTestRun(undefined, {
             test_id: testId,
@@ -124,4 +143,4 @@ function tmpFile(prefix = 'tmp.') {
   return path.join(tmpdir, prefix + crypto.randomBytes(16).toString('hex'));
 }
 
-module.exports = TestomatioReporter;
+module.exports = PlaywrightReporter;

package/lib/adapter/webdriver.js

@@ -1,7 +1,7 @@
 // eslint-disable-next-line global-require, import/no-extraneous-dependencies
 const WDIOReporter = require('@wdio/reporter').default;
 const TestomatClient = require('../client');
-const { parseTest } = require('../util');
+const { parseTest } = require('../utils/utils');
 
 class WebdriverReporter extends WDIOReporter {
   constructor(options) {

package/lib/bin/reportXml.js

@@ -23,6 +23,7 @@ program
     }
     let { javaTests, lang } = opts;
     if (opts.envFile) {
+      console.log(APP_PREFIX, 'Loading env file:', opts.envFile);
       debug('Loading env file: %s', opts.envFile) 
       require('dotenv').config({ path: opts.envFile }); // eslint-disable-line
     }