@testomatio/reporter 0.6.9 → 0.7.0-beta

package/Changelog.md

@@ -1,3 +1,12 @@
+# 0.7.0
+
+* Support for `@cucumber/cucumber` (>= 7.0) added
+* Initial support for C# and NUnit  
+
+# 0.6.10
+
+* Fixed uploading multilpe artifacts in Playwright
+
 # 0.6.9
 
 * Fixed pending tests reports for Cypress

package/README.md

@@ -224,6 +224,7 @@ Tested Frameworks:
 * Python (Pytest)
 * Minitest (Ruby)
 * PHPUnit (PHP)
+* NUnit (C#)
 
 
 To import JUnit reports into Testomat.io **NodeJS >=14 is required**.
@@ -231,14 +232,9 @@ To import JUnit reports into Testomat.io **NodeJS >=14 is required**.
 Package `@testomatio/reporter` should be installed:
 
 ```
-npm i @testomatio/reporter@latest
+npm i @testomatio/reporter@latest --save-dev
 ```
 
-For local development it is recommended to install @testomatio/reporter globally:
-
-```
-npm i -g @testomatio/reporter@latest
-```
 
 Run your test framework and generate a JUnit report.
 
@@ -249,7 +245,12 @@ TESTOMATIO={API_KEY} npx report-xml "{pattern}" --lang={lang}
 ```
 
 * `pattern` - is a glob pattern to match all XML files from report. For instance, `"test/report/**.xml"` or just `report.xml`
-* `--lang` option can be specified to identify source code of the project. Example: `--lang=Ruby` or `--lang=Java` or `--lang=Python`.
+* `--lang` option can be specified to identify source code of the project. Example: `--lang=Ruby` or `--lang=Java` or `--lang=Python`. Possible values:
+  * `c#`
+  * `java`
+  * `ruby`
+  * `python`
+  * `php`
 * `--java-tests` option is avaiable for Java projects, and can be set if path to tests is different then `src/test/java`. When this option is enable, `lang` option is automatically set to `java`
 * `--env-file <envfile>` option to load environment variables from .env file. Inside env file TESTOMATIO credentials like `TESTOMATIO` api key or [S3 config](#attaching-test-artifacts) can be stored.
 
@@ -288,6 +289,14 @@ TESTOMATIO={API_KEY} npx report-xml "target/surefire-reports/*.xml" --java-tests
 
 > You can specify `--java-test` option to set a path to tests if they are located in path other than `src/test/java`
 
+#### Example: NUnit
+
+Generate NUnit XML report and run the following code:
+
+```
+TESTOMATIO={API_KEY} npx report-xml "report.xml" --lang="c#"
+```
+
 #### Example: Ruby on Rails with Minitest
 
 ```ruby

package/lib/adapter/codecept.js

@@ -1,5 +1,5 @@
 if (!global.codeceptjs) {
-  // eslint-disable-next-line global-require
+  // eslint-disable-next-line global-require, import/no-extraneous-dependencies
   global.codeceptjs = require('codeceptjs');
 }
 

package/lib/adapter/cucumber/current.js

@@ -0,0 +1,185 @@
+// eslint-disable-next-line global-require, import/no-extraneous-dependencies
+const { Formatter, formatterHelpers } = require("@cucumber/cucumber");
+const chalk = require('chalk');
+const fs = require('fs');
+const { TestomatClient, TRConstants } = require('../../reporter');
+
+const { GherkinDocumentParser, PickleParser } = formatterHelpers
+const {
+  getGherkinScenarioLocationMap,
+  getGherkinStepMap,
+} = GherkinDocumentParser
+const { getPickleStepMap } = PickleParser
+
+class CucumberReporter extends Formatter {
+  constructor(options) {
+    super(options);
+    options.eventBroadcaster.on('envelope', this.parseEnvelope.bind(this))
+    this.failures = [];
+    this.cases = [];
+
+    const apiKey = process.env.TESTOMATIO;
+    if (!apiKey || apiKey === '') {
+      console.log(chalk.red('TESTOMATIO key is empty, ignoring reports'));
+      return;
+    }
+
+    this.client = new TestomatClient({ apiKey });
+    this.status = TRConstants.PASSED;
+
+  }
+
+  parseEnvelope(envelope) {
+    if (envelope.testCaseStarted && this.client) this.client.createRun()
+    if (envelope.testCaseFinished) this.onTestCaseFinished(envelope.testCaseFinished);
+    if (envelope.testRunFinished) this.onTestRunFinished(envelope);
+  }
+
+  onTestCaseFinished(testCaseFinished) {
+    const testCaseAttempt = this.eventDataCollector.getTestCaseAttempt(testCaseFinished.testCaseStartedId);
+
+    let example;
+    let status = TRConstants.PASSED;
+    let color = 'green';
+    let message;
+
+    this.cases.push(testCaseAttempt);
+
+    if (testCaseAttempt.worstTestStepResult) {
+      if (testCaseAttempt.worstTestStepResult.status === 'SKIPPED') {
+        status = TRConstants.SKIPPED;
+      }
+      if (testCaseAttempt.worstTestStepResult.status === 'UNDEFINED') {
+        status = TRConstants.SKIPPED;
+        message = 'Undefined steps. Implement missing steps and rerun this scenario';
+      }
+      if (testCaseAttempt.worstTestStepResult.status === 'FAILED') {
+        message = testCaseAttempt?.worstTestStepResult?.message;
+        status = TRConstants.FAILED;
+      }
+      color = getStatusColor(testCaseAttempt.worstTestStepResult.status);
+      if (status !== TRConstants.PASSED) this.failures.push(testCaseAttempt);
+    }
+
+    if (testCaseAttempt.pickle.astNodeIds.length > 1) {
+      example = getExample(testCaseAttempt);
+      testCaseAttempt.example = example;
+    }
+
+    const scenario = testCaseAttempt.pickle.name;
+    const testId = testCaseAttempt.pickle.id;
+
+    let exampleString = '';
+    if (example) exampleString = ` ${example.join(' | ')}`
+    let cliMessage = `${chalk.bold(scenario)}${exampleString}: ${chalk[color].bold(status.toUpperCase())} `;
+
+    if (message) cliMessage += chalk.gray(message.split('\n')[0]);
+    console.log(cliMessage);
+
+    if (status !== TRConstants.PASSED && status !== TRConstants.SKIPPED) {
+      this.status = TRConstants.FAILED;
+    }
+
+    const time = Object.values(testCaseAttempt.stepResults)
+      .map(t => t.duration)
+      .reduce((sum, duration) => sum + (duration.seconds * 1000)  + (duration.nanos / 1000000), 0)
+
+    if (!this.client) return;
+
+    this.client.addTestRun(testId, status, {
+      // error: testCaseAttempt.worstTestStepResult.message,
+      message,
+      steps: getSteps(testCaseAttempt).map(s => s.toString()).join('\n').trim(),
+      example: { ...example},
+      title: scenario,
+      time,
+    });
+  }
+
+  onTestRunFinished(envelope) {
+    if (this.failures.length > 0) {
+      console.log(chalk.bold('\nSUMMARY:\n\n'));
+
+      this.failures.forEach((tc, i) => {
+        let message = `  ${i+1}) ${tc.pickle.name}\n`;
+
+        const steps = getSteps(tc);
+
+        steps.forEach(s => {
+          message += `     ${s.toString()}\n`
+        })
+
+        console.log(message);
+        if (tc?.worstTestStepResult?.message) {
+          console.log(chalk.red(tc?.worstTestStepResult?.message));
+        }
+        console.log();
+
+      })
+    }
+
+    const { testRunFinished } = envelope;
+
+    const bgColor = testRunFinished.success ? 'bgGreen' : 'bgRed';
+    const prefixSummary = `${chalk.bold(testRunFinished.success ? ' SUCCESS ' : ' FALIURE ')}`;
+    console.log();
+    console.log(chalk[bgColor](` ${prefixSummary} | Total Scenarios: ${chalk.bold(this.cases.length)} `));
+
+    if (!this.client) return;
+
+    this.client.updateRunStatus(testRunFinished.success ? TRConstants.PASSED : TRConstants.FAILED)
+  }
+};
+
+function getSteps(tc) {
+  const stepIds = Object.keys(tc.stepResults);
+  const pickleSteps = getPickleStepMap(tc.pickle)
+  return stepIds.map(stepId => {
+    const ts = tc.testCase.testSteps.find(t => t.id === stepId);
+    if (!ts) return;
+    if (!ts.pickleStepId) return;
+    const result = tc.stepResults[stepId];
+    const pickleStep = pickleSteps[ts.pickleStepId];
+    const sourceStepId = pickleStep.astNodeIds[0];
+    const step = {
+      text: pickleStep.text,
+      duration: result.duration,
+      status: result.status,
+    }
+    const color = getStatusColor(result.status);
+    if (sourceStepId && getGherkinStepMap(tc.gherkinDocument)[sourceStepId]) {
+      step.keyword = getGherkinStepMap(tc.gherkinDocument)[sourceStepId].keyword;
+    }
+    step.toString = function toString() {
+      const duration = step.duration.seconds * 1000 + (step.duration.nanos / 1000000)
+      const durationString = ` ${  chalk.gray(`(${Number(duration).toFixed(2)}ms)`)}`;
+      const stepString = `${chalk.bold(this.keyword)}${this.text}`.trim();
+      if (color === 'red') return chalk.red(stepString) + durationString;
+      if (color === 'yellow') return chalk.yellow(stepString) + durationString;
+      return stepString + durationString;
+    }
+    return step;
+  }).filter(s => !!s)
+}
+
+function getStatusColor(status) {
+  if (status === 'UNDEFINED') return 'yellow';
+  if (status === 'SKIPPED') return 'yellow';
+  if (status === 'FAILED') return 'red';
+  return 'green'
+}
+
+function getExample(testCaseAttempt) {
+  const nodesMap = getGherkinScenarioLocationMap(testCaseAttempt.gherkinDocument);
+  const exampleNodeId = testCaseAttempt.pickle.astNodeIds[1];
+  if (!nodesMap[exampleNodeId]) return;
+  const featureDoc = fs.readFileSync(testCaseAttempt.gherkinDocument.uri).toString();
+  const { line } = nodesMap[exampleNodeId];
+  const example = featureDoc.split('\n')[line-1];
+  if (example) {
+    return example.trim().split('|').filter(r => !!r).map(r => r.trim());
+  };
+
+}
+
+module.exports = CucumberReporter;

package/lib/adapter/cucumber/legacy.js

@@ -0,0 +1,136 @@
+// eslint-disable-next-line import/no-unresolved
+const { Formatter } = require('cucumber');
+const chalk = require('chalk');
+const { TestomatClient, TRConstants } = require('../../reporter');
+const { parseTest } = require('../../util');
+
+const createTestomatFormatter = apiKey => {
+  if (!apiKey || apiKey === '') {
+    console.log(chalk.red('TESTOMATIO key is empty, ignoring reports'));
+  }
+
+  const documents = {};
+  const dataTableMap = {};
+
+  const addDocument = gherkinDocument => {
+    documents[gherkinDocument.uri] = gherkinDocument.document;
+  };
+
+  const getTitle = scenario => {
+    let { name } = scenario;
+    if (scenario.tags.length) {
+      let tags = '';
+      for (const tag of scenario.tags) {
+        tags = `${tags} ${tag.name}`;
+      }
+      name = `${name}${tags}`;
+    }
+    return name;
+  };
+
+  const getFeature = uri => documents[uri].feature;
+
+  const getScenario = location => {
+    const { children } = getFeature(location.uri);
+    for (const scenario of children) {
+      if (scenario.type === 'Scenario' && scenario.location.line === location.line) {
+        return scenario;
+      }
+      if (scenario.type === 'ScenarioOutline') {
+        for (const example of scenario.examples) {
+          for (const tableBody of example.tableBody) {
+            if (tableBody.location.line === location.line) {
+              return scenario;
+            }
+          }
+        }
+      }
+    }
+
+    return null;
+  };
+
+  const loadDataTable = (scenario, uri) => {
+    if (scenario.type === 'ScenarioOutline') {
+      for (const example of scenario.examples) {
+        for (const tableBody of example.tableBody) {
+          const dataMap = example.tableHeader.cells.reduce((acc, cell, index) => {
+            acc[cell.value] = tableBody.cells[index].value;
+            return acc;
+          }, {});
+
+          dataTableMap[`${uri}:${tableBody.location.line}`] = JSON.stringify(dataMap);
+        }
+      }
+    }
+  };
+
+  const getDataTableMap = (scenario, sourceLocation) => {
+    const key = `${sourceLocation.uri}:${sourceLocation.line}`;
+    if (!dataTableMap[key]) {
+      loadDataTable(scenario, sourceLocation.uri);
+    }
+
+    return dataTableMap[key] || '';
+  };
+
+  const getTestId = scenario => {
+    if (scenario) {
+      for (const tag of scenario.tags) {
+        const testId = parseTest(tag.name);
+        if (testId) return testId;
+      }
+    }
+
+    return null;
+  };
+
+  return class TestomatFormatter extends Formatter {
+    constructor(options) {
+      super(options);
+
+      if (!apiKey) return;
+
+      this.client = new TestomatClient({ apiKey });
+      this.status = TRConstants.PASSED;
+
+      options.eventBroadcaster.on('gherkin-document', addDocument);
+      options.eventBroadcaster.on('test-run-started', () => this.client.createRun());
+      options.eventBroadcaster.on('test-case-finished', this.onTestCaseFinished.bind(this));
+      options.eventBroadcaster.on('test-run-finished', () => this.client.updateRunStatus(this.status));
+    }
+
+    onTestCaseFinished(event) {
+      const scenario = getScenario(event.sourceLocation);
+      const testId = getTestId(scenario);
+      const status = event.result.status === 'undefined' ? TRConstants.SKIPPED : event.result.status;
+
+      let example = getDataTableMap(scenario, event.sourceLocation);
+      if (example) example = JSON.parse(example);
+
+      if (!scenario.name) return;
+
+      let message = '';
+      let cliMessage = `- ${scenario.name}: ${chalk.bold(status.toUpperCase())}`;
+
+      if (event.result.status === 'undefined') {
+        cliMessage += chalk.yellow(
+          ' (undefined steps. Run Cucumber without this formatter and implement missing steps)',
+        );
+        message = 'Undefined steps. Implement missing steps and rerun this scenario';
+      }
+      console.log(cliMessage);
+      if (status !== TRConstants.PASSED && status !== TRConstants.SKIPPED) {
+        this.status = TRConstants.FAILED;
+      }
+      this.client.addTestRun(testId, status, {
+        error: event.result.exception,
+        message,
+        example,
+        title: getTitle(scenario),
+      });
+    }
+  };
+};
+
+module.exports = createTestomatFormatter(process.env.TESTOMATIO);

package/lib/adapter/cucumber.js

@@ -1,136 +1,16 @@
-// eslint-disable-next-line import/no-unresolved
-const { Formatter } = require('cucumber');
-const chalk = require('chalk');
-const { TestomatClient, TRConstants } = require('../reporter');
-const { parseTest } = require('../util');
-
-const createTestomatFormatter = apiKey => {
-  if (!apiKey || apiKey === '') {
-    console.log(chalk.red('TESTOMATIO key is empty, ignoring reports'));
+try {
+  // eslint-disable-next-line import/no-unresolved
+  require.resolve("@cucumber/cucumber");
+  // eslint-disable-next-line global-require, import/no-extraneous-dependencies
+  module.exports = require('./cucumber/current');
+} catch (_e) {
+  try {
+    // eslint-disable-next-line global-require, import/no-extraneous-dependencies
+    require.resolve('cucumber');
+    // eslint-disable-next-line global-require, import/no-extraneous-dependencies
+    module.exports = require('./cucumber/legacy');
+  } catch (_err) {
+    console.error('Cucumber packages: "@cucumber/cucumber" or "cucumber" were not detected. Report won\'t be sent');
+    module.exports = {}
   }
-
-  const documents = {};
-  const dataTableMap = {};
-
-  const addDocument = gherkinDocument => {
-    documents[gherkinDocument.uri] = gherkinDocument.document;
-  };
-
-  const getTitle = scenario => {
-    let { name } = scenario;
-    if (scenario.tags.length) {
-      let tags = '';
-      for (const tag of scenario.tags) {
-        tags = `${tags} ${tag.name}`;
-      }
-      name = `${name}${tags}`;
-    }
-    return name;
-  };
-
-  const getFeature = uri => documents[uri].feature;
-
-  const getScenario = location => {
-    const { children } = getFeature(location.uri);
-    for (const scenario of children) {
-      if (scenario.type === 'Scenario' && scenario.location.line === location.line) {
-        return scenario;
-      }
-      if (scenario.type === 'ScenarioOutline') {
-        for (const example of scenario.examples) {
-          for (const tableBody of example.tableBody) {
-            if (tableBody.location.line === location.line) {
-              return scenario;
-            }
-          }
-        }
-      }
-    }
-
-    return null;
-  };
-
-  const loadDataTable = (scenario, uri) => {
-    if (scenario.type === 'ScenarioOutline') {
-      for (const example of scenario.examples) {
-        for (const tableBody of example.tableBody) {
-          const dataMap = example.tableHeader.cells.reduce((acc, cell, index) => {
-            acc[cell.value] = tableBody.cells[index].value;
-            return acc;
-          }, {});
-
-          dataTableMap[`${uri}:${tableBody.location.line}`] = JSON.stringify(dataMap);
-        }
-      }
-    }
-  };
-
-  const getDataTableMap = (scenario, sourceLocation) => {
-    const key = `${sourceLocation.uri}:${sourceLocation.line}`;
-    if (!dataTableMap[key]) {
-      loadDataTable(scenario, sourceLocation.uri);
-    }
-
-    return dataTableMap[key] || '';
-  };
-
-  const getTestId = scenario => {
-    if (scenario) {
-      for (const tag of scenario.tags) {
-        const testId = parseTest(tag.name);
-        if (testId) return testId;
-      }
-    }
-
-    return null;
-  };
-
-  return class TestomatFormatter extends Formatter {
-    constructor(options) {
-      super(options);
-
-      if (!apiKey) return;
-
-      this.client = new TestomatClient({ apiKey });
-      this.status = TRConstants.PASSED;
-
-      options.eventBroadcaster.on('gherkin-document', addDocument);
-      options.eventBroadcaster.on('test-run-started', () => this.client.createRun());
-      options.eventBroadcaster.on('test-case-finished', this.onTestCaseFinished.bind(this));
-      options.eventBroadcaster.on('test-run-finished', () => this.client.updateRunStatus(this.status));
-    }
-
-    onTestCaseFinished(event) {
-      const scenario = getScenario(event.sourceLocation);
-      const testId = getTestId(scenario);
-      const status = event.result.status === 'undefined' ? TRConstants.SKIPPED : event.result.status;
-
-      let example = getDataTableMap(scenario, event.sourceLocation);
-      if (example) example = JSON.parse(example);
-
-      if (!scenario.name) return;
-
-      let message = '';
-      let cliMessage = `- ${scenario.name}: ${chalk.bold(status.toUpperCase())}`;
-
-      if (event.result.status === 'undefined') {
-        cliMessage += chalk.yellow(
-          ' (undefined steps. Run Cucumber without this formatter and implement missing steps)',
-        );
-        message = 'Undefined steps. Implement missing steps and rerun this scenario';
-      }
-      console.log(cliMessage);
-      if (status !== TRConstants.PASSED && status !== TRConstants.SKIPPED) {
-        this.status = TRConstants.FAILED;
-      }
-      this.client.addTestRun(testId, status, {
-        error: event.result.exception,
-        message,
-        example,
-        title: getTitle(scenario),
-      });
-    }
-  };
-};
-
-module.exports = createTestomatFormatter(process.env.TESTOMATIO);
+}

package/lib/adapter/cypress-plugin/index.js

@@ -55,11 +55,14 @@ const testomatioReporter = on => {
 
       const files = [...videos, ...screenshots];
 
+      let state;
       switch (test.state) {
         case 'passed': state = TRConstants.PASSED; break;
         case 'failed': state = TRConstants.FAILED; break;
         case 'skipped':
-        case 'pending': state = TRConstants.SKIPPED;
+        case 'pending': 
+        default:
+          state = TRConstants.SKIPPED;
       }
 
       addSpecTestsPromises.push(client.addTestRun(testId, state, {

package/lib/adapter/mocha.js

@@ -1,3 +1,4 @@
+// eslint-disable-next-line global-require, import/no-extraneous-dependencies
 const Mocha = require('mocha');
 const TestomatClient = require('../client');
 const TRConstants = require('../constants');

package/lib/adapter/webdriver.js

@@ -1,3 +1,4 @@
+// eslint-disable-next-line global-require, import/no-extraneous-dependencies
 const WDIOReporter = require('@wdio/reporter').default;
 const TestomatClient = require('../client');
 const { parseTest } = require('../util');

package/lib/fileUploader.js

@@ -2,6 +2,7 @@ const AWS = require('aws-sdk');
 const fs = require('fs');
 const path = require('path');
 const chalk = require('chalk');
+const { randomUUID } = require('crypto');
 const memoize = require('lodash.memoize');
 const { APP_PREFIX } = require('./constants');
 
@@ -55,7 +56,7 @@ const uploadUsingS3 = async (filePath, runId) => {
   const s3 = new AWS.S3(config);
   const file = fs.readFileSync(filePath);
 
-  Key = `${runId}/${Key || path.basename(filePath)}`;
+  Key = `${runId}/${randomUUID()}-${Key || path.basename(filePath)}`;
   const ACL = TESTOMATIO_PRIVATE_ARTIFACTS ? 'private' : 'public-read';
 
   try {

package/lib/junit-adapter/csharp.js

@@ -0,0 +1,18 @@
+const Adapter = require('./adapter');
+
+class CSharpAdapter extends Adapter {
+
+  formatTest(t) {
+    const title = t.title.replace(/\(.*?\)/, '').trim();
+    const example = t.title.match(/\((.*?)\)/);
+    if (example) t.example = { ...example[1].split(',')};
+    const suite = t.suite_title.split('.')
+    t.suite_title = suite.pop();
+    t.file = suite.join('/');
+    t.title = title.trim();
+    return t;
+  }
+}
+
+
+module.exports = CSharpAdapter;