@testomatio/reporter 0.5.9 → 0.6.0

package/.eslintrc.js

@@ -11,7 +11,7 @@ module.exports = {
     SharedArrayBuffer: 'readonly',
   },
   parserOptions: {
-    ecmaVersion: 2018,
+    ecmaVersion: 2020,
   },
   rules: {
     'max-len': ['error', { code: 120 }],
@@ -25,6 +25,7 @@ module.exports = {
     'no-use-before-define': 0,
     'object-curly-newline': 0,
     'consistent-return': 0,
+    'no-await-in-loop': 0,
     'no-param-reassign': 0,
     'operator-linebreak': 0,
     'prefer-destructuring': 0,

package/.github/workflows/node.js.yml

@@ -15,7 +15,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [12.x, 14.x, 16.x]
+        node-version: [14.x, 16.x]
         # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
 
     steps:

package/Changelog.md

@@ -1,3 +1,8 @@
+# 0.5.10
+
+* Fixed reporting Scenario Outline in Cypress-Cucumber
+* Fixed error reports for Cypress when running in Chrome
+
 # 0.5.9
 
 * Added environment on Cypress report

package/README.md

@@ -205,6 +205,109 @@ Run the following command from you project folder:
 TESTOMATIO={API_KEY} npx @testomatio/reporter -c 'npx wdio wdio.conf.js'
 ```
 
+## JUnit Reports
+
+> **Notice** JUnit reports are supported since 0.6.0
+
+Other frameworks and languages are supported via JUnit reports.
+
+JUnit XML format is standard among test runners on various platforms. Testomat.io can load XML reports from test runners and create tests in a project from it. If your framework is not supported yet, generate JUnit report and upload it into Testomat.io
+
+Testomat.io will not only create a run report but also collect all information about tests, including source code, and create tests in a system as regular importer do. The only difference that normal process of Testomat.io import does not require executing tests on import, while importing via repoter requires to have tests executed and XML report provided.
+
+Tested Frameworks:
+
+* JUnit (JUnit)
+* Python (Pytest)
+* Minitest (Ruby)
+* PHPUnit (PHP)
+
+
+To import JUnit reports into Testomat.io **NodeJS >=14 is required**.
+
+Package `@testomatio/reporter` should be installed:
+
+```
+npm i @testomatio/reporter@latest
+```
+
+For local development it is recommended to install @testomatio/reporter globally:
+
+```
+npm i -g @testomatio/reporter@latest
+```
+
+Run your test framework and generate a JUnit report.
+
+Then import XML report into Testomat.io
+
+```
+TESTOMATIO={API_KEY} npx report-xml "{pattern}" --lang={lang}
+```
+
+* `pattern` - is a glob pattern to match all XML files from report. For instance, `"test/report/**.xml"` or just `report.xml`
+* `--lang` option should be specified to identify source code of the project. Example: `--lang=Ruby` or `--lang=Java` or `--lang=Python`.
+* `--java-tests` option is avaiable for Java projects, and can be set if path to tests is different then `src/test/java`. When this option is enable, `lang` option is automatically set to `java`
+
+
+> *Notice:* All options from [Advanced Usage](#advanced-usage) are also available for JUnit reporter
+
+Check the list of examples:
+
+#### Example: Pytest
+
+Run pytest tests and generate a report to `report.xml`:
+
+```
+pytest --junit-xml report.xml
+```
+
+Import report with this command
+
+```
+TESTOMATIO={API_KEY} npx report-xml report.xml --lang=python
+```
+
+#### Example: JUnit with Maven
+
+Run tests via Maven, make sure JUnit report was configured in `pom.xml`.
+
+```
+maven clean test
+```
+
+Import report with this command:
+
+```
+TESTOMATIO={API_KEY} npx report-xml "target/surefire-reports/*.xml" --java-tests
+```
+
+> You can specify `--java-test` option to set a path to tests if they are located in path other than `src/test/java`
+
+#### Example: Ruby on Rails with Minitest
+
+```ruby
+# test_helper.rb:
+
+reporters = [Minitest::Reporters::DefaultReporter.new(color: true)]
+# enable JUnit reporter
+reporters << Minitest::Reporters::JUnitReporter.new
+
+Minitest::Reporters.use! reporters
+```
+
+Launch tests:
+
+```
+rails test
+```
+
+Import reports from `test/reports` directory:
+
+```
+TESTOMATIO={API_KEY} npx report-xml "test/reports/*.xml" --lang ruby
+```
+
 ## Advanced Usage
 
 ### Create Unmatched Tests

package/lib/adapter/cypress-plugin/index.js

@@ -3,6 +3,10 @@ const { parseTest, parseSuite } = require('../../util');
 const TestomatClient = require('../../client');
 
 const testomatioReporter = on => {
+  if (!process.env.TESTOMATIO) {
+    console.log('TESTOMATIO key is empty, ignoring reports');
+    return
+  }
   const client = new TestomatClient({ apiKey: process.env.TESTOMATIO });
 
   on('before:run', async (run) => {
@@ -24,15 +28,23 @@ const testomatioReporter = on => {
       const time = latestAttempt.duration;
       const error = latestAttempt.error;
 
-      const title = test.title.pop();
+      let title = test.title.pop();
+      const examples = title.match(/\(example (#\d+)\)/);
+      let example = null;
+      if (examples && examples[1]) example = { example: examples[1] };
+      title = title.replace(/\(example #\d+\)/, '').trim();
+
       const suiteTitle = test.title.pop();
 
       const testId = parseTest(title);
       const suiteId = parseSuite(suiteTitle);
 
       if (error) {
-        error.inspect = function() {
-          return this.codeFrame.frame;
+        error.inspect = function() { // eslint-disable-line func-names
+          if (this && this.codeFrame) {
+            return this.codeFrame.frame;
+          }
+          return '';
         }
       }
 
@@ -46,7 +58,7 @@ const testomatioReporter = on => {
       const state = test.state === 'passed' ? TRConstants.PASSED : TRConstants.FAILED;
 
       addSpecTestsPromises.push(client.addTestRun(testId, state, {
-        title, time, error, files, suite_title: suiteTitle, suite_id: suiteId
+        title, time, example, error, files, suite_title: suiteTitle, suite_id: suiteId
       }));
     }
 

package/lib/bin/reportXml.js

@@ -0,0 +1,51 @@
+#!/usr/bin/env node
+const program = require("commander");
+// const chalk = require("chalk");
+const glob = require('glob');
+
+const { APP_PREFIX } = require('../constants');
+const XmlReader = require("../xmlReader");
+
+// const apiKey = process.env['INPUT_TESTOMATIO-KEY'] || process.env.TESTOMATIO;
+// const title = process.env.TESTOMATIO_TITLE;
+
+// console.log(chalk.cyan.bold(` 🤩 xmlReader by Testomat.io v${version}`));
+
+program
+  .arguments("<pattern>")
+  .option("-d, --dir <dir>", "Project directory")
+  .option("--java-tests [java-path]", "Load Java tests from path, by default: src/test/java")
+  .option("--lang <lang>", "Language used (python, ruby, java)")
+  .action(async (pattern, opts) => {
+    if (!pattern.endsWith('.xml')) {
+      pattern += '.xml';
+    }
+    let { javaTests, lang } = opts;
+    if (javaTests === true) javaTests = 'src/test/java';
+    lang = lang?.toLowerCase();
+    const runReader = new XmlReader({ javaTests, lang });
+    const files = glob.sync(pattern, { cwd: opts.dir || process.cwd() });
+    if (!files.length) {
+      console.log(APP_PREFIX,`Report can't be created. No XML files found 😥`);
+      process.exitCode = 1;
+      return;
+    }
+
+    for (const file of files) {
+      console.log(APP_PREFIX,`Parsed ${file}`);
+      runReader.parse(file);
+    }
+    try {
+      await runReader.createRun();
+      await runReader.uploadData();
+    } catch (err) {
+      console.log(APP_PREFIX, 'Error updating status, skipping...', err);
+    }
+  });
+
+
+if (process.argv.length < 1) {
+  program.outputHelp();
+}
+
+program.parse(process.argv);

package/lib/client.js

@@ -234,7 +234,7 @@ class TestomatClient {
 
   formatError(error, message) {
     if (!message) message = error.message;
-    if (error.inspect) message = error.inspect();
+    if (error.inspect) message = error.inspect() || '';
 
     let stack = `\n${chalk.bold(message)}\n`;
 

package/lib/junit-adapter/adapter.js

@@ -0,0 +1,25 @@
+class Adapter {
+
+  constructor(opts) {
+    this.opts = opts;
+  }
+
+  getFilePath(t) {
+    return t.file;
+  }
+
+  formatTest(t) {
+    return t;
+  }
+
+  formatStack(t) {
+    return t.stack || '';
+  }
+
+  formatMessage(t) {
+    return t.message;
+  }
+
+}
+
+module.exports = Adapter;

package/lib/junit-adapter/java.js

@@ -0,0 +1,40 @@
+const path = require('path');
+const Adapter = require('./adapter');
+
+class JavaAdapter extends Adapter {
+
+  getFilePath(t) {
+    const fileName = namespaceToFileName(t.suite_title)
+    return this.opts.javaTests + path.sep + fileName;
+  }
+
+  formatTest(t) {
+    const fileParts = t.suite_title.split('.')
+    const example = t.title.match(/\[(.*)\]/)?.[1];
+    if (example) t.example = { "#": example }
+
+    t.file = namespaceToFileName(t.suite_title);
+    t.title = t.title.split('(')[0];
+    t.suite_title = fileParts[fileParts.length - 1].replace(/\$/g, ' | ')
+    return t;
+  }
+
+  formatStack(t) {
+    const stack = super.formatStack(t);
+
+    const file = t.suite_title.split('.');
+
+    const fileLine = `at .*${file[file.length - 1]}\.java:(\\d+)` // eslint-disable-line no-useless-escape
+    const regexp = new RegExp(fileLine,"g")
+    return stack.replace(regexp, `${this.opts.javaTests}${path.sep}${namespaceToFileName(t.suite_title)}:$1:`);
+  }
+}
+
+function namespaceToFileName(fileName) {
+    const fileParts = fileName.split('.')
+    fileParts[fileParts.length - 1] = fileParts[fileParts.length - 1]?.replace(/\$.*/, '')
+    return `${fileParts.join(path.sep)  }.java`;
+
+}
+
+module.exports = JavaAdapter;

package/lib/junit-adapter/javascript.js

@@ -0,0 +1,31 @@
+const createCallsiteRecord = require('callsite-record');
+const path = require('path');
+const Adapter = require('./adapter');
+
+class JavaScriptAdapter extends Adapter {
+
+  formatStack(t) {
+    let stack = super.formatStack(t);
+
+    try {
+      const error = new Error(stack.split('\n')[0])
+      error.stack = stack;
+      const record = createCallsiteRecord({
+        forError: error,
+      });
+      if (record && !record.filename.startsWith('http')) {
+        stack += record.renderSync({
+          stackFilter: frame =>
+            frame.getFileName().indexOf(path.sep) > -1 &&
+            frame.getFileName().indexOf('node_modules') < 0 &&
+            frame.getFileName().indexOf('internal') < 0,
+        });
+      }
+      return stack;
+    } catch (err) {
+      return stack;
+    }
+  }
+}
+
+module.exports = JavaScriptAdapter;

package/lib/junit-adapter/python.js

@@ -0,0 +1,42 @@
+const path = require('path');
+const fs = require('fs');
+const Adapter = require('./adapter');
+
+class PythonAdapter extends Adapter {
+
+  getFilePath(t) {
+    const fileName = namespaceToFileName(t.suite_title)
+    return fileName;
+  }
+
+  formatTest(t) {
+    const fileParts = t.suite_title.split('.')
+    const example = t.title.match(/\[(.*)\]/)?.[1];
+    if (example) t.example = { "#": example }
+
+    t.file = namespaceToFileName(t.suite_title);
+    t.title = t.title.split('[')[0];
+    t.suite_title = fileParts[fileParts.length - 1].replace(/\$/g, ' | ')
+    return t;
+  }
+
+  formatMessage(t) {
+    return t.message.split('
')[0];
+  }
+
+}
+
+function namespaceToFileName(fileName) {
+    const fileParts = fileName.split('.')
+
+    while (fileParts.length > 0) {
+      const file = `${fileParts.join(path.sep)  }.py`;
+      if (fs.existsSync(`${fileParts.join(path.sep)  }.py`)) {
+        return file;
+      }
+      fileParts.pop();
+    }
+    return null;
+}
+
+module.exports = PythonAdapter;

package/lib/junit-adapter/ruby.js

@@ -0,0 +1,11 @@
+const Adapter = require('./adapter');
+
+class RubyAdapter extends Adapter {
+
+  formatStack(t) {
+    const stack = super.formatStack(t);
+    return stack.replace(/\[(.*?:.\d*)\]/g, '\n$1')
+  }
+}
+
+module.exports = RubyAdapter;

package/lib/util.js

@@ -1,4 +1,8 @@
 const { URL } = require('url');
+const { sep } = require('path');
+const chalk = require('chalk');
+const fs = require('fs');
+const isValid = require('is-valid-path');
 
 /**
  * @param {String} testTitle - Test title
@@ -48,9 +52,98 @@ const isValidUrl = s => {
   }
 };
 
+const fetchFilesFromStackTrace = (stack = '') => {
+  const files = stack.matchAll(/file:?\/(\/.*?\.(png|avi|webm|jpg|html|txt))/g);
+  return Array.from(files).map(f => f[1]).filter(f => fs.existsSync(f));
+}
+
+const fetchSourceCodeFromStackTrace = (stack = '') => {
+  const stackLines = stack.split('\n')
+    .filter(l => l.includes(':'))
+    // .map(l => l.match(/\[(.*?)\]/)?.[1] || l) // minitest format
+    // .map(l => l.split(':')[0])
+    .map(l => l.trim())
+    .map(l => l.split(' ').find(p => p.includes(':')))
+    .filter(l => isValid(l.split(':')[0]))
+
+    // // filter out 3rd party libs
+    .filter(l => !l.includes(`vendor${  sep}`))
+    .filter(l => !l.includes(`node_modules${  sep}`))
+    .filter(l => fs.existsSync(l.split(':')[0]))
+    .filter(l => fs.lstatSync(l.split(':')[0]).isFile())
+
+  if (!stackLines.length) return '';
+
+  const [file, line] = stackLines[0].split(':');
+
+  const prepend = 3;
+  const source = fetchSourceCode(fs.readFileSync(file).toString(), { line, prepend, limit: 7 })
+
+  return source.split('\n')
+    .map((l, i) => {
+      if (i === prepend) return `${line} > ${chalk.bold(l)}`;
+      return `${line - prepend + i} | ${l}`
+    }).join('\n')
+}
+
+const fetchSourceCode = (contents, opts = {}) => {
+  if (!opts.title && !opts.line) return '';
+
+    // code fragment is 20 lines
+  const limit = opts.limit || 50;
+  let lineIndex;
+  if (opts.line) lineIndex = opts.line - 1;
+  const lines = contents.split('\n')
+
+  // remove special chars from title
+  if (!lineIndex && opts.title) {
+    const title = opts.title.replace(/[([@].*/g, '')
+    lineIndex = lines.findIndex(l => l.includes(title))
+  }
+
+  if (opts.prepend) {
+    lineIndex -= opts.prepend;
+  }
+
+  if (lineIndex) {
+    const result = [];
+    for (let i = lineIndex; i < (lineIndex + limit); i++) {
+      if (lines[i] === undefined) continue;
+
+      if (i > lineIndex + 2 && !opts.prepend) {
+        // annotation
+        if (opts.lang === 'php' && lines[i].trim().startsWith('#[')) break;
+        if (opts.lang === 'php' && lines[i].includes(' private function ')) break;
+        if (opts.lang === 'php' && lines[i].includes(' protected function ')) break;
+        if (opts.lang === 'php' && lines[i].includes(' public function ')) break;
+        if (opts.lang === 'python' && lines[i].trim().match(/^@\w+/)) break;
+        if (opts.lang === 'python' && lines[i].includes(' def ')) break;
+        if (opts.lang === 'ruby' && lines[i].includes(' def ')) break;
+        if (opts.lang === 'ruby' && lines[i].includes(' test ')) break;
+        if (opts.lang === 'ruby' && lines[i].includes(' it ')) break;
+        if (opts.lang === 'ruby' && lines[i].includes(' specify ')) break;
+        if (opts.lang === 'ruby' && lines[i].includes(' context ')) break;
+        if (opts.lang === 'ts' && lines[i].includes(' it(')) break;
+        if (opts.lang === 'ts' && lines[i].includes(' test(')) break;
+        if (opts.lang === 'js' && lines[i].includes(' it(')) break;
+        if (opts.lang === 'js' && lines[i].includes(' test(')) break;
+        if (opts.lang === 'java' && lines[i].trim().match(/^@\w+/)) break;
+        if (opts.lang === 'java' && lines[i].includes(' public void ')) break;
+        if (opts.lang === 'java' && lines[i].includes(' class ')) break;
+      }
+      result.push(lines[i])
+    }
+    return result.join('\n');
+  }
+
+}
+
 module.exports = {
   parseTest,
   parseSuite,
   ansiRegExp,
   isValidUrl,
+  fetchSourceCode,
+  fetchSourceCodeFromStackTrace,
+  fetchFilesFromStackTrace,
 };