@testomatio/reporter 0.5.10 → 0.6.1

package/.eslintrc.js

@@ -11,7 +11,7 @@ module.exports = {
     SharedArrayBuffer: 'readonly',
   },
   parserOptions: {
-    ecmaVersion: 2018,
+    ecmaVersion: 2020,
   },
   rules: {
     'max-len': ['error', { code: 120 }],
@@ -25,6 +25,7 @@ module.exports = {
     'no-use-before-define': 0,
     'object-curly-newline': 0,
     'consistent-return': 0,
+    'no-await-in-loop': 0,
     'no-param-reassign': 0,
     'operator-linebreak': 0,
     'prefer-destructuring': 0,

package/.github/workflows/node.js.yml

@@ -15,7 +15,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [12.x, 14.x, 16.x]
+        node-version: [14.x, 16.x]
         # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
 
     steps:

package/Changelog.md

@@ -1,3 +1,11 @@
+# 0.6.1
+
+* Fixed creating RunGroup with JUnit reporter
+
+# 0.6.0
+
+* JUnit reporter support
+
 # 0.5.10
 
 * Fixed reporting Scenario Outline in Cypress-Cucumber

package/README.md

@@ -205,6 +205,136 @@ Run the following command from you project folder:
 TESTOMATIO={API_KEY} npx @testomatio/reporter -c 'npx wdio wdio.conf.js'
 ```
 
+## JUnit Reports
+
+> **Notice** JUnit reports are supported since 0.6.0
+
+Other frameworks and languages are supported via JUnit reports.
+
+JUnit XML format is standard among test runners on various platforms. Testomat.io can load XML reports from test runners and create tests in a project from it. If your framework is not supported yet, generate JUnit report and upload it into Testomat.io
+
+Testomat.io will not only create a run report but also collect all information about tests, including source code, and create tests in a system as regular importer do. The only difference that normal process of Testomat.io import does not require executing tests on import, while importing via repoter requires to have tests executed and XML report provided.
+
+Tested Frameworks:
+
+* JUnit (JUnit)
+* Python (Pytest)
+* Minitest (Ruby)
+* PHPUnit (PHP)
+
+
+To import JUnit reports into Testomat.io **NodeJS >=14 is required**.
+
+Package `@testomatio/reporter` should be installed:
+
+```
+npm i @testomatio/reporter@latest
+```
+
+For local development it is recommended to install @testomatio/reporter globally:
+
+```
+npm i -g @testomatio/reporter@latest
+```
+
+Run your test framework and generate a JUnit report.
+
+Then import XML report into Testomat.io
+
+```
+TESTOMATIO={API_KEY} npx report-xml "{pattern}" --lang={lang}
+```
+
+* `pattern` - is a glob pattern to match all XML files from report. For instance, `"test/report/**.xml"` or just `report.xml`
+* `--lang` option can be specified to identify source code of the project. Example: `--lang=Ruby` or `--lang=Java` or `--lang=Python`.
+* `--java-tests` option is avaiable for Java projects, and can be set if path to tests is different then `src/test/java`. When this option is enable, `lang` option is automatically set to `java`
+
+
+> *Notice:* All options from [Advanced Usage](#advanced-usage) are also available for JUnit reporter
+
+Check the list of examples:
+
+#### Example: Pytest
+
+Run pytest tests and generate a report to `report.xml`:
+
+```
+pytest --junit-xml report.xml
+```
+
+Import report with this command
+
+```
+TESTOMATIO={API_KEY} npx report-xml report.xml --lang=python
+```
+
+#### Example: JUnit with Maven
+
+Run tests via Maven, make sure JUnit report was configured in `pom.xml`.
+
+```
+mvn clean test
+```
+
+Import report with this command:
+
+```
+TESTOMATIO={API_KEY} npx report-xml "target/surefire-reports/*.xml" --java-tests
+```
+
+> You can specify `--java-test` option to set a path to tests if they are located in path other than `src/test/java`
+
+#### Example: Ruby on Rails with Minitest
+
+```ruby
+# test_helper.rb:
+
+reporters = [Minitest::Reporters::DefaultReporter.new(color: true)]
+# enable JUnit reporter
+reporters << Minitest::Reporters::JUnitReporter.new
+
+Minitest::Reporters.use! reporters
+```
+
+Launch tests:
+
+```
+rails test
+```
+
+Import reports from `test/reports` directory:
+
+```
+TESTOMATIO={API_KEY} npx report-xml "test/reports/*.xml" --lang ruby
+```
+
+### Artifacts
+
+Screenshots or videos from tests are uploaded if a test contains output with a path to file of following format:
+
+```
+file://path/to/screenshot.png
+```
+
+For instance, inside Java test you can use `System.out.println` to print a path to file that should be uploaded as a screenshot.
+
+```java
+System.out.println("file://" + pathToScreenshot);
+```
+
+This will produce XML report which contains path to a file:
+
+```xml
+<testcase>
+  <system-out><![CDATA[
+    file://path/to/scrrenshot.png
+  ]]></system-out>
+</testcase>
+```
+
+When uploaded XML report, all files from `file://` will be uploaded to corresponding tests.
+
+
 ## Advanced Usage
 
 ### Create Unmatched Tests

package/lib/adapter/cypress-plugin/index.js

@@ -29,7 +29,7 @@ const testomatioReporter = on => {
       const error = latestAttempt.error;
 
       let title = test.title.pop();
-      let examples = title.match(/\(example (#\d+)\)/);
+      const examples = title.match(/\(example (#\d+)\)/);
       let example = null;
       if (examples && examples[1]) example = { example: examples[1] };
       title = title.replace(/\(example #\d+\)/, '').trim();
@@ -40,7 +40,7 @@ const testomatioReporter = on => {
       const suiteId = parseSuite(suiteTitle);
 
       if (error) {
-        error.inspect = function() {
+        error.inspect = function() { // eslint-disable-line func-names
           if (this && this.codeFrame) {
             return this.codeFrame.frame;
           }

package/lib/bin/reportXml.js

@@ -0,0 +1,51 @@
+#!/usr/bin/env node
+const program = require("commander");
+// const chalk = require("chalk");
+const glob = require('glob');
+
+const { APP_PREFIX } = require('../constants');
+const XmlReader = require("../xmlReader");
+
+// const apiKey = process.env['INPUT_TESTOMATIO-KEY'] || process.env.TESTOMATIO;
+// const title = process.env.TESTOMATIO_TITLE;
+
+// console.log(chalk.cyan.bold(` 🤩 xmlReader by Testomat.io v${version}`));
+
+program
+  .arguments("<pattern>")
+  .option("-d, --dir <dir>", "Project directory")
+  .option("--java-tests [java-path]", "Load Java tests from path, by default: src/test/java")
+  .option("--lang <lang>", "Language used (python, ruby, java)")
+  .action(async (pattern, opts) => {
+    if (!pattern.endsWith('.xml')) {
+      pattern += '.xml';
+    }
+    let { javaTests, lang } = opts;
+    if (javaTests === true) javaTests = 'src/test/java';
+    lang = lang?.toLowerCase();
+    const runReader = new XmlReader({ javaTests, lang });
+    const files = glob.sync(pattern, { cwd: opts.dir || process.cwd() });
+    if (!files.length) {
+      console.log(APP_PREFIX,`Report can't be created. No XML files found 😥`);
+      process.exitCode = 1;
+      return;
+    }
+
+    for (const file of files) {
+      console.log(APP_PREFIX,`Parsed ${file}`);
+      runReader.parse(file);
+    }
+    try {
+      await runReader.createRun();
+      await runReader.uploadData();
+    } catch (err) {
+      console.log(APP_PREFIX, 'Error updating status, skipping...', err);
+    }
+  });
+
+
+if (process.argv.length < 1) {
+  program.outputHelp();
+}
+
+program.parse(process.argv);

package/lib/junit-adapter/adapter.js

@@ -0,0 +1,25 @@
+class Adapter {
+
+  constructor(opts) {
+    this.opts = opts;
+  }
+
+  getFilePath(t) {
+    return t.file;
+  }
+
+  formatTest(t) {
+    return t;
+  }
+
+  formatStack(t) {
+    return t.stack || '';
+  }
+
+  formatMessage(t) {
+    return t.message;
+  }
+
+}
+
+module.exports = Adapter;

package/lib/junit-adapter/java.js

@@ -0,0 +1,40 @@
+const path = require('path');
+const Adapter = require('./adapter');
+
+class JavaAdapter extends Adapter {
+
+  getFilePath(t) {
+    const fileName = namespaceToFileName(t.suite_title)
+    return this.opts.javaTests + path.sep + fileName;
+  }
+
+  formatTest(t) {
+    const fileParts = t.suite_title.split('.')
+    const example = t.title.match(/\[(.*)\]/)?.[1];
+    if (example) t.example = { "#": example }
+
+    t.file = namespaceToFileName(t.suite_title);
+    t.title = t.title.split('(')[0];
+    t.suite_title = fileParts[fileParts.length - 1].replace(/\$/g, ' | ')
+    return t;
+  }
+
+  formatStack(t) {
+    const stack = super.formatStack(t);
+
+    const file = t.suite_title.split('.');
+
+    const fileLine = `at .*${file[file.length - 1]}\.java:(\\d+)` // eslint-disable-line no-useless-escape
+    const regexp = new RegExp(fileLine,"g")
+    return stack.replace(regexp, `${this.opts.javaTests}${path.sep}${namespaceToFileName(t.suite_title)}:$1:`);
+  }
+}
+
+function namespaceToFileName(fileName) {
+    const fileParts = fileName.split('.')
+    fileParts[fileParts.length - 1] = fileParts[fileParts.length - 1]?.replace(/\$.*/, '')
+    return `${fileParts.join(path.sep)  }.java`;
+
+}
+
+module.exports = JavaAdapter;

package/lib/junit-adapter/javascript.js

@@ -0,0 +1,31 @@
+const createCallsiteRecord = require('callsite-record');
+const path = require('path');
+const Adapter = require('./adapter');
+
+class JavaScriptAdapter extends Adapter {
+
+  formatStack(t) {
+    let stack = super.formatStack(t);
+
+    try {
+      const error = new Error(stack.split('\n')[0])
+      error.stack = stack;
+      const record = createCallsiteRecord({
+        forError: error,
+      });
+      if (record && !record.filename.startsWith('http')) {
+        stack += record.renderSync({
+          stackFilter: frame =>
+            frame.getFileName().indexOf(path.sep) > -1 &&
+            frame.getFileName().indexOf('node_modules') < 0 &&
+            frame.getFileName().indexOf('internal') < 0,
+        });
+      }
+      return stack;
+    } catch (err) {
+      return stack;
+    }
+  }
+}
+
+module.exports = JavaScriptAdapter;

package/lib/junit-adapter/python.js

@@ -0,0 +1,42 @@
+const path = require('path');
+const fs = require('fs');
+const Adapter = require('./adapter');
+
+class PythonAdapter extends Adapter {
+
+  getFilePath(t) {
+    const fileName = namespaceToFileName(t.suite_title)
+    return fileName;
+  }
+
+  formatTest(t) {
+    const fileParts = t.suite_title.split('.')
+    const example = t.title.match(/\[(.*)\]/)?.[1];
+    if (example) t.example = { "#": example }
+
+    t.file = namespaceToFileName(t.suite_title);
+    t.title = t.title.split('[')[0];
+    t.suite_title = fileParts[fileParts.length - 1].replace(/\$/g, ' | ')
+    return t;
+  }
+
+  formatMessage(t) {
+    return t.message.split('
')[0];
+  }
+
+}
+
+function namespaceToFileName(fileName) {
+    const fileParts = fileName.split('.')
+
+    while (fileParts.length > 0) {
+      const file = `${fileParts.join(path.sep)  }.py`;
+      if (fs.existsSync(`${fileParts.join(path.sep)  }.py`)) {
+        return file;
+      }
+      fileParts.pop();
+    }
+    return null;
+}
+
+module.exports = PythonAdapter;

package/lib/junit-adapter/ruby.js

@@ -0,0 +1,11 @@
+const Adapter = require('./adapter');
+
+class RubyAdapter extends Adapter {
+
+  formatStack(t) {
+    const stack = super.formatStack(t);
+    return stack.replace(/\[(.*?:.\d*)\]/g, '\n$1')
+  }
+}
+
+module.exports = RubyAdapter;

package/lib/util.js

@@ -1,4 +1,8 @@
 const { URL } = require('url');
+const { sep } = require('path');
+const chalk = require('chalk');
+const fs = require('fs');
+const isValid = require('is-valid-path');
 
 /**
  * @param {String} testTitle - Test title
@@ -48,9 +52,98 @@ const isValidUrl = s => {
   }
 };
 
+const fetchFilesFromStackTrace = (stack = '') => {
+  const files = stack.matchAll(/file:?\/(\/.*?\.(png|avi|webm|jpg|html|txt))/g);
+  return Array.from(files).map(f => f[1]).filter(f => fs.existsSync(f));
+}
+
+const fetchSourceCodeFromStackTrace = (stack = '') => {
+  const stackLines = stack.split('\n')
+    .filter(l => l.includes(':'))
+    // .map(l => l.match(/\[(.*?)\]/)?.[1] || l) // minitest format
+    // .map(l => l.split(':')[0])
+    .map(l => l.trim())
+    .map(l => l.split(' ').find(p => p.includes(':')))
+    .filter(l => isValid(l.split(':')[0]))
+
+    // // filter out 3rd party libs
+    .filter(l => !l.includes(`vendor${  sep}`))
+    .filter(l => !l.includes(`node_modules${  sep}`))
+    .filter(l => fs.existsSync(l.split(':')[0]))
+    .filter(l => fs.lstatSync(l.split(':')[0]).isFile())
+
+  if (!stackLines.length) return '';
+
+  const [file, line] = stackLines[0].split(':');
+
+  const prepend = 3;
+  const source = fetchSourceCode(fs.readFileSync(file).toString(), { line, prepend, limit: 7 })
+
+  return source.split('\n')
+    .map((l, i) => {
+      if (i === prepend) return `${line} > ${chalk.bold(l)}`;
+      return `${line - prepend + i} | ${l}`
+    }).join('\n')
+}
+
+const fetchSourceCode = (contents, opts = {}) => {
+  if (!opts.title && !opts.line) return '';
+
+    // code fragment is 20 lines
+  const limit = opts.limit || 50;
+  let lineIndex;
+  if (opts.line) lineIndex = opts.line - 1;
+  const lines = contents.split('\n')
+
+  // remove special chars from title
+  if (!lineIndex && opts.title) {
+    const title = opts.title.replace(/[([@].*/g, '')
+    lineIndex = lines.findIndex(l => l.includes(title))
+  }
+
+  if (opts.prepend) {
+    lineIndex -= opts.prepend;
+  }
+
+  if (lineIndex) {
+    const result = [];
+    for (let i = lineIndex; i < (lineIndex + limit); i++) {
+      if (lines[i] === undefined) continue;
+
+      if (i > lineIndex + 2 && !opts.prepend) {
+        // annotation
+        if (opts.lang === 'php' && lines[i].trim().startsWith('#[')) break;
+        if (opts.lang === 'php' && lines[i].includes(' private function ')) break;
+        if (opts.lang === 'php' && lines[i].includes(' protected function ')) break;
+        if (opts.lang === 'php' && lines[i].includes(' public function ')) break;
+        if (opts.lang === 'python' && lines[i].trim().match(/^@\w+/)) break;
+        if (opts.lang === 'python' && lines[i].includes(' def ')) break;
+        if (opts.lang === 'ruby' && lines[i].includes(' def ')) break;
+        if (opts.lang === 'ruby' && lines[i].includes(' test ')) break;
+        if (opts.lang === 'ruby' && lines[i].includes(' it ')) break;
+        if (opts.lang === 'ruby' && lines[i].includes(' specify ')) break;
+        if (opts.lang === 'ruby' && lines[i].includes(' context ')) break;
+        if (opts.lang === 'ts' && lines[i].includes(' it(')) break;
+        if (opts.lang === 'ts' && lines[i].includes(' test(')) break;
+        if (opts.lang === 'js' && lines[i].includes(' it(')) break;
+        if (opts.lang === 'js' && lines[i].includes(' test(')) break;
+        if (opts.lang === 'java' && lines[i].trim().match(/^@\w+/)) break;
+        if (opts.lang === 'java' && lines[i].includes(' public void ')) break;
+        if (opts.lang === 'java' && lines[i].includes(' class ')) break;
+      }
+      result.push(lines[i])
+    }
+    return result.join('\n');
+  }
+
+}
+
 module.exports = {
   parseTest,
   parseSuite,
   ansiRegExp,
   isValidUrl,
+  fetchSourceCode,
+  fetchSourceCodeFromStackTrace,
+  fetchFilesFromStackTrace,
 };