@testomatio/reporter 0.5.10 → 0.6.0-beta.1

package/README.md

@@ -205,6 +205,24 @@ Run the following command from you project folder:
 TESTOMATIO={API_KEY} npx @testomatio/reporter -c 'npx wdio wdio.conf.js'
 ```
 
+## JUnit Reports
+
+Other frameworks and languages are supported via JUnit reports.
+
+JUnit XML format is standard among test runners on various platforms. Testomat.io can load XML reports from test runners and create tests in a project from it. If your framework is not supported yet, generate JUnit report and upload it into Testomat.io
+
+```
+TESTOMATIO={API_KEY} npx @testomatio/reporter -c 'npx wdio wdio.conf.js'
+```
+
+Tested Frameworks:
+
+* JUnit (JUnit)
+* Python (Pytest)
+* Minitest (Ruby)
+* PHPUnit (PHP)
+
+
 ## Advanced Usage
 
 ### Create Unmatched Tests

package/lib/bin/reportXml.js

@@ -0,0 +1,50 @@
+#!/usr/bin/env node
+const program = require("commander");
+// const chalk = require("chalk");
+const glob = require('glob');
+
+const { APP_PREFIX } = require('../constants');
+const XmlReader = require("../xmlReader");
+
+// const apiKey = process.env['INPUT_TESTOMATIO-KEY'] || process.env.TESTOMATIO;
+// const title = process.env.TESTOMATIO_TITLE;
+
+// console.log(chalk.cyan.bold(` 🤩 xmlReader by Testomat.io v${version}`));
+
+program
+  .arguments("<pattern>")
+  .option("-d, --dir <dir>", "Project directory")
+  .option("--java-tests [java-path]", "Load Java tests from path, by default: src/test/java")
+  .option("--lang <lang>", "Language used (python, ruby, java)")
+  .action(async (pattern, opts) => {
+    if (!pattern.endsWith('.xml')) {
+      pattern += '.xml';
+    }
+    let { javaTests, lang } = opts;
+    if (javaTests === true) javaTests = 'src/test/java';
+    const runReader = new XmlReader({ javaTests, lang });
+    const files = glob.sync(pattern, { cwd: opts.dir || process.cwd() });
+    if (!files.length) {
+      console.log(APP_PREFIX,`Report can't be created. No XML files found 😥`);
+      process.exitCode = 1;
+      return;
+    }
+
+    for (const file of files) {
+      console.log(APP_PREFIX,`Parsed ${file}`);
+      runReader.parse(file);
+    }
+    try {
+      await runReader.createRun();
+      await runReader.uploadData();
+    } catch (err) {
+      console.log(APP_PREFIX, 'Error updating status, skipping...', err);
+    }
+  });
+
+
+if (process.argv.length < 1) {
+  program.outputHelp();
+}
+
+program.parse(process.argv);

package/lib/junit-adapter/adapter.js

@@ -0,0 +1,25 @@
+class Adapter {
+
+  constructor(opts) {
+    this.opts = opts;
+  }
+
+  getFilePath(t) {
+    return t.file;
+  }
+
+  formatTest(t) {
+    return t;
+  }
+
+  formatStack(t) {
+    return t.stack || '';
+  }
+
+  formatMessage(t) {
+    return t.message;
+  }
+
+}
+
+module.exports = Adapter;

package/lib/junit-adapter/java.js

@@ -0,0 +1,40 @@
+const path = require('path');
+const Adapter = require('./adapter');
+
+class JavaAdapter extends Adapter {
+
+  getFilePath(t) {
+    const fileName = namespaceToFileName(t.suite_title)
+    return this.opts.javaTests + path.sep + fileName;
+  }
+
+  formatTest(t) {
+    const fileParts = t.suite_title.split('.')
+    const example = t.title.match(/\[(.*)\]/)?.[1];
+    if (example) t.example = { "#": example }
+
+    t.file = namespaceToFileName(t.suite_title);
+    t.title = t.title.split('(')[0];
+    t.suite_title = fileParts[fileParts.length - 1].replace(/\$/g, ' | ')
+    return t;
+  }
+
+  formatStack(t) {
+    let stack = super.formatStack(t);
+
+    const file = t.suite_title.split('.');
+
+    const fileLine = `at .*${file[file.length - 1]}\.java\:(\\d+)`
+    const regexp = new RegExp(fileLine,"g")
+    return stack.replace(regexp, `${this.opts.javaTests}${path.sep}${namespaceToFileName(t.suite_title)}:$1:`);
+  }
+}
+
+function namespaceToFileName(fileName) {
+    let fileParts = fileName.split('.')
+    fileParts[fileParts.length - 1] = fileParts[fileParts.length - 1]?.replace(/\$.*/, '')
+    return fileParts.join(path.sep) + '.java';
+
+}
+
+module.exports = JavaAdapter;

package/lib/junit-adapter/javascript.js

@@ -0,0 +1,31 @@
+const createCallsiteRecord = require('callsite-record');
+const path = require('path');
+const Adapter = require('./adapter');
+
+class JavaScriptAdapter extends Adapter {
+
+  formatStack(t) {
+    let stack = super.formatStack(t);
+
+    try {
+      const error = new Error(stack.split('\n')[0])
+      error.stack = stack;
+      const record = createCallsiteRecord({
+        forError: error,
+      });
+      if (record && !record.filename.startsWith('http')) {
+        stack += record.renderSync({
+          stackFilter: frame =>
+            frame.getFileName().indexOf(path.sep) > -1 &&
+            frame.getFileName().indexOf('node_modules') < 0 &&
+            frame.getFileName().indexOf('internal') < 0,
+        });
+      }
+      return stack;
+    } catch (err) {
+      return stack;
+    }
+  }
+}
+
+module.exports = JavaScriptAdapter;

package/lib/junit-adapter/python.js

@@ -0,0 +1,42 @@
+const path = require('path');
+const Adapter = require('./adapter');
+const fs = require('fs');
+
+class PythonAdapter extends Adapter {
+
+  getFilePath(t) {
+    const fileName = namespaceToFileName(t.suite_title)
+    return fileName;
+  }
+
+  formatTest(t) {
+    const fileParts = t.suite_title.split('.')
+    const example = t.title.match(/\[(.*)\]/)?.[1];
+    if (example) t.example = { "#": example }
+
+    t.file = namespaceToFileName(t.suite_title);
+    t.title = t.title.split('[')[0];
+    t.suite_title = fileParts[fileParts.length - 1].replace(/\$/g, ' | ')
+    return t;
+  }
+
+  formatMessage(t) {
+    return t.message.split('
')[0];
+  }
+
+}
+
+function namespaceToFileName(fileName) {
+    let fileParts = fileName.split('.')
+
+    while (fileParts.length > 0) {
+      const file = fileParts.join(path.sep) + '.py';
+      if (fs.existsSync(fileParts.join(path.sep) + '.py')) {
+        return file;
+      }
+      fileParts.pop();
+    }
+    return null;
+}
+
+module.exports = PythonAdapter;

package/lib/junit-adapter/ruby.js

@@ -0,0 +1,13 @@
+const path = require('path');
+const Adapter = require('./adapter');
+const fs = require('fs');
+
+class RubyAdapter extends Adapter {
+
+  formatStack(t) {
+    const stack = super.formatStack(t);
+    return stack.replace(/\[(.*?\:.\d*)\]/g, '\n$1')
+  }
+}
+
+module.exports = RubyAdapter;

package/lib/util.js

@@ -1,4 +1,10 @@
+const { fstat } = require('fs');
+const { includes, split } = require('lodash');
 const { URL } = require('url');
+const { sep } = require('path');
+const chalk = require('chalk');
+const fs = require('fs');
+const isValid = require('is-valid-path');
 
 /**
  * @param {String} testTitle - Test title
@@ -48,9 +54,98 @@ const isValidUrl = s => {
   }
 };
 
+const fetchFilesFromStackTrace = (stack = '') => {
+  const files = stack.matchAll(/file:?\/(\/.*?\.(png|avi|webm|jpg|html|txt))/g);
+  return Array.from(files).map(f => f[1]).filter(f => fs.existsSync(f));
+}
+
+const fetchSourceCodeFromStackTrace = (stack = '') => {
+  const stackLines = stack.split('\n')
+    .filter(l => l.includes(':'))
+    // .map(l => l.match(/\[(.*?)\]/)?.[1] || l) // minitest format
+    // .map(l => l.split(':')[0])
+    .map(l => l.trim())
+    .map(l => l.split(' ').find(p => p.includes(':')))
+    .filter(l => isValid(l.split(':')[0]))
+
+    // // filter out 3rd party libs
+    .filter(l => !l.includes('vendor' + sep))
+    .filter(l => !l.includes('node_modules' + sep))
+    .filter(l => fs.existsSync(l.split(':')[0]))
+    .filter(l => fs.lstatSync(l.split(':')[0]).isFile())
+
+  if (!stackLines.length) return '';
+
+  const [file, line] = stackLines[0].split(':');
+
+  const prepend = 3;
+  const source = fetchSourceCode(fs.readFileSync(file).toString(), { line, prepend, limit: 7 })
+
+  return source.split('\n')
+    .map((l, i) => {
+      if (i === prepend) return `${line} > ${chalk.bold(l)}`;
+      return `${line - prepend + i} | ${l}`
+    }).join('\n')
+}
+
+const fetchSourceCode = (contents, opts = {}) => {
+  if (!opts.title && !opts.line) return '';
+
+    // code fragment is 20 lines
+  const limit = opts.limit || 50;
+  let lineIndex;
+  if (opts.line) lineIndex = opts.line - 1;
+  const lines = contents.split('\n')
+
+  // remove special chars from title
+  if (!lineIndex && opts.title) {
+    const title = opts.title.replace(/[\(\[\@].*/g, '')
+    lineIndex = lines.findIndex(l => l.includes(title))
+  }
+
+  if (opts.prepend) {
+    lineIndex -= opts.prepend;
+  }
+
+  if (lineIndex) {
+    const result = [];
+    for (let i = lineIndex; i < (lineIndex + limit); i++) {
+      if (lines[i] === undefined) continue;
+
+      if (i > lineIndex + 2 && !opts.prepend) {
+        // annotation
+        if (opts.lang === 'php' && lines[i].trim().startsWith('#[')) break;
+        if (opts.lang === 'php' && lines[i].includes(' private function ')) break;
+        if (opts.lang === 'php' && lines[i].includes(' protected function ')) break;
+        if (opts.lang === 'php' && lines[i].includes(' public function ')) break;
+        if (opts.lang === 'python' && lines[i].trim().match(/^\@\w+/)) break;
+        if (opts.lang === 'python' && lines[i].includes(' def ')) break;
+        if (opts.lang === 'ruby' && lines[i].includes(' def ')) break;
+        if (opts.lang === 'ruby' && lines[i].includes(' test ')) break;
+        if (opts.lang === 'ruby' && lines[i].includes(' it ')) break;
+        if (opts.lang === 'ruby' && lines[i].includes(' specify ')) break;
+        if (opts.lang === 'ruby' && lines[i].includes(' context ')) break;
+        if (opts.lang === 'ts' && lines[i].includes(' it(')) break;
+        if (opts.lang === 'ts' && lines[i].includes(' test(')) break;
+        if (opts.lang === 'js' && lines[i].includes(' it(')) break;
+        if (opts.lang === 'js' && lines[i].includes(' test(')) break;
+        if (opts.lang === 'java' && lines[i].trim().match(/^\@\w+/)) break;
+        if (opts.lang === 'java' && lines[i].includes(' public void ')) break;
+        if (opts.lang === 'java' && lines[i].includes(' class ')) break;
+      }
+      result.push(lines[i])
+    }
+    return result.join('\n');
+  }
+
+}
+
 module.exports = {
   parseTest,
   parseSuite,
   ansiRegExp,
   isValidUrl,
+  fetchSourceCode,
+  fetchSourceCodeFromStackTrace,
+  fetchFilesFromStackTrace,
 };

package/lib/xmlReader.js

@@ -0,0 +1,348 @@
+const path = require("path");
+const axios = require('axios');
+const chalk = require('chalk');
+const fs = require("fs");
+
+// const util = require("util"); // you can see a result
+const { XMLParser } = require("fast-xml-parser");
+const { APP_PREFIX } = require('./constants');
+const { isValidUrl, fetchFilesFromStackTrace, fetchSourceCode, fetchSourceCodeFromStackTrace } = require('./util');
+const upload = require('./fileUploader');
+
+const Adapter = require('./junit-adapter/adapter');
+const JavaScriptAdapter = require('./junit-adapter/javascript');
+const JavaAdapter = require('./junit-adapter/java');
+const PythonAdapter = require('./junit-adapter/python');
+const RubyAdapter = require('./junit-adapter/ruby');
+
+const TESTOMATIO_URL = process.env.TESTOMATIO_URL || "https://app.testomat.io";
+const TESTOMATIO = process.env.TESTOMATIO; // key?
+const { TESTOMATIO_RUNGROUP_TITLE, TESTOMATIO_TITLE, TESTOMATIO_ENV, TESTOMATIO_RUN } = process.env;
+
+const options = {
+  ignoreDeclaration: true,
+  ignoreAttributes: false,
+  alwaysCreateTextNode: false,
+  attributeNamePrefix: "",
+  parseTagValue: true,
+};
+
+class XmlReader {
+  
+  constructor(opts = {}) {
+    this.requestParams = {
+      apiKey: opts.apiKey || TESTOMATIO,
+      url: opts.url || TESTOMATIO_URL,
+      title: TESTOMATIO_TITLE,
+      env: TESTOMATIO_ENV,
+      group_title: TESTOMATIO_RUNGROUP_TITLE,
+    };
+    this.runId = opts.runId || TESTOMATIO_RUN;
+    this.adapter = new Adapter(opts)
+    this.opts = opts;
+    this.axios = axios.create();
+    this.parser = new XMLParser(options);
+    this.tests = []
+    this.stats = {}
+    this.stats.language = opts.lang?.toLowerCase();
+    this.filesToUpload = {}
+  }
+
+  connectAdapter() {
+    if (this.opts.javaTests || this.stats.language === 'java') {
+      this.adapter = new JavaAdapter(this.opts);
+    }
+    if (this.stats.language == 'js') {
+      this.adapter = new JavaScriptAdapter(this.opts);
+    }
+    if (this.stats.language === 'python') {
+      this.adapter = new PythonAdapter(this.opts);
+    }
+    if (this.stats.language === 'ruby') {
+      this.adapter = new RubyAdapter(this.opts);
+    }
+  }
+
+  parse(fileName) {       
+    const xmlData = fs.readFileSync(path.resolve(fileName));
+    const jsonResult = this.parser.parse(xmlData);
+    let jsonSuite;
+
+    if (jsonResult.testsuites) {
+      jsonSuite = jsonResult.testsuites;
+    } else if (jsonResult.testsuite) {
+      jsonSuite = jsonResult;
+    } else {
+      console.log(jsonResult)
+      throw new Error("Format can't be parsed")
+    }
+    const { testsuite, name, tests, failures, errors, filepath } = jsonSuite;
+
+    const resultTests = processTestSuite(testsuite);
+    
+    const hasFailures = resultTests.filter(t => t.status === 'failed').length > 0;
+    const status = ( failures > 0 || errors > 0 || hasFailures) ? 'failed' : 'passed';
+
+    this.tests = this.tests.concat(resultTests);
+
+    return {
+      status,
+      create_tests: true,
+      name,
+      tests_count: parseInt(tests, 10),
+      passed_count: parseInt(tests - failures, 10),
+      failed_count: parseInt(failures, 10),
+      skipped_count: 0,
+      tests: resultTests,
+    };
+  }
+
+  calculateStats() {
+    this.stats = {
+      ...this.stats,
+      status: 'passed',
+      create_tests: true,
+      tests_count: 0,
+      passed_count: 0,
+      failed_count: 0,
+      skipped_count: 0,
+    }
+    this.tests.forEach(t => {
+      this.stats.tests_count++;
+      if (t.status === 'passed') this.stats.passed_count++;
+      if (t.status === 'failed') this.stats.failed_count++;
+    })
+    if (this.stats.failed_count) this.stats.status = 'failed';
+
+    return this.stats;
+  }
+
+  fetchSourceCode() {
+    this.tests.forEach(t => {
+      try {
+        let file = this.adapter.getFilePath(t)
+        if (!file) return;
+
+        if (!this.stats.language) {
+          if (file.endsWith('.php')) this.stats.language = 'php';
+          if (file.endsWith('.py')) this.stats.language = 'python';
+          if (file.endsWith('.java')) this.stats.language = 'java';
+          if (file.endsWith('.rb')) this.stats.language = 'ruby';
+          if (file.endsWith('.js')) this.stats.language  = 'js';
+          if (file.endsWith('.ts')) this.stats.language = 'ts';
+        }
+
+        const contents = fs.readFileSync(file).toString();
+        t.code = fetchSourceCode(contents, { ...t, lang: this.stats.language })
+      } catch (err) {
+        if (process.env['DEBUG']) {
+            console.log(err)
+        }
+      }
+    });
+  }
+
+  formatTests() {
+    this.tests.forEach(t => {
+      if (t.file) {
+        t.file = t.file.replace(process.cwd() + path.sep, '')
+      }
+
+      this.adapter.formatTest(t)
+
+      t.title = t.title
+      // insert a space before all caps
+        .replace(/([A-Z])/g, ' $1')
+      // _ chars to spaces
+        .replace(/_/g, ' ')
+      // uppercase the first character
+        .replace(/^(.)|\s(.)/g, $1 => $1.toLowerCase())
+
+        // remove standard prefixes
+        .replace(/^test\s/, '')
+        .replace(/^should\s/, '')
+    });
+  }
+
+  formatErrors() {
+    this.tests.filter(t => !!t.stack).forEach(t => {
+      t.stack = this.formatStack(t)
+      t.message = this.adapter.formatMessage(t);
+    });
+
+  }
+
+  formatStack(t) {
+    let stack = this.adapter.formatStack(t);
+
+    const sourcePart = fetchSourceCodeFromStackTrace(stack);
+
+    if (!sourcePart) return stack;
+
+    return `${stack}\n\n${chalk.bold.red('################[ Failure ]################')}\n${fetchSourceCodeFromStackTrace(stack)}`;
+  }
+
+  async uploadArtifacts() {
+    if (!this.runId) return;
+    for (const t of this.tests.filter(t => !!t.stack)) {
+      const files = fetchFilesFromStackTrace(t.stack);
+      t.artifacts = await Promise.all(files.map(f => upload.uploadFileByPath(f, this.runId)));
+    }
+  }
+
+  async createRun() {
+    if (this.runId) return;
+
+    const runParams = {
+      api_key: this.requestParams.apiKey,
+      title: this.requestParams.title,
+      env: this.requestParams.env,
+      group_title: this.requestParams.runGroup,
+    };    
+
+    try {
+      const resp = await this.axios.post(this.url, runParams, {
+        maxContentLength: Infinity,
+        maxBodyLength: Infinity,
+        headers: {
+          // Overwrite Axios's automatically set Content-Type
+          'Content-Type': 'application/json',
+        }
+      });
+      if (resp.status >= 400) {
+        const data = resp.data || { message: '' };
+        console.log(
+          APP_PREFIX,
+          `Report couldn't be processed: (${resp.status}) ${data.message}`,
+        );
+        return;
+      }
+      this.runId = resp.data.uid;
+      this.runUrl = `${TESTOMATIO_URL}/${resp.data.url.split('/').splice(3).join('/')}`;
+    } catch(err) {
+      if (process.env.DEBUG) console.log(err)
+      const data = err?.response?.data || { message: '' };
+      console.log(APP_PREFIX, 'Error creating run, skipping...', err?.response?.statusText, data);
+    }
+  }
+
+  async uploadData() {
+    await this.uploadArtifacts();
+    this.calculateStats();
+    this.connectAdapter();
+    this.fetchSourceCode();
+    this.formatErrors();
+    this.formatTests();
+
+    if (process.env['DEBUG']) {
+      console.log({
+        ...this.stats,
+        tests: this.tests,
+      })
+    }
+
+    const dataString = JSON.stringify({
+      ...this.stats,
+      api_key: this.requestParams.apiKey,
+      tests: this.tests,
+    });
+
+    try {
+      const resp = await this.axios.put(`${this.url}/${this.runId}`, dataString, {
+        maxContentLength: Infinity,
+        maxBodyLength: Infinity,
+        headers: {
+          // Overwrite Axios's automatically set Content-Type
+          'Content-Type': 'application/json',
+        }
+      });
+
+      if (resp.status >= 400) {
+        const data = resp.data || { message: '' };
+        console.log(
+          APP_PREFIX,
+          `Report couldn't be processed: (${resp.status}) ${data.message}`,
+        );
+        return;
+      }
+
+      if (this.runUrl) {
+        console.log(APP_PREFIX, '📊 Report Saved. Report URL:', chalk.magenta(this.runUrl));
+      }
+      process.env.runId = this.runId;
+      return resp;
+    } catch (err) {
+      // if (process.env.DEBUG) console.log(err.response)
+      const data = err.response.data || { message: '' };
+      console.log(APP_PREFIX, 'Error uploading, skipping...', err.response.statusText, data);
+      return err.response;
+    }
+
+  }
+
+  get url() {
+    if (!isValidUrl(this.requestParams.url)) {
+      console.log(
+        APP_PREFIX,
+        chalk.red(`Error creating report on Testomat.io, report url '${TESTOMAT_URL}' is invalid`),
+      );
+      return;
+    }
+
+    return `${this.requestParams.url}/api/reporter`;
+  }
+}
+
+module.exports = XmlReader;
+
+
+function reduceTestCases(prev, item) {
+  let testCases = item.testcase;
+  if (!Array.isArray(testCases)) {
+    testCases = [testCases]
+  }
+  testCases.filter(t => !!t).forEach(testCaseItem => {
+    let file = testCaseItem.file || item.filepath || '';
+
+    let stack = '';
+    let message = '';
+    if (testCaseItem.error) stack = testCaseItem.error;
+    if (testCaseItem.failure) stack = testCaseItem.failure;
+    if (testCaseItem?.failure?.message) message = testCaseItem.failure.message;
+    if (testCaseItem?.error?.message) message = testCaseItem.error.message;
+
+    if (testCaseItem.failure && testCaseItem.failure['#text']) stack = testCaseItem.failure['#text'];
+    if (testCaseItem.error && testCaseItem.error['#text']) stack = testCaseItem.error['#text'];
+    if (!message) message = stack.trim().split('\n')[0];
+
+    // prepend system output
+    stack = `${testCaseItem['system-out'] || testCaseItem['log'] || ''}\n\n${stack}`.trim()
+
+    prev.push({
+      create: true,
+      file,
+      stack,
+      message,
+      line: testCaseItem.lineno,
+      run_time: testCaseItem.time,
+      status: (testCaseItem.failure || testCaseItem.error) ? 'failed' : 'passed',
+      title: testCaseItem.name,
+      suite_title: testCaseItem.classname || item.name,
+    })
+  });
+  return prev;
+}
+
+function processTestSuite(testsuite) {
+  if (testsuite.testsuite) return processTestSuite(testsuite.testsuite)
+
+  let suites = testsuite;
+  if (!Array.isArray(testsuite)) {
+    suites = [testsuite];
+  }
+
+  const res = suites.reduce(reduceTestCases, []);
+
+  return res;
+}
+