@testomatio/reporter 0.5.8 → 0.6.0-beta.1

package/Changelog.md

@@ -1,3 +1,12 @@
+# 0.5.10
+
+* Fixed reporting Scenario Outline in Cypress-Cucumber
+* Fixed error reports for Cypress when running in Chrome
+
+# 0.5.9
+
+* Added environment on Cypress report
+
 # 0.5.8
 
 * Fixed Cypress.io reporting

package/README.md

@@ -205,6 +205,24 @@ Run the following command from you project folder:
 TESTOMATIO={API_KEY} npx @testomatio/reporter -c 'npx wdio wdio.conf.js'
 ```
 
+## JUnit Reports
+
+Other frameworks and languages are supported via JUnit reports.
+
+JUnit XML format is standard among test runners on various platforms. Testomat.io can load XML reports from test runners and create tests in a project from it. If your framework is not supported yet, generate JUnit report and upload it into Testomat.io
+
+```
+TESTOMATIO={API_KEY} npx @testomatio/reporter -c 'npx wdio wdio.conf.js'
+```
+
+Tested Frameworks:
+
+* JUnit (JUnit)
+* Python (Pytest)
+* Minitest (Ruby)
+* PHPUnit (PHP)
+
+
 ## Advanced Usage
 
 ### Create Unmatched Tests

package/lib/adapter/cypress-plugin/index.js

@@ -3,13 +3,13 @@ const { parseTest, parseSuite } = require('../../util');
 const TestomatClient = require('../../client');
 
 const testomatioReporter = on => {
+  if (!process.env.TESTOMATIO) {
+    console.log('TESTOMATIO key is empty, ignoring reports');
+    return
+  }
   const client = new TestomatClient({ apiKey: process.env.TESTOMATIO });
 
   on('before:run', async (run) => {
-    if (!client.title) {
-      client.title = `${run.config.projectName} tests`
-      if (client.title.includes('undefined')) delete client.title;
-    }
     if (!client.env) {
       client.env = `${run.browser.displayName},${run.system.osName}`
     }
@@ -28,7 +28,12 @@ const testomatioReporter = on => {
       const time = latestAttempt.duration;
       const error = latestAttempt.error;
 
-      const title = test.title.pop();
+      let title = test.title.pop();
+      let examples = title.match(/\(example (#\d+)\)/);
+      let example = null;
+      if (examples && examples[1]) example = { example: examples[1] };
+      title = title.replace(/\(example #\d+\)/, '').trim();
+
       const suiteTitle = test.title.pop();
 
       const testId = parseTest(title);
@@ -36,7 +41,10 @@ const testomatioReporter = on => {
 
       if (error) {
         error.inspect = function() {
-          return this.codeFrame.frame;
+          if (this && this.codeFrame) {
+            return this.codeFrame.frame;
+          }
+          return '';
         }
       }
 
@@ -50,7 +58,7 @@ const testomatioReporter = on => {
       const state = test.state === 'passed' ? TRConstants.PASSED : TRConstants.FAILED;
 
       addSpecTestsPromises.push(client.addTestRun(testId, state, {
-        title, time, error, files, suite_title: suiteTitle, suite_id: suiteId
+        title, time, example, error, files, suite_title: suiteTitle, suite_id: suiteId
       }));
     }
 

package/lib/bin/reportXml.js

@@ -0,0 +1,50 @@
+#!/usr/bin/env node
+const program = require("commander");
+// const chalk = require("chalk");
+const glob = require('glob');
+
+const { APP_PREFIX } = require('../constants');
+const XmlReader = require("../xmlReader");
+
+// const apiKey = process.env['INPUT_TESTOMATIO-KEY'] || process.env.TESTOMATIO;
+// const title = process.env.TESTOMATIO_TITLE;
+
+// console.log(chalk.cyan.bold(` 🤩 xmlReader by Testomat.io v${version}`));
+
+program
+  .arguments("<pattern>")
+  .option("-d, --dir <dir>", "Project directory")
+  .option("--java-tests [java-path]", "Load Java tests from path, by default: src/test/java")
+  .option("--lang <lang>", "Language used (python, ruby, java)")
+  .action(async (pattern, opts) => {
+    if (!pattern.endsWith('.xml')) {
+      pattern += '.xml';
+    }
+    let { javaTests, lang } = opts;
+    if (javaTests === true) javaTests = 'src/test/java';
+    const runReader = new XmlReader({ javaTests, lang });
+    const files = glob.sync(pattern, { cwd: opts.dir || process.cwd() });
+    if (!files.length) {
+      console.log(APP_PREFIX,`Report can't be created. No XML files found 😥`);
+      process.exitCode = 1;
+      return;
+    }
+
+    for (const file of files) {
+      console.log(APP_PREFIX,`Parsed ${file}`);
+      runReader.parse(file);
+    }
+    try {
+      await runReader.createRun();
+      await runReader.uploadData();
+    } catch (err) {
+      console.log(APP_PREFIX, 'Error updating status, skipping...', err);
+    }
+  });
+
+
+if (process.argv.length < 1) {
+  program.outputHelp();
+}
+
+program.parse(process.argv);

package/lib/client.js

@@ -25,6 +25,7 @@ class TestomatClient {
     this.apiKey = params.apiKey || process.env.TESTOMATIO;
     this.title = params.title || process.env.TESTOMATIO_TITLE;
     this.createNewTests = !!process.env.TESTOMATIO_CREATE
+    this.env  = TESTOMATIO_ENV;
     this.parallel = params.parallel;
     this.runId = process.env.runId;
     this.queue = Promise.resolve();
@@ -45,8 +46,8 @@ class TestomatClient {
       api_key: this.apiKey.trim(),
       title: this.title,
       parallel: this.parallel,
+      env: this.env,
       group_title: TESTOMATIO_RUNGROUP_TITLE,
-      env: TESTOMATIO_ENV,
     };
 
     global.testomatioArtifacts = [];
@@ -233,7 +234,7 @@ class TestomatClient {
 
   formatError(error, message) {
     if (!message) message = error.message;
-    if (error.inspect) message = error.inspect();
+    if (error.inspect) message = error.inspect() || '';
 
     let stack = `\n${chalk.bold(message)}\n`;
 

package/lib/junit-adapter/adapter.js

@@ -0,0 +1,25 @@
+class Adapter {
+
+  constructor(opts) {
+    this.opts = opts;
+  }
+
+  getFilePath(t) {
+    return t.file;
+  }
+
+  formatTest(t) {
+    return t;
+  }
+
+  formatStack(t) {
+    return t.stack || '';
+  }
+
+  formatMessage(t) {
+    return t.message;
+  }
+
+}
+
+module.exports = Adapter;

package/lib/junit-adapter/java.js

@@ -0,0 +1,40 @@
+const path = require('path');
+const Adapter = require('./adapter');
+
+class JavaAdapter extends Adapter {
+
+  getFilePath(t) {
+    const fileName = namespaceToFileName(t.suite_title)
+    return this.opts.javaTests + path.sep + fileName;
+  }
+
+  formatTest(t) {
+    const fileParts = t.suite_title.split('.')
+    const example = t.title.match(/\[(.*)\]/)?.[1];
+    if (example) t.example = { "#": example }
+
+    t.file = namespaceToFileName(t.suite_title);
+    t.title = t.title.split('(')[0];
+    t.suite_title = fileParts[fileParts.length - 1].replace(/\$/g, ' | ')
+    return t;
+  }
+
+  formatStack(t) {
+    let stack = super.formatStack(t);
+
+    const file = t.suite_title.split('.');
+
+    const fileLine = `at .*${file[file.length - 1]}\.java\:(\\d+)`
+    const regexp = new RegExp(fileLine,"g")
+    return stack.replace(regexp, `${this.opts.javaTests}${path.sep}${namespaceToFileName(t.suite_title)}:$1:`);
+  }
+}
+
+function namespaceToFileName(fileName) {
+    let fileParts = fileName.split('.')
+    fileParts[fileParts.length - 1] = fileParts[fileParts.length - 1]?.replace(/\$.*/, '')
+    return fileParts.join(path.sep) + '.java';
+
+}
+
+module.exports = JavaAdapter;

package/lib/junit-adapter/javascript.js

@@ -0,0 +1,31 @@
+const createCallsiteRecord = require('callsite-record');
+const path = require('path');
+const Adapter = require('./adapter');
+
+class JavaScriptAdapter extends Adapter {
+
+  formatStack(t) {
+    let stack = super.formatStack(t);
+
+    try {
+      const error = new Error(stack.split('\n')[0])
+      error.stack = stack;
+      const record = createCallsiteRecord({
+        forError: error,
+      });
+      if (record && !record.filename.startsWith('http')) {
+        stack += record.renderSync({
+          stackFilter: frame =>
+            frame.getFileName().indexOf(path.sep) > -1 &&
+            frame.getFileName().indexOf('node_modules') < 0 &&
+            frame.getFileName().indexOf('internal') < 0,
+        });
+      }
+      return stack;
+    } catch (err) {
+      return stack;
+    }
+  }
+}
+
+module.exports = JavaScriptAdapter;

package/lib/junit-adapter/python.js

@@ -0,0 +1,42 @@
+const path = require('path');
+const Adapter = require('./adapter');
+const fs = require('fs');
+
+class PythonAdapter extends Adapter {
+
+  getFilePath(t) {
+    const fileName = namespaceToFileName(t.suite_title)
+    return fileName;
+  }
+
+  formatTest(t) {
+    const fileParts = t.suite_title.split('.')
+    const example = t.title.match(/\[(.*)\]/)?.[1];
+    if (example) t.example = { "#": example }
+
+    t.file = namespaceToFileName(t.suite_title);
+    t.title = t.title.split('[')[0];
+    t.suite_title = fileParts[fileParts.length - 1].replace(/\$/g, ' | ')
+    return t;
+  }
+
+  formatMessage(t) {
+    return t.message.split('
')[0];
+  }
+
+}
+
+function namespaceToFileName(fileName) {
+    let fileParts = fileName.split('.')
+
+    while (fileParts.length > 0) {
+      const file = fileParts.join(path.sep) + '.py';
+      if (fs.existsSync(fileParts.join(path.sep) + '.py')) {
+        return file;
+      }
+      fileParts.pop();
+    }
+    return null;
+}
+
+module.exports = PythonAdapter;

package/lib/junit-adapter/ruby.js

@@ -0,0 +1,13 @@
+const path = require('path');
+const Adapter = require('./adapter');
+const fs = require('fs');
+
+class RubyAdapter extends Adapter {
+
+  formatStack(t) {
+    const stack = super.formatStack(t);
+    return stack.replace(/\[(.*?\:.\d*)\]/g, '\n$1')
+  }
+}
+
+module.exports = RubyAdapter;

package/lib/util.js

@@ -1,4 +1,10 @@
+const { fstat } = require('fs');
+const { includes, split } = require('lodash');
 const { URL } = require('url');
+const { sep } = require('path');
+const chalk = require('chalk');
+const fs = require('fs');
+const isValid = require('is-valid-path');
 
 /**
  * @param {String} testTitle - Test title
@@ -48,9 +54,98 @@ const isValidUrl = s => {
   }
 };
 
+const fetchFilesFromStackTrace = (stack = '') => {
+  const files = stack.matchAll(/file:?\/(\/.*?\.(png|avi|webm|jpg|html|txt))/g);
+  return Array.from(files).map(f => f[1]).filter(f => fs.existsSync(f));
+}
+
+const fetchSourceCodeFromStackTrace = (stack = '') => {
+  const stackLines = stack.split('\n')
+    .filter(l => l.includes(':'))
+    // .map(l => l.match(/\[(.*?)\]/)?.[1] || l) // minitest format
+    // .map(l => l.split(':')[0])
+    .map(l => l.trim())
+    .map(l => l.split(' ').find(p => p.includes(':')))
+    .filter(l => isValid(l.split(':')[0]))
+
+    // // filter out 3rd party libs
+    .filter(l => !l.includes('vendor' + sep))
+    .filter(l => !l.includes('node_modules' + sep))
+    .filter(l => fs.existsSync(l.split(':')[0]))
+    .filter(l => fs.lstatSync(l.split(':')[0]).isFile())
+
+  if (!stackLines.length) return '';
+
+  const [file, line] = stackLines[0].split(':');
+
+  const prepend = 3;
+  const source = fetchSourceCode(fs.readFileSync(file).toString(), { line, prepend, limit: 7 })
+
+  return source.split('\n')
+    .map((l, i) => {
+      if (i === prepend) return `${line} > ${chalk.bold(l)}`;
+      return `${line - prepend + i} | ${l}`
+    }).join('\n')
+}
+
+const fetchSourceCode = (contents, opts = {}) => {
+  if (!opts.title && !opts.line) return '';
+
+    // code fragment is 20 lines
+  const limit = opts.limit || 50;
+  let lineIndex;
+  if (opts.line) lineIndex = opts.line - 1;
+  const lines = contents.split('\n')
+
+  // remove special chars from title
+  if (!lineIndex && opts.title) {
+    const title = opts.title.replace(/[\(\[\@].*/g, '')
+    lineIndex = lines.findIndex(l => l.includes(title))
+  }
+
+  if (opts.prepend) {
+    lineIndex -= opts.prepend;
+  }
+
+  if (lineIndex) {
+    const result = [];
+    for (let i = lineIndex; i < (lineIndex + limit); i++) {
+      if (lines[i] === undefined) continue;
+
+      if (i > lineIndex + 2 && !opts.prepend) {
+        // annotation
+        if (opts.lang === 'php' && lines[i].trim().startsWith('#[')) break;
+        if (opts.lang === 'php' && lines[i].includes(' private function ')) break;
+        if (opts.lang === 'php' && lines[i].includes(' protected function ')) break;
+        if (opts.lang === 'php' && lines[i].includes(' public function ')) break;
+        if (opts.lang === 'python' && lines[i].trim().match(/^\@\w+/)) break;
+        if (opts.lang === 'python' && lines[i].includes(' def ')) break;
+        if (opts.lang === 'ruby' && lines[i].includes(' def ')) break;
+        if (opts.lang === 'ruby' && lines[i].includes(' test ')) break;
+        if (opts.lang === 'ruby' && lines[i].includes(' it ')) break;
+        if (opts.lang === 'ruby' && lines[i].includes(' specify ')) break;
+        if (opts.lang === 'ruby' && lines[i].includes(' context ')) break;
+        if (opts.lang === 'ts' && lines[i].includes(' it(')) break;
+        if (opts.lang === 'ts' && lines[i].includes(' test(')) break;
+        if (opts.lang === 'js' && lines[i].includes(' it(')) break;
+        if (opts.lang === 'js' && lines[i].includes(' test(')) break;
+        if (opts.lang === 'java' && lines[i].trim().match(/^\@\w+/)) break;
+        if (opts.lang === 'java' && lines[i].includes(' public void ')) break;
+        if (opts.lang === 'java' && lines[i].includes(' class ')) break;
+      }
+      result.push(lines[i])
+    }
+    return result.join('\n');
+  }
+
+}
+
 module.exports = {
   parseTest,
   parseSuite,
   ansiRegExp,
   isValidUrl,
+  fetchSourceCode,
+  fetchSourceCodeFromStackTrace,
+  fetchFilesFromStackTrace,
 };