@testomatio/reporter 1.0.12 → 1.0.14-beta

package/lib/bin/reportXml.js

@@ -23,6 +23,7 @@ program
     }
     let { javaTests, lang } = opts;
     if (opts.envFile) {
+      console.log(APP_PREFIX, 'Loading env file:', opts.envFile);
       debug('Loading env file: %s', opts.envFile) 
       require('dotenv').config({ path: opts.envFile }); // eslint-disable-line
     }

package/lib/client.js

@@ -36,6 +36,7 @@ class Client {
    * @returns {Promise<any>} - resolves to Run id which should be used to update / add test
    */
   createRun() {
+    debug('Creating run...');
     // all pipes disabled, skipping
     if (!this.pipes?.filter(p => p.isEnabled).length) return Promise.resolve();
 
@@ -59,6 +60,7 @@ class Client {
    * @returns {Promise<PipeResult[]>}
    */
   async addTestRun(status, testData, storeArtifacts = []) {
+    debug('Adding test run for test', testData?.test_id || 'unknown test');
     // all pipes disabled, skipping
     if (!this.pipes?.filter(p => p.isEnabled).length) return [];
 
@@ -170,6 +172,7 @@ class Client {
    * @returns {Promise<any>} - A Promise that resolves when finishes the run.
    */
   updateRunStatus(status, isParallel = false) {
+    debug('Updating run status...');
     // all pipes disabled, skipping
     if (!this.pipes?.filter(p => p.isEnabled).length) return Promise.resolve();
 

package/lib/pipe/testomatio.js

@@ -33,7 +33,9 @@ class TestomatioPipe {
     this.sharedRun = !!process.env.TESTOMATIO_SHARED_RUN;
     this.groupTitle = params.groupTitle || process.env.TESTOMATIO_RUNGROUP_TITLE;
     this.env = process.env.TESTOMATIO_ENV;
-    this.axios = axios.create();
+    this.axios = axios.create({
+      timeout: 30000,
+    });
     this.isEnabled = true;
     // do not finish this run (for parallel testing)
     this.proceed = process.env.TESTOMATIO_PROCEED;
@@ -51,6 +53,7 @@ class TestomatioPipe {
    * @returns Promise<void>
    */
   async createRun() {
+    debug('Creating run...');
     if (!this.isEnabled) return;
 
     let buildUrl = process.env.BUILD_URL || process.env.CI_JOB_URL || process.env.CIRCLE_BUILD_URL;
@@ -58,7 +61,7 @@ class TestomatioPipe {
     // GitHub Actions Url
     if (!buildUrl && process.env.GITHUB_RUN_ID) {
       // eslint-disable-next-line max-len
-      buildUrl = `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`
+      buildUrl = `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`;
     }
 
     // Azure DevOps Url
@@ -83,7 +86,7 @@ class TestomatioPipe {
         env: this.env,
         title: this.title,
         shared_run: this.sharedRun,
-      }).filter(([, value]) => !!value)
+      }).filter(([, value]) => !!value),
     );
 
     if (this.runId) {
@@ -110,57 +113,65 @@ class TestomatioPipe {
       console.error(
         APP_PREFIX,
         'Error creating Testomat.io report, please check if your API key is valid. Skipping report',
+        err,
       );
     }
+    debug('Run created');
   }
 
   /**
-   * 
-   * @param testData data 
-   * @returns 
+   *
+   * @param testData data
+   * @returns
    */
   addTest(data) {
+    debug('Adding test...');
     if (!this.isEnabled) return;
     if (!this.runId) return;
     data.api_key = this.apiKey;
     data.create = this.createNewTests;
     const json = JsonCycle.stringify(data);
 
-    return this.axios.post(`${this.url}/api/reporter/${this.runId}/testrun`, json, {
-      maxContentLength: Infinity,
-      maxBodyLength: Infinity,
-      headers: {
-        // Overwrite Axios's automatically set Content-Type
-        'Content-Type': 'application/json',
-      },
-    })
-    .catch((err) => {
-      if (err.response) {
-        if (err.response.status >= 400) {
-          const responseData = err.response.data || { message: '' };
+    return this.axios
+      .post(`${this.url}/api/reporter/${this.runId}/testrun`, json, {
+        maxContentLength: Infinity,
+        maxBodyLength: Infinity,
+        headers: {
+          // Overwrite Axios's automatically set Content-Type
+          'Content-Type': 'application/json',
+        },
+      })
+      .catch(err => {
+        if (err.response) {
+          if (err.response.status >= 400) {
+            const responseData = err.response.data || { message: '' };
+            console.log(
+              APP_PREFIX,
+              chalk.yellow(`Warning: ${responseData.message} (${err.response.status})`),
+              chalk.grey(data?.title || ''),
+            );
+            if (err.response.data.message.includes('could not be matched')) {
+              this.hasUnmatchedTests = true;
+            }
+            return;
+          }
           console.log(
             APP_PREFIX,
-            chalk.yellow(`Warning: ${responseData.message} (${err.response.status})`),
-            chalk.grey(data?.title || ''),
+            chalk.yellow(`Warning: ${data?.title || ''} (${err.response?.status})`),
+            `Report couldn't be processed: ${err?.response?.data?.message}`,
           );
-          if (err.response.data.message.includes('could not be matched')) {
-            this.hasUnmatchedTests = true;
-          }          
-          return;
+        } else {
+          console.log(APP_PREFIX, chalk.blue(data?.title || ''), "Report couldn't be processed", err);
         }
-        // eslint-disable-next-line max-len
-        console.log(APP_PREFIX, chalk.blue(data?.title || ''), `Report couldn't be processed: ${err.response.data.message}`);
-      } else {
-        console.log(APP_PREFIX, chalk.blue(data?.title || ''), "Report couldn't be processed", err);
-      }
-    });
+      });
   }
 
   /**
-   * @param {import('../../types').RunData} params 
-   * @returns 
+   * @param {import('../../types').RunData} params
+   * @returns
    */
   async finishRun(params) {
+    debug('Finishing run...');
     if (!this.isEnabled) return;
 
     const { status, parallel } = params;
@@ -185,7 +196,6 @@ class TestomatioPipe {
         if (this.runPublicUrl) {
           console.log(APP_PREFIX, '🌟 Public URL:', chalk.magenta(this.runPublicUrl));
         }
-        
       }
       if (this.runUrl && this.proceed) {
         const notFinishedMessage = chalk.yellow.bold('Run was not finished because of $TESTOMATIO_PROCEED');
@@ -197,19 +207,30 @@ class TestomatioPipe {
         // eslint-disable-next-line max-len
         console.log(APP_PREFIX, chalk.yellow.bold('⚠️ Some reported tests were not found in Testomat.io project'));
         // eslint-disable-next-line max-len
-        console.log(APP_PREFIX, `If you use Testomat.io as a reporter only, please re-run tests using ${chalk.bold('TESTOMATIO_CREATE=1')}`);
+        console.log(
+          APP_PREFIX,
+          `If you use Testomat.io as a reporter only, please re-run tests using ${chalk.bold('TESTOMATIO_CREATE=1')}`,
+        );
         // eslint-disable-next-line max-len
-        console.log(APP_PREFIX, `But to keep your tests consistent it is recommended to ${chalk.bold('import tests first')}`);
+        console.log(
+          APP_PREFIX,
+          `But to keep your tests consistent it is recommended to ${chalk.bold('import tests first')}`,
+        );
         console.log(APP_PREFIX, 'If tests were imported but still not matched, assign test IDs to your tests.');
         console.log(APP_PREFIX, 'You can do that automatically via command line tools:');
         console.log(APP_PREFIX, chalk.bold('npx check-tests ... --update-ids'), 'See: https://bit.ly/js-update-ids');
         console.log(APP_PREFIX, 'or for Cucumber:');
         // eslint-disable-next-line max-len
-        console.log(APP_PREFIX, chalk.bold('npx check-cucumber ... --update-ids'), 'See: https://bit.ly/bdd-update-ids');
+        console.log(
+          APP_PREFIX,
+          chalk.bold('npx check-cucumber ... --update-ids'),
+          'See: https://bit.ly/bdd-update-ids',
+        );
       }
     } catch (err) {
       console.log(APP_PREFIX, 'Error updating status, skipping...', err);
     }
+    debug('Run finished');
   }
 
   toString() {
@@ -219,17 +240,16 @@ class TestomatioPipe {
 
 module.exports = TestomatioPipe;
 
-
-function setS3Credentials(artifacts) {  
+function setS3Credentials(artifacts) {
   if (!Object.keys(artifacts).length) return;
 
   console.log(APP_PREFIX, 'S3 were credentials obtained from Testomat.io...');
 
-  if (artifacts.ACCESS_KEY_ID) process.env.S3_ACCESS_KEY_ID = artifacts.ACCESS_KEY_ID; 
-  if (artifacts.SECRET_ACCESS_KEY) process.env.S3_SECRET_ACCESS_KEY = artifacts.SECRET_ACCESS_KEY; 
-  if (artifacts.REGION) process.env.S3_REGION = artifacts.REGION; 
-  if (artifacts.BUCKET) process.env.S3_BUCKET = artifacts.BUCKET; 
-  if (artifacts.ENDPOINT) process.env.S3_ENDPOINT = artifacts.ENDPOINT; 
-  if (artifacts.presign) process.env.TESTOMATIO_PRIVATE_ARTIFACTS = '1'; 
+  if (artifacts.ACCESS_KEY_ID) process.env.S3_ACCESS_KEY_ID = artifacts.ACCESS_KEY_ID;
+  if (artifacts.SECRET_ACCESS_KEY) process.env.S3_SECRET_ACCESS_KEY = artifacts.SECRET_ACCESS_KEY;
+  if (artifacts.REGION) process.env.S3_REGION = artifacts.REGION;
+  if (artifacts.BUCKET) process.env.S3_BUCKET = artifacts.BUCKET;
+  if (artifacts.ENDPOINT) process.env.S3_ENDPOINT = artifacts.ENDPOINT;
+  if (artifacts.presign) process.env.TESTOMATIO_PRIVATE_ARTIFACTS = '1';
   resetConfig();
 }

package/lib/util.js

@@ -55,11 +55,21 @@ const isValidUrl = s => {
   }
 };
 
+const fileMatchRegex = /file:(\/\/?[^:\s]+?\.(png|avi|webm|jpg|html|txt))/ig;
+
 const fetchFilesFromStackTrace = (stack = '') => {
-  const files = stack.matchAll(/file:?\/(\/.*?\.(png|avi|webm|jpg|html|txt))/g);
-  return Array.from(files)
-    .map(f => f[1])
-    .filter(f => fs.existsSync(f));
+
+  const files = Array.from(stack.matchAll(fileMatchRegex))
+  .map(f => f[1].trim())
+  .map(f => f.startsWith('//') ? f.substring(1) : f )
+  
+  debug('Found files in stack trace: ', files);
+
+  return files.filter(f => {
+      const isFile = fs.existsSync(f);
+      if (!isFile) debug('File %s could not be found and uploaded as artifact', f);
+      return isFile;
+    });
 };
 
 const fetchSourceCodeFromStackTrace = (stack = '') => {
@@ -96,6 +106,29 @@ const fetchSourceCodeFromStackTrace = (stack = '') => {
     .join('\n');
 };
 
+const TEST_ID_REGEX = /@T([\w\d]{8})/;
+
+const fetchIdFromCode = (code, opts = {}) => {
+  const comments = code.split('\n').map(l => l.trim()).filter(l => {
+    switch (opts.lang) {
+      case 'ruby':
+      case 'python':
+        return l.startsWith('# ');
+      default:
+        return l.startsWith('// ');
+    }
+  });
+
+  return comments.find(c => c.match(TEST_ID_REGEX))?.match(TEST_ID_REGEX)?.[1];
+}
+
+
+const fetchIdFromOutput = (output) => {
+  const lines = output.split('\n').map(l => l.trim()).filter(l => l.startsWith('tid://'));
+
+  return lines.find(c => c.match(TEST_ID_REGEX))?.match(TEST_ID_REGEX)?.[1];
+}
+
 const fetchSourceCode = (contents, opts = {}) => {
   if (!opts.title && !opts.line) return '';
 
@@ -252,6 +285,8 @@ module.exports = {
   isSameTest,
   fetchSourceCode,
   fetchSourceCodeFromStackTrace,
+  fetchIdFromCode,
+  fetchIdFromOutput,
   fetchFilesFromStackTrace,
   fileSystem,
   getCurrentDateTime,

package/lib/xmlReader.js

@@ -4,7 +4,12 @@ const chalk = require('chalk');
 const fs = require("fs");
 const { XMLParser } = require("fast-xml-parser");
 const { APP_PREFIX, STATUS } = require('./constants');
-const { fetchFilesFromStackTrace, fetchSourceCode, fetchSourceCodeFromStackTrace } = require('./util');
+const { fetchFilesFromStackTrace, 
+        fetchIdFromOutput, 
+        fetchSourceCode, 
+        fetchSourceCodeFromStackTrace, 
+        fetchIdFromCode 
+      } = require('./util');
 const upload = require('./fileUploader');
 const pipesFactory = require('./pipe');
 const adapterFactory = require('./junit-adapter');
@@ -25,7 +30,7 @@ const options = {
 const reduceOptions = {};
 
 class XmlReader {
-  
+
   constructor(opts = {}) {
     this.requestParams = {
       apiKey: opts.apiKey || TESTOMATIO,
@@ -39,8 +44,8 @@ class XmlReader {
     if (!this.adapter) throw new Error('XML adapter for this format not found');
 
     this.opts = opts || {};
-    const store = {}
-    this.pipes = pipesFactory(opts, store);
+    this.store = {}
+    this.pipes = pipesFactory(opts, this.store);
 
     this.parser = new XMLParser(options);
     this.tests = []
@@ -49,7 +54,7 @@ class XmlReader {
     this.filesToUpload = {}
 
     this.version = JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'package.json')).toString()).version;
-    console.log(APP_PREFIX, `Testomatio Reporter v${this.version}`);    
+    console.log(APP_PREFIX, `Testomatio Reporter v${this.version}`);
   }
 
   connectAdapter() {
@@ -61,21 +66,21 @@ class XmlReader {
     return this.adapter;
   }
 
-  parse(fileName) {       
+  parse(fileName) {
     const xmlData = fs.readFileSync(path.resolve(fileName));
     const jsonResult = this.parser.parse(xmlData);
     let jsonSuite;
-    
+
     if (jsonResult.testsuites) {
       jsonSuite = jsonResult.testsuites;
     } else if (jsonResult.testsuite) {
       jsonSuite = jsonResult;
     } else if (jsonResult.TestRun) {
-      return this.processTRX(jsonResult);      
+      return this.processTRX(jsonResult);
     } else if (jsonResult['test-run']) {
       return this.processNUnit(jsonResult['test-run']);
     } else if (jsonResult.assemblies) {
-      return this.processXUnit(jsonResult.assemblies);      
+      return this.processXUnit(jsonResult.assemblies);
     } else {
       console.log(jsonResult)
       throw new Error("Format can't be parsed")
@@ -89,9 +94,9 @@ class XmlReader {
 
     reduceOptions.preferClassname = this.stats.language === 'python';
     const resultTests = processTestSuite(testsuite);
-    
+
     const hasFailures = resultTests.filter(t => t.status === 'failed').length > 0;
-    const status = ( failures > 0 || errors > 0 || hasFailures) ? 'failed' : 'passed';
+    const status = (failures > 0 || errors > 0 || hasFailures) ? 'failed' : 'passed';
 
     this.tests = this.tests.concat(resultTests);
 
@@ -124,7 +129,7 @@ class XmlReader {
       skipped_count: parseInt(inconclusive + skipped, 10),
       tests: resultTests,
     };
-  }  
+  }
 
   processTRX(jsonSuite) {
     let defs = jsonSuite?.TestRun?.TestDefinitions?.UnitTest;
@@ -133,10 +138,10 @@ class XmlReader {
     const tests = defs.map(td => {
       const title = td.name.replace(/\(.*?\)/, '').trim();
       let example = td.name.match(/\((.*?)\)/);
-      if (example) example = { ...example[1].split(',')};
+      if (example) example = { ...example[1].split(',') };
       const suite = td.TestMethod.className.split(', ')[0].split('.');
       const suite_title = suite.pop();
-      return { 
+      return {
         title,
         example,
         file: suite.join('/'),
@@ -149,18 +154,18 @@ class XmlReader {
     let result = jsonSuite?.TestRun?.Results?.UnitTestResult;
     if (!Array.isArray(result)) result = [result].filter(d => !!d);
 
-    const results = result.map(td => ({ 
+    const results = result.map(td => ({
       id: td.executionId,
       // seconds are used in junit reports, but ms are used by testomatio
       run_time: parseFloat(td.duration) * 1000,
-      status: td.outcome, 
+      status: td.outcome,
       stack: td.Output.StdOut,
       files: td?.ResultFiles?.ResultFile?.map(rf => rf.path)
     }));
 
-    
+
     results.forEach(r => {
-      const test = tests.find(t => t.id === r.id) || { };
+      const test = tests.find(t => t.id === r.id) || {};
       r.suite_title = test.suite_title;
       r.title = test.title?.trim();
       if (test.code) r.code = test.code;
@@ -175,7 +180,7 @@ class XmlReader {
     });
 
     debug(results);
-    
+
     const counters = jsonSuite?.TestRun?.ResultSummary?.Counters || {};
 
     const failed_count = parseInt(counters.failed, 10) + parseInt(counters.error, 10);
@@ -193,7 +198,7 @@ class XmlReader {
       skipped_count: parseInt(counters.notExecuted, 10),
       failed_count,
       tests: results,
-    };    
+    };
   }
 
   processXUnit(assemblies) {
@@ -227,7 +232,7 @@ class XmlReader {
           let status = STATUS.PASSED;
           if (result === 'Pass') status = STATUS.PASSED;
           if (result === 'Fail') status = STATUS.FAILED;
-          if (result === 'Skip') status = STATUS.SKIPPED;          
+          if (result === 'Skip') status = STATUS.SKIPPED;
 
           const pathParts = type.split('.');
           const suite_title = pathParts[pathParts.length - 1];
@@ -300,12 +305,19 @@ class XmlReader {
           if (file.endsWith('.py')) this.stats.language = 'python';
           if (file.endsWith('.java')) this.stats.language = 'java';
           if (file.endsWith('.rb')) this.stats.language = 'ruby';
-          if (file.endsWith('.js')) this.stats.language  = 'js';
+          if (file.endsWith('.js')) this.stats.language = 'js';
           if (file.endsWith('.ts')) this.stats.language = 'ts';
         }
 
+        if (!fs.existsSync(file)) {
+          debug('Failed to open file with the source code', file)
+          return;
+        }
         const contents = fs.readFileSync(file).toString();
         t.code = fetchSourceCode(contents, { ...t, lang: this.stats.language })
+        if (t.code) debug('Fetched code for test %s', t.title);
+        t.test_id = fetchIdFromCode(t.code, { lang: this.stats.language })
+        if (t.test_id) debug('Fetched test id %s for test %s', t.test_id, t.title);
       } catch (err) {
         debug(err)
       }
@@ -349,8 +361,12 @@ class XmlReader {
       let files = [];
       if (test.files?.length) files = test.files.map(f => path.join(process.cwd(), f))
       files = [...files, ...fetchFilesFromStackTrace(test.stack)];
-      debug('Uploading files', files)
-      test.artifacts = await Promise.all(files.map(f => upload.uploadFileByPath(f, this.runId)));
+
+      if (!files.length) continue;
+
+      const runId = this.runId || this.store.runId || Date.now().toString();
+      test.artifacts = await Promise.all(files.map(f => upload.uploadFileByPath(f, runId)));
+      console.log(APP_PREFIX, `🗄️ Uploaded ${chalk.bold(`${files.length} artifacts`)} for test ${test.title}`);
     }
   }
 
@@ -360,7 +376,7 @@ class XmlReader {
       title: this.requestParams.title,
       env: this.requestParams.env,
       group_title: this.requestParams.group_title,
-    };    
+    };
 
     debug("Run", runParams);
 
@@ -378,9 +394,9 @@ class XmlReader {
     debug(
       'Uploading data',
       {
-      ...this.stats,
-      tests: this.tests,
-    })
+        ...this.stats,
+        tests: this.tests,
+      })
 
     const dataString = {
       ...this.stats,
@@ -402,6 +418,8 @@ function reduceTestCases(prev, item) {
   if (!Array.isArray(testCases)) {
     testCases = [testCases]
   }
+  const suiteOutput = item['system-out'] || item.output || item.log || '';
+  const suiteErr = item['system-err'] || item.output || item.log || '';
   testCases.filter(t => !!t).forEach(testCaseItem => {
     const file = testCaseItem.file || item.filepath || '';
 
@@ -417,8 +435,9 @@ function reduceTestCases(prev, item) {
     if (testCaseItem.error && testCaseItem.error['#text']) stack = testCaseItem.error['#text'];
     if (!message) message = stack.trim().split('\n')[0];
 
-    // prepend system output
-    stack = `${testCaseItem['system-out'] || testCaseItem.output || testCaseItem.log || ''}\n\n${stack}`.trim()
+    // eslint-disable-next-line
+    stack = `${testCaseItem['system-out'] || testCaseItem.output || testCaseItem.log || ''}\n\n${stack}\n\n${suiteOutput}\n\n${suiteErr}`.trim()
+    const testId = fetchIdFromOutput(stack);
 
     let status = STATUS.PASSED.toString();
     if ('failure' in testCaseItem || 'error' in testCaseItem) status = STATUS.FAILED;
@@ -428,6 +447,7 @@ function reduceTestCases(prev, item) {
       create: true,
       file,
       stack,
+      test_id: testId,
       message,
       line: testCaseItem.lineno,
       // seconds are used in junit reports, but ms are used by testomatio

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@testomatio/reporter",
-  "version": "1.0.12",
+  "version": "1.0.14-beta",
   "description": "Testomatio Reporter Client",
   "main": "./lib/reporter.js",
   "typings": "typings/index.d.ts",