@testomatio/reporter 0.8.0-beta.20 → 0.8.0-beta.30

package/Changelog.md

@@ -1,3 +1,15 @@
+# 0.7.6
+
+* Updated to use AWS S3 3.0 SDK for uploading
+
+# 0.7.5
+
+* Fixed reporting skipped tests in mocha
+
+# 0.7.4
+
+* Fixed parsing source code in JUnit files
+
 # 0.7.3
 
 * CodeceptJS: Upload all traces and videos from artifacts

package/README.md

@@ -20,6 +20,19 @@ For testcafe use testcafe reporter:
 npm i testcafe-reporter-testomatio
 ```
 
+For newman use:
+
+```bash
+npm i newman-reporter-testomatio --save-dev
+```
+
+>`newman` and `newman-reporter-testomatio` should be installed in the same directory.
+\
+If you run your tests using globally installed newman (`newman run ...`), intall `newman-reporter-testomatio` globally too (`npm i newman-reporter-testomatio -g`).
+\
+If you use locally installed newman (within the project) (`npx newman run ...`), install `newman-reporter-testomatio` locally (`npm i newman-reporter-testomatio`).
+You can verify installed packages via `npm list` or `npm list -g`.
+
 ## Usage
 
 ### CodeceptJS
@@ -121,6 +134,15 @@ Run the following command from you project folder:
 TESTOMATIO={API_KEY} npx testcafe chrome -r testomatio
 ```
 
+### Newman (Postman)
+
+Run collection and specify `testomatio` as reporter:
+
+`TESTOMATIO={API_KEY} npx newman run {collection_name.json} -r testomatio`
+
+> _`check-test` not supported for newman for now, tests will be created on testomatio by default_
+
+
 ### Cypress
 
 Load the test using using `check-test`.
@@ -212,13 +234,16 @@ TESTOMATIO={API_KEY} npx start-test-run -c 'npx wdio wdio.conf.js'
 Pipes allow you to get report inside different systems (e.g. Pull request comment, database etc)
 For now next pipes available:
 
-### Github
+### GitHub
 This pipe adds comment with run report to GitHub Pull Request.
 
 To use it:
 1. run your tests using github actions in Pull Request
 2. pass `GH_PAT` (GitHub Personal Access Token) as environment variable.
 
+> Last report (comment) will be replaced with the new one.
+To leave previous report pass `GITHUB_KEEP_OUTDATED_REPORTS=1` env variable.
+
 ### GitLab
 This pipe adds comment with run report to GitLab Merge Request.
 
@@ -226,6 +251,9 @@ To use it:
 1. run your tests in Merge Request (pipeline trigger should be `merge_request`)
 2. pass `GITLAB_PAT` (GitLab Personal Access Token) as environment variable.
 
+> Last report (comment) will be replaced with the new one.
+To leave previous report pass `GITLAB_KEEP_OUTDATED_REPORTS=1` env variable.
+
 ## JUnit Reports
 
 > **Notice** JUnit reports are supported since 0.6.0

package/lib/adapter/codecept.js

@@ -67,7 +67,7 @@ function CodeceptReporter(config) {
   });
 
   event.dispatcher.on(event.all.result, async () => {
-    if (videos.length && upload.isArtifactsEnabled) {
+    if (videos.length && upload.isArtifactsEnabled()) {
       console.log(TRConstants.APP_PREFIX, `🎞️  Uploading ${videos.length} videos...`);
 
       const promises = [];

package/lib/adapter/mocha.js

@@ -5,14 +5,19 @@ const TestomatClient = require('../client');
 const TRConstants = require('../constants');
 const { parseTest } = require('../util');
 
-const { EVENT_RUN_BEGIN, EVENT_RUN_END, EVENT_TEST_FAIL, EVENT_TEST_PASS } = Mocha.Runner.constants;
+const { EVENT_RUN_BEGIN, EVENT_RUN_END, EVENT_TEST_FAIL, EVENT_TEST_PASS, EVENT_TEST_PENDING } = Mocha.Runner.constants;
 
 function MochaReporter(runner, opts) {
   Mocha.reporters.Base.call(this, runner);
-  let passes = 0;
-  let failures = 0;
+  let passes = 0; let failures = 0; let skipped = 0; // eslint-disable-line no-unused-vars
 
-  const client = new TestomatClient({ apiKey: opts?.reporterOptions?.apiKey });
+  const apiKey = opts?.reporterOptions?.apiKey || process.env.TESTOMATIO;
+
+  if (!apiKey) {
+    console.log('TESTOMATIO key is empty, ignoring reports');
+    return;
+  }
+  const client = new TestomatClient({ apiKey });
 
   runner.on(EVENT_RUN_BEGIN, () => {
     client.createRun();
@@ -28,6 +33,16 @@ function MochaReporter(runner, opts) {
     });
   });
 
+  runner.on(EVENT_TEST_PENDING, test => {
+    skipped += 1;
+    console.log('skip: %s', test.fullTitle());
+    const testId = parseTest(test.title);
+    client.addTestRun(testId, TRConstants.SKIPPED, {
+      title: test.title,
+      time: test.duration,
+    });
+  });
+
   runner.on(EVENT_TEST_FAIL, (test, err) => {
     failures += 1;
     console.log(chalk.bold.red('✖'), test.fullTitle(), chalk.gray(err.message));

package/lib/adapter/playwright.js

@@ -69,7 +69,7 @@ class TestomatioReporter {
   async onEnd(result) {
     if (!this.client) return;
 
-    if (this.videos.length && upload.isArtifactsEnabled) {
+    if (this.videos.length && upload.isArtifactsEnabled()) {
       console.log(Status.APP_PREFIX, `🎞️  Uploading ${this.videos.length} videos...`);
 
       const promises = [];

package/lib/bin/reportXml.js

@@ -1,8 +1,9 @@
 #!/usr/bin/env node
+
 const program = require("commander");
 const chalk = require("chalk");
 const glob = require('glob');
-
+const debug = require('debug')('@testomatio/reporter:xml-cli');
 const { APP_PREFIX } = require('../constants');
 const XmlReader = require("../xmlReader");
 
@@ -22,7 +23,10 @@ program
       pattern += '.xml';
     }
     let { javaTests, lang } = opts;
-    if (opts.envFile) require('dotenv').config(opts.envFile); // eslint-disable-line
+    if (opts.envFile) {
+      debug('Loading env file: %s', opts.envFile) 
+      require('dotenv').config({ path: opts.envFile }); // eslint-disable-line
+    }
     if (javaTests === true) javaTests = 'src/test/java';
     lang = lang?.toLowerCase();
     const runReader = new XmlReader({ javaTests, lang });

package/lib/client.js

@@ -8,7 +8,6 @@ const pipesFactory = require('./pipe');
 
 const { TESTOMATIO_ENV } = process.env;
 
-
 class TestomatClient {
   /**
    * Create a Testomat client instance
@@ -17,9 +16,11 @@ class TestomatClient {
    */
   constructor(params = {}) {
     this.title = params.title || process.env.TESTOMATIO_TITLE;
-    this.env  = TESTOMATIO_ENV;
+    this.createNewTests = params.createNewTests ?? !!process.env.TESTOMATIO_CREATE;
+    this.proceed = process.env.TESTOMATIO_PROCEED;
+    this.env = TESTOMATIO_ENV;
     this.parallel = params.parallel;
-    const store = {}
+    const store = {};
     this.pipes = pipesFactory(params, store);
     this.queue = Promise.resolve();
     this.totalUploaded = 0;
@@ -44,7 +45,9 @@ class TestomatClient {
 
     global.testomatioArtifacts = [];
 
-    this.queue = this.queue.then(() => Promise.all(this.pipes.map(p => p.createRun(runParams)))).catch(err => console.log(APP_PREFIX, err));;
+    this.queue = this.queue
+      .then(() => Promise.all(this.pipes.map(p => p.createRun(runParams))))
+      .catch(err => console.log(APP_PREFIX, err));
 
     return this.queue;
   }
@@ -100,7 +103,7 @@ class TestomatClient {
       uploadedFiles.push(upload.uploadFileAsBuffer(buffer, fileName, this.runId));
     }
 
-    const artifacts = await Promise.all(uploadedFiles)
+    const artifacts = await Promise.all(uploadedFiles);
 
     global.testomatioArtifacts = [];
 
@@ -112,7 +115,7 @@ class TestomatClient {
       status,
       stack,
       example,
-      title,      
+      title,
       suite_title,
       suite_id,
       test_id,
@@ -121,7 +124,9 @@ class TestomatClient {
       artifacts,
     };
 
-    this.queue = this.queue.then(() => Promise.all(this.pipes.map(p => p.addTest(data)))).catch(err => console.log(APP_PREFIX, err));;
+    this.queue = this.queue
+      .then(() => Promise.all(this.pipes.map(p => p.addTest(data))))
+      .catch(err => console.log(APP_PREFIX, err));
     return this.queue;
   }
 
@@ -139,11 +144,13 @@ class TestomatClient {
     if (status === PASSED) statusEvent = 'pass';
     if (status === FAILED) statusEvent = 'fail';
     if (isParallel) statusEvent += '_parallel';
-    const runParams = { status, statusEvent }
+    const runParams = { status, statusEvent };
 
-    this.queue = this.queue.then(() => Promise.all(this.pipes.map(p => p.finishRun(runParams)))).catch(err => console.log(APP_PREFIX, err));;
+    this.queue = this.queue
+      .then(() => Promise.all(this.pipes.map(p => p.finishRun(runParams))))
+      .catch(err => console.log(APP_PREFIX, err));
 
-    if (upload.isArtifactsEnabled && this.totalUploaded > 0) {
+    if (upload.isArtifactsEnabled() && this.totalUploaded > 0) {
       console.log(
         APP_PREFIX,
         `🗄️  Total ${this.totalUploaded} artifacts ${

package/lib/fileUploader.js

@@ -1,4 +1,6 @@
-const AWS = require('aws-sdk');
+const debug = require('debug')('@testomatio/reporter:upload');
+const { S3 } = require('@aws-sdk/client-s3');
+const { Upload } = require("@aws-sdk/lib-storage");
 const fs = require('fs');
 const path = require('path');
 const chalk = require('chalk');
@@ -6,18 +8,38 @@ const { randomUUID } = require('crypto');
 const memoize = require('lodash.memoize');
 const { APP_PREFIX } = require('./constants');
 
-const {
-  S3_ENDPOINT,
-  S3_REGION,
-  S3_BUCKET,
-  S3_ACCESS_KEY_ID,
-  S3_SECRET_ACCESS_KEY,
-  TESTOMATIO_DISABLE_ARTIFACTS,
-  TESTOMATIO_PRIVATE_ARTIFACTS,
-  S3_FORCE_PATH_STYLE,
-} = process.env;
-
-const isArtifactsEnabled = S3_BUCKET && !TESTOMATIO_DISABLE_ARTIFACTS;
+const keys = [
+  'S3_ENDPOINT',
+  'S3_REGION',
+  'S3_BUCKET',
+  'S3_ACCESS_KEY_ID',
+  'S3_SECRET_ACCESS_KEY',
+  'TESTOMATIO_DISABLE_ARTIFACTS',
+  'TESTOMATIO_PRIVATE_ARTIFACTS',
+  'S3_FORCE_PATH_STYLE',
+];
+
+let config;
+
+function getConfig() {
+  if (config) return config;
+  config = keys.reduce((acc, key) => {
+    acc[key] = process.env[key];
+    return acc;
+  }, {});
+  debug('Config', config);
+  return config;
+}
+
+let isEnabled;
+
+const isArtifactsEnabled = () => {
+  if (isEnabled !== undefined) return isEnabled;
+  const { S3_BUCKET, TESTOMATIO_DISABLE_ARTIFACTS } = getConfig();
+  isEnabled = !!(S3_BUCKET && !TESTOMATIO_DISABLE_ARTIFACTS);
+  debug(`Upload is ${isEnabled ? 'enabled' : 'disabled'}`);
+  return isEnabled;
+};
 
 const _getFileExtBase64 = str => {
   const type = str.charAt(0);
@@ -32,6 +54,25 @@ const _getFileExtBase64 = str => {
   );
 };
 
+const _getS3Config = () => {
+  const { S3_REGION, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_FORCE_PATH_STYLE, S3_ENDPOINT } = getConfig();
+
+  const cfg = {
+    region: S3_REGION,
+    credentials: {
+      accessKeyId: S3_ACCESS_KEY_ID,
+      secretAccessKey: S3_SECRET_ACCESS_KEY,
+      s3ForcePathStyle: S3_FORCE_PATH_STYLE,
+    }
+  };
+
+  if (S3_ENDPOINT) {
+    cfg.endpoint = S3_ENDPOINT;
+  }
+
+  return cfg;
+}
+
 const uploadUsingS3 = async (filePath, runId) => {
   let ContentType;
   let Key;
@@ -42,40 +83,37 @@ const uploadUsingS3 = async (filePath, runId) => {
     Key = filePath.name;
   }
 
-  const config = {
-    region: S3_REGION,
-    accessKeyId: S3_ACCESS_KEY_ID,
-    secretAccessKey: S3_SECRET_ACCESS_KEY,
-    s3ForcePathStyle: S3_FORCE_PATH_STYLE,
-  };
-
-  if (S3_ENDPOINT) {
-    config.endpoint = new AWS.Endpoint(S3_ENDPOINT);
-  }
-
-  const s3 = new AWS.S3(config);
   if (!fs.existsSync(filePath)) {
     console.error(chalk.yellow(`Artifacts file ${filePath} does not exist. Skipping...`));
     return;
   }
+
+  const {  S3_REGION, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_ENDPOINT, TESTOMATIO_PRIVATE_ARTIFACTS, S3_BUCKET } = getConfig();
   
   const file = fs.readFileSync(filePath);
 
   Key = `${runId}/${randomUUID()}-${Key || path.basename(filePath)}`;
   const ACL = TESTOMATIO_PRIVATE_ARTIFACTS ? 'private' : 'public-read';
 
+  const s3 = new S3(_getS3Config());
+
   try {
-    const out = await s3
-      .upload({
+    const out = new Upload({
+      client: s3,
+
+      params: {
         Bucket: S3_BUCKET,
         Key,
         Body: file,
         ContentType,
         ACL,
-      })
-      .promise();
+      }
+    });
 
-    return out.Location;
+    await out.done();
+    debug('Uploaded', out.singleUploadResult.Location)
+
+    return out.singleUploadResult.Location;
   } catch (e) {
     console.log(APP_PREFIX, chalk.bold.red(`Failed uploading '${Key}'. Please check S3 credentials`), {
       accessKeyId: S3_ACCESS_KEY_ID,
@@ -100,35 +138,30 @@ const uploadUsingS3 = async (filePath, runId) => {
 };
 
 const uploadUsingS3AsBuffer = async (buffer, fileName, runId) => {
-  const config = {
-    region: S3_REGION,
-    accessKeyId: S3_ACCESS_KEY_ID,
-    secretAccessKey: S3_SECRET_ACCESS_KEY,
-    s3ForcePathStyle: S3_FORCE_PATH_STYLE,
-  };
-
-  if (S3_ENDPOINT) {
-    config.endpoint = new AWS.Endpoint(S3_ENDPOINT);
-  }
 
-  const s3 = new AWS.S3(config);
+  const {  S3_REGION, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_ENDPOINT, TESTOMATIO_PRIVATE_ARTIFACTS, S3_BUCKET } = getConfig();
 
   const ACL = TESTOMATIO_PRIVATE_ARTIFACTS ? 'private' : 'public-read';
 
   const fileExtension = _getFileExtBase64(buffer.toString('base64'));
   const Key = `${runId}/${fileName}${fileExtension}`;
 
+  const s3 = new S3(_getS3Config());
+
   try {
-    const out = await s3
-      .upload({
+    const out = new Upload({
+      client: s3,
+
+      params: {
         Bucket: S3_BUCKET,
         Key,
         Body: buffer,
         ACL,
-      })
-      .promise();
+      }
+    });
+    await out.done();
 
-    return out.Location;
+    return out.singleUploadResult.Location;
   } catch (e) {
     console.log(APP_PREFIX, chalk.bold.red(`Failed uploading '${Key}'. Please check S3 credentials`), {
       accessKeyId: S3_ACCESS_KEY_ID,
@@ -154,7 +187,7 @@ const uploadUsingS3AsBuffer = async (buffer, fileName, runId) => {
 
 const uploadFileByPath = async (filePath, runId) => {
   try {
-    if (isArtifactsEnabled) {
+    if (isArtifactsEnabled()) {
       return uploadUsingS3(filePath, runId);
     }
   } catch (e) {
@@ -164,7 +197,7 @@ const uploadFileByPath = async (filePath, runId) => {
 
 const uploadFileAsBuffer = async (buffer, fileName, runId) => {
   try {
-    if (isArtifactsEnabled) {
+    if (isArtifactsEnabled()) {
       return uploadUsingS3AsBuffer(buffer, fileName, runId);
     }
   } catch (e) {
@@ -175,5 +208,5 @@ const uploadFileAsBuffer = async (buffer, fileName, runId) => {
 module.exports = {
   uploadFileByPath: memoize(uploadFileByPath),
   uploadFileAsBuffer: memoize(uploadFileAsBuffer),
-  isArtifactsEnabled,
+  isArtifactsEnabled: memoize(isArtifactsEnabled),
 };

package/lib/pipe/csv.js

@@ -1,5 +1,5 @@
-const path = require("path");
-const fs = require("fs");
+const path = require('path');
+const fs = require('fs');
 const csvWriter = require('csv-writer');
 const chalk = require('chalk');
 const merge = require('lodash.merge');
@@ -7,105 +7,113 @@ const { isSameTest, getCurrentDateTime } = require('../util');
 const { CSV_HEADERS } = require('../constants');
 
 class CsvPipe {
+  constructor(params, store = {}) {
+    this.store = store;
+    this.title = params.title || process.env.TESTOMATIO_TITLE;
+    this.isEnabled = true;
+    this.results = [];
 
-    constructor(params, store = {}) {
-        this.store = store;
-        this.title = params.title || process.env.TESTOMATIO_TITLE;
-        this.isEnabled = true;
-        this.results = [];
-
-        this.outputDir = "export";
-        this.csvFilename = process.env.TESTOMATIO_CSV_FILENAME;
-        this.isCsvSave = false;
-
-        if (this.csvFilename !== undefined && this.csvFilename.split(".").length > 0 ) {
-            this.isCsvSave = true;
-
-            if(this.csvFilename.split(".")[0] === "report") {
-                this.outputFile = path.resolve(process.cwd(), this.outputDir, "report.csv");
-            }
-            else {
-                this.outputFile = path.resolve(process.cwd(), this.outputDir, getCurrentDateTime() + "_" + this.csvFilename.split(".")[0] + ".csv");
-            }
-        }        
+    this.outputDir = 'export';
+    this.csvFilename = process.env.TESTOMATIO_CSV_FILENAME;
+    this.isCsvSave = false;
+
+    if (this.csvFilename !== undefined && this.csvFilename.split('.').length > 0) {
+      this.isCsvSave = true;
+
+      if (this.csvFilename.split('.')[0] === 'report') {
+        this.outputFile = path.resolve(process.cwd(), this.outputDir, 'report.csv');
+      } else {
+        this.outputFile = path.resolve(
+          process.cwd(),
+          this.outputDir,
+          `${getCurrentDateTime()}_${this.csvFilename.split('.')[0]}.csv`,
+        );
+      }
     }
+  }
 
-    async createRun() {}
+  async createRun() {
+    // empty
+  }
 
-    updateRun() {}
+  updateRun() {}
 
-    /**
-     * Create a folder that will contain the exported files
-     */
-    checkExportDir() {
-        if (!fs.existsSync(this.outputDir)) {
-            return fs.mkdirSync(this.outputDir);
-        }
-    }
-    /**
-     * Save data to the csv file.
-     * @param {Object} data - data that will be added to the CSV file. Example: [{suite_title: "Suite #1", test: "Test-case-1", message: "Test msg"}]
-     * @param {Object} headers - csv file headers. Example: [{ id: 'suite_title', title: 'Suite_title' }]
-     */
-    async saveToCsv(data, headers) {
-        // First, we check whether the export directory exists: if yes - OK, no - create it.
-        this.checkExportDir();
-
-        console.log(chalk.yellow(`The test results will be added to the csv. It will take some time...`));
-        //Create csv writer object
-        const writer = csvWriter.createObjectCsvWriter({
-            path: this.outputFile,
-            header: headers
-        });
-        //Save csv file based on the current data
-        try {
-            await writer.writeRecords(data);
-            console.log(chalk.green(`Recording completed! You can check the result in file = ${this.outputFile}`));
-        } catch(e) {
-            console.log('Unknown error: ', e);
-        }        
+  /**
+   * Create a folder that will contain the exported files
+   */
+  checkExportDir() {
+    if (!fs.existsSync(this.outputDir)) {
+      return fs.mkdirSync(this.outputDir);
     }
-    /**
-     * Add test data to the result array for saving. As a result of this function, we get a result object to save.
-     * @param {Object} test - object which includes each test entry.
-     */
-    addTest(test) {
-        if (!this.isEnabled) return;
-        
-        const index = this.results.findIndex(t => isSameTest(t, test)); 
-        // update if they were already added
-        if (index >= 0) {
-            this.results[index] = merge(this.results[index], test);
-            return;
-        }
-
-        const {suite_title, title, status, message, stack} = test;
-        
-        this.results.push({
-            suite_title,
-            title,
-            status,
-            message,
-            stack
-        });
+  }
+
+  /**
+   * Save data to the csv file.
+   * @param {Object} data - data that will be added to the CSV file.
+   * Example: [{suite_title: "Suite #1", test: "Test-case-1", message: "Test msg"}]
+   * @param {Object} headers - csv file headers. Example: [{ id: 'suite_title', title: 'Suite_title' }]
+   */
+  async saveToCsv(data, headers) {
+    // First, we check whether the export directory exists: if yes - OK, no - create it.
+    this.checkExportDir();
+
+    console.log(chalk.yellow(`The test results will be added to the csv. It will take some time...`));
+    // Create csv writer object
+    const writer = csvWriter.createObjectCsvWriter({
+      path: this.outputFile,
+      header: headers,
+    });
+    // Save csv file based on the current data
+    try {
+      await writer.writeRecords(data);
+      console.log(chalk.green(`Recording completed! You can check the result in file = ${this.outputFile}`));
+    } catch (e) {
+      console.log('Unknown error: ', e);
     }
-    /**
-     * Save short tests data to the csv file.
-     * @param {Object} runParams - run params.
-     */
-    async finishRun(runParams) {
-        if (!this.isEnabled) return;
-
-        if (runParams.tests) runParams.tests.forEach(t => this.addTest(t));
-        //Save results based on the default headers
-        if(this.isCsvSave) {
-            this.saveToCsv(this.results, CSV_HEADERS);
-        }       
+  }
+
+  /**
+   * Add test data to the result array for saving. As a result of this function, we get a result object to save.
+   * @param {Object} test - object which includes each test entry.
+   */
+  addTest(test) {
+    if (!this.isEnabled) return;
+
+    const index = this.results.findIndex(t => isSameTest(t, test));
+    // update if they were already added
+    if (index >= 0) {
+      this.results[index] = merge(this.results[index], test);
+      return;
     }
 
-    toString() {
-        return 'csv exporter';
+    const { suite_title, title, status, message, stack } = test;
+
+    this.results.push({
+      suite_title,
+      title,
+      status,
+      message,
+      stack,
+    });
+  }
+
+  /**
+   * Save short tests data to the csv file.
+   * @param {Object} runParams - run params.
+   */
+  async finishRun(runParams) {
+    if (!this.isEnabled) return;
+
+    if (runParams.tests) runParams.tests.forEach(t => this.addTest(t));
+    // Save results based on the default headers
+    if (this.isCsvSave) {
+      this.saveToCsv(this.results, CSV_HEADERS);
     }
+  }
+
+  toString() {
+    return 'csv exporter';
+  }
 }
 
-module.exports = CsvPipe;
+module.exports = CsvPipe;