@testomatio/reporter 0.8.0-beta.3 → 0.8.0-beta.30

package/Changelog.md

@@ -1,3 +1,15 @@
+# 0.7.6
+
+* Updated to use AWS S3 3.0 SDK for uploading
+
+# 0.7.5
+
+* Fixed reporting skipped tests in mocha
+
+# 0.7.4
+
+* Fixed parsing source code in JUnit files
+
 # 0.7.3
 
 * CodeceptJS: Upload all traces and videos from artifacts

package/README.md

@@ -20,6 +20,19 @@ For testcafe use testcafe reporter:
 npm i testcafe-reporter-testomatio
 ```
 
+For newman use:
+
+```bash
+npm i newman-reporter-testomatio --save-dev
+```
+
+>`newman` and `newman-reporter-testomatio` should be installed in the same directory.
+\
+If you run your tests using globally installed newman (`newman run ...`), intall `newman-reporter-testomatio` globally too (`npm i newman-reporter-testomatio -g`).
+\
+If you use locally installed newman (within the project) (`npx newman run ...`), install `newman-reporter-testomatio` locally (`npm i newman-reporter-testomatio`).
+You can verify installed packages via `npm list` or `npm list -g`.
+
 ## Usage
 
 ### CodeceptJS
@@ -121,6 +134,15 @@ Run the following command from you project folder:
 TESTOMATIO={API_KEY} npx testcafe chrome -r testomatio
 ```
 
+### Newman (Postman)
+
+Run collection and specify `testomatio` as reporter:
+
+`TESTOMATIO={API_KEY} npx newman run {collection_name.json} -r testomatio`
+
+> _`check-test` not supported for newman for now, tests will be created on testomatio by default_
+
+
 ### Cypress
 
 Load the test using using `check-test`.
@@ -208,6 +230,30 @@ Run the following command from you project folder:
 TESTOMATIO={API_KEY} npx start-test-run -c 'npx wdio wdio.conf.js'
 ```
 
+## Pipes
+Pipes allow you to get report inside different systems (e.g. Pull request comment, database etc)
+For now next pipes available:
+
+### GitHub
+This pipe adds comment with run report to GitHub Pull Request.
+
+To use it:
+1. run your tests using github actions in Pull Request
+2. pass `GH_PAT` (GitHub Personal Access Token) as environment variable.
+
+> Last report (comment) will be replaced with the new one.
+To leave previous report pass `GITHUB_KEEP_OUTDATED_REPORTS=1` env variable.
+
+### GitLab
+This pipe adds comment with run report to GitLab Merge Request.
+
+To use it:
+1. run your tests in Merge Request (pipeline trigger should be `merge_request`)
+2. pass `GITLAB_PAT` (GitLab Personal Access Token) as environment variable.
+
+> Last report (comment) will be replaced with the new one.
+To leave previous report pass `GITLAB_KEEP_OUTDATED_REPORTS=1` env variable.
+
 ## JUnit Reports
 
 > **Notice** JUnit reports are supported since 0.6.0
@@ -408,6 +454,23 @@ Add environments to run by providing `TESTOMATIO_ENV` as comma seperated values:
 TESTOMATIO={API_KEY} TESTOMATIO_ENV="Windows, Chrome" <actual run command>
 ```
 
+### Save test results to .csv file
+Add an env to run by specifying the `TESTOMATIO_CSV_FILENAME` variable.
+
+1) using default report name:
+
+```bash
+TESTOMATIO={API_KEY} TESTOMATIO_CSV_FILENAME="report.csv" <actual run command>
+```
+
+2) using unique report name:
+
+```bash
+TESTOMATIO={API_KEY} TESTOMATIO_CSV_FILENAME="test.csv" <actual run command>
+```
+_It's create a new /export folder with csv files_
+
+
 ### Attaching Test Artifacts
 
 To save a test artifacts (screenshots and videos) of a failed test use S3 storage.

package/lib/adapter/codecept.js

@@ -35,11 +35,6 @@ function CodeceptReporter(config) {
   const testTimeMap = {};
   const { apiKey } = config;
 
-  if (!apiKey) {
-    console.log('TESTOMATIO key is empty, ignoring reports');
-    return;
-  }
-
   const getDuration = test => {
     if (testTimeMap[test.id]) {
       return Date.now() - testTimeMap[test.id];
@@ -72,7 +67,7 @@ function CodeceptReporter(config) {
   });
 
   event.dispatcher.on(event.all.result, async () => {
-    if (videos.length && upload.isArtifactsEnabled) {
+    if (videos.length && upload.isArtifactsEnabled()) {
       console.log(TRConstants.APP_PREFIX, `🎞️  Uploading ${videos.length} videos...`);
 
       const promises = [];

package/lib/adapter/cucumber/current.js

@@ -18,13 +18,7 @@ class CucumberReporter extends Formatter {
     this.failures = [];
     this.cases = [];
 
-    const apiKey = process.env.TESTOMATIO;
-    if (!apiKey || apiKey === '') {
-      console.log(chalk.red('TESTOMATIO key is empty, ignoring reports'));
-      return;
-    }
-
-    this.client = new TestomatClient({ apiKey });
+    this.client = new TestomatClient();
     this.status = TRConstants.PASSED;
 
   }

package/lib/adapter/jasmine.js

@@ -5,14 +5,7 @@ const { PASSED, FAILED } = require('../constants');
 class JasmineReporter {
   constructor(options) {
     this.testTimeMap = {};
-    const { apiKey } = options;
-
-    if (!apiKey) {
-      console.log('TESTOMATIO key is empty, ignoring reports');
-      return;
-    }
-
-    this.client = new TestomatClient({ apiKey });
+    this.client = new TestomatClient({ apiKey: options?.apiKey });
     this.client.createRun();
   }
 

package/lib/adapter/jest.js

@@ -6,14 +6,8 @@ class JestReporter {
   constructor(globalConfig, options) {
     this._globalConfig = globalConfig;
     this._options = options;
-    const { apiKey } = options;
 
-    if (!apiKey) {
-      console.log('TESTOMATIO key is empty, ignoring reports');
-      return;
-    }
-
-    this.client = new TestomatClient({ apiKey });
+    this.client = new TestomatClient({ apiKey: options?.apiKey });
     this.client.createRun();
   }
 

package/lib/adapter/mocha.js

@@ -1,17 +1,23 @@
 // eslint-disable-next-line global-require, import/no-extraneous-dependencies
 const Mocha = require('mocha');
+const chalk = require('chalk');
 const TestomatClient = require('../client');
 const TRConstants = require('../constants');
 const { parseTest } = require('../util');
 
-const { EVENT_RUN_BEGIN, EVENT_RUN_END, EVENT_TEST_FAIL, EVENT_TEST_PASS } = Mocha.Runner.constants;
+const { EVENT_RUN_BEGIN, EVENT_RUN_END, EVENT_TEST_FAIL, EVENT_TEST_PASS, EVENT_TEST_PENDING } = Mocha.Runner.constants;
 
 function MochaReporter(runner, opts) {
   Mocha.reporters.Base.call(this, runner);
-  let passes = 0;
-  let failures = 0;
+  let passes = 0; let failures = 0; let skipped = 0; // eslint-disable-line no-unused-vars
 
-  const client = new TestomatClient({ apiKey: opts?.reporterOptions?.apiKey });
+  const apiKey = opts?.reporterOptions?.apiKey || process.env.TESTOMATIO;
+
+  if (!apiKey) {
+    console.log('TESTOMATIO key is empty, ignoring reports');
+    return;
+  }
+  const client = new TestomatClient({ apiKey });
 
   runner.on(EVENT_RUN_BEGIN, () => {
     client.createRun();
@@ -19,7 +25,7 @@ function MochaReporter(runner, opts) {
 
   runner.on(EVENT_TEST_PASS, test => {
     passes += 1;
-    console.log('pass: %s', test.fullTitle());
+    console.log(chalk.bold.green('✔'), test.fullTitle());
     const testId = parseTest(test.title);
     client.addTestRun(testId, TRConstants.PASSED, {
       title: test.title,
@@ -27,9 +33,19 @@ function MochaReporter(runner, opts) {
     });
   });
 
+  runner.on(EVENT_TEST_PENDING, test => {
+    skipped += 1;
+    console.log('skip: %s', test.fullTitle());
+    const testId = parseTest(test.title);
+    client.addTestRun(testId, TRConstants.SKIPPED, {
+      title: test.title,
+      time: test.duration,
+    });
+  });
+
   runner.on(EVENT_TEST_FAIL, (test, err) => {
     failures += 1;
-    console.log('fail: %s -- error: %s', test.fullTitle(), err.message);
+    console.log(chalk.bold.red('✖'), test.fullTitle(), chalk.gray(err.message));
     const testId = parseTest(test.title);
     client.addTestRun(testId, TRConstants.FAILED, {
       error: err,
@@ -39,8 +55,8 @@ function MochaReporter(runner, opts) {
   });
 
   runner.on(EVENT_RUN_END, () => {
-    console.log('end: %d/%d', passes, passes + failures);
     const status = failures === 0 ? TRConstants.PASSED : TRConstants.FAILED;
+    console.log(chalk.bold(status), `${passes} passed, ${failures} failed`);
     client.updateRunStatus(status);
   });
 }

package/lib/adapter/playwright.js

@@ -10,13 +10,7 @@ const { parseTest } = require('../util');
 
 class TestomatioReporter {
   constructor(config = {}) {
-    const { apiKey } = config;
-
-    if (!apiKey) {
-      console.log('API Key is not provided. Testomat.io report is disabled');
-    } else {
-      this.client = new TestomatioClient({ apiKey });
-    }
+    this.client = new TestomatioClient({ apiKey: config?.apiKey });
 
     this.videos = [];
   }
@@ -75,7 +69,7 @@ class TestomatioReporter {
   async onEnd(result) {
     if (!this.client) return;
 
-    if (this.videos.length && upload.isArtifactsEnabled) {
+    if (this.videos.length && upload.isArtifactsEnabled()) {
       console.log(Status.APP_PREFIX, `🎞️  Uploading ${this.videos.length} videos...`);
 
       const promises = [];

package/lib/adapter/webdriver.js

@@ -7,10 +7,7 @@ class WebdriverReporter extends WDIOReporter {
   constructor(options) {
     super(options);
 
-    const { apiKey } = options;
-    if (!apiKey) return;
-
-    this.client = new TestomatClient({ apiKey });
+    this.client = new TestomatClient({ apiKey: options?.apiKey });
     options = Object.assign(options, { stdout: true });
 
     this._addTestPromises = [];

package/lib/bin/reportXml.js

@@ -1,8 +1,9 @@
 #!/usr/bin/env node
+
 const program = require("commander");
 const chalk = require("chalk");
 const glob = require('glob');
-
+const debug = require('debug')('@testomatio/reporter:xml-cli');
 const { APP_PREFIX } = require('../constants');
 const XmlReader = require("../xmlReader");
 
@@ -22,7 +23,10 @@ program
       pattern += '.xml';
     }
     let { javaTests, lang } = opts;
-    if (opts.envFile) require('dotenv').config(opts.envFile); // eslint-disable-line
+    if (opts.envFile) {
+      debug('Loading env file: %s', opts.envFile) 
+      require('dotenv').config({ path: opts.envFile }); // eslint-disable-line
+    }
     if (javaTests === true) javaTests = 'src/test/java';
     lang = lang?.toLowerCase();
     const runReader = new XmlReader({ javaTests, lang });

package/lib/client.js

@@ -1,4 +1,3 @@
-const axios = require('axios');
 const createCallsiteRecord = require('callsite-record');
 const { sep, join } = require('path');
 const fs = require('fs');
@@ -6,8 +5,8 @@ const chalk = require('chalk');
 const upload = require('./fileUploader');
 const { PASSED, FAILED, FINISHED, APP_PREFIX } = require('./constants');
 const pipesFactory = require('./pipe');
-const { TESTOMATIO_ENV } = process.env;
 
+const { TESTOMATIO_ENV } = process.env;
 
 class TestomatClient {
   /**
@@ -15,17 +14,18 @@ class TestomatClient {
    *
    * @param {*} params
    */
-  constructor(params) {
-    this.apiKey = params.apiKey || process.env.TESTOMATIO;
+  constructor(params = {}) {
     this.title = params.title || process.env.TESTOMATIO_TITLE;
-    this.env  = TESTOMATIO_ENV;
+    this.createNewTests = params.createNewTests ?? !!process.env.TESTOMATIO_CREATE;
+    this.proceed = process.env.TESTOMATIO_PROCEED;
+    this.env = TESTOMATIO_ENV;
     this.parallel = params.parallel;
-    const store = {}
+    const store = {};
     this.pipes = pipesFactory(params, store);
     this.queue = Promise.resolve();
-    this.axios = axios.create();
     this.totalUploaded = 0;
     this.version = JSON.parse(fs.readFileSync(join(__dirname, '..', 'package.json')).toString()).version;
+    console.log(APP_PREFIX, `Testomatio Reporter v${this.version}`);
   }
 
   /**
@@ -34,7 +34,9 @@ class TestomatClient {
    * @returns {Promise} - resolves to Run id which should be used to update / add test
    */
   createRun() {
-    const { runId } = process.env;
+    // all pipes disabled, skipping
+    if (!this.pipes.filter(p => p.isEnabled).length) return;
+
     const runParams = {
       title: this.title,
       parallel: this.parallel,
@@ -43,7 +45,9 @@ class TestomatClient {
 
     global.testomatioArtifacts = [];
 
-    this.queue = this.queue.then(() => Promise.all(this.pipes.map(p => p.createRun(runParams))));
+    this.queue = this.queue
+      .then(() => Promise.all(this.pipes.map(p => p.createRun(runParams))))
+      .catch(err => console.log(APP_PREFIX, err));
 
     return this.queue;
   }
@@ -54,6 +58,9 @@ class TestomatClient {
    * @returns {Promise}
    */
   async addTestRun(testId, status, testData = {}) {
+    // all pipes disabled, skipping
+    if (!this.pipes.filter(p => p.isEnabled).length) return;
+
     const {
       error = '',
       time = '',
@@ -96,7 +103,7 @@ class TestomatClient {
       uploadedFiles.push(upload.uploadFileAsBuffer(buffer, fileName, this.runId));
     }
 
-    const artifacts = await Promise.all(uploadedFiles)
+    const artifacts = await Promise.all(uploadedFiles);
 
     global.testomatioArtifacts = [];
 
@@ -108,7 +115,7 @@ class TestomatClient {
       status,
       stack,
       example,
-      title,      
+      title,
       suite_title,
       suite_id,
       test_id,
@@ -117,7 +124,9 @@ class TestomatClient {
       artifacts,
     };
 
-    this.queue = this.queue.then(() => Promise.all(this.pipes.map(p => p.addTest(data))));
+    this.queue = this.queue
+      .then(() => Promise.all(this.pipes.map(p => p.addTest(data))))
+      .catch(err => console.log(APP_PREFIX, err));
     return this.queue;
   }
 
@@ -127,16 +136,21 @@ class TestomatClient {
    * @returns {Promise}
    */
   updateRunStatus(status, isParallel) {
+    // all pipes disabled, skipping
+    if (!this.pipes.filter(p => p.isEnabled).length) return;
+
     let statusEvent;
     if (status === FINISHED) statusEvent = 'finish';
     if (status === PASSED) statusEvent = 'pass';
     if (status === FAILED) statusEvent = 'fail';
     if (isParallel) statusEvent += '_parallel';
-    const runParams = { status, statusEvent }
+    const runParams = { status, statusEvent };
 
-    this.queue = this.queue.then(() => Promise.all(this.pipes.map(p => p.finishRun(runParams))));
+    this.queue = this.queue
+      .then(() => Promise.all(this.pipes.map(p => p.finishRun(runParams))))
+      .catch(err => console.log(APP_PREFIX, err));
 
-    if (upload.isArtifactsEnabled && this.totalUploaded > 0) {
+    if (upload.isArtifactsEnabled() && this.totalUploaded > 0) {
       console.log(
         APP_PREFIX,
         `🗄️  Total ${this.totalUploaded} artifacts ${

package/lib/constants.js

@@ -2,10 +2,19 @@ const chalk = require('chalk');
 
 const APP_PREFIX = chalk.gray('[TESTOMATIO]');
 
+const CSV_HEADERS = [
+  { id: 'suite_title', title: 'Suite_title' },
+  { id: 'title', title: 'Title' },
+  { id: 'status', title: 'Status' },
+  { id: 'message', title: 'Message' },
+  { id: 'stack', title: 'Stack' },
+];
+
 module.exports = Object.freeze({
   PASSED: 'passed',
   FAILED: 'failed',
   SKIPPED: 'skipped',
   FINISHED: 'finished',
   APP_PREFIX,
+  CSV_HEADERS
 });

package/lib/fileUploader.js

@@ -1,4 +1,6 @@
-const AWS = require('aws-sdk');
+const debug = require('debug')('@testomatio/reporter:upload');
+const { S3 } = require('@aws-sdk/client-s3');
+const { Upload } = require("@aws-sdk/lib-storage");
 const fs = require('fs');
 const path = require('path');
 const chalk = require('chalk');
@@ -6,18 +8,38 @@ const { randomUUID } = require('crypto');
 const memoize = require('lodash.memoize');
 const { APP_PREFIX } = require('./constants');
 
-const {
-  S3_ENDPOINT,
-  S3_REGION,
-  S3_BUCKET,
-  S3_ACCESS_KEY_ID,
-  S3_SECRET_ACCESS_KEY,
-  TESTOMATIO_DISABLE_ARTIFACTS,
-  TESTOMATIO_PRIVATE_ARTIFACTS,
-  S3_FORCE_PATH_STYLE,
-} = process.env;
-
-const isArtifactsEnabled = S3_BUCKET && !TESTOMATIO_DISABLE_ARTIFACTS;
+const keys = [
+  'S3_ENDPOINT',
+  'S3_REGION',
+  'S3_BUCKET',
+  'S3_ACCESS_KEY_ID',
+  'S3_SECRET_ACCESS_KEY',
+  'TESTOMATIO_DISABLE_ARTIFACTS',
+  'TESTOMATIO_PRIVATE_ARTIFACTS',
+  'S3_FORCE_PATH_STYLE',
+];
+
+let config;
+
+function getConfig() {
+  if (config) return config;
+  config = keys.reduce((acc, key) => {
+    acc[key] = process.env[key];
+    return acc;
+  }, {});
+  debug('Config', config);
+  return config;
+}
+
+let isEnabled;
+
+const isArtifactsEnabled = () => {
+  if (isEnabled !== undefined) return isEnabled;
+  const { S3_BUCKET, TESTOMATIO_DISABLE_ARTIFACTS } = getConfig();
+  isEnabled = !!(S3_BUCKET && !TESTOMATIO_DISABLE_ARTIFACTS);
+  debug(`Upload is ${isEnabled ? 'enabled' : 'disabled'}`);
+  return isEnabled;
+};
 
 const _getFileExtBase64 = str => {
   const type = str.charAt(0);
@@ -32,6 +54,25 @@ const _getFileExtBase64 = str => {
   );
 };
 
+const _getS3Config = () => {
+  const { S3_REGION, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_FORCE_PATH_STYLE, S3_ENDPOINT } = getConfig();
+
+  const cfg = {
+    region: S3_REGION,
+    credentials: {
+      accessKeyId: S3_ACCESS_KEY_ID,
+      secretAccessKey: S3_SECRET_ACCESS_KEY,
+      s3ForcePathStyle: S3_FORCE_PATH_STYLE,
+    }
+  };
+
+  if (S3_ENDPOINT) {
+    cfg.endpoint = S3_ENDPOINT;
+  }
+
+  return cfg;
+}
+
 const uploadUsingS3 = async (filePath, runId) => {
   let ContentType;
   let Key;
@@ -42,40 +83,37 @@ const uploadUsingS3 = async (filePath, runId) => {
     Key = filePath.name;
   }
 
-  const config = {
-    region: S3_REGION,
-    accessKeyId: S3_ACCESS_KEY_ID,
-    secretAccessKey: S3_SECRET_ACCESS_KEY,
-    s3ForcePathStyle: S3_FORCE_PATH_STYLE,
-  };
-
-  if (S3_ENDPOINT) {
-    config.endpoint = new AWS.Endpoint(S3_ENDPOINT);
-  }
-
-  const s3 = new AWS.S3(config);
   if (!fs.existsSync(filePath)) {
     console.error(chalk.yellow(`Artifacts file ${filePath} does not exist. Skipping...`));
     return;
   }
+
+  const {  S3_REGION, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_ENDPOINT, TESTOMATIO_PRIVATE_ARTIFACTS, S3_BUCKET } = getConfig();
   
   const file = fs.readFileSync(filePath);
 
   Key = `${runId}/${randomUUID()}-${Key || path.basename(filePath)}`;
   const ACL = TESTOMATIO_PRIVATE_ARTIFACTS ? 'private' : 'public-read';
 
+  const s3 = new S3(_getS3Config());
+
   try {
-    const out = await s3
-      .upload({
+    const out = new Upload({
+      client: s3,
+
+      params: {
         Bucket: S3_BUCKET,
         Key,
         Body: file,
         ContentType,
         ACL,
-      })
-      .promise();
+      }
+    });
 
-    return out.Location;
+    await out.done();
+    debug('Uploaded', out.singleUploadResult.Location)
+
+    return out.singleUploadResult.Location;
   } catch (e) {
     console.log(APP_PREFIX, chalk.bold.red(`Failed uploading '${Key}'. Please check S3 credentials`), {
       accessKeyId: S3_ACCESS_KEY_ID,
@@ -100,35 +138,30 @@ const uploadUsingS3 = async (filePath, runId) => {
 };
 
 const uploadUsingS3AsBuffer = async (buffer, fileName, runId) => {
-  const config = {
-    region: S3_REGION,
-    accessKeyId: S3_ACCESS_KEY_ID,
-    secretAccessKey: S3_SECRET_ACCESS_KEY,
-    s3ForcePathStyle: S3_FORCE_PATH_STYLE,
-  };
-
-  if (S3_ENDPOINT) {
-    config.endpoint = new AWS.Endpoint(S3_ENDPOINT);
-  }
 
-  const s3 = new AWS.S3(config);
+  const {  S3_REGION, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_ENDPOINT, TESTOMATIO_PRIVATE_ARTIFACTS, S3_BUCKET } = getConfig();
 
   const ACL = TESTOMATIO_PRIVATE_ARTIFACTS ? 'private' : 'public-read';
 
   const fileExtension = _getFileExtBase64(buffer.toString('base64'));
   const Key = `${runId}/${fileName}${fileExtension}`;
 
+  const s3 = new S3(_getS3Config());
+
   try {
-    const out = await s3
-      .upload({
+    const out = new Upload({
+      client: s3,
+
+      params: {
         Bucket: S3_BUCKET,
         Key,
         Body: buffer,
         ACL,
-      })
-      .promise();
+      }
+    });
+    await out.done();
 
-    return out.Location;
+    return out.singleUploadResult.Location;
   } catch (e) {
     console.log(APP_PREFIX, chalk.bold.red(`Failed uploading '${Key}'. Please check S3 credentials`), {
       accessKeyId: S3_ACCESS_KEY_ID,
@@ -154,7 +187,7 @@ const uploadUsingS3AsBuffer = async (buffer, fileName, runId) => {
 
 const uploadFileByPath = async (filePath, runId) => {
   try {
-    if (isArtifactsEnabled) {
+    if (isArtifactsEnabled()) {
       return uploadUsingS3(filePath, runId);
     }
   } catch (e) {
@@ -164,7 +197,7 @@ const uploadFileByPath = async (filePath, runId) => {
 
 const uploadFileAsBuffer = async (buffer, fileName, runId) => {
   try {
-    if (isArtifactsEnabled) {
+    if (isArtifactsEnabled()) {
       return uploadUsingS3AsBuffer(buffer, fileName, runId);
     }
   } catch (e) {
@@ -175,5 +208,5 @@ const uploadFileAsBuffer = async (buffer, fileName, runId) => {
 module.exports = {
   uploadFileByPath: memoize(uploadFileByPath),
   uploadFileAsBuffer: memoize(uploadFileAsBuffer),
-  isArtifactsEnabled,
+  isArtifactsEnabled: memoize(isArtifactsEnabled),
 };