@testomatio/reporter 0.8.0-beta.3 → 0.8.0-beta.31

package/Changelog.md

@@ -1,3 +1,15 @@
+# 0.7.6
+
+* Updated to use AWS S3 3.0 SDK for uploading
+
+# 0.7.5
+
+* Fixed reporting skipped tests in mocha
+
+# 0.7.4
+
+* Fixed parsing source code in JUnit files
+
 # 0.7.3
 
 * CodeceptJS: Upload all traces and videos from artifacts

package/README.md

@@ -20,11 +20,24 @@ For testcafe use testcafe reporter:
 npm i testcafe-reporter-testomatio
 ```
 
+For newman use:
+
+```bash
+npm i newman-reporter-testomatio --save-dev
+```
+
+>`newman` and `newman-reporter-testomatio` should be installed in the same directory.
+\
+If you run your tests using globally installed newman (`newman run ...`), intall `newman-reporter-testomatio` globally too (`npm i newman-reporter-testomatio -g`).
+\
+If you use locally installed newman (within the project) (`npx newman run ...`), install `newman-reporter-testomatio` locally (`npm i newman-reporter-testomatio`).
+You can verify installed packages via `npm list` or `npm list -g`.
+
 ## Usage
 
 ### CodeceptJS
 
-Make sure you load all your tests using [check-test](https://github.com/testomatio/check-tests#cli).
+Make sure you load all your tests using [check-tests](https://github.com/testomatio/check-tests#cli).
 
 Add plugin to [codecept conf](https://github.com/testomatio/reporter/blob/master/example/codecept/codecept.conf.js#L23):
 
@@ -81,7 +94,7 @@ TESTOMATIO={API_KEY} npx playwright test
 
 ### Mocha
 
-Load the test using using `check-test` if not done already. Get the test id from testomat account and add it to your mocha test like in this [example](https://github.com/testomatio/reporter/blob/master/example/mocha/test/index.test.js#L4).
+Load the test using using `check-tests` if not done already. Get the test id from testomat account and add it to your mocha test like in this [example](https://github.com/testomatio/reporter/blob/master/example/mocha/test/index.test.js#L4).
 
 Run the following command from you project folder:
 
@@ -91,7 +104,7 @@ mocha --reporter ./node_modules/testomat-reporter/lib/adapter/mocha.js --reporte
 
 ### Jest
 
-Load the test using using `check-test`. Add the test id to your tests like in this [example](https://github.com/testomatio/reporter/blob/master/example/jest/index.test.js#L1).
+Load the test using using `check-tests`. Add the test id to your tests like in this [example](https://github.com/testomatio/reporter/blob/master/example/jest/index.test.js#L1).
 
 Add the following line to [jest.config.js](https://github.com/testomatio/reporter/blob/master/example/jest/jest.config.js#L100):
 
@@ -113,7 +126,7 @@ TESTOMATIO={API_KEY} ./node_modules/.bin/cucumber-js --format ./node_modules/@te
 
 ### TestCafe
 
-Load the test using using `check-test`.
+Load the test using using `check-tests`.
 
 Run the following command from you project folder:
 
@@ -121,9 +134,20 @@ Run the following command from you project folder:
 TESTOMATIO={API_KEY} npx testcafe chrome -r testomatio
 ```
 
+### Newman (Postman)
+
+Run collection and specify `testomatio` as reporter:
+
+```bash
+TESTOMATIO={API_KEY} npx newman run {collection_name.json} -r testomatio
+```
+
+> _`check-tests` not supported for newman for now, tests will be created on testomatio by default_
+
+
 ### Cypress
 
-Load the test using using `check-test`.
+Load the test using using `check-tests`.
 
 Register our `cypress-plugin` in `cypress/plugins/index.js`:
 
@@ -151,7 +175,7 @@ TESTOMATIO={API_KEY} npx cypress run
 
 ### Protractor
 
-Load the test using using `check-test`.
+Load the test using using `check-tests`.
 
 Add the following lines to [conf.js](https://github.com/angular/protractor/blob/5.4.1/example/conf.js):
 
@@ -173,7 +197,7 @@ TESTOMATIO={API_KEY} npx start-test-run -c 'npx protractor conf.js'
 
 ### WebdriverIO
 
-Load the test using using `check-test`.
+Load the test using using `check-tests`.
 
 Add the following lines to [wdio.conf.js](https://webdriver.io/docs/configurationfile/):
 
@@ -208,6 +232,30 @@ Run the following command from you project folder:
 TESTOMATIO={API_KEY} npx start-test-run -c 'npx wdio wdio.conf.js'
 ```
 
+## Pipes
+Pipes allow you to get report inside different systems (e.g. Pull request comment, database etc)
+For now next pipes available:
+
+### GitHub
+This pipe adds comment with run report to GitHub Pull Request.
+
+To use it:
+1. run your tests using github actions in Pull Request
+2. pass `GH_PAT` (GitHub Personal Access Token) as environment variable.
+
+> Last report (comment) will be replaced with the new one.
+To leave previous report pass `GITHUB_KEEP_OUTDATED_REPORTS=1` env variable.
+
+### GitLab
+This pipe adds comment with run report to GitLab Merge Request.
+
+To use it:
+1. run your tests in Merge Request (pipeline trigger should be `merge_request`)
+2. pass `GITLAB_PAT` (GitLab Personal Access Token) as environment variable.
+
+> Last report (comment) will be replaced with the new one.
+To leave previous report pass `GITLAB_KEEP_OUTDATED_REPORTS=1` env variable.
+
 ## JUnit Reports
 
 > **Notice** JUnit reports are supported since 0.6.0
@@ -408,6 +456,23 @@ Add environments to run by providing `TESTOMATIO_ENV` as comma seperated values:
 TESTOMATIO={API_KEY} TESTOMATIO_ENV="Windows, Chrome" <actual run command>
 ```
 
+### Save test results to .csv file
+Add an env to run by specifying the `TESTOMATIO_CSV_FILENAME` variable.
+
+1) using default report name:
+
+```bash
+TESTOMATIO={API_KEY} TESTOMATIO_CSV_FILENAME="report.csv" <actual run command>
+```
+
+2) using unique report name:
+
+```bash
+TESTOMATIO={API_KEY} TESTOMATIO_CSV_FILENAME="test.csv" <actual run command>
+```
+_It's create a new /export folder with csv files_
+
+
 ### Attaching Test Artifacts
 
 To save a test artifacts (screenshots and videos) of a failed test use S3 storage.
@@ -520,3 +585,8 @@ If you want to finish a run started by `--launch` use `--finish` option. `TESTOM
 ```bash
 TESTOMATIO={API_KEY} TESTOMATIO_RUN={RUN_ID} npx start-test-run --finish
 ```
+
+### Debug logs
+Pass `DEBUG` variable with module name e.g. `DEBUG=@testomatio/reporter:pipe:github`.
+(Module name could be taken directly from the required module code).
+To log all debug info pass `DEBUG=*`.

package/lib/ArtifactStorage.js

@@ -0,0 +1,132 @@
+const debug = require('debug')('@testomatio/reporter:storage');
+const { join, resolve } = require('path');
+const fs = require('fs');
+const os = require('os');
+const uuid = require('uuid');
+
+const { TESTOMAT_ARTIFACT_SUFFIX } = require('./constants');
+const { specificTestInfo } = require('./util');
+
+class ArtifactStorage {
+    _tmpPrefix = "tsmt_reporter";
+     
+    constructor(params) { 
+        this.isFile = false;
+        this.isMemory = true;
+        this.storage = params?.toFile;
+
+        if (this.storage) {
+            this.isFile = true;
+            this.isMemory = false;
+            this.tmpDirFullpath = this.createTestomatTmpDir(this._tmpPrefix);
+            debug('SAVE to tmp folder mode enabled!');
+        }
+    }
+
+    static async storeToFile(tmpDirName, artifact, testSuffix = "test") {
+        const suffix = TESTOMAT_ARTIFACT_SUFFIX + uuid.v4() + testSuffix;
+        const dirpath = join(os.tmpdir(), tmpDirName);
+        const filepath = resolve(dirpath, suffix + ".json");
+
+        return await fs.promises.appendFile(filepath, JSON.stringify(artifact));
+    }
+
+    static async artifact(artifact, context) {
+        //TODO: mode for different pre-hooks. As variant - "all-tests" || "only_fail"
+        // const mode = process.env.TESTOMAT_ARTIFACTS || "only_fail";
+
+        if (Array.isArray(global.testomatioArtifacts)) {
+            debug("Saving artifacts to global storage");
+
+            global.testomatioArtifacts.push(artifact);
+        }
+
+        if (global?.testomatioArtifacts === undefined) {
+            const tmpDirNames = ArtifactStorage.tmpTestomatDirNames();
+
+            const testSuffix = specificTestInfo(context.test);
+
+            if (Array.isArray(tmpDirNames) && tmpDirNames.length) {
+                debug("Saving artifacts to memory tmp folder");
+
+                return await ArtifactStorage.storeToFile(tmpDirNames[tmpDirNames.length - 1], artifact, testSuffix);
+            }
+        }
+    }
+
+    async artifactByTestName(test) {
+        const list = [];
+
+        if (this.isFile && this.tmpDirFullpath) {
+            const files = fs.readdirSync(this.tmpDirFullpath);
+
+            for (const file of files) {
+                if (file.includes(test)) {
+                    const buff = await fs.promises.readFile(this.tmpDirFullpath + "/" + file);
+                    
+                    list.push(JSON.parse(buff.toString()));
+                }                
+            }
+        }
+
+        return list;
+    }
+
+    async tmpContents() {
+        const contents = [];
+
+        if (this.isFile && this.tmpDirFullpath) {            
+            const files = fs.readdirSync(this.tmpDirFullpath);
+
+            for (const file of files) {
+                const buff = await fs.promises.readFile(this.tmpDirFullpath + "/" + file);
+                const content = buff.toString();
+
+                contents.push(content);
+            }
+        }
+
+        return contents;
+    }
+
+    createTestomatTmpDir(clientPrefix) {
+        return fs.mkdtempSync(join(os.tmpdir(), clientPrefix));
+    }
+
+    clearTmpDirByName(name) {
+        const tmpDirPath = join(os.tmpdir(), name);
+
+        if (fs.existsSync(tmpDirPath)) {
+            fs.rmSync(tmpDirPath, { recursive: true });
+            debug(` Testomat tmpDir = ${tmpDirPath} was deleted successfully!`);
+            
+            return;
+        }
+    }
+
+    static tmpTestomatDirNames() {
+        const subname = this._tmpPrefix || "tsmt_reporter";
+
+        return fs.readdirSync(os.tmpdir(), { withFileTypes: true })
+                        .filter((item) => item.isDirectory())
+                        .map((item) => item.name)
+                        .filter((name) => name.includes(subname));
+    }
+
+    cleanup() {
+        if (this.isFile && this.tmpDirFullpath) {
+            const tmpDirNames = ArtifactStorage.tmpTestomatDirNames();
+
+            if (Array.isArray(tmpDirNames) && tmpDirNames.length) {
+                for (const name of tmpDirNames) {
+                    this.clearTmpDirByName(name);
+                }
+            }
+        }
+        else {
+            debug("The tmp folder has not been created! Nothing to delete");
+        }
+    }
+}
+
+module.exports = ArtifactStorage;

package/lib/adapter/codecept.js

@@ -35,11 +35,6 @@ function CodeceptReporter(config) {
   const testTimeMap = {};
   const { apiKey } = config;
 
-  if (!apiKey) {
-    console.log('TESTOMATIO key is empty, ignoring reports');
-    return;
-  }
-
   const getDuration = test => {
     if (testTimeMap[test.id]) {
       return Date.now() - testTimeMap[test.id];
@@ -72,7 +67,7 @@ function CodeceptReporter(config) {
   });
 
   event.dispatcher.on(event.all.result, async () => {
-    if (videos.length && upload.isArtifactsEnabled) {
+    if (videos.length && upload.isArtifactsEnabled()) {
       console.log(TRConstants.APP_PREFIX, `🎞️  Uploading ${videos.length} videos...`);
 
       const promises = [];

package/lib/adapter/cucumber/current.js

@@ -18,13 +18,7 @@ class CucumberReporter extends Formatter {
     this.failures = [];
     this.cases = [];
 
-    const apiKey = process.env.TESTOMATIO;
-    if (!apiKey || apiKey === '') {
-      console.log(chalk.red('TESTOMATIO key is empty, ignoring reports'));
-      return;
-    }
-
-    this.client = new TestomatClient({ apiKey });
+    this.client = new TestomatClient();
     this.status = TRConstants.PASSED;
 
   }

package/lib/adapter/jasmine.js

@@ -5,14 +5,7 @@ const { PASSED, FAILED } = require('../constants');
 class JasmineReporter {
   constructor(options) {
     this.testTimeMap = {};
-    const { apiKey } = options;
-
-    if (!apiKey) {
-      console.log('TESTOMATIO key is empty, ignoring reports');
-      return;
-    }
-
-    this.client = new TestomatClient({ apiKey });
+    this.client = new TestomatClient({ apiKey: options?.apiKey });
     this.client.createRun();
   }
 

package/lib/adapter/jest.js

@@ -6,14 +6,8 @@ class JestReporter {
   constructor(globalConfig, options) {
     this._globalConfig = globalConfig;
     this._options = options;
-    const { apiKey } = options;
 
-    if (!apiKey) {
-      console.log('TESTOMATIO key is empty, ignoring reports');
-      return;
-    }
-
-    this.client = new TestomatClient({ apiKey });
+    this.client = new TestomatClient({ apiKey: options?.apiKey });
     this.client.createRun();
   }
 

package/lib/adapter/mocha.js

@@ -1,47 +1,97 @@
 // eslint-disable-next-line global-require, import/no-extraneous-dependencies
+const debug = require('debug')('@testomatio/reporter:adapter:mocha');
 const Mocha = require('mocha');
+const chalk = require('chalk');
 const TestomatClient = require('../client');
 const TRConstants = require('../constants');
-const { parseTest } = require('../util');
+const { parseTest, specificTestInfo } = require('../util');
+const ArtifactStorage = require('../ArtifactStorage');
 
-const { EVENT_RUN_BEGIN, EVENT_RUN_END, EVENT_TEST_FAIL, EVENT_TEST_PASS } = Mocha.Runner.constants;
+const { EVENT_RUN_BEGIN, EVENT_RUN_END, EVENT_TEST_FAIL, EVENT_TEST_PASS, EVENT_TEST_PENDING } = Mocha.Runner.constants;
 
 function MochaReporter(runner, opts) {
   Mocha.reporters.Base.call(this, runner);
-  let passes = 0;
-  let failures = 0;
+  let passes = 0; let failures = 0; let skipped = 0;
+  let artifactStore;
 
-  const client = new TestomatClient({ apiKey: opts?.reporterOptions?.apiKey });
+  const apiKey = opts?.reporterOptions?.apiKey || process.env.TESTOMATIO;
+
+  if (!apiKey) {
+    debug('TESTOMATIO key is empty, ignoring reports');
+    return;
+  }
+  const client = new TestomatClient({ apiKey });
 
   runner.on(EVENT_RUN_BEGIN, () => {
     client.createRun();
+
+    const params = { 
+      "toFile": true
+    };
+
+    (runner._workerReporter !== undefined)
+      ? artifactStore = new ArtifactStorage(params)
+      : artifactStore = new ArtifactStorage();
   });
 
-  runner.on(EVENT_TEST_PASS, test => {
+  runner.on(EVENT_TEST_PASS, async(test) => {
     passes += 1;
-    console.log('pass: %s', test.fullTitle());
+    console.log(chalk.bold.green('✔'), test.fullTitle());
+    const testId = parseTest(test.title);
+
+    const specificTest =  specificTestInfo(test);
+    const content = await artifactStore.artifactByTestName(specificTest);
+
+    debug(`test=${specificTest} content = `, content);
+
+    client.addTestRun(
+      testId, 
+      TRConstants.PASSED, 
+      {
+        title: test.title,
+        time: test.duration,
+      },
+      content
+    );
+  });
+
+  runner.on(EVENT_TEST_PENDING, test => {
+    skipped += 1;
+    console.log('skip: %s', test.fullTitle());
     const testId = parseTest(test.title);
-    client.addTestRun(testId, TRConstants.PASSED, {
+    client.addTestRun(testId, TRConstants.SKIPPED, {
       title: test.title,
       time: test.duration,
     });
   });
 
-  runner.on(EVENT_TEST_FAIL, (test, err) => {
+  runner.on(EVENT_TEST_FAIL, async(test, err) => {
     failures += 1;
-    console.log('fail: %s -- error: %s', test.fullTitle(), err.message);
+    console.log(chalk.bold.red('✖'), test.fullTitle(), chalk.gray(err.message));
     const testId = parseTest(test.title);
-    client.addTestRun(testId, TRConstants.FAILED, {
-      error: err,
-      title: test.title,
-      time: test.duration,
-    });
+
+    const specificTest =  specificTestInfo(test);
+    const content = await artifactStore.artifactByTestName(specificTest);
+
+    debug(`fail test=${specificTest} content = `, content);
+
+    client.addTestRun(
+      testId,
+      TRConstants.FAILED, {
+        error: err,
+        title: test.title,
+        time: test.duration,
+      }, 
+      content
+    );
   });
 
   runner.on(EVENT_RUN_END, () => {
-    console.log('end: %d/%d', passes, passes + failures);
     const status = failures === 0 ? TRConstants.PASSED : TRConstants.FAILED;
+    console.log(chalk.bold(status), `${passes} passed, ${failures} failed, ${skipped} skipped`);
     client.updateRunStatus(status);
+
+    artifactStore.cleanup();
   });
 }
 

package/lib/adapter/playwright.js

@@ -10,13 +10,7 @@ const { parseTest } = require('../util');
 
 class TestomatioReporter {
   constructor(config = {}) {
-    const { apiKey } = config;
-
-    if (!apiKey) {
-      console.log('API Key is not provided. Testomat.io report is disabled');
-    } else {
-      this.client = new TestomatioClient({ apiKey });
-    }
+    this.client = new TestomatioClient({ apiKey: config?.apiKey });
 
     this.videos = [];
   }
@@ -75,7 +69,7 @@ class TestomatioReporter {
   async onEnd(result) {
     if (!this.client) return;
 
-    if (this.videos.length && upload.isArtifactsEnabled) {
+    if (this.videos.length && upload.isArtifactsEnabled()) {
       console.log(Status.APP_PREFIX, `🎞️  Uploading ${this.videos.length} videos...`);
 
       const promises = [];

package/lib/adapter/webdriver.js

@@ -7,10 +7,7 @@ class WebdriverReporter extends WDIOReporter {
   constructor(options) {
     super(options);
 
-    const { apiKey } = options;
-    if (!apiKey) return;
-
-    this.client = new TestomatClient({ apiKey });
+    this.client = new TestomatClient({ apiKey: options?.apiKey });
     options = Object.assign(options, { stdout: true });
 
     this._addTestPromises = [];

package/lib/bin/reportXml.js

@@ -1,8 +1,9 @@
 #!/usr/bin/env node
+
 const program = require("commander");
 const chalk = require("chalk");
 const glob = require('glob');
-
+const debug = require('debug')('@testomatio/reporter:xml-cli');
 const { APP_PREFIX } = require('../constants');
 const XmlReader = require("../xmlReader");
 
@@ -22,7 +23,10 @@ program
       pattern += '.xml';
     }
     let { javaTests, lang } = opts;
-    if (opts.envFile) require('dotenv').config(opts.envFile); // eslint-disable-line
+    if (opts.envFile) {
+      debug('Loading env file: %s', opts.envFile) 
+      require('dotenv').config({ path: opts.envFile }); // eslint-disable-line
+    }
     if (javaTests === true) javaTests = 'src/test/java';
     lang = lang?.toLowerCase();
     const runReader = new XmlReader({ javaTests, lang });