@testomatio/reporter 0.8.0-beta.9 → 0.8.2-beta.1

package/Changelog.md

@@ -0,0 +1,251 @@
+# 0.7.6
+
+* Updated to use AWS S3 3.0 SDK for uploading
+
+# 0.7.5
+
+* Fixed reporting skipped tests in mocha
+
+# 0.7.4
+
+* Fixed parsing source code in JUnit files
+
+# 0.7.3
+
+* CodeceptJS: Upload all traces and videos from artifacts
+* Fixed reporting skipped test in XML
+* added `--timelimit` option to `report-xml` command line
+
+# 0.7.2
+
+* Fixed uploading non-existing file
+
+# 0.7.1
+
+* Support for NUnit XML v3 format
+
+# 0.7.0
+
+* Support for `@cucumber/cucumber` (>= 7.0) added
+* Initial support for C# and NUnit  
+
+# 0.6.10
+
+* Fixed uploading multilpe artifacts in Playwright
+
+# 0.6.9
+
+* Fixed pending tests reports for Cypress
+
+# 0.6.8
+# 0.6.7
+
+* Pytest: fixed creating suites from reports
+
+# 0.6.6
+
+* JUnit reporter: prefer suite title over testcase classname in a report
+
+# 0.6.5
+
+* Fixed test statuses for runs in JUnit reporter
+
+# 0.6.4
+
+* Added `TESTOMATIO_PROCEED=1` param to not close current run
+* Fixed priority of commands from `npx @testomatio/reporter`
+
+# 0.6.3
+
+* Fixed `npx start-test-run` to launch commands
+
+# 0.6.2
+
+* Added `--env-file` option to load env variables from env file
+
+# 0.6.1
+
+* Fixed creating RunGroup with JUnit reporter
+
+# 0.6.0
+
+* JUnit reporter support
+
+# 0.5.10
+
+* Fixed reporting Scenario Outline in Cypress-Cucumber
+* Fixed error reports for Cypress when running in Chrome
+
+# 0.5.9
+
+* Added environment on Cypress report
+
+# 0.5.8
+
+* Fixed Cypress.io reporting
+
+# 0.5.7
+
+* Fixed webdriverio artifacts
+
+# 0.5.6
+
+* Unmark failed CodeceptJS tests as skipped
+
+# 0.5.5
+
+* Fixed `BeforeSuite` failures in CodeceptJS
+
+# 0.5.4
+
+Added `TESTOMATIO_CREATE=1` option to create unmatched tests on report
+
+```
+TESTOMATIO_CREATE=1 TESTOMATIO=apiKey npx codeceptjs run
+```
+
+# 0.5.3
+
+* Fixed parsing suites
+
+# 0.5.2
+
+* Fixed multiple upload of artifacts in Cypress.io
+
+# 0.5.1
+
+* Fixed Cypress.io to report tests inside nested suites
+
+# 0.5.0
+
+* Added Cypress.io plugin
+* Added artifacts upload to webdriverio
+
+# 0.4.6
+
+- Fixed CodeceptJS reporter to report tests failed in hooks
+
+# 0.4.5
+
+- Fixed "Total XX artifacts publicly uploaded to S3 bucket" when no S3 bucket is configured
+- Improved S3 connection error messages
+
+# 0.4.4
+
+- Fixed returning 0 exit code when a process fails when running tests in parallel via `start-test-run`. Previously was using the last exit code returned by a process. Currently prefers the highest exit code that was returned by a process.
+
+# 0.4.3
+
+- Added `TESTOMATIO_DISABLE_ARTIFACTS` env variable to disable publishing artifacts.
+
+# 0.4.2
+
+- print version of reporter
+- print number of uploaded artifacts
+- print access mode for uploaded artifacts
+
+# 0.4.1
+
+Added `global.testomatioArtifacts = []` array which can be used to add arbitrary artifacts to a report.
+
+```js
+// inside a running test:
+global.testomatioArtifacts.push('file/to/upload.png');
+```
+
+# 0.4.0
+
+- Playwright: Introduced playwright/test support with screenshots and video artifacts
+
+> Known issues: reporting using projects configured in Playwright does not work yet
+
+- CodeceptJS: added video uploads
+
+# 0.3.16
+
+- CodeceptJS: fixed reporting tests with empty steps (on retry)
+
+# 0.3.15
+
+- Finish Run via API:
+
+```
+TESTOMATIO={apiKey} TESTOMATIO_RUN={runId} npx @testomatio/reporter@latest --finish
+```
+
+# 0.3.14
+
+- Create an empty Run via API:
+
+```
+TESTOMATIO={apiKey} npx @testomatio/reporter@latest --launch
+```
+
+# 0.3.13
+
+- Checking for a valid report URL
+- Sending unlimited data on test report
+
+# 0.3.12
+
+- Fixed submitting arbitrary data on a test run
+- Jest: fixed sending errors with stack traces
+- Cypress: fixed sending reports
+
+# 0.3.11
+
+- Fixed circular JSON reference when submitting data to Testomatio
+
+# 0.3.10
+
+- Minor fixes
+
+# 0.3.9
+
+- Making all reporters to run without API key
+
+# 0.3.8
+
+- Fixed `npx start-test-run` to work with empty API keys
+
+# 0.3.7
+
+- Fixed release
+
+# 0.3.6
+
+- Update title and rungroup on start for scheduled runs.
+
+# 0.3.5
+
+- Added `TESTOMATIO_RUN` environment variable to pass id of a specific run to report
+
+# 0.3.4
+
+- Minor fixes
+
+# 0.3.3
+
+- [CodeceptJS] Fixed stack trace reporting
+- [CodeceptJS] Fixed displaying of nested steps
+- [CodeceptJS][mocha] Added assertion diff to report
+
+# 0.3.2
+
+- Fixed error message for S3 uploading
+
+# 0.3.1
+
+- [CodeceptJS] Better formatter for nested structures and BDD tests
+
+# 0.3.0
+
+- Added `TESTOMATIO_TITLE` env variable to set a name for Run
+- Added `TESTOMATIO_RUNGROUP_TITLE` env variable to attach Run to RunGroup
+- Added `TESTOMATIO_ENV` env variable to attach additional env values to report
+- [CodeceptJS] **CodeceptJS v3 support**
+- [CodeceptJS] Dropped support for CodeceptJS 2
+- [CodeceptJS] Added support for before hooks
+- [CodeceptJS] Log of steps
+- [CodeceptJS] Upload screenshots of failed tests to S3
+- [CodeceptJS] Updated to use with parallel execution

package/README.md

@@ -20,11 +20,24 @@ For testcafe use testcafe reporter:
 npm i testcafe-reporter-testomatio
 ```
 
+For newman use:
+
+```bash
+npm i newman-reporter-testomatio --save-dev
+```
+
+>`newman` and `newman-reporter-testomatio` should be installed in the same directory.
+\
+If you run your tests using globally installed newman (`newman run ...`), intall `newman-reporter-testomatio` globally too (`npm i newman-reporter-testomatio -g`).
+\
+If you use locally installed newman (within the project) (`npx newman run ...`), install `newman-reporter-testomatio` locally (`npm i newman-reporter-testomatio`).
+You can verify installed packages via `npm list` or `npm list -g`.
+
 ## Usage
 
 ### CodeceptJS
 
-Make sure you load all your tests using [check-test](https://github.com/testomatio/check-tests#cli).
+Make sure you load all your tests using [check-tests](https://github.com/testomatio/check-tests#cli).
 
 Add plugin to [codecept conf](https://github.com/testomatio/reporter/blob/master/example/codecept/codecept.conf.js#L23):
 
@@ -81,7 +94,7 @@ TESTOMATIO={API_KEY} npx playwright test
 
 ### Mocha
 
-Load the test using using `check-test` if not done already. Get the test id from testomat account and add it to your mocha test like in this [example](https://github.com/testomatio/reporter/blob/master/example/mocha/test/index.test.js#L4).
+Load the test using using `check-tests` if not done already. Get the test id from testomat account and add it to your mocha test like in this [example](https://github.com/testomatio/reporter/blob/master/example/mocha/test/index.test.js#L4).
 
 Run the following command from you project folder:
 
@@ -91,7 +104,7 @@ mocha --reporter ./node_modules/testomat-reporter/lib/adapter/mocha.js --reporte
 
 ### Jest
 
-Load the test using using `check-test`. Add the test id to your tests like in this [example](https://github.com/testomatio/reporter/blob/master/example/jest/index.test.js#L1).
+Load the test using using `check-tests`. Add the test id to your tests like in this [example](https://github.com/testomatio/reporter/blob/master/example/jest/index.test.js#L1).
 
 Add the following line to [jest.config.js](https://github.com/testomatio/reporter/blob/master/example/jest/jest.config.js#L100):
 
@@ -113,7 +126,7 @@ TESTOMATIO={API_KEY} ./node_modules/.bin/cucumber-js --format ./node_modules/@te
 
 ### TestCafe
 
-Load the test using using `check-test`.
+Load the test using using `check-tests`.
 
 Run the following command from you project folder:
 
@@ -121,9 +134,20 @@ Run the following command from you project folder:
 TESTOMATIO={API_KEY} npx testcafe chrome -r testomatio
 ```
 
+### Newman (Postman)
+
+Run collection and specify `testomatio` as reporter:
+
+```bash
+TESTOMATIO={API_KEY} npx newman run {collection_name.json} -r testomatio
+```
+
+> _`check-tests` not supported for newman for now, tests will be created on testomatio by default_
+
+
 ### Cypress
 
-Load the test using using `check-test`.
+Load the test using using `check-tests`.
 
 Register our `cypress-plugin` in `cypress/plugins/index.js`:
 
@@ -151,7 +175,7 @@ TESTOMATIO={API_KEY} npx cypress run
 
 ### Protractor
 
-Load the test using using `check-test`.
+Load the test using using `check-tests`.
 
 Add the following lines to [conf.js](https://github.com/angular/protractor/blob/5.4.1/example/conf.js):
 
@@ -173,7 +197,7 @@ TESTOMATIO={API_KEY} npx start-test-run -c 'npx protractor conf.js'
 
 ### WebdriverIO
 
-Load the test using using `check-test`.
+Load the test using using `check-tests`.
 
 Add the following lines to [wdio.conf.js](https://webdriver.io/docs/configurationfile/):
 
@@ -208,6 +232,30 @@ Run the following command from you project folder:
 TESTOMATIO={API_KEY} npx start-test-run -c 'npx wdio wdio.conf.js'
 ```
 
+## Pipes
+Pipes allow you to get report inside different systems (e.g. Pull request comment, database etc)
+For now next pipes available:
+
+### GitHub
+This pipe adds comment with run report to GitHub Pull Request.
+
+To use it:
+1. run your tests using github actions in Pull Request
+2. pass `GH_PAT` (GitHub Personal Access Token) as environment variable.
+
+> Last report (comment) will be replaced with the new one.
+To leave previous report pass `GITHUB_KEEP_OUTDATED_REPORTS=1` env variable.
+
+### GitLab
+This pipe adds comment with run report to GitLab Merge Request.
+
+To use it:
+1. run your tests in Merge Request (pipeline trigger should be `merge_request`)
+2. pass `GITLAB_PAT` (GitLab Personal Access Token) as environment variable.
+
+> Last report (comment) will be replaced with the new one.
+To leave previous report pass `GITLAB_KEEP_OUTDATED_REPORTS=1` env variable.
+
 ## JUnit Reports
 
 > **Notice** JUnit reports are supported since 0.6.0
@@ -408,6 +456,23 @@ Add environments to run by providing `TESTOMATIO_ENV` as comma seperated values:
 TESTOMATIO={API_KEY} TESTOMATIO_ENV="Windows, Chrome" <actual run command>
 ```
 
+### Save test results to .csv file
+Add an env to run by specifying the `TESTOMATIO_CSV_FILENAME` variable.
+
+1) using default report name:
+
+```bash
+TESTOMATIO={API_KEY} TESTOMATIO_CSV_FILENAME="report.csv" <actual run command>
+```
+
+2) using unique report name:
+
+```bash
+TESTOMATIO={API_KEY} TESTOMATIO_CSV_FILENAME="test.csv" <actual run command>
+```
+_It's create a new /export folder with csv files_
+
+
 ### Attaching Test Artifacts
 
 To save a test artifacts (screenshots and videos) of a failed test use S3 storage.
@@ -520,3 +585,8 @@ If you want to finish a run started by `--launch` use `--finish` option. `TESTOM
 ```bash
 TESTOMATIO={API_KEY} TESTOMATIO_RUN={RUN_ID} npx start-test-run --finish
 ```
+
+### Debug logs
+Pass `DEBUG` variable with module name e.g. `DEBUG=@testomatio/reporter:pipe:github`.
+(Module name could be taken directly from the required module code).
+To log all debug info pass `DEBUG=*`.

package/lib/ArtifactStorage.js

@@ -0,0 +1,131 @@
+const debug = require('debug')('@testomatio/reporter:storage');
+const { join, resolve } = require('path');
+const fs = require('fs');
+const os = require('os');
+const uuid = require('uuid');
+const { TESTOMAT_ARTIFACT_SUFFIX } = require('./constants');
+const { specificTestInfo } = require('./util');
+
+class ArtifactStorage {
+
+  constructor(params) {
+    this._tmpPrefix = "tsmt_reporter";
+    this.isFile = false;
+    this.isMemory = true;
+    this.storage = params?.toFile;
+
+    if (this.storage) {
+      this.isFile = true;
+      this.isMemory = false;
+      this.tmpDirFullpath = this.createTestomatTmpDir(this._tmpPrefix);
+      debug('SAVE to tmp folder mode enabled!');
+    }
+  }
+
+  static async storeToFile(tmpDirName, artifact, testSuffix = "test") {
+    const suffix = TESTOMAT_ARTIFACT_SUFFIX + uuid.v4() + testSuffix;
+    const dirpath = join(os.tmpdir(), tmpDirName);
+    const filepath = resolve(dirpath, `${suffix}.json`);
+
+    return fs.promises.appendFile(filepath, JSON.stringify(artifact));
+  }
+
+  static async artifact(artifact, context) {
+    // TODO: mode for different pre-hooks. As variant - "all-tests" || "only_fail"
+    // const mode = process.env.TESTOMAT_ARTIFACTS || "only_fail";
+
+    if (Array.isArray(global.testomatioArtifacts)) {
+      debug("Saving artifacts to global storage");
+
+      global.testomatioArtifacts.push(artifact);
+    }
+
+    if (global?.testomatioArtifacts === undefined) {
+      const tmpDirNames = ArtifactStorage.tmpTestomatDirNames();
+
+      const testSuffix = specificTestInfo(context.test);
+
+      if (Array.isArray(tmpDirNames) && tmpDirNames.length) {
+        debug("Saving artifacts to memory tmp folder");
+
+        return ArtifactStorage.storeToFile(tmpDirNames[tmpDirNames.length - 1], artifact, testSuffix);
+      }
+    }
+  }
+
+  async artifactByTestName(test) {
+    const list = [];
+
+    if (this.isFile && this.tmpDirFullpath) {
+      const files = fs.readdirSync(this.tmpDirFullpath);
+
+      for (const file of files) {
+        if (file.includes(test)) {
+          const buff = await fs.promises.readFile(`${this.tmpDirFullpath  }/${  file}`);
+
+          list.push(JSON.parse(buff.toString()));
+        }
+      }
+    }
+
+    return list;
+  }
+
+  async tmpContents() {
+    const contents = [];
+
+    if (this.isFile && this.tmpDirFullpath) {
+      const files = fs.readdirSync(this.tmpDirFullpath);
+
+      for (const file of files) {
+        const buff = await fs.promises.readFile(`${this.tmpDirFullpath  }/${  file}`);
+        const content = buff.toString();
+
+        contents.push(content);
+      }
+    }
+
+    return contents;
+  }
+
+  createTestomatTmpDir(clientPrefix) {
+    return fs.mkdtempSync(join(os.tmpdir(), clientPrefix));
+  }
+
+  clearTmpDirByName(name) {
+    const tmpDirPath = join(os.tmpdir(), name);
+
+    if (fs.existsSync(tmpDirPath)) {
+      fs.rmSync(tmpDirPath, { recursive: true });
+      debug(` Testomat tmpDir = ${tmpDirPath} was deleted successfully!`);
+
+      
+    }
+  }
+
+  static tmpTestomatDirNames() {
+    const subname = this._tmpPrefix || "tsmt_reporter";
+
+    return fs.readdirSync(os.tmpdir(), { withFileTypes: true })
+      .filter((item) => item.isDirectory())
+      .map((item) => item.name)
+      .filter((name) => name.includes(subname));
+  }
+
+  cleanup() {
+    if (this.isFile && this.tmpDirFullpath) {
+      const tmpDirNames = ArtifactStorage.tmpTestomatDirNames();
+
+      if (Array.isArray(tmpDirNames) && tmpDirNames.length) {
+        for (const name of tmpDirNames) {
+          this.clearTmpDirByName(name);
+        }
+      }
+    }
+    else {
+      debug("The tmp folder has not been created! Nothing to delete");
+    }
+  }
+}
+
+module.exports = ArtifactStorage;

package/lib/adapter/codecept.js

@@ -1,15 +1,15 @@
+const chalk = require('chalk');
+const TestomatClient = require('../client');
+const { STATUS, APP_PREFIX } = require('../constants');
+const upload = require('../fileUploader');
+const Output = require('../output');
+
 if (!global.codeceptjs) {
   // eslint-disable-next-line global-require, import/no-extraneous-dependencies
   global.codeceptjs = require('codeceptjs');
 }
 
 const { event, recorder, codecept } = global.codeceptjs;
-const chalk = require('chalk');
-const TestomatClient = require('../client');
-const { FAILED } = require('../constants');
-const TRConstants = require('../constants');
-const upload = require('../fileUploader');
-const Output = require('../output');
 
 let currentMetaStep = [];
 let error;
@@ -35,11 +35,6 @@ function CodeceptReporter(config) {
   const testTimeMap = {};
   const { apiKey } = config;
 
-  if (!apiKey) {
-    console.log('TESTOMATIO key is empty, ignoring reports');
-    return;
-  }
-
   const getDuration = test => {
     if (testTimeMap[test.id]) {
       return Date.now() - testTimeMap[test.id];
@@ -72,16 +67,17 @@ function CodeceptReporter(config) {
   });
 
   event.dispatcher.on(event.all.result, async () => {
-    if (videos.length && upload.isArtifactsEnabled) {
-      console.log(TRConstants.APP_PREFIX, `🎞️  Uploading ${videos.length} videos...`);
+    if (videos.length && upload.isArtifactsEnabled()) {
+      console.log(APP_PREFIX, `🎞️  Uploading ${videos.length} videos...`);
 
       const promises = [];
       for (const video of videos) {
         const { testId, title, path, type } = video;
         const file = { path, type, title };
         promises.push(
-          client.addTestRun(testId, undefined, {
+          client.addTestRun(undefined, {
             ...stripExampleFromTitle(title),
+            test_id: testId,
             files: [file],
           }),
         );
@@ -89,7 +85,7 @@ function CodeceptReporter(config) {
       await Promise.all(promises);
     }
 
-    const status = failedTests.length === 0 ? TRConstants.PASSED : TRConstants.FAILED;
+    const status = failedTests.length === 0 ? STATUS.PASSED : STATUS.FAILED;
     client.updateRunStatus(status);
   });
 
@@ -100,12 +96,13 @@ function CodeceptReporter(config) {
     }
     const testId = parseTest(tags);
     const testObj = getTestAndMessage(title);
-    client.addTestRun(testId, TRConstants.PASSED, {
+    client.addTestRun(STATUS.PASSED, {
       ...stripExampleFromTitle(title),
       suite_title: test.parent && test.parent.title,
       message: testObj.message,
       time: getDuration(test),
       steps: output.text(),
+      test_id: testId,
     });
     output.stop();
   });
@@ -124,9 +121,10 @@ function CodeceptReporter(config) {
       failedTests.push(id || title);
       const testId = parseTest(tags);
 
-      client.addTestRun(testId, TRConstants.FAILED, {
+      client.addTestRun(STATUS.FAILED, {
         ...stripExampleFromTitle(title),
         suite_title: suite.title,
+        test_id: testId,
         error,
         time: 0,
       });
@@ -135,7 +133,7 @@ function CodeceptReporter(config) {
   });
 
   event.dispatcher.on(event.test.after, test => {
-    if (test.state && test.state !== FAILED) return;
+    if (test.state && test.state !== STATUS.FAILED) return;
     if (test.err) error = test.err;
     const { id, tags, title, artifacts } = test;
     failedTests.push(id || title);
@@ -154,8 +152,9 @@ function CodeceptReporter(config) {
       if (aid.startsWith('trace')) files.push({ testId: id, title, path: artifacts[aid], type: 'application/zip' }); 
     }
 
-    client.addTestRun(testId, TRConstants.FAILED, {
+    client.addTestRun(STATUS.FAILED, {
       ...stripExampleFromTitle(title),
+      test_id: testId,
       suite_title: test.parent && test.parent.title,
       error,
       message: testObj.message,
@@ -172,8 +171,9 @@ function CodeceptReporter(config) {
 
     const testId = parseTest(tags);
     const testObj = getTestAndMessage(title);
-    client.addTestRun(testId, TRConstants.SKIPPED, {
+    client.addTestRun(STATUS.SKIPPED, {
       ...stripExampleFromTitle(title),
+      test_id: testId,
       suite_title: test.parent && test.parent.title,
       message: testObj.message,
       time: getDuration(test),
@@ -212,12 +212,13 @@ function CodeceptReporter(config) {
     currentMetaStep = metaSteps;
     stepShift = 2 * shift;
 
-    let duration = new Date() - stepStart;
-    if (duration) {
-      duration = repeat(1) + chalk.grey(`(${duration}ms)`);
+    const durationMs = (+new Date() - (+stepStart));
+    let duration = '';
+    if (durationMs) {
+      duration = repeat(1) + chalk.grey(`(${durationMs}ms)`);
     }
 
-    if (step.status === TRConstants.FAILED) {
+    if (step.status === STATUS.FAILED) {
       output.push(repeat(stepShift) + chalk.red(step.toString()) + duration);
     } else {
       output.push(repeat(stepShift) + step.toString() + duration);