@testomatio/reporter 1.1.0-beta → 1.1.0-beta-3

package/Changelog.md

@@ -0,0 +1,355 @@
+<!-- pending release updates -->
+# 1.0.17
+
+Renamed `TESTOMATIO_STACK_FILTER` to `TESTOMATIO_STACK_IGNORE`
+
+# 1.0.16
+
+* Addded [stack trace configuration](./docs/stacktrace.md) and documentation:
+
+```
+TESTOMATIO_STACK_IGNORE="tests/support/**.js" <actual-run-command>
+```
+* Jest: fixed reporting tests without a suite title
+
+# 1.0.15
+
+* Attach Run to Jira Issue via `TESTOMATIO_JIRA_ID` env variable:
+
+```
+TESTOMATIO_JIRA_ID=TST-12 <actual run command>
+```
+
+* Mocha - removed requirement of TESTOMATIO API Key
+
+# 1.0.14
+
+* Execute tests by tag names. Use this filter to select tests associated with specific tags.
+  
+  ```bash
+  TESTOMATIO={API_KEY} npx start-test-run -c 'actual run command' --filter 'testomatio:tag-name=smoke'
+  ```
+
+  * more instruction you can find in docs/pipes/testomatio.md
+
+# 1.0.13
+
+* JUnit improvements
+  * Match test from source code by adding Test ID as a comment:
+
+  ```java
+  // @T8acca9eb
+  ```
+  * Match test from output by adding Test ID as output:
+
+  ```java
+  System.out.println("tid://@T8acca9eb");
+  ```
+  * Support for suite before and after output
+  * Improved support for artifacts
+
+# 1.0.12
+
+& Logger refactoring by @olexandr13 in #208
+* fix undefined logs by @olexandr13 in #210
+
+# 1.0.11
+
+* fix steps duplication for codecept report by @olexandr13 in #209
+
+# 1.0.10
+
+* Added `TESTOMATIO_PUBLISH=1` variable to automatically publish run report
+
+# 1.0.9
+
+* Support XUnit format
+* Improved support for parametrized Java tests
+
+# 1.0.8 
+
+* Fixed `Can't read push of undefined` when logging steps
+
+# 1.0.6
+
+* Testomat.io. Auto-detect current build url and report it to Testomat.io. Manually url can be set with `BUILD_URL` variable:
+
+```
+BUILD_URL=https://.... TESTOMATIO=apiKey <actual test command>
+```
+
+# 1.0.5 
+
+* Fix "create tests" params processing for testomatio pipe
+
+# 1.0.4
+
+* Fixed parallel run
+
+# 1.0.3
+
+* Fixed reporting parallel runs
+
+# 1.0.0
+
+
+* Added [`TESTOMATIO_SHARED_RUN` option](https://github.com/testomatio/reporter/blob/master/docs/pipes.md#reporting-parallel-execution-to-to-same-run) to use a shared run for parallel executions
+* Reworked [documentation](https://github.com/testomatio/reporter/tree/master#readme).
+* Added an option to obtain [S3 configuration](https://github.com/testomatio/reporter/blob/master/docs/artifacts.md#configuration) from Testomat.io
+* Introduced [pipes](https://github.com/testomatio/reporter/blob/master/docs/pipes.md):
+  * GitHub
+  * GitLab
+  * CSV Pipe
+
+
+# 0.7.6
+
+* Updated to use AWS S3 3.0 SDK for uploading
+
+# 0.7.5
+
+* Fixed reporting skipped tests in mocha
+
+# 0.7.4
+
+* Fixed parsing source code in JUnit files
+
+# 0.7.3
+
+* CodeceptJS: Upload all traces and videos from artifacts
+* Fixed reporting skipped test in XML
+* added `--timelimit` option to `report-xml` command line
+
+# 0.7.2
+
+* Fixed uploading non-existing file
+
+# 0.7.1
+
+* Support for NUnit XML v3 format
+
+# 0.7.0
+
+* Support for `@cucumber/cucumber` (>= 7.0) added
+* Initial support for C# and NUnit  
+
+# 0.6.10
+
+* Fixed uploading multilpe artifacts in Playwright
+
+# 0.6.9
+
+* Fixed pending tests reports for Cypress
+
+# 0.6.8
+# 0.6.7
+
+* Pytest: fixed creating suites from reports
+
+# 0.6.6
+
+* JUnit reporter: prefer suite title over testcase classname in a report
+
+# 0.6.5
+
+* Fixed test statuses for runs in JUnit reporter
+
+# 0.6.4
+
+* Added `TESTOMATIO_PROCEED=1` param to not close current run
+* Fixed priority of commands from `npx @testomatio/reporter`
+
+# 0.6.3
+
+* Fixed `npx start-test-run` to launch commands
+
+# 0.6.2
+
+* Added `--env-file` option to load env variables from env file
+
+# 0.6.1
+
+* Fixed creating RunGroup with JUnit reporter
+
+# 0.6.0
+
+* JUnit reporter support
+
+# 0.5.10
+
+* Fixed reporting Scenario Outline in Cypress-Cucumber
+* Fixed error reports for Cypress when running in Chrome
+
+# 0.5.9
+
+* Added environment on Cypress report
+
+# 0.5.8
+
+* Fixed Cypress.io reporting
+
+# 0.5.7
+
+* Fixed webdriverio artifacts
+
+# 0.5.6
+
+* Unmark failed CodeceptJS tests as skipped
+
+# 0.5.5
+
+* Fixed `BeforeSuite` failures in CodeceptJS
+
+# 0.5.4
+
+Added `TESTOMATIO_CREATE=1` option to create unmatched tests on report
+
+```
+TESTOMATIO_CREATE=1 TESTOMATIO=apiKey npx codeceptjs run
+```
+
+# 0.5.3
+
+* Fixed parsing suites
+
+# 0.5.2
+
+* Fixed multiple upload of artifacts in Cypress.io
+
+# 0.5.1
+
+* Fixed Cypress.io to report tests inside nested suites
+
+# 0.5.0
+
+* Added Cypress.io plugin
+* Added artifacts upload to webdriverio
+
+# 0.4.6
+
+- Fixed CodeceptJS reporter to report tests failed in hooks
+
+# 0.4.5
+
+- Fixed "Total XX artifacts publicly uploaded to S3 bucket" when no S3 bucket is configured
+- Improved S3 connection error messages
+
+# 0.4.4
+
+- Fixed returning 0 exit code when a process fails when running tests in parallel via `start-test-run`. Previously was using the last exit code returned by a process. Currently prefers the highest exit code that was returned by a process.
+
+# 0.4.3
+
+- Added `TESTOMATIO_DISABLE_ARTIFACTS` env variable to disable publishing artifacts.
+
+# 0.4.2
+
+- print version of reporter
+- print number of uploaded artifacts
+- print access mode for uploaded artifacts
+
+# 0.4.1
+
+Added `global.testomatioArtifacts = []` array which can be used to add arbitrary artifacts to a report.
+
+```js
+// inside a running test:
+global.testomatioArtifacts.push('file/to/upload.png');
+```
+
+# 0.4.0
+
+- Playwright: Introduced playwright/test support with screenshots and video artifacts
+
+> Known issues: reporting using projects configured in Playwright does not work yet
+
+- CodeceptJS: added video uploads
+
+# 0.3.16
+
+- CodeceptJS: fixed reporting tests with empty steps (on retry)
+
+# 0.3.15
+
+- Finish Run via API:
+
+```
+TESTOMATIO={apiKey} TESTOMATIO_RUN={runId} npx @testomatio/reporter@latest --finish
+```
+
+# 0.3.14
+
+- Create an empty Run via API:
+
+```
+TESTOMATIO={apiKey} npx @testomatio/reporter@latest --launch
+```
+
+# 0.3.13
+
+- Checking for a valid report URL
+- Sending unlimited data on test report
+
+# 0.3.12
+
+- Fixed submitting arbitrary data on a test run
+- Jest: fixed sending errors with stack traces
+- Cypress: fixed sending reports
+
+# 0.3.11
+
+- Fixed circular JSON reference when submitting data to Testomatio
+
+# 0.3.10
+
+- Minor fixes
+
+# 0.3.9
+
+- Making all reporters to run without API key
+
+# 0.3.8
+
+- Fixed `npx start-test-run` to work with empty API keys
+
+# 0.3.7
+
+- Fixed release
+
+# 0.3.6
+
+- Update title and rungroup on start for scheduled runs.
+
+# 0.3.5
+
+- Added `TESTOMATIO_RUN` environment variable to pass id of a specific run to report
+
+# 0.3.4
+
+- Minor fixes
+
+# 0.3.3
+
+- [CodeceptJS] Fixed stack trace reporting
+- [CodeceptJS] Fixed displaying of nested steps
+- [CodeceptJS][mocha] Added assertion diff to report
+
+# 0.3.2
+
+- Fixed error message for S3 uploading
+
+# 0.3.1
+
+- [CodeceptJS] Better formatter for nested structures and BDD tests
+
+# 0.3.0
+
+- Added `TESTOMATIO_TITLE` env variable to set a name for Run
+- Added `TESTOMATIO_RUNGROUP_TITLE` env variable to attach Run to RunGroup
+- Added `TESTOMATIO_ENV` env variable to attach additional env values to report
+- [CodeceptJS] **CodeceptJS v3 support**
+- [CodeceptJS] Dropped support for CodeceptJS 2
+- [CodeceptJS] Added support for before hooks
+- [CodeceptJS] Log of steps
+- [CodeceptJS] Upload screenshots of failed tests to S3
+- [CodeceptJS] Updated to use with parallel execution

package/README.md

@@ -11,12 +11,12 @@ Testomat.io Reporter (this npm package) supports:
 
 * 🏄 Integarion with all popular [JavaScript/TypeScript frameworks](./docs/frameworks.md)
 * 🗄️ Screenshots, videos, traces [uploaded into S3 bucket](./docs/artifacts.md)
-* 🔎 Stack traces and error messages
-* 🐙 [GitHub](./docs/pipes.md#github-pipe) & [GitLab](./docs/pipes.md#gitlab-pipe) integration
+* 🔎 [Stack traces](./docs/stacktrace.md) and error messages
+* 🐙 [GitHub](./docs/pipes/github.md) & [GitLab](./docs/pipes/gitlab.md) integration
 * 🚅 Realtime reports
 * 🗃️ Other test frameworks supported via [JUNit XML](./docs/junit.md)
 * 🚶‍♀️ Steps *(work in progress)*
-* 📄 Logger *(work in progress, supports Jest for now)*
+* 📄 [Logger](./docs/logger.md) *(work in progress, supports Jest for now)*
 * ☁️ Custom properties and metadata *(work in progress)*
 * 💯 Free & open-source.
 * 📊 Public and private Run reports on cloud via [Testomat.io App](https://testomat.io) 👇
@@ -85,13 +85,13 @@ or any [other via JUnit](./docs/junit.md) report....
 
 ### 2️⃣ Configure Reports
 
-* [Create report on Testomat.io](./docs/pipes.md#testomatio-pipe).
-* [Create brief summary report for GitHub Pull Request](./docs/pipes.md#github-pipe) 👇
-* [Create brief summary report for GitLab Merge Request](./docs/pipes.md#gitlab-pipe).
-* [Configure other pipes](./docs/pipes.md) for other ways to process test results output.
+* [Create report on Testomat.io](./docs/pipes/testomatio.md).
+* [Create brief summary report for GitHub Pull Request](./docs/pipes/github.md) 👇
+* [Create brief summary report for GitLab Merge Request](./docs/pipes/gitlab.md).
+* [Configure other pipes](./docs/pipes/md) for other ways to process test results output.
 
 
-![GitHub report](./docs/images/github.png)
+![GitHub report](./docs/pipes/images/github.png)
 
 GitHub report published as a comment to Pull Request:
 
@@ -121,10 +121,14 @@ Bring this reporter on CI and never lose test results again!
 
 * 🛠️ [Frameworks](./docs/frameworks.md)
 * ⛲ [Pipes](./docs/pipes.md)
+  * [Testomat.io](./docs/pipes/testomatio.md)
+  * [GitHub](./docs/pipes/github.md)
+  * [Gitlab](./docs/pipes/gitlab.md)
+  * [CSV](./docs/pipes/csv.md)
 * 📓 [JUnit](./docs/junit.md)
 * 🗄️ [Artifacts](./docs/artifacts.md)
 * 🔂 [Workflows](./docs/workflows.md)
-* 🔂 [Logger](./docs/logger.md)
+* 🖊️ [Logger](./docs/logger.md)
 
 ## Development
 

package/lib/adapter/codecept.js

@@ -1,9 +1,9 @@
 const debug = require('debug')('@testomatio/reporter:adapter:codeceptjs');
 const chalk = require('chalk');
 const TestomatClient = require('../client');
-const { STATUS, APP_PREFIX } = require('../constants');
+const { STATUS, APP_PREFIX, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
 const upload = require('../fileUploader');
-const Output = require('../output');
+const { parseTest: getIdFromTestTitle, fileSystem } = require('../utils/utils');
 
 if (!global.codeceptjs) {
   // eslint-disable-next-line global-require, import/no-extraneous-dependencies
@@ -16,9 +16,9 @@ let currentMetaStep = [];
 let error;
 let stepShift = 0;
 
-const output = new Output({
-  filterFn: stack => !stack.includes('codeceptjs/lib/output'), // output from codeceptjs
-});
+// const output = new Output({
+//   filterFn: stack => !stack.includes('codeceptjs/lib/output'), // output from codeceptjs
+// });
 
 let stepStart = new Date();
 
@@ -51,24 +51,36 @@ function CodeceptReporter(config) {
 
   recorder.startUnlessRunning();
 
+  global.testomatioRunningEnvironment = 'codeceptjs';
+
   // Listening to events
   event.dispatcher.on(event.all.before, () => {
+    // clear tmp dir
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
+
     recorder.add('Creating new run', () => client.createRun());
     videos = [];
     traces = [];
+
+    if (!global.testomatioDataStore) global.testomatioDataStore = {};
   });
 
   event.dispatcher.on(event.test.before, () => {
     recorder.add(() => {
       currentMetaStep = [];
-      output.reset();
-      output.start();
+      // output.reset();
+      // output.start();
       stepShift = 0;
     });
+
+    if (!global.testomatioDataStore) global.testomatioDataStore = {};
+    // reset steps
+    global.testomatioDataStore.steps = [];
   });
 
   event.dispatcher.on(event.test.started, test => {
     testTimeMap[test.id] = Date.now();
+    if (global.testomatioDataStore) global.testomatioDataStore.currentlyRunningTestId = getIdFromTestTitle(test.title);
   });
 
   event.dispatcher.on(event.all.result, async () => {
@@ -97,10 +109,10 @@ function CodeceptReporter(config) {
       suite_title: test.parent && test.parent.title,
       message: testObj.message,
       time: getDuration(test),
-      steps: output.text(),
+      steps: global.testomatioDataStore.steps.join('\n') || null,
       test_id: testId,
     });
-    output.stop();
+    // output.stop();
   });
 
   event.dispatcher.on(event.test.failed, (test, err) => {
@@ -125,7 +137,7 @@ function CodeceptReporter(config) {
         time: 0,
       });
     }
-    output.stop();
+    // output.stop();
   });
 
   event.dispatcher.on(event.test.after, test => {
@@ -141,32 +153,33 @@ function CodeceptReporter(config) {
 
     const files = [];
     if (artifacts.screenshot) files.push({ path: artifacts.screenshot, type: 'image/png' });
-    // todo: video must be uploaded later....    
+    // todo: video must be uploaded later....
 
-    
-    const reportTestPromise = client.addTestRun(STATUS.FAILED, {
-      ...stripExampleFromTitle(title),
-      test_id: testId,
-      suite_title: test.parent && test.parent.title,
-      error,
-      message: testObj.message,
-      time: getDuration(test),
-      files,
-      steps: output.text(),
-    }).then(pipes => {
-      testId = pipes.filter(p => p.pipe.includes('Testomatio'))[0]?.result?.data?.test_id;
-
-      debug("artifacts", artifacts);
-      
-      for (const aid in artifacts) {
-        if (aid.startsWith('video')) videos.push({ testId, title, path: artifacts[aid], type: 'video/webm' });
-        if (aid.startsWith('trace')) traces.push({ testId, title, path: artifacts[aid], type: 'application/zip' });
-      }
-    });
+    const reportTestPromise = client
+      .addTestRun(STATUS.FAILED, {
+        ...stripExampleFromTitle(title),
+        test_id: testId,
+        suite_title: test.parent && test.parent.title,
+        error,
+        message: testObj.message,
+        time: getDuration(test),
+        files,
+        steps: global.testomatioDataStore?.steps?.join('\n') || null,
+      })
+      .then(pipes => {
+        testId = pipes.filter(p => p.pipe.includes('Testomatio'))[0]?.result?.data?.test_id;
+
+        debug('artifacts', artifacts);
+
+        for (const aid in artifacts) {
+          if (aid.startsWith('video')) videos.push({ testId, title, path: artifacts[aid], type: 'video/webm' });
+          if (aid.startsWith('trace')) traces.push({ testId, title, path: artifacts[aid], type: 'application/zip' });
+        }
+      });
 
     reportTestPromises.push(reportTestPromise);
 
-    output.stop();
+    // output.stop();
   });
 
   event.dispatcher.on(event.test.skipped, test => {
@@ -182,7 +195,7 @@ function CodeceptReporter(config) {
       message: testObj.message,
       time: getDuration(test),
     });
-    output.stop();
+    // output.stop();
   });
 
   event.dispatcher.on(event.step.started, step => {
@@ -207,30 +220,37 @@ function CodeceptReporter(config) {
         // eslint-disable-next-line no-continue
         if (!metaSteps[i]) continue;
         if (metaSteps[i].isBDD()) {
-          output.push(repeat(stepShift) + chalk.bold(metaSteps[i].toString()) + metaSteps[i].comment);
+          // output.push(repeat(stepShift) + chalk.bold(metaSteps[i].toString()) + metaSteps[i].comment);
+          global.testomatioDataStore?.steps?.push(
+            repeat(stepShift) + chalk.bold(metaSteps[i].toString()) + metaSteps[i].comment,
+          );
         } else {
-          output.push(repeat(stepShift) + chalk.green.bold(metaSteps[i].toString()));
+          // output.push(repeat(stepShift) + chalk.green.bold(metaSteps[i].toString()));
+          global.testomatioDataStore?.steps?.push(repeat(stepShift) + chalk.green.bold(metaSteps[i].toString()));
         }
       }
     }
     currentMetaStep = metaSteps;
     stepShift = 2 * shift;
 
-    const durationMs = (+new Date() - (+stepStart));
+    const durationMs = +new Date() - (+stepStart);
     let duration = '';
     if (durationMs) {
       duration = repeat(1) + chalk.grey(`(${durationMs}ms)`);
     }
 
     if (step.status === STATUS.FAILED) {
-      output.push(repeat(stepShift) + chalk.red(step.toString()) + duration);
+      // output.push(repeat(stepShift) + chalk.red(step.toString()) + duration);
+      global.testomatioDataStore?.steps?.push(repeat(stepShift) + chalk.red(step.toString()) + duration);
     } else {
-      output.push(repeat(stepShift) + step.toString() + duration);
+      // output.push(repeat(stepShift) + step.toString() + duration);
+      global.testomatioDataStore?.steps?.push(repeat(stepShift) + step.toString() + duration);
     }
   });
 
   event.dispatcher.on(event.step.comment, step => {
-    output.push(chalk.cyan.bold(step.toString()));
+    // output.push(chalk.cyan.bold(step.toString()));
+    global.testomatioDataStore?.steps?.push(chalk.cyan.bold(step.toString()));
   });
 }