@testomatio/reporter 1.0.16 → 1.0.18-beta.1

package/Changelog.md

@@ -0,0 +1,359 @@
+<!-- pending release updates -->
+# 1.0.18
+
+* Fixed stack traces for CodeceptJS
+
+# 1.0.17
+
+Renamed `TESTOMATIO_STACK_FILTER` to `TESTOMATIO_STACK_IGNORE`
+
+# 1.0.16
+
+* Addded [stack trace configuration](./docs/stacktrace.md) and documentation:
+
+```
+TESTOMATIO_STACK_IGNORE="tests/support/**.js" <actual-run-command>
+```
+* Jest: fixed reporting tests without a suite title
+
+# 1.0.15
+
+* Attach Run to Jira Issue via `TESTOMATIO_JIRA_ID` env variable:
+
+```
+TESTOMATIO_JIRA_ID=TST-12 <actual run command>
+```
+
+* Mocha - removed requirement of TESTOMATIO API Key
+
+# 1.0.14
+
+* Execute tests by tag names. Use this filter to select tests associated with specific tags.
+  
+  ```bash
+  TESTOMATIO={API_KEY} npx start-test-run -c 'actual run command' --filter 'testomatio:tag-name=smoke'
+  ```
+
+  * more instruction you can find in docs/pipes/testomatio.md
+
+# 1.0.13
+
+* JUnit improvements
+  * Match test from source code by adding Test ID as a comment:
+
+  ```java
+  // @T8acca9eb
+  ```
+  * Match test from output by adding Test ID as output:
+
+  ```java
+  System.out.println("tid://@T8acca9eb");
+  ```
+  * Support for suite before and after output
+  * Improved support for artifacts
+
+# 1.0.12
+
+& Logger refactoring by @olexandr13 in #208
+* fix undefined logs by @olexandr13 in #210
+
+# 1.0.11
+
+* fix steps duplication for codecept report by @olexandr13 in #209
+
+# 1.0.10
+
+* Added `TESTOMATIO_PUBLISH=1` variable to automatically publish run report
+
+# 1.0.9
+
+* Support XUnit format
+* Improved support for parametrized Java tests
+
+# 1.0.8 
+
+* Fixed `Can't read push of undefined` when logging steps
+
+# 1.0.6
+
+* Testomat.io. Auto-detect current build url and report it to Testomat.io. Manually url can be set with `BUILD_URL` variable:
+
+```
+BUILD_URL=https://.... TESTOMATIO=apiKey <actual test command>
+```
+
+# 1.0.5 
+
+* Fix "create tests" params processing for testomatio pipe
+
+# 1.0.4
+
+* Fixed parallel run
+
+# 1.0.3
+
+* Fixed reporting parallel runs
+
+# 1.0.0
+
+
+* Added [`TESTOMATIO_SHARED_RUN` option](https://github.com/testomatio/reporter/blob/master/docs/pipes.md#reporting-parallel-execution-to-to-same-run) to use a shared run for parallel executions
+* Reworked [documentation](https://github.com/testomatio/reporter/tree/master#readme).
+* Added an option to obtain [S3 configuration](https://github.com/testomatio/reporter/blob/master/docs/artifacts.md#configuration) from Testomat.io
+* Introduced [pipes](https://github.com/testomatio/reporter/blob/master/docs/pipes.md):
+  * GitHub
+  * GitLab
+  * CSV Pipe
+
+
+# 0.7.6
+
+* Updated to use AWS S3 3.0 SDK for uploading
+
+# 0.7.5
+
+* Fixed reporting skipped tests in mocha
+
+# 0.7.4
+
+* Fixed parsing source code in JUnit files
+
+# 0.7.3
+
+* CodeceptJS: Upload all traces and videos from artifacts
+* Fixed reporting skipped test in XML
+* added `--timelimit` option to `report-xml` command line
+
+# 0.7.2
+
+* Fixed uploading non-existing file
+
+# 0.7.1
+
+* Support for NUnit XML v3 format
+
+# 0.7.0
+
+* Support for `@cucumber/cucumber` (>= 7.0) added
+* Initial support for C# and NUnit  
+
+# 0.6.10
+
+* Fixed uploading multilpe artifacts in Playwright
+
+# 0.6.9
+
+* Fixed pending tests reports for Cypress
+
+# 0.6.8
+# 0.6.7
+
+* Pytest: fixed creating suites from reports
+
+# 0.6.6
+
+* JUnit reporter: prefer suite title over testcase classname in a report
+
+# 0.6.5
+
+* Fixed test statuses for runs in JUnit reporter
+
+# 0.6.4
+
+* Added `TESTOMATIO_PROCEED=1` param to not close current run
+* Fixed priority of commands from `npx @testomatio/reporter`
+
+# 0.6.3
+
+* Fixed `npx start-test-run` to launch commands
+
+# 0.6.2
+
+* Added `--env-file` option to load env variables from env file
+
+# 0.6.1
+
+* Fixed creating RunGroup with JUnit reporter
+
+# 0.6.0
+
+* JUnit reporter support
+
+# 0.5.10
+
+* Fixed reporting Scenario Outline in Cypress-Cucumber
+* Fixed error reports for Cypress when running in Chrome
+
+# 0.5.9
+
+* Added environment on Cypress report
+
+# 0.5.8
+
+* Fixed Cypress.io reporting
+
+# 0.5.7
+
+* Fixed webdriverio artifacts
+
+# 0.5.6
+
+* Unmark failed CodeceptJS tests as skipped
+
+# 0.5.5
+
+* Fixed `BeforeSuite` failures in CodeceptJS
+
+# 0.5.4
+
+Added `TESTOMATIO_CREATE=1` option to create unmatched tests on report
+
+```
+TESTOMATIO_CREATE=1 TESTOMATIO=apiKey npx codeceptjs run
+```
+
+# 0.5.3
+
+* Fixed parsing suites
+
+# 0.5.2
+
+* Fixed multiple upload of artifacts in Cypress.io
+
+# 0.5.1
+
+* Fixed Cypress.io to report tests inside nested suites
+
+# 0.5.0
+
+* Added Cypress.io plugin
+* Added artifacts upload to webdriverio
+
+# 0.4.6
+
+- Fixed CodeceptJS reporter to report tests failed in hooks
+
+# 0.4.5
+
+- Fixed "Total XX artifacts publicly uploaded to S3 bucket" when no S3 bucket is configured
+- Improved S3 connection error messages
+
+# 0.4.4
+
+- Fixed returning 0 exit code when a process fails when running tests in parallel via `start-test-run`. Previously was using the last exit code returned by a process. Currently prefers the highest exit code that was returned by a process.
+
+# 0.4.3
+
+- Added `TESTOMATIO_DISABLE_ARTIFACTS` env variable to disable publishing artifacts.
+
+# 0.4.2
+
+- print version of reporter
+- print number of uploaded artifacts
+- print access mode for uploaded artifacts
+
+# 0.4.1
+
+Added `global.testomatioArtifacts = []` array which can be used to add arbitrary artifacts to a report.
+
+```js
+// inside a running test:
+global.testomatioArtifacts.push('file/to/upload.png');
+```
+
+# 0.4.0
+
+- Playwright: Introduced playwright/test support with screenshots and video artifacts
+
+> Known issues: reporting using projects configured in Playwright does not work yet
+
+- CodeceptJS: added video uploads
+
+# 0.3.16
+
+- CodeceptJS: fixed reporting tests with empty steps (on retry)
+
+# 0.3.15
+
+- Finish Run via API:
+
+```
+TESTOMATIO={apiKey} TESTOMATIO_RUN={runId} npx @testomatio/reporter@latest --finish
+```
+
+# 0.3.14
+
+- Create an empty Run via API:
+
+```
+TESTOMATIO={apiKey} npx @testomatio/reporter@latest --launch
+```
+
+# 0.3.13
+
+- Checking for a valid report URL
+- Sending unlimited data on test report
+
+# 0.3.12
+
+- Fixed submitting arbitrary data on a test run
+- Jest: fixed sending errors with stack traces
+- Cypress: fixed sending reports
+
+# 0.3.11
+
+- Fixed circular JSON reference when submitting data to Testomatio
+
+# 0.3.10
+
+- Minor fixes
+
+# 0.3.9
+
+- Making all reporters to run without API key
+
+# 0.3.8
+
+- Fixed `npx start-test-run` to work with empty API keys
+
+# 0.3.7
+
+- Fixed release
+
+# 0.3.6
+
+- Update title and rungroup on start for scheduled runs.
+
+# 0.3.5
+
+- Added `TESTOMATIO_RUN` environment variable to pass id of a specific run to report
+
+# 0.3.4
+
+- Minor fixes
+
+# 0.3.3
+
+- [CodeceptJS] Fixed stack trace reporting
+- [CodeceptJS] Fixed displaying of nested steps
+- [CodeceptJS][mocha] Added assertion diff to report
+
+# 0.3.2
+
+- Fixed error message for S3 uploading
+
+# 0.3.1
+
+- [CodeceptJS] Better formatter for nested structures and BDD tests
+
+# 0.3.0
+
+- Added `TESTOMATIO_TITLE` env variable to set a name for Run
+- Added `TESTOMATIO_RUNGROUP_TITLE` env variable to attach Run to RunGroup
+- Added `TESTOMATIO_ENV` env variable to attach additional env values to report
+- [CodeceptJS] **CodeceptJS v3 support**
+- [CodeceptJS] Dropped support for CodeceptJS 2
+- [CodeceptJS] Added support for before hooks
+- [CodeceptJS] Log of steps
+- [CodeceptJS] Upload screenshots of failed tests to S3
+- [CodeceptJS] Updated to use with parallel execution

package/lib/adapter/codecept.js

@@ -1,7 +1,7 @@
 const debug = require('debug')('@testomatio/reporter:adapter:codeceptjs');
 const chalk = require('chalk');
 const TestomatClient = require('../client');
-const { STATUS, APP_PREFIX, TESTOMAT_TMP_STORAGE } = require('../constants');
+const { STATUS, APP_PREFIX, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
 const upload = require('../fileUploader');
 const { parseTest: getIdFromTestTitle, fileSystem } = require('../utils/utils');
 
@@ -56,7 +56,7 @@ function CodeceptReporter(config) {
   // Listening to events
   event.dispatcher.on(event.all.before, () => {
     // clear tmp dir
-    fileSystem.clearDir(TESTOMAT_TMP_STORAGE.mainDir);
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
 
     recorder.add('Creating new run', () => client.createRun());
     videos = [];
@@ -147,9 +147,6 @@ function CodeceptReporter(config) {
     failedTests.push(id || title);
     let testId = parseTest(tags);
     const testObj = getTestAndMessage(title);
-    if (error && error.stack && test.steps && test.steps.length) {
-      error.stack = test.steps[test.steps.length - 1].line();
-    }
 
     const files = [];
     if (artifacts.screenshot) files.push({ path: artifacts.screenshot, type: 'image/png' });

package/lib/adapter/cucumber/current.js

@@ -2,9 +2,9 @@
 const { Formatter, formatterHelpers } = require('@cucumber/cucumber');
 const chalk = require('chalk');
 const fs = require('fs');
-const { STATUS, TESTOMAT_TMP_STORAGE } = require('../../constants');
+const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../../constants');
 const TestomatClient = require('../../client');
-const logger = require('../../logger');
+const logger = require('../../storages/logger');
 const { parseTest, fileSystem } = require('../../utils/utils');
 
 const { GherkinDocumentParser, PickleParser } = formatterHelpers;
@@ -37,7 +37,7 @@ class CucumberReporter extends Formatter {
 
   parseEnvelope(envelope) {
     if (envelope.testRunStarted) {
-      fileSystem.clearDir(TESTOMAT_TMP_STORAGE.mainDir);
+      fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
     }
     if (envelope.testCaseStarted && this.client) {
       this.client.createRun();

package/lib/adapter/cucumber/legacy.js

@@ -2,7 +2,7 @@
 const { Formatter } = require('cucumber');
 const chalk = require('chalk');
 const { parseTest, fileSystem } = require('../../utils/utils');
-const { STATUS, TESTOMAT_TMP_STORAGE } = require('../../constants');
+const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../../constants');
 const TestomatClient = require('../../client');
 
 const createTestomatFormatter = apiKey => {
@@ -97,7 +97,7 @@ const createTestomatFormatter = apiKey => {
 
       options.eventBroadcaster.on('gherkin-document', addDocument);
       options.eventBroadcaster.on('test-run-started', () => {
-        fileSystem.clearDir(TESTOMAT_TMP_STORAGE.mainDir);
+        fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
         this?.client?.createRun();
       });
       options.eventBroadcaster.on('test-case-finished', this.onTestCaseFinished.bind(this));

package/lib/adapter/jest.js

@@ -1,5 +1,5 @@
 const TestomatClient = require('../client');
-const { STATUS, TESTOMAT_TMP_STORAGE } = require('../constants');
+const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
 const { parseTest, ansiRegExp, fileSystem } = require('../utils/utils');
 
 class JestReporter {
@@ -11,20 +11,9 @@ class JestReporter {
     this.client.createRun();
   }
 
-  static getIdOfCurrentlyRunningTest() {
-    if (!process.env.JEST_WORKER_ID) return null;
-    try {
-      // @ts-expect-error "expect" could only be defined inside Jest environement (forbidden to import it outside)
-      // eslint-disable-next-line no-undef
-      if (expect && expect?.getState()?.currentTestName) return parseTest(expect?.getState()?.currentTestName); 
-    } catch (e) {
-      return null; 
-    }
-  }
-
   onRunStart() {
     // clear tmp dir
-    fileSystem.clearDir(TESTOMAT_TMP_STORAGE.mainDir);
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
   }
 
   onTestResult(test, testResult) {
@@ -72,9 +61,6 @@ class JestReporter {
     const { numFailedTests } = results;
     const status = numFailedTests === 0 ? STATUS.PASSED : STATUS.FAILED;
     this.client.updateRunStatus(status);
-
-    // clear tmp dir
-    fileSystem.clearDir(TESTOMAT_TMP_STORAGE.mainDir);
   }
 }
 

package/lib/adapter/mocha.js

@@ -1,11 +1,9 @@
 // eslint-disable-next-line global-require, import/no-extraneous-dependencies
 const Mocha = require('mocha');
-const debug = require('debug')('@testomatio/reporter:adapter:mocha');
 const chalk = require('chalk');
 const TestomatClient = require('../client');
-const { STATUS, TESTOMAT_TMP_STORAGE } = require('../constants');
-const { parseTest, specificTestInfo, fileSystem } = require('../utils/utils');
-const ArtifactStorage = require('../_ArtifactStorageOld');
+const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
+const { parseTest, fileSystem } = require('../utils/utils');
 
 const { EVENT_RUN_BEGIN, EVENT_RUN_END, EVENT_TEST_FAIL, EVENT_TEST_PASS, EVENT_TEST_PENDING } =
   Mocha.Runner.constants;
@@ -15,7 +13,7 @@ function MochaReporter(runner, opts) {
   let passes = 0;
   let failures = 0;
   let skipped = 0;
-  let artifactStore;
+  // let artifactStore;
 
   const apiKey = opts?.reporterOptions?.apiKey || process.env.TESTOMATIO;
 
@@ -24,13 +22,7 @@ function MochaReporter(runner, opts) {
   runner.on(EVENT_RUN_BEGIN, () => {
     client.createRun();
 
-    const params = {
-      toFile: true,
-    };
-
-    artifactStore = runner._workerReporter !== undefined ? new ArtifactStorage(params) : new ArtifactStorage();
-
-    fileSystem.clearDir(TESTOMAT_TMP_STORAGE.mainDir);
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
   });
 
   runner.on(EVENT_TEST_PASS, async test => {
@@ -38,20 +30,11 @@ function MochaReporter(runner, opts) {
     console.log(chalk.bold.green('✔'), test.fullTitle());
     const testId = parseTest(test.title);
 
-    const specificTest = specificTestInfo(test);
-    const content = await artifactStore.artifactByTestName(specificTest);
-
-    debug(`test=${specificTest} content = `, content);
-
-    client.addTestRun(
-      STATUS.PASSED,
-      {
-        test_id: testId,
-        title: test.title,
-        time: test.duration,
-      },
-      content,
-    );
+    client.addTestRun(STATUS.PASSED, {
+      test_id: testId,
+      title: test.title,
+      time: test.duration,
+    });
   });
 
   runner.on(EVENT_TEST_PENDING, test => {
@@ -70,29 +53,18 @@ function MochaReporter(runner, opts) {
     console.log(chalk.bold.red('✖'), test.fullTitle(), chalk.gray(err.message));
     const testId = parseTest(test.title);
 
-    const specificTest = specificTestInfo(test);
-    const content = await artifactStore.artifactByTestName(specificTest);
-
-    debug(`fail test=${specificTest} content = `, content);
-
-    client.addTestRun(
-      STATUS.FAILED,
-      {
-        error: err,
-        test_id: testId,
-        title: test.title,
-        time: test.duration,
-      },
-      content,
-    );
+    client.addTestRun(STATUS.FAILED, {
+      error: err,
+      test_id: testId,
+      title: test.title,
+      time: test.duration,
+    });
   });
 
   runner.on(EVENT_RUN_END, () => {
     const status = failures === 0 ? STATUS.PASSED : STATUS.FAILED;
     console.log(chalk.bold(status), `${passes} passed, ${failures} failed, ${skipped} skipped`);
     client.updateRunStatus(status);
-
-    artifactStore.cleanup();
   });
 }
 

package/lib/adapter/playwright.js

@@ -3,7 +3,7 @@ const crypto = require('crypto');
 const os = require('os');
 const path = require('path');
 const fs = require('fs');
-const { APP_PREFIX, STATUS: Status, TESTOMAT_TMP_STORAGE } = require('../constants');
+const { APP_PREFIX, STATUS: Status, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
 const TestomatioClient = require('../client');
 const { isArtifactsEnabled } = require('../fileUploader');
 const { parseTest, fileSystem } = require('../utils/utils');
@@ -19,15 +19,17 @@ class PlaywrightReporter {
 
   onBegin(_config, suite) {
     // clean data storage
-    fileSystem.clearDir(TESTOMAT_TMP_STORAGE.mainDir);
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
     if (!this.client) return;
     this.suite = suite;
     this.client.createRun();
   }
 
-  // onTestBegin(test) {
-  //   const testId = parseTest(test.title);
-  // }
+  /*
+  onTestBegin(test) {
+    // does not work; value is not to the storage context
+    global.testTitle = test.title;
+  } */
 
   onTestEnd(test, result) {
     if (!this.client) return;
@@ -44,23 +46,33 @@ class PlaywrightReporter {
       appendStep(step, steps);
     }
 
-    const logs = `\n\n${chalk.bold('Logs:')}\n${chalk.red(result.stderr.join(''))}\n${result.stdout.join('')}`;
-
-    const reportTestPromise = this.client.addTestRun(checkStatus(result.status), {
-      error,
-      test_id: testId,
-      suite_title,
-      title,
-      steps: steps.join('\n'),
-      time: duration,
-      stack: logs,
-    }).then(pipes => {
-      testId = pipes?.filter(p => p.pipe.includes('Testomatio'))[0]?.result?.data?.test_id;
-
-      this.uploads.push({
-        testId, title, suite_title, files: result.attachments.filter((a) => a.body || a.path)
+    let logs = '';
+    if (result.stderr.length || result.stdout.length) {
+      logs = `\n\n${chalk.bold('Logs:')}\n${chalk.red(result.stderr.join(''))}\n${result.stdout.join('')}`;
+    }
+
+    const reportTestPromise = this.client
+      .addTestRun(checkStatus(result.status), {
+        error,
+        test_id: testId,
+        suite_title,
+        title,
+        steps: steps.join('\n'),
+        time: duration,
+        stack: logs,
+      })
+      .then(pipes => {
+        testId = pipes?.filter(p => p.pipe.includes('Testomatio'))[0]?.result?.data?.test_id;
+
+        this.uploads.push({
+          testId,
+          title,
+          suite_title,
+          files: result.attachments.filter(a => a.body || a.path),
+        });
+        // remove empty uploads
+        this.uploads = this.uploads.filter(upload => upload.files.length);
       });
-    });
 
     reportTestPromises.push(reportTestPromise);
   }

package/lib/bin/startTest.js

@@ -53,6 +53,12 @@ program
 
     let exitCode = 0;
 
+    if (!command.split) {
+      process.exitCode = 255;
+      console.log(APP_PREFIX, `No command provided. Use -c option to launch a test runner.`);
+      return;
+    }
+
     const client = new TestomatClient({ apiKey, title, parallel: true });
 
     if(filter) {
@@ -74,12 +80,6 @@ program
       }
     }
 
-    if (!command.split) {
-      process.exitCode = 255;
-      console.log(APP_PREFIX, `No command provided. Use -c option to launch a test runner.`);
-      return;
-    }
-
     const testCmds = command.split(' ');
     console.log(APP_PREFIX, `🚀 Running`, chalk.green(command));