@testomatio/reporter 1.0.3 → 1.0.5

package/README.md

@@ -12,7 +12,7 @@ Testomat.io Reporter (this npm package) supports:
 * 🏄 Integarion with all popular [JavaScript/TypeScript frameworks](./docs/frameworks.md)
 * 🗄️ Screenshots, videos, traces [uploaded into S3 bucket](./docs/artifacts.md)
 * 🔎 Stack traces and error messages
-* 🐙 [GitHub](./docs/pipes.md#github-pipe) & [GitLab](./docs/pipes.md#gitlab-pipe) integration
+* 🐙 [GitHub](./docs/pipes/github.d) & [GitLab](./docs/pipes/gitlab.d) integration
 * 🚅 Realtime reports
 * 🗃️ Other test frameworks supported via [JUNit XML](./docs/junit.md)
 * 🚶‍♀️ Steps *(work in progress)*
@@ -85,13 +85,13 @@ or any [other via JUnit](./docs/junit.md) report....
 
 ### 2️⃣ Configure Reports
 
-* [Create report on Testomat.io](./docs/pipes.md#testomatio-pipe).
-* [Create brief summary report for GitHub Pull Request](./docs/pipes.md#github-pipe) 👇
-* [Create brief summary report for GitLab Merge Request](./docs/pipes.md#gitlab-pipe).
-* [Configure other pipes](./docs/pipes.md) for other ways to process test results output.
+* [Create report on Testomat.io](./docs/pipes/testomatio.md).
+* [Create brief summary report for GitHub Pull Request](./docs/pipes/github.md) 👇
+* [Create brief summary report for GitLab Merge Request](./docs/pipes/gitlab.md).
+* [Configure other pipes](./docs/pipes/md) for other ways to process test results output.
 
 
-![GitHub report](./docs/images/github.png)
+![GitHub report](./docs/pipes/images/github.png)
 
 GitHub report published as a comment to Pull Request:
 
@@ -121,6 +121,10 @@ Bring this reporter on CI and never lose test results again!
 
 * 🛠️ [Frameworks](./docs/frameworks.md)
 * ⛲ [Pipes](./docs/pipes.md)
+  * [Testomat.io](./docs/pipes/testomatio.md)
+  * [GitHub](./docs/pipes/github.md)
+  * [Gitlab](./docs/pipes/gitlab.md)
+  * [CSV](./docs/pipes/csv.md)
 * 📓 [JUnit](./docs/junit.md)
 * 🗄️ [Artifacts](./docs/artifacts.md)
 * 🔂 [Workflows](./docs/workflows.md)

package/lib/pipe/github.js

@@ -21,7 +21,8 @@ class GitHubPipe {
     this.token = params.GH_PAT || process.env.GH_PAT;
     this.ref = process.env.GITHUB_REF;
     this.repo = process.env.GITHUB_REPOSITORY;
-    this.hiddenCommentData = `<!--- testomat.io report ${process.env.GITHUB_WORKFLOW || ''} -->`;
+    this.jobKey = `${process.env.GITHUB_WORKFLOW || ''} / ${process.env.GITHUB_JOB || ''}`;
+    this.hiddenCommentData = `<!--- testomat.io report ${this.jobKey} -->`;
 
     debug('GitHub Pipe: ', this.token ? 'TOKEN' : '*no token*', 'Ref:', this.ref, 'Repo:', this.repo);
 
@@ -73,35 +74,32 @@ class GitHubPipe {
     let summary = `${this.hiddenCommentData}
 
 | [![Testomat.io Report](https://avatars.githubusercontent.com/u/59105116?s=36&v=4)](https://testomat.io)  | ${
-  statusEmoji(runParams.status,)} ${runParams.status.toUpperCase()} ${statusEmoji(runParams.status)} |
+  statusEmoji(runParams.status,)} ${`${process.env.GITHUB_JOB} ${runParams.status}`.toUpperCase()} |
 | --- | --- |        
 | Tests | ✔️  **${this.tests.length}** tests run  |
-| Summary | ${statusEmoji('failed')} **${failedCount}** failed; ${statusEmoji(
+| Summary | ${failedCount ? `${statusEmoji('failed')} **${failedCount}** failed; ` : ''} ${statusEmoji(
       'passed',
     )} **${passedCount}** passed; **${statusEmoji('skipped')}** ${skippedCount} skipped |
-| Duration | 🕐  **${humanizeDuration(parseInt(this.tests.reduce((a, t) => a + (t.run_time || 0), 0), 10), {
-      maxDecimalPoints: 0,
-    })}** |
-`;
+| Duration | 🕐  **${humanizeDuration(
+      parseInt(
+        this.tests.reduce((a, t) => a + (t.run_time || 0), 0),
+        10,
+      ),
+      {
+        maxDecimalPoints: 0,
+      },
+    )}** |`;
+
     if (this.store.runUrl) {
-      summary += `| Testomat.io Report | 📊 [Run #${this.store.runId}](${this.store.runUrl})  | `;
+      summary += `\n| Testomat.io Report | 📊 [Run #${this.store.runId}](${this.store.runUrl})  | `;
     }
     if (process.env.GITHUB_WORKFLOW) {
-      summary += `| Workflow | 🗂️  ${process.env.GITHUB_WORKFLOW} | `;
+      summary += `\n| Job | 🗂️  [${this.jobKey}](${process.env.GITHUB_SERVER_URL || 'https://github.com'}/${
+        this.repo
+      }/actions/runs/${process.env.GITHUB_RUN_ID}) | `;
     }
     if (process.env.RUNNER_OS) {
-      summary += `| Operating System | 🖥️ \`${process.env.RUNNER_OS}\` ${process.env.RUNNER_ARCH || ''} | `;
-    }
-    if (process.env.GITHUB_HEAD_REF) {
-      summary += `| Branch | 🌳  \`${process.env.GITHUB_HEAD_REF}\` | `;
-    }
-    if (process.env.GITHUB_RUN_ATTEMPT) {
-      summary += `| Run Attempt | 🌒  \`${process.env.GITHUB_RUN_ATTEMPT}\` | `;
-    }
-    if (process.env.GITHUB_RUN_ID) {
-      summary += `| Build Log | ✒️  ${process.env.GITHUB_SERVER_URL || 'https://github.com'}/${
-        this.repo
-      }/actions/runs/${process.env.GITHUB_RUN_ID} | `;
+      summary += `\n| Operating System | 🖥️ \`${process.env.RUNNER_OS}\` ${process.env.RUNNER_ARCH || ''} | `;
     }
 
     const failures = this.tests

package/lib/pipe/testomatio.js

@@ -38,7 +38,8 @@ class TestomatioPipe {
     // do not finish this run (for parallel testing)
     this.proceed = process.env.TESTOMATIO_PROCEED;
     this.runId = params.runId || process.env.runId;
-    this.createNewTests = !!process.env.TESTOMATIO_CREATE;
+    this.createNewTests = params.createNewTests ?? !!process.env.TESTOMATIO_CREATE;
+    this.hasUnmatchedTests = false;
 
     if (!isValidUrl(this.url.trim())) {
       this.isEnabled = false;
@@ -115,14 +116,17 @@ class TestomatioPipe {
           const responseData = err.response.data || { message: '' };
           console.log(
             APP_PREFIX,
-            chalk.blue(this.title),
-            `Report couldn't be processed: (${err.response.status}) ${responseData.message}`,
+            chalk.yellow(`Warning: ${responseData.message} (${err.response.status})`),
+            chalk.grey(data?.title || ''),
           );
+          if (err.response.data.message.includes('could not be matched')) {
+            this.hasUnmatchedTests = true;
+          }          
           return;
         }
-        console.log(APP_PREFIX, chalk.blue(this.title), `Report couldn't be processed: ${err.response.data.message}`);
+        console.log(APP_PREFIX, chalk.blue(data?.title || ''), `Report couldn't be processed: ${err.response.data.message}`);
       } else {
-        console.log(APP_PREFIX, chalk.blue(this.title), "Report couldn't be processed", err);
+        console.log(APP_PREFIX, chalk.blue(data?.title || ''), "Report couldn't be processed", err);
       }
     });
   }
@@ -159,6 +163,17 @@ class TestomatioPipe {
         console.log(APP_PREFIX, `📊 ${notFinishedMessage}. Report URL: ${chalk.magenta(this.runUrl)}`);
         console.log(APP_PREFIX, `🛬 Run to finish it: TESTOMATIO_RUN=${this.runId} npx start-test-run --finish`);
       }
+      if (this.hasUnmatchedTests) {
+        console.log('');
+        console.log(APP_PREFIX, chalk.yellow.bold('⚠️ Some reported tests were not found in Testomat.io project'));
+        console.log(APP_PREFIX, `If you use Testomat.io as a reporter only, please re-run tests using ${chalk.bold('TESTOMATIO_CREATE=1')}`);
+        console.log(APP_PREFIX, `But to keep your tests consistent it is recommended to ${chalk.bold('import tests first')}`);
+        console.log(APP_PREFIX, 'If tests were imported but still not matched, assign test IDs to your tests.');
+        console.log(APP_PREFIX, 'You can do that automatically via command line tools:');
+        console.log(APP_PREFIX, chalk.bold('npx check-tests ... --update-ids'), 'See: https://bit.ly/js-update-ids');
+        console.log(APP_PREFIX, 'or for Cucumber:');
+        console.log(APP_PREFIX, chalk.bold('npx check-cucumber ... --update-ids'), 'See: https://bit.ly/bdd-update-ids');
+      }
     } catch (err) {
       console.log(APP_PREFIX, 'Error updating status, skipping...', err);
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@testomatio/reporter",
-  "version": "1.0.3",
+  "version": "1.0.5",
   "description": "Testomatio Reporter Client",
   "main": "./lib/reporter.js",
   "typings": "typings/index.d.ts",

package/Changelog.md

@@ -1,269 +0,0 @@
-<!-- pending release updates -->
-
-# 1.0.3
-
-* Fixed reporting parallel runs
-
-# 1.0.0
-
-
-* Added [`TESTOMATIO_SHARED_RUN` option](https://github.com/testomatio/reporter/blob/master/docs/pipes.md#reporting-parallel-execution-to-to-same-run) to use a shared run for parallel executions
-* Reworked [documentation](https://github.com/testomatio/reporter/tree/master#readme).
-* Added an option to obtain [S3 configuration](https://github.com/testomatio/reporter/blob/master/docs/artifacts.md#configuration) from Testomat.io
-* Introduced [pipes](https://github.com/testomatio/reporter/blob/master/docs/pipes.md):
-  * GitHub
-  * GitLab
-  * CSV Pipe
-
-
-# 0.7.6
-
-* Updated to use AWS S3 3.0 SDK for uploading
-
-# 0.7.5
-
-* Fixed reporting skipped tests in mocha
-
-# 0.7.4
-
-* Fixed parsing source code in JUnit files
-
-# 0.7.3
-
-* CodeceptJS: Upload all traces and videos from artifacts
-* Fixed reporting skipped test in XML
-* added `--timelimit` option to `report-xml` command line
-
-# 0.7.2
-
-* Fixed uploading non-existing file
-
-# 0.7.1
-
-* Support for NUnit XML v3 format
-
-# 0.7.0
-
-* Support for `@cucumber/cucumber` (>= 7.0) added
-* Initial support for C# and NUnit  
-
-# 0.6.10
-
-* Fixed uploading multilpe artifacts in Playwright
-
-# 0.6.9
-
-* Fixed pending tests reports for Cypress
-
-# 0.6.8
-# 0.6.7
-
-* Pytest: fixed creating suites from reports
-
-# 0.6.6
-
-* JUnit reporter: prefer suite title over testcase classname in a report
-
-# 0.6.5
-
-* Fixed test statuses for runs in JUnit reporter
-
-# 0.6.4
-
-* Added `TESTOMATIO_PROCEED=1` param to not close current run
-* Fixed priority of commands from `npx @testomatio/reporter`
-
-# 0.6.3
-
-* Fixed `npx start-test-run` to launch commands
-
-# 0.6.2
-
-* Added `--env-file` option to load env variables from env file
-
-# 0.6.1
-
-* Fixed creating RunGroup with JUnit reporter
-
-# 0.6.0
-
-* JUnit reporter support
-
-# 0.5.10
-
-* Fixed reporting Scenario Outline in Cypress-Cucumber
-* Fixed error reports for Cypress when running in Chrome
-
-# 0.5.9
-
-* Added environment on Cypress report
-
-# 0.5.8
-
-* Fixed Cypress.io reporting
-
-# 0.5.7
-
-* Fixed webdriverio artifacts
-
-# 0.5.6
-
-* Unmark failed CodeceptJS tests as skipped
-
-# 0.5.5
-
-* Fixed `BeforeSuite` failures in CodeceptJS
-
-# 0.5.4
-
-Added `TESTOMATIO_CREATE=1` option to create unmatched tests on report
-
-```
-TESTOMATIO_CREATE=1 TESTOMATIO=apiKey npx codeceptjs run
-```
-
-# 0.5.3
-
-* Fixed parsing suites
-
-# 0.5.2
-
-* Fixed multiple upload of artifacts in Cypress.io
-
-# 0.5.1
-
-* Fixed Cypress.io to report tests inside nested suites
-
-# 0.5.0
-
-* Added Cypress.io plugin
-* Added artifacts upload to webdriverio
-
-# 0.4.6
-
-- Fixed CodeceptJS reporter to report tests failed in hooks
-
-# 0.4.5
-
-- Fixed "Total XX artifacts publicly uploaded to S3 bucket" when no S3 bucket is configured
-- Improved S3 connection error messages
-
-# 0.4.4
-
-- Fixed returning 0 exit code when a process fails when running tests in parallel via `start-test-run`. Previously was using the last exit code returned by a process. Currently prefers the highest exit code that was returned by a process.
-
-# 0.4.3
-
-- Added `TESTOMATIO_DISABLE_ARTIFACTS` env variable to disable publishing artifacts.
-
-# 0.4.2
-
-- print version of reporter
-- print number of uploaded artifacts
-- print access mode for uploaded artifacts
-
-# 0.4.1
-
-Added `global.testomatioArtifacts = []` array which can be used to add arbitrary artifacts to a report.
-
-```js
-// inside a running test:
-global.testomatioArtifacts.push('file/to/upload.png');
-```
-
-# 0.4.0
-
-- Playwright: Introduced playwright/test support with screenshots and video artifacts
-
-> Known issues: reporting using projects configured in Playwright does not work yet
-
-- CodeceptJS: added video uploads
-
-# 0.3.16
-
-- CodeceptJS: fixed reporting tests with empty steps (on retry)
-
-# 0.3.15
-
-- Finish Run via API:
-
-```
-TESTOMATIO={apiKey} TESTOMATIO_RUN={runId} npx @testomatio/reporter@latest --finish
-```
-
-# 0.3.14
-
-- Create an empty Run via API:
-
-```
-TESTOMATIO={apiKey} npx @testomatio/reporter@latest --launch
-```
-
-# 0.3.13
-
-- Checking for a valid report URL
-- Sending unlimited data on test report
-
-# 0.3.12
-
-- Fixed submitting arbitrary data on a test run
-- Jest: fixed sending errors with stack traces
-- Cypress: fixed sending reports
-
-# 0.3.11
-
-- Fixed circular JSON reference when submitting data to Testomatio
-
-# 0.3.10
-
-- Minor fixes
-
-# 0.3.9
-
-- Making all reporters to run without API key
-
-# 0.3.8
-
-- Fixed `npx start-test-run` to work with empty API keys
-
-# 0.3.7
-
-- Fixed release
-
-# 0.3.6
-
-- Update title and rungroup on start for scheduled runs.
-
-# 0.3.5
-
-- Added `TESTOMATIO_RUN` environment variable to pass id of a specific run to report
-
-# 0.3.4
-
-- Minor fixes
-
-# 0.3.3
-
-- [CodeceptJS] Fixed stack trace reporting
-- [CodeceptJS] Fixed displaying of nested steps
-- [CodeceptJS][mocha] Added assertion diff to report
-
-# 0.3.2
-
-- Fixed error message for S3 uploading
-
-# 0.3.1
-
-- [CodeceptJS] Better formatter for nested structures and BDD tests
-
-# 0.3.0
-
-- Added `TESTOMATIO_TITLE` env variable to set a name for Run
-- Added `TESTOMATIO_RUNGROUP_TITLE` env variable to attach Run to RunGroup
-- Added `TESTOMATIO_ENV` env variable to attach additional env values to report
-- [CodeceptJS] **CodeceptJS v3 support**
-- [CodeceptJS] Dropped support for CodeceptJS 2
-- [CodeceptJS] Added support for before hooks
-- [CodeceptJS] Log of steps
-- [CodeceptJS] Upload screenshots of failed tests to S3
-- [CodeceptJS] Updated to use with parallel execution