@testomatio/reporter 0.4.0 → 0.4.4

package/Changelog.md

@@ -1,3 +1,26 @@
+# 0.4.4
+
+* Fixed returning 0 exit code when a process fails when running tests in parallel via `start-test-run`. Previously was using the last exit code returned by a process. Currently prefers the highest exit code that was returned by a process.
+
+# 0.4.3
+
+* Added `TESTOMATIO_DISABLE_ARTIFACTS` env variable to disable publishing artifacts.
+
+# 0.4.2
+
+* print version of reporter
+* print number of uploaded artifacts
+* print access mode for uploaded artifacts
+
+# 0.4.1
+
+Added `global.testomatioArtifacts = []` array which can be used to add arbitrary artifacts to a report.
+
+```js
+// inside a running test:
+global.testomatioArtifacts.push('file/to/upload.png');
+```
+
 # 0.4.0
 
 * Playwright: Introduced playwright/test support with screenshots and video artifacts

package/README.md

@@ -169,9 +169,9 @@ Add environments to run by providing `TESTOMATIO_ENV` as comma seperated values:
 TESTOMATIO={apiKey} TESTOMATIO_ENV="Windows, Chrome" <actual run command>
 ```
 
-## Attaching Screenshots
+## Attaching Test Artifacts
 
-To save a screenshot of a failed test use S3 storage.
+To save a test artifacts (screenshots and videos) of a failed test use S3 storage.
 Please note, that the **storage is not connected to Testomatio**.
 This allows you to store your artifacts on your own account and not expose S3 credentials.
 
@@ -183,12 +183,77 @@ To save screenshots provide a configuration for S3 bucket via environment variab
 - **S3_SECRET_ACCESS_KEY** - Secret.
 - **S3_ENDPOINT** - for providers other than AWS
 
-> AWS S3 bucket is required to have `public-read` permission to operate. Public links are used by Testomatio to display screenshots in UI.
+By default tests artifacts are uploaded to bucket with `public-read` permission. 
+In this case uploaded files will be publicly accessible in Internet. 
+These public links will be used by Testomat.io to display images and videos.
+
+To upload files with `private` access bucket add `TESTOMATIO_PRIVATE_ARTIFACTS=1` environment value. 
+Then update provide the same S3 credentials in "Settings > Artifacts" section of a Testomat.io project,
+so Testomat.io could connect to the same bucket and fetch uploaded artifacts. 
+Links to files will be pre-signed and expires automatically in 10 minutes.
+
+Example upload configuration in environment variables
+
+##### AWS
+
+```
+TESTOMATIO_PRIVATE_ARTIFACTS=1
+S3_ACCESS_KEY_ID=11111111111111111111
+S3_SECRET_ACCESS_KEY=2222222222222222222222222222222222222222222
+S3_BUCKET=artifacts
+S3_REGION=us-west-1
+```
+
+##### DigitalOcean
+
+```
+TESTOMATIO_PRIVATE_ARTIFACTS=1
+S3_ENDPOINT=https://ams3.digitaloceanspaces.com
+S3_ACCESS_KEY_ID=11111111111111111111
+S3_SECRET_ACCESS_KEY=2222222222222222222222222222222222222222222
+S3_BUCKET=artifacts
+S3_REGION=ams3
+```
+
+##### Minio
+
+```
+S3_ENDPOINT=http://company.storage.com
+S3_ACCESS_KEY_ID=minio
+S3_SECRET_ACCESS_KEY=minio123
+S3_BUCKET=testomatio
+S3_FORCE_PATH_STYLE=true
+```
+
+> It is important to add S3_FORCE_PATH_STYLE var for minio setup
+
 
 For local testing, it is recommended to store this configuration in `.env` file and load it with [dotenv](https://www.npmjs.com/package/dotenv) library.
 
 On CI set environment variables in CI config.
 
+Test artifacts are automatically uploaded for these test runners:
+
+* CodeceptJS 
+* Playwright
+
+To manually attach an artifact and upload it for a test use `global.testomatioArtifacts` array:
+
+```js
+// attach a picture inside a test
+global.testomatioArtifacts.push('img/file.png');
+// attach a picture and add a name to it
+global.testomatioArtifacts.push({ name: 'Screenshot', path: 'img/file.png' });
+```
+
+Artifacts will be uploaded for the current test when it is finished.
+
+To disable uploading artifacts add `TESTOMATIO_DISABLE_ARTIFACTS` environment variable:
+
+```
+TESTOMATIO_DISABLE_ARTIFACTS=1
+```
+
 
 ## Starting an Empty Run
 

package/lib/adapter/codecept.js

@@ -88,7 +88,7 @@ module.exports = (config) => {
 
   event.dispatcher.on(event.all.result, async () => {
     if (videos.length) {
-      console.log(TRConstants.APP_PREFIX, `🎞️ Uploading ${videos.length} videos...`);
+      console.log(TRConstants.APP_PREFIX, `🎞️  Uploading ${videos.length} videos...`);
 
       let promises = [];
       for (const video of videos) {

package/lib/adapter/playwright.js

@@ -76,7 +76,7 @@ class TestomatioReporter {
     if (!this.client) return;
 
     if (this.videos.length) {
-      console.log(Status.APP_PREFIX, `🎞️ Uploading ${this.videos.length} videos...`);
+      console.log(Status.APP_PREFIX, `🎞️  Uploading ${this.videos.length} videos...`);
 
       let promises = [];
       for (const video of this.videos) {

package/lib/bin/startTest.js

@@ -43,6 +43,8 @@ program
     return;
   }
 
+  let exitCode = 0;
+
   const testCmds = command.split(" ");
   console.log(APP_PREFIX, `🚀 Running`, chalk.green(command));
 
@@ -55,7 +57,9 @@ program
         "⚠️ ",
         `Runner exited with ${chalk.bold(code)}, report is ignored`
       );
-      process.exitCode = code;
+
+      if (code > exitCode) exitCode = code;
+      process.exitCode = exitCode;
     });
 
     return;
@@ -71,7 +75,9 @@ program
       console.log(APP_PREFIX, emoji, `Runner exited with ${chalk.bold(code)}`);
       const status = code === 0 ? "passed" : "failed";
       client.updateRunStatus(status, true);
-      process.exitCode = code;
+
+      if (code > exitCode) exitCode = code;
+      process.exitCode = exitCode;
     });
   });
 });

package/lib/client.js

@@ -2,14 +2,15 @@ const axios = require("axios");
 const JsonCycle = require("json-cycle");
 const { URL } = require("url");
 const createCallsiteRecord = require("callsite-record");
-
-const { sep } = require("path");
+const { sep, join } = require("path");
+const fs = require("fs");
 const chalk = require("chalk");
-const { uploadFile } = require("./fileUploader");
+const { isPrivate, uploadFile } = require("./fileUploader");
 const { PASSED, FAILED, FINISHED, APP_PREFIX } = require("./constants");
 
 const TESTOMAT_URL = process.env.TESTOMATIO_URL || "https://app.testomat.io";
-const { TESTOMATIO_RUNGROUP_TITLE, TESTOMATIO_ENV, TESTOMATIO_RUN } = process.env;
+const { TESTOMATIO_RUNGROUP_TITLE, TESTOMATIO_ENV, TESTOMATIO_RUN } =
+  process.env;
 
 if (TESTOMATIO_RUN) {
   process.env.runId = TESTOMATIO_RUN;
@@ -28,6 +29,8 @@ class TestomatClient {
     this.runId = process.env.runId;
     this.queue = Promise.resolve();
     this.axios = axios.create();
+    this.totalUploaded = 0;
+    this.version = JSON.parse(fs.readFileSync(join(__dirname, '..', 'package.json')).toString()).version;
   }
 
   /**
@@ -37,7 +40,7 @@ class TestomatClient {
    */
   createRun() {
     const { runId } = process.env;
-    if (!this.apiKey) throw new Error('No API key is set, can\'t create run');
+    if (!this.apiKey) throw new Error("No API key is set, can't create run");
     const runParams = {
       api_key: this.apiKey.trim(),
       title: this.title,
@@ -45,10 +48,15 @@ class TestomatClient {
       group_title: TESTOMATIO_RUNGROUP_TITLE,
       env: TESTOMATIO_ENV,
     };
+
+    global.testomatioArtifacts = [];
+
     if (!isValidUrl(TESTOMAT_URL.trim())) {
       console.log(
         APP_PREFIX,
-        chalk.red(`Error creating report on Testomat.io, report url '${TESTOMAT_URL}' is invalid`)
+        chalk.red(
+          `Error creating report on Testomat.io, report url '${TESTOMAT_URL}' is invalid`
+        )
       );
       return;
     }
@@ -56,7 +64,8 @@ class TestomatClient {
     if (runId) {
       this.runId = runId;
       this.queue = this.queue.then(() =>
-        axios.put(`${TESTOMAT_URL.trim()}/api/reporter/${runId}`, runParams));
+        axios.put(`${TESTOMAT_URL.trim()}/api/reporter/${runId}`, runParams)
+      );
       return Promise.resolve(runId);
     }
 
@@ -73,10 +82,12 @@ class TestomatClient {
             console.log(
               APP_PREFIX,
               "📊 Report created. Report ID:",
-              this.runId
+              this.runId,
+              chalk.gray(`v${this.version}`)
             );
             process.env.runId = this.runId;
-          }))
+          })
+      )
       .catch(() => {
         console.log(
           APP_PREFIX,
@@ -113,50 +124,18 @@ class TestomatClient {
     let stack = "";
 
     if (error) {
-      if (!message) message = error.message;
-      if (error.inspect) message = error.inspect();
-
-      stack = `\n${chalk.bold(message)}\n`;
-
-      // diffs for mocha, cypress, codeceptjs style
-      if (error.actual && error.expected) {
-        stack += `\n\n${chalk.bold.green("+ expected")} ${chalk.bold.red(
-          "- actual"
-        )}`;
-        stack += `\n${chalk.red(
-          `- ${error.actual.toString().split("\n").join("\n- ")}`
-        )}`;
-        stack += `\n${chalk.green(
-          `+ ${error.expected.toString().split("\n").join("\n+ ")}`
-        )}`;
-        stack += "\n\n";
-      }
-
-      try {
-        const record = createCallsiteRecord({
-          forError: error,
-        });
-        if (record) {
-          stack += record.renderSync({
-            stackFilter: (frame) =>
-              frame.getFileName().indexOf(sep) > -1 &&
-              frame.getFileName().indexOf("node_modules") < 0 &&
-              frame.getFileName().indexOf("internal") < 0,
-          });
-        }
-      } catch (e) {
-        console.log(e);
-      }
+      stack = this.formatError(error);
     }
     if (steps) {
-      stack = stack
-        ? `${steps}\n\n${chalk.bold.red(
-          "################[ Failure ]################"
-        )}\n${stack}`
-        : steps;
+      stack = this.formatSteps(stack, steps);
     }
 
     if (this.runId) {
+      if (Array.isArray(global.testomatioArtifacts)) {
+        files.push(...global.testomatioArtifacts);
+        global.testomatioArtifacts = [];
+      }
+
       for (const file of files) {
         uploadedFiles.push(uploadFile(file, this.runId));
       }
@@ -165,6 +144,11 @@ class TestomatClient {
     this.queue = this.queue
       .then(async () => {
         if (!this.runId) return;
+
+        global.testomatioArtifacts = [];
+
+        this.totalUploaded += uploadedFiles.filter((n) => n).length;
+
         const json = JsonCycle.stringify({
           api_key: this.apiKey,
           files,
@@ -196,7 +180,7 @@ class TestomatClient {
       .catch((err) => {
         if (err.response) {
           if (err.response.status >= 400) {
-            const data = err.response.data || { message: '' };
+            const data = err.response.data || { message: "" };
             console.log(
               APP_PREFIX,
               chalk.blue(title),
@@ -210,7 +194,12 @@ class TestomatClient {
             `Report couldn't be processed: ${err.response.data.message}`
           );
         } else {
-          console.log(APP_PREFIX, chalk.blue(title), "Report couldn't be processed", err);
+          console.log(
+            APP_PREFIX,
+            chalk.blue(title),
+            "Report couldn't be processed",
+            err
+          );
         }
       });
 
@@ -243,6 +232,15 @@ class TestomatClient {
               chalk.magenta(this.runUrl)
             );
           }
+
+          if (this.totalUploaded > 0) {
+            console.log(
+              APP_PREFIX,
+              `🗄️  Total ${this.totalUploaded} artifacts ${
+                isPrivate ? "privately" : chalk.bold("publicly")
+              } uploaded to S3 bucket  `
+            );
+          }
         }
       })
       .catch((err) => {
@@ -250,6 +248,50 @@ class TestomatClient {
       });
     return this.queue;
   }
+
+  formatSteps(stack, steps) {
+    return stack
+      ? `${steps}\n\n${chalk.bold.red("################[ Failure ]################")}\n${stack}`
+      : steps;
+  }
+
+  formatError(error, message) {
+    if (!message) message = error.message;
+    if (error.inspect) message = error.inspect();
+
+    let stack = `\n${chalk.bold(message)}\n`;
+
+    // diffs for mocha, cypress, codeceptjs style
+    if (error.actual && error.expected) {
+      stack += `\n\n${chalk.bold.green("+ expected")} ${chalk.bold.red(
+        "- actual"
+      )}`;
+      stack += `\n${chalk.red(
+        `- ${error.actual.toString().split("\n").join("\n- ")}`
+      )}`;
+      stack += `\n${chalk.green(
+        `+ ${error.expected.toString().split("\n").join("\n+ ")}`
+      )}`;
+      stack += "\n\n";
+    }
+
+    try {
+      const record = createCallsiteRecord({
+        forError: error,
+      });
+      if (record) {
+        stack += record.renderSync({
+          stackFilter: (frame) =>
+            frame.getFileName().indexOf(sep) > -1 &&
+            frame.getFileName().indexOf("node_modules") < 0 &&
+            frame.getFileName().indexOf("internal") < 0,
+        });
+      }
+      return stack;
+    } catch (e) {
+      console.log(e);
+    }
+  }
 }
 
 module.exports = TestomatClient;

package/lib/fileUploader.js

@@ -3,11 +3,16 @@ const fs = require("fs");
 const path = require("path");
 const chalk = require("chalk");
 
+const isPrivate = process.env.TESTOMATIO_PRIVATE_ARTIFACTS;
+
 const uploadUsingS3 = async (filePath, runId) => {
   const region = process.env.S3_REGION;
   const bucket = process.env.S3_BUCKET;
   const accessKeyId = process.env.S3_ACCESS_KEY_ID;
   const secretAccessKey = process.env.S3_SECRET_ACCESS_KEY;
+  const isDisabled = process.env.TESTOMATIO_DISABLE_ARTIFACTS;
+
+  if (isDisabled) return;
 
   let contentType;
   let fileName;
@@ -34,13 +39,12 @@ const uploadUsingS3 = async (filePath, runId) => {
       Key: fileName,
       Body: file,
       ContentType: contentType,
-      ACL: "public-read",
+      ACL: isPrivate ? "private" : "public-read",
     })
     .promise();
   return out.Location;
 };
 
-
 const uploadFile = async (filePath, runId) => {
   try {
     if (process.env.S3_BUCKET) {
@@ -54,4 +58,5 @@ const uploadFile = async (filePath, runId) => {
 
 module.exports = {
   uploadFile,
+  isPrivate,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@testomatio/reporter",
-  "version": "0.4.0",
+  "version": "0.4.4",
   "description": "Testomatio Reporter Client",
   "main": "./lib/reporter.js",
   "repository": "git@github.com:testomatio/reporter.git",