@testomatio/reporter 1.1.0-beta-3 → 1.1.0-beta.artifact-count-retry-msg-1

package/lib/adapter/codecept.js

@@ -1,7 +1,7 @@
 const debug = require('debug')('@testomatio/reporter:adapter:codeceptjs');
 const chalk = require('chalk');
 const TestomatClient = require('../client');
-const { STATUS, APP_PREFIX, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
+const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
 const upload = require('../fileUploader');
 const { parseTest: getIdFromTestTitle, fileSystem } = require('../utils/utils');
 
@@ -89,7 +89,7 @@ function CodeceptReporter(config) {
     await Promise.all(reportTestPromises);
 
     if (upload.isArtifactsEnabled()) {
-      uploadAttachments(client, videos, '🎞️  Uploading', 'video');
+      uploadAttachments(client, videos, '🎞️ Uploading', 'video');
       uploadAttachments(client, traces, '📁 Uploading', 'trace');
     }
 
@@ -147,9 +147,6 @@ function CodeceptReporter(config) {
     failedTests.push(id || title);
     let testId = parseTest(tags);
     const testObj = getTestAndMessage(title);
-    if (error && error.stack && test.steps && test.steps.length) {
-      error.stack = test.steps[test.steps.length - 1].line();
-    }
 
     const files = [];
     if (artifacts.screenshot) files.push({ path: artifacts.screenshot, type: 'image/png' });
@@ -256,7 +253,7 @@ function CodeceptReporter(config) {
 
 async function uploadAttachments(client, attachments, messagePrefix, attachmentType) {
   if (attachments.length > 0) {
-    console.log(APP_PREFIX, `Attachments: ${messagePrefix} ${attachments.length} ${attachmentType}/-s ...`);
+    debug(`Attachments: ${messagePrefix} ${attachments.length} ${attachmentType}/-s ...`);
 
     const promises = attachments.map(async (attachment) => {
       const { testId, title, path, type } = attachment;

package/lib/adapter/cypress-plugin/index.js

@@ -10,6 +10,7 @@ const testomatioReporter = on => {
   const client = new TestomatClient({ apiKey: process.env.TESTOMATIO });
 
   on('before:run', async (run) => {
+    // TODO: looks like client.env does not exist
     if (!client.env) {
       client.env = `${run.browser.displayName},${run.system.osName}`
     }
@@ -25,8 +26,9 @@ const testomatioReporter = on => {
       const lastAttemptIndex = test.attempts.length - 1;
       const latestAttempt = test.attempts[lastAttemptIndex];
 
-      const time = latestAttempt.duration;
-      const error = latestAttempt.error;
+      // latestAttempt.duration && latestAttempt.error were available in adapters version up to 13 JFYI
+      const time = latestAttempt.duration || latestAttempt.wallClockDuration || test.duration;
+      let error = latestAttempt.error;
 
       let title = test.title.pop();
       const examples = title.match(/\(example (#\d+)\)/);
@@ -39,19 +41,30 @@ const testomatioReporter = on => {
       const testId = parseTest(title);
       const suiteId = parseSuite(suiteTitle);
 
-      if (error) {
+      if (!error && test.displayError) {
+        error = { message: test.displayError };
         error.inspect = function() { // eslint-disable-line func-names
-          if (this && this.codeFrame) {
-            return this.codeFrame.frame;
-          }
-          return '';
-        }
+          return this.message;
+        };
       }
 
-      const screenshots = results.screenshots
-        .filter(screenshot => screenshot.path.includes(title))
-        .filter(screenshot => screenshot.testAttemptIndex === lastAttemptIndex)
-        .map(screenshot => screenshot.path);
+      const formattedError = error
+        ? {
+            message: error.message,
+            inspect: error.inspect || function() { return this.message; },
+          }
+        : '';
+
+      const screenshots = Array.isArray(results.screenshots)
+        ? results.screenshots
+            .filter(
+              screenshot =>
+                screenshot?.path &&
+                screenshot?.path.includes(title) &&
+                screenshot?.takenAt
+            )
+            .map(screenshot => screenshot.path)
+        : [];
 
       const files = [...videos, ...screenshots];
 
@@ -65,8 +78,16 @@ const testomatioReporter = on => {
           state = STATUS.SKIPPED;
       }
 
-      addSpecTestsPromises.push(client.addTestRun(state, {
-        title, time, example, error, files, suite_title: suiteTitle, test_id: testId, suite_id: suiteId
+      addSpecTestsPromises.push(
+        client.addTestRun(state, {
+          title, 
+          time, 
+          example, 
+          error: formattedError, 
+          files, 
+          suite_title: suiteTitle, 
+          test_id: testId, 
+          suite_id: suiteId
       }));
     }
 

package/lib/adapter/playwright.js

@@ -1,9 +1,10 @@
+const debug = require('debug')('@testomatio/reporter:adapter:playwright');
 const chalk = require('chalk');
 const crypto = require('crypto');
 const os = require('os');
 const path = require('path');
 const fs = require('fs');
-const { APP_PREFIX, STATUS: Status, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
+const { STATUS: Status, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
 const TestomatioClient = require('../client');
 const { isArtifactsEnabled } = require('../fileUploader');
 const { parseTest, fileSystem } = require('../utils/utils');
@@ -83,7 +84,7 @@ class PlaywrightReporter {
     await Promise.all(reportTestPromises);
 
     if (this.uploads.length && isArtifactsEnabled()) {
-      console.log(APP_PREFIX, `🎞️  Uploading ${this.uploads.length} files...`);
+      debug(`Attachments: 📁 Uploading ${this.uploads.length} files...`);
 
       const promises = [];
 
@@ -95,7 +96,11 @@ class PlaywrightReporter {
             const fileName = tmpFile();
             fs.writeFileSync(fileName, attachment.body);
           }
-          return { path: attachment.path, title, type: attachment.contentType };
+          return {
+            path: attachment.path,
+            title,
+            type: attachment.contentType 
+          };
         });
 
         promises.push(

package/lib/bin/startTest.js

@@ -53,6 +53,12 @@ program
 
     let exitCode = 0;
 
+    if (!command.split) {
+      process.exitCode = 255;
+      console.log(APP_PREFIX, `No command provided. Use -c option to launch a test runner.`);
+      return;
+    }
+
     const client = new TestomatClient({ apiKey, title, parallel: true });
 
     if(filter) {
@@ -74,12 +80,6 @@ program
       }
     }
 
-    if (!command.split) {
-      process.exitCode = 255;
-      console.log(APP_PREFIX, `No command provided. Use -c option to launch a test runner.`);
-      return;
-    }
-
     const testCmds = command.split(' ');
     console.log(APP_PREFIX, `🚀 Running`, chalk.green(command));
 

package/lib/client.js

@@ -169,19 +169,19 @@ class Client {
 
     for (const file of files) {
       uploadedFiles.push(upload.uploadFileByPath(file, this.uuid));
+      this.totalUploaded++;
     }
 
     for (const [idx, buffer] of filesBuffers.entries()) {
       const fileName = `${idx + 1}-${title.replace(/\s+/g, '-')}`;
       uploadedFiles.push(upload.uploadFileAsBuffer(buffer, fileName, this.uuid));
+      this.totalUploaded++;
     }
 
     const artifacts = await Promise.all(uploadedFiles);
 
     // global.testomatioArtifacts = [];
 
-    this.totalUploaded += uploadedFiles.filter(n => n).length;
-
     const data = {
       files,
       steps,
@@ -204,8 +204,13 @@ class Client {
         this.pipes.map(async p => {
           try {
             const result = await p.addTest(data);
-            return { pipe: p.toString(), result };
-          } catch (err) {
+
+            return {
+              pipe: p.toString(),
+              result
+            };
+          } 
+          catch (err) {
             console.log(APP_PREFIX, p.toString(), err);
           }
         }),
@@ -233,14 +238,27 @@ class Client {
       .then(() => Promise.all(this.pipes.map(p => p.finishRun(runParams))))
       .then(() => {
         debug('TOTAL artifacts', this.totalUploaded);
-        if (this.totalUploaded && !upload.isArtifactsEnabled())
+        if (this.totalUploaded && !upload.isArtifactsEnabled()) {
           debug(`${this.totalUploaded} artifacts are not uploaded, because artifacts uploading is not enabled`);
+          console.log(APP_PREFIX, `To ${chalk.bold('disable')} artifact uploads set: TESTOMATIO_DISABLE_ARTIFACTS=1`);
+        }
 
         if (this.totalUploaded && upload.isArtifactsEnabled()) {
+          const isPrivate = process.env.TESTOMATIO_PRIVATE_ARTIFACTS;
+
+          if (!isPrivate) {
+            console.log(APP_PREFIX, `To enable ${chalk.bold('PRIVATE')} uploads set: TESTOMATIO_PRIVATE_ARTIFACTS=1`);
+          }          
+          console.log(
+            APP_PREFIX,
+            `To enable ${chalk.bold('PUBLIC')} uploads remove TESTOMATIO_PRIVATE_ARTIFACTS env variable`,
+          );
+          console.log(APP_PREFIX, '---------------');
+
           console.log(
             APP_PREFIX,
             `🗄️ ${this.totalUploaded} artifacts ${
-              process.env.TESTOMATIO_PRIVATE_ARTIFACTS ? 'privately' : chalk.bold('publicly')
+              isPrivate ? 'privately' : chalk.bold('publicly')
             } uploaded to S3 bucket`,
           );
         }

package/lib/config.js

@@ -0,0 +1,24 @@
+// This file is used to read environment variables from .env file and process.env
+
+// ! uncommenting next line leads ro reading vars from .env file
+// require('dotenv').config();
+const debug = require('debug')('@testomatio/reporter:config');
+
+/* for possibility to use multiple env files (reading different paths)
+const dotenv = require('dotenv');
+const envFileVars = dotenv.config({ path: '.env' }).parsed; */
+
+// select only TESTOMATIO related variables (only to print them in debug)
+const testomatioEnvVars =
+  Object.keys(process.env)
+    .filter(key => key.startsWith('TESTOMATIO') || key.startsWith('S3_'))
+    .reduce((obj, key) => {
+      obj[key] = process.env[key];
+      return obj;
+    }, {}) || {};
+debug('TESTOMATIO variables:', testomatioEnvVars);
+
+// includes variables from .env file and process.env
+const config = process.env;
+
+module.exports = config;

package/lib/constants.js

@@ -20,10 +20,17 @@ const STATUS = {
   SKIPPED: 'skipped',
   FINISHED: 'finished',
 };
+// html pipe var
+const HTML_REPORT = {
+  FOLDER: "html-report",
+  REPORT_DEFAULT_NAME: "testomatio-report.html",
+  TEMPLATE_NAME: 'testomatio.hbs'
+};
 
 module.exports = {
   APP_PREFIX,
   TESTOMAT_TMP_STORAGE_DIR,
   CSV_HEADERS,
   STATUS,
+  HTML_REPORT
 }

package/lib/fileUploader.js

@@ -3,10 +3,16 @@ const { S3 } = require('@aws-sdk/client-s3');
 const { Upload } = require('@aws-sdk/lib-storage');
 
 const fs = require('fs');
+const util = require('util');
 const path = require('path');
+const promiseRetry = require('promise-retry');
+
+const readFile = util.promisify(fs.readFile);
+const stat = util.promisify(fs.stat);
 const chalk = require('chalk');
 const { randomUUID } = require('crypto');
 const memoize = require('lodash.memoize');
+
 const { APP_PREFIX } = require('./constants');
 
 const keys = [
@@ -93,62 +99,63 @@ const uploadUsingS3 = async (filePath, runId) => {
     Key = filePath.name;
   }
 
-  if (!fs.existsSync(filePath)) {
-    console.error(chalk.yellow(`Artifacts file ${filePath} does not exist. Skipping...`));
-    return;
-  }
-
   const {
-    TESTOMATIO_PRIVATE_ARTIFACTS, S3_BUCKET
+    TESTOMATIO_PRIVATE_ARTIFACTS,
+    S3_BUCKET
   } = getConfig();
 
-  debug('S3 config', getMaskedConfig());
-  debug('Uploading', filePath, 'to', S3_BUCKET);
-  
-const fileData = fs.readFileSync(filePath);
-
-  Key = `${runId}/${randomUUID()}-${Key || path.basename(filePath)}`;
-  const ACL = TESTOMATIO_PRIVATE_ARTIFACTS ? 'private' : 'public-read';
+  try {
+    debug('S3 config', getMaskedConfig());
+    debug('Started upload', filePath, 'to ', S3_BUCKET);
 
-  const s3 = new S3(_getS3Config());
+    // Verification that the file was actually created: 20 attempts of 0.5 second => 10sec
+    const isFileExist = await checkFileExists(filePath, 20, 500);
 
-  const params = {
-    Bucket: S3_BUCKET,
-    Key,
-    Body: fileData,
-    ContentType,
-    ACL,
-  };
+    if (!isFileExist) {
+      console.error(chalk.yellow(`Artifacts file ${filePath} does not exist. Skipping...`));
+      return;
+    }
 
-  try {
-    const out = new Upload({
-      client: s3,
-      params
-    });
+    debug('File: ', filePath, ' exists');
+    
+    const fileData = await readFile(filePath);
 
-    await out.done();
-    debug('Uploaded', out.singleUploadResult.Location)
+    Key = `${runId}/${randomUUID()}-${Key || path.basename(filePath)}`;
 
-    return out.singleUploadResult.Location;
-  } catch (e) {
-    console.log(e);
-    console.log(APP_PREFIX, chalk.red(`Failed uploading '${Key}'. Please check S3 credentials`), getMaskedConfig());
+    const ACL = TESTOMATIO_PRIVATE_ARTIFACTS ? 'private' : 'public-read';
 
-    console.log(APP_PREFIX, `To ${chalk.bold('disable')} artifact uploads set: TESTOMATIO_DISABLE_ARTIFACTS=1`);
-    if (!TESTOMATIO_PRIVATE_ARTIFACTS) {
-      console.log(APP_PREFIX, `To enable ${chalk.bold('PRIVATE')} uploads set: TESTOMATIO_PRIVATE_ARTIFACTS=1`);
-    } else {
+    if (!S3_BUCKET || !fileData) {
       console.log(
         APP_PREFIX,
-        `To enable ${chalk.bold('PUBLIC')} uploads remove TESTOMATIO_PRIVATE_ARTIFACTS env variable`,
-      );
+        chalk.bold.red(`Failed uploading '${Key}'. Please check S3 credentials`), getMaskedConfig());
+      return;
     }
+
+    const s3 = new S3(_getS3Config());
+
+    const params = {
+      Bucket: S3_BUCKET,
+      Key,
+      Body: fileData,
+      ContentType,
+      ACL,
+    };
+  
+    const out = new Upload({
+      client: s3,
+      params
+    });
+
+    return await getS3LocationLink(out);
+  } 
+  catch (e) {
+    debug('S3 file uploading error: ', e);
+    console.log(APP_PREFIX, 'S3 file uploading error: ', e);
     console.log(APP_PREFIX, '---------------');
   } 
 };
 
 const uploadUsingS3AsBuffer = async (buffer, fileName, runId) => {
-
   const {
     S3_REGION, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_ENDPOINT, TESTOMATIO_PRIVATE_ARTIFACTS, S3_BUCKET
   } = getConfig();
@@ -158,6 +165,18 @@ const uploadUsingS3AsBuffer = async (buffer, fileName, runId) => {
   const fileExtension = _getFileExtBase64(buffer.toString('base64'));
   const Key = `${runId}/${fileName}${fileExtension}`;
 
+  if (!S3_BUCKET || !buffer) {
+    console.log(APP_PREFIX, chalk.bold.red(`Failed uploading '${Key}'. Please check S3 credentials`), {
+      accessKeyId: S3_ACCESS_KEY_ID,
+      secretAccessKey: S3_SECRET_ACCESS_KEY ? '**** (hidden) ***' : '(empty)',
+      region: S3_REGION,
+      bucket: S3_BUCKET,
+      acl: ACL,
+      endpoint: S3_ENDPOINT,
+    });
+    return;
+  }
+
   const s3 = new S3(_getS3Config());
 
   try {
@@ -171,28 +190,13 @@ const uploadUsingS3AsBuffer = async (buffer, fileName, runId) => {
         ACL,
       }
     });
-    await out.done();
 
-    return out.singleUploadResult.Location;
-  } catch (e) {
-    console.log(APP_PREFIX, chalk.bold.red(`Failed uploading '${Key}'. Please check S3 credentials`), {
-      accessKeyId: S3_ACCESS_KEY_ID,
-      secretAccessKey: S3_SECRET_ACCESS_KEY ? '**** (hidden) ***' : '(empty)',
-      region: S3_REGION,
-      bucket: S3_BUCKET,
-      acl: ACL,
-      endpoint: S3_ENDPOINT,
-    });
+    return await getS3LocationLink(out);
+  } 
+  catch (e) {
+    debug('S3 buffer uploading error: ', e);
 
-    console.log(APP_PREFIX, `To ${chalk.bold('disable')} artifact uploads set: TESTOMATIO_DISABLE_ARTIFACTS=1`);
-    if (!TESTOMATIO_PRIVATE_ARTIFACTS) {
-      console.log(APP_PREFIX, `To enable ${chalk.bold('PRIVATE')} uploads set: TESTOMATIO_PRIVATE_ARTIFACTS=1`);
-    } else {
-      console.log(
-        APP_PREFIX,
-        `To enable ${chalk.bold('PUBLIC')} uploads remove TESTOMATIO_PRIVATE_ARTIFACTS env variable`,
-      );
-    }
+    console.log(APP_PREFIX, 'S3 buffer uploading error: ', e);
     console.log(APP_PREFIX, '---------------');
   }
 };
@@ -203,7 +207,9 @@ const uploadFileByPath = async (filePath, runId) => {
       return uploadUsingS3(filePath, runId);
     }
   } catch (e) {
-    console.error(chalk.red('Error occurred while uploading artifacts'), e);
+    debug(e);
+
+    console.error(chalk.red('Error occurred while uploading artifacts! '), e);
   }
 };
 
@@ -213,10 +219,61 @@ const uploadFileAsBuffer = async (buffer, fileName, runId) => {
       return uploadUsingS3AsBuffer(buffer, fileName, runId);
     }
   } catch (e) {
-    console.error(chalk.red('Error occurred while uploading artifacts'), e);
+    debug(e);
+
+    console.error(chalk.red('Error occurred while uploading artifacts! '), e);
   }
 };
 
+const checkFileExists = async (filePath, attempts = 5, intervalMs = 500) => {
+  const checkFile = async () => {
+    const fileStats = await stat(filePath);
+    if (fileStats.isFile()) {
+      return true;
+    }
+
+    throw new Error('File not found');
+  };
+
+  try {
+    await promiseRetry(
+      { 
+        retries: attempts,
+        minTimeout: intervalMs 
+      },
+      checkFile
+    );
+
+    return true;
+  } catch (err) {
+    console.error(
+      chalk.yellow(`File ${filePath} was not found or did not have time to be generated...`)
+    );
+
+    return false;
+  }
+};
+
+const getS3LocationLink = async (out) => {
+  const response = await out.done();
+
+  let s3Location = response?.Location;
+
+  debug('Uploaded response.Location', s3Location);
+
+  if (!s3Location) {
+    // TODO: out: a fallback case  - remove after deeper testing
+    s3Location = out?.singleUploadResult?.Location;
+    debug('Uploaded singleUploadResult.Location', s3Location);
+
+    if (!s3Location) {
+      throw new Error("Problems getting the S3 artifact's link. Please check S3 permissions!");
+    }
+  }
+
+  return s3Location;
+};
+
 module.exports = {
   uploadFileByPath: memoize(uploadFileByPath),
   uploadFileAsBuffer: memoize(uploadFileAsBuffer),

package/lib/pipe/gitlab.js

@@ -23,7 +23,7 @@ class GitLabPipe {
     this.store = store;
     this.tests = [];
     // GitLab PAT looks like glpat-nKGdja3jsG4850sGksh7
-    this.token = params.GITLAB_PAT || this.ENV.GITLAB_PAT;
+    this.token = params.GITLAB_PAT || process.env.GITLAB_PAT || this.ENV.GITLAB_PAT;
     this.hiddenCommentData = `<!--- testomat.io report ${process.env.CI_JOB_NAME || ''} -->`;
 
     debug(