@testomatio/reporter 1.1.0-beta-3 → 1.1.0-beta-codecept-logger

package/lib/adapter/codecept.js

@@ -147,9 +147,6 @@ function CodeceptReporter(config) {
     failedTests.push(id || title);
     let testId = parseTest(tags);
     const testObj = getTestAndMessage(title);
-    if (error && error.stack && test.steps && test.steps.length) {
-      error.stack = test.steps[test.steps.length - 1].line();
-    }
 
     const files = [];
     if (artifacts.screenshot) files.push({ path: artifacts.screenshot, type: 'image/png' });

package/lib/adapter/cucumber/current.js

@@ -6,6 +6,7 @@ const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../../constants');
 const TestomatClient = require('../../client');
 const logger = require('../../storages/logger');
 const { parseTest, fileSystem } = require('../../utils/utils');
+const { TESTOMATIO } = require('../../config');
 
 const { GherkinDocumentParser, PickleParser } = formatterHelpers;
 const { getGherkinScenarioLocationMap, getGherkinStepMap } = GherkinDocumentParser;
@@ -29,7 +30,7 @@ class CucumberReporter extends Formatter {
     this.failures = [];
     this.cases = [];
 
-    this.client = new TestomatClient({ apiKey: options.apiKey || process.env.TESTOMATIO });
+    this.client = new TestomatClient({ apiKey: options.apiKey || TESTOMATIO });
     this.status = STATUS.PASSED;
 
     global.testomatioRunningEnvironment = 'cucumber:current';

package/lib/adapter/cucumber/legacy.js

@@ -4,6 +4,7 @@ const chalk = require('chalk');
 const { parseTest, fileSystem } = require('../../utils/utils');
 const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../../constants');
 const TestomatClient = require('../../client');
+const { TESTOMATIO } = require('../../config');
 
 const createTestomatFormatter = apiKey => {
   if (!apiKey || apiKey === '') {
@@ -151,4 +152,4 @@ const createTestomatFormatter = apiKey => {
   };
 };
 
-module.exports = createTestomatFormatter(process.env.TESTOMATIO);
+module.exports = createTestomatFormatter(TESTOMATIO);

package/lib/adapter/cypress-plugin/index.js

@@ -1,17 +1,19 @@
 const { STATUS } = require('../../constants');
 const { parseTest, parseSuite } = require('../../utils/utils');
 const TestomatClient = require('../../client');
+const { TESTOMATIO } = require('../../config');
 
 const testomatioReporter = on => {
-  if (!process.env.TESTOMATIO) {
+  if (!TESTOMATIO) {
     console.log('TESTOMATIO key is empty, ignoring reports');
-    return
+    return;
   }
-  const client = new TestomatClient({ apiKey: process.env.TESTOMATIO });
+  const client = new TestomatClient({ apiKey: TESTOMATIO });
 
-  on('before:run', async (run) => {
+  on('before:run', async run => {
+    // TODO: looks like client.env does not exist
     if (!client.env) {
-      client.env = `${run.browser.displayName},${run.system.osName}`
+      client.env = `${run.browser.displayName},${run.system.osName}`;
     }
     await client.createRun();
   });
@@ -25,8 +27,9 @@ const testomatioReporter = on => {
       const lastAttemptIndex = test.attempts.length - 1;
       const latestAttempt = test.attempts[lastAttemptIndex];
 
-      const time = latestAttempt.duration;
-      const error = latestAttempt.error;
+      // latestAttempt.duration && latestAttempt.error were available in adapters version up to 13 JFYI
+      const time = latestAttempt.duration || latestAttempt.wallClockDuration || test.duration;
+      let error = latestAttempt.error;
 
       let title = test.title.pop();
       const examples = title.match(/\(example (#\d+)\)/);
@@ -39,35 +42,59 @@ const testomatioReporter = on => {
       const testId = parseTest(title);
       const suiteId = parseSuite(suiteTitle);
 
-      if (error) {
-        error.inspect = function() { // eslint-disable-line func-names
-          if (this && this.codeFrame) {
-            return this.codeFrame.frame;
-          }
-          return '';
-        }
+      if (!error && test.displayError) {
+        error = { message: test.displayError };
+        error.inspect = function () {
+          // eslint-disable-line func-names
+          return this.message;
+        };
       }
 
-      const screenshots = results.screenshots
-        .filter(screenshot => screenshot.path.includes(title))
-        .filter(screenshot => screenshot.testAttemptIndex === lastAttemptIndex)
-        .map(screenshot => screenshot.path);
+      const formattedError = error
+        ? {
+            message: error.message,
+            inspect:
+              error.inspect ||
+              function () {
+                return this.message;
+              },
+          }
+        : '';
+
+      const screenshots = Array.isArray(results.screenshots)
+        ? results.screenshots
+            .filter(screenshot => screenshot?.path && screenshot?.path.includes(title) && screenshot?.takenAt)
+            .map(screenshot => screenshot.path)
+        : [];
 
       const files = [...videos, ...screenshots];
 
       let state;
       switch (test.state) {
-        case 'passed': state = STATUS.PASSED; break;
-        case 'failed': state = STATUS.FAILED; break;
+        case 'passed':
+          state = STATUS.PASSED;
+          break;
+        case 'failed':
+          state = STATUS.FAILED;
+          break;
         case 'skipped':
-        case 'pending': 
+        case 'pending':
         default:
           state = STATUS.SKIPPED;
       }
 
-      addSpecTestsPromises.push(client.addTestRun(state, {
-        title, time, example, error, files, suite_title: suiteTitle, test_id: testId, suite_id: suiteId
-      }));
+      addSpecTestsPromises.push(
+        client.addTestRun(state, {
+          title,
+          time,
+          example,
+          error: formattedError,
+          files,
+          suite_title: suiteTitle,
+          test_id: testId,
+          suite_id: suiteId,
+        }),
+      );
     }
 
     await Promise.all(addSpecTestsPromises);

package/lib/adapter/mocha.js

@@ -4,6 +4,7 @@ const chalk = require('chalk');
 const TestomatClient = require('../client');
 const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
 const { parseTest, fileSystem } = require('../utils/utils');
+const { TESTOMATIO } = require('../config');
 
 const { EVENT_RUN_BEGIN, EVENT_RUN_END, EVENT_TEST_FAIL, EVENT_TEST_PASS, EVENT_TEST_PENDING } =
   Mocha.Runner.constants;
@@ -15,7 +16,7 @@ function MochaReporter(runner, opts) {
   let skipped = 0;
   // let artifactStore;
 
-  const apiKey = opts?.reporterOptions?.apiKey || process.env.TESTOMATIO;
+  const apiKey = opts?.reporterOptions?.apiKey || TESTOMATIO;
 
   const client = new TestomatClient({ apiKey });
 

package/lib/bin/startTest.js

@@ -5,6 +5,7 @@ const chalk = require('chalk');
 const TestomatClient = require('../client');
 const { APP_PREFIX, STATUS } = require('../constants');
 const { version } = require('../../package.json');
+const { TESTOMATIO } = require('../config');
 
 console.log(chalk.cyan.bold(` 🤩 Testomat.io Reporter v${version}`));
 
@@ -20,7 +21,7 @@ program
 
     if (opts.envFile) require('dotenv').config(opts.envFile); // eslint-disable-line
 
-    const apiKey = process.env['INPUT_TESTOMATIO-KEY'] || process.env.TESTOMATIO;
+    const apiKey = process.env['INPUT_TESTOMATIO-KEY'] || TESTOMATIO;
     const title = process.env.TESTOMATIO_TITLE;
 
     if (launch) {
@@ -53,6 +54,12 @@ program
 
     let exitCode = 0;
 
+    if (!command.split) {
+      process.exitCode = 255;
+      console.log(APP_PREFIX, `No command provided. Use -c option to launch a test runner.`);
+      return;
+    }
+
     const client = new TestomatClient({ apiKey, title, parallel: true });
 
     if(filter) {
@@ -74,12 +81,6 @@ program
       }
     }
 
-    if (!command.split) {
-      process.exitCode = 255;
-      console.log(APP_PREFIX, `No command provided. Use -c option to launch a test runner.`);
-      return;
-    }
-
     const testCmds = command.split(' ');
     console.log(APP_PREFIX, `🚀 Running`, chalk.green(command));
 

package/lib/config.js

@@ -0,0 +1,29 @@
+// This file is used to read environment variables from .env file and process.env
+
+// ! uncommenting next line leads ro reading vars from .env file
+// require('dotenv').config();
+const debug = require('debug')('@testomatio/reporter:config');
+
+/* for possibility to use multiple env files (reading different paths)
+const dotenv = require('dotenv');
+const envFileVars = dotenv.config({ path: '.env' }).parsed; */
+
+const TESTOMATIO = process.env.TESTOMATIO || process.env.TESTOMATIO_API_KEY || process.env.TESTOMATIO_TOKEN || '';
+if (TESTOMATIO === 'undefined') console.error('TESTOMATIO is "undefined". Something went wrong. Contact dev team.');
+process.env.TESTOMATIO = TESTOMATIO;
+
+// select only TESTOMATIO related variables (only to print them in debug)
+const testomatioEnvVars =
+  Object.keys(process.env)
+    .filter(key => key.startsWith('TESTOMATIO') || key.startsWith('S3_'))
+    .reduce((obj, key) => {
+      obj[key] = process.env[key];
+      return obj;
+    }, {}) || {};
+debug('TESTOMATIO variables:', testomatioEnvVars);
+
+// includes variables from .env file and process.env
+const config = process.env;
+
+module.exports = config;
+module.exports.TESTOMATIO = TESTOMATIO;

package/lib/constants.js

@@ -20,10 +20,17 @@ const STATUS = {
   SKIPPED: 'skipped',
   FINISHED: 'finished',
 };
+// html pipe var
+const HTML_REPORT = {
+  FOLDER: "html-report",
+  REPORT_DEFAULT_NAME: "testomatio-report.html",
+  TEMPLATE_NAME: 'testomatio.hbs'
+};
 
 module.exports = {
   APP_PREFIX,
   TESTOMAT_TMP_STORAGE_DIR,
   CSV_HEADERS,
   STATUS,
+  HTML_REPORT
 }

package/lib/fileUploader.js

@@ -3,10 +3,16 @@ const { S3 } = require('@aws-sdk/client-s3');
 const { Upload } = require('@aws-sdk/lib-storage');
 
 const fs = require('fs');
+const util = require('util');
 const path = require('path');
+const promiseRetry = require('promise-retry');
+
+const readFile = util.promisify(fs.readFile);
+const stat = util.promisify(fs.stat);
 const chalk = require('chalk');
 const { randomUUID } = require('crypto');
 const memoize = require('lodash.memoize');
+
 const { APP_PREFIX } = require('./constants');
 
 const keys = [
@@ -93,51 +99,64 @@ const uploadUsingS3 = async (filePath, runId) => {
     Key = filePath.name;
   }
 
-  if (!fs.existsSync(filePath)) {
-    console.error(chalk.yellow(`Artifacts file ${filePath} does not exist. Skipping...`));
-    return;
-  }
-
   const {
-    TESTOMATIO_PRIVATE_ARTIFACTS, S3_BUCKET
+    TESTOMATIO_PRIVATE_ARTIFACTS,
+    S3_BUCKET
   } = getConfig();
 
-  debug('S3 config', getMaskedConfig());
-  debug('Uploading', filePath, 'to', S3_BUCKET);
-  
-const fileData = fs.readFileSync(filePath);
+  try {
+    debug('S3 config', getMaskedConfig());
+    debug('Started upload', filePath, 'to ', S3_BUCKET);
 
-  Key = `${runId}/${randomUUID()}-${Key || path.basename(filePath)}`;
-  const ACL = TESTOMATIO_PRIVATE_ARTIFACTS ? 'private' : 'public-read';
+    // Verification that the file was actually created: 20 attempts of 0.5 second => 10sec
+    const isFileExist = await checkFileExists(filePath, 20, 500);
 
-  const s3 = new S3(_getS3Config());
+    if (!isFileExist) {
+      console.error(chalk.yellow(`Artifacts file ${filePath} does not exist. Skipping...`));
+      return;
+    }
 
-  const params = {
-    Bucket: S3_BUCKET,
-    Key,
-    Body: fileData,
-    ContentType,
-    ACL,
-  };
+    debug('File: ', filePath, ' exists');
+    
+    const fileData = await readFile(filePath);
 
-  try {
+    Key = `${runId}/${randomUUID()}-${Key || path.basename(filePath)}`;
+
+    const ACL = TESTOMATIO_PRIVATE_ARTIFACTS ? 'private' : 'public-read';
+
+    if (!S3_BUCKET || !fileData) {
+      console.log(
+        APP_PREFIX,
+        chalk.bold.red(`Failed uploading '${Key}'. Please check S3 credentials`), getMaskedConfig());
+      return;
+    }
+
+    const s3 = new S3(_getS3Config());
+
+    const params = {
+      Bucket: S3_BUCKET,
+      Key,
+      Body: fileData,
+      ContentType,
+      ACL,
+    };
+  
     const out = new Upload({
       client: s3,
       params
     });
 
-    await out.done();
-    debug('Uploaded', out.singleUploadResult.Location)
-
-    return out.singleUploadResult.Location;
-  } catch (e) {
-    console.log(e);
-    console.log(APP_PREFIX, chalk.red(`Failed uploading '${Key}'. Please check S3 credentials`), getMaskedConfig());
-
+    return await getS3LocationLink(out);
+  } 
+  catch (e) {
+    debug('S3 file uploading error: ', e);
+   
     console.log(APP_PREFIX, `To ${chalk.bold('disable')} artifact uploads set: TESTOMATIO_DISABLE_ARTIFACTS=1`);
+
     if (!TESTOMATIO_PRIVATE_ARTIFACTS) {
       console.log(APP_PREFIX, `To enable ${chalk.bold('PRIVATE')} uploads set: TESTOMATIO_PRIVATE_ARTIFACTS=1`);
-    } else {
+    } 
+    else {
       console.log(
         APP_PREFIX,
         `To enable ${chalk.bold('PUBLIC')} uploads remove TESTOMATIO_PRIVATE_ARTIFACTS env variable`,
@@ -148,7 +167,6 @@ const fileData = fs.readFileSync(filePath);
 };
 
 const uploadUsingS3AsBuffer = async (buffer, fileName, runId) => {
-
   const {
     S3_REGION, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_ENDPOINT, TESTOMATIO_PRIVATE_ARTIFACTS, S3_BUCKET
   } = getConfig();
@@ -158,6 +176,18 @@ const uploadUsingS3AsBuffer = async (buffer, fileName, runId) => {
   const fileExtension = _getFileExtBase64(buffer.toString('base64'));
   const Key = `${runId}/${fileName}${fileExtension}`;
 
+  if (!S3_BUCKET || !buffer) {
+    console.log(APP_PREFIX, chalk.bold.red(`Failed uploading '${Key}'. Please check S3 credentials`), {
+      accessKeyId: S3_ACCESS_KEY_ID,
+      secretAccessKey: S3_SECRET_ACCESS_KEY ? '**** (hidden) ***' : '(empty)',
+      region: S3_REGION,
+      bucket: S3_BUCKET,
+      acl: ACL,
+      endpoint: S3_ENDPOINT,
+    });
+    return;
+  }
+
   const s3 = new S3(_getS3Config());
 
   try {
@@ -171,20 +201,14 @@ const uploadUsingS3AsBuffer = async (buffer, fileName, runId) => {
         ACL,
       }
     });
-    await out.done();
 
-    return out.singleUploadResult.Location;
-  } catch (e) {
-    console.log(APP_PREFIX, chalk.bold.red(`Failed uploading '${Key}'. Please check S3 credentials`), {
-      accessKeyId: S3_ACCESS_KEY_ID,
-      secretAccessKey: S3_SECRET_ACCESS_KEY ? '**** (hidden) ***' : '(empty)',
-      region: S3_REGION,
-      bucket: S3_BUCKET,
-      acl: ACL,
-      endpoint: S3_ENDPOINT,
-    });
+    return await getS3LocationLink(out);
+  } 
+  catch (e) {
+    debug('S3 buffer uploading error: ', e);
 
     console.log(APP_PREFIX, `To ${chalk.bold('disable')} artifact uploads set: TESTOMATIO_DISABLE_ARTIFACTS=1`);
+
     if (!TESTOMATIO_PRIVATE_ARTIFACTS) {
       console.log(APP_PREFIX, `To enable ${chalk.bold('PRIVATE')} uploads set: TESTOMATIO_PRIVATE_ARTIFACTS=1`);
     } else {
@@ -203,7 +227,9 @@ const uploadFileByPath = async (filePath, runId) => {
       return uploadUsingS3(filePath, runId);
     }
   } catch (e) {
-    console.error(chalk.red('Error occurred while uploading artifacts'), e);
+    debug(e);
+
+    console.error(chalk.red('Error occurred while uploading artifacts! '), e);
   }
 };
 
@@ -213,10 +239,61 @@ const uploadFileAsBuffer = async (buffer, fileName, runId) => {
       return uploadUsingS3AsBuffer(buffer, fileName, runId);
     }
   } catch (e) {
-    console.error(chalk.red('Error occurred while uploading artifacts'), e);
+    debug(e);
+
+    console.error(chalk.red('Error occurred while uploading artifacts! '), e);
   }
 };
 
+const checkFileExists = async (filePath, attempts = 5, intervalMs = 500) => {
+  const checkFile = async () => {
+    const fileStats = await stat(filePath);
+    if (fileStats.isFile()) {
+      return true;
+    }
+
+    throw new Error('File not found');
+  };
+
+  try {
+    await promiseRetry(
+      { 
+        retries: attempts,
+        minTimeout: intervalMs 
+      },
+      checkFile
+    );
+
+    return true;
+  } catch (err) {
+    console.error(
+      chalk.yellow(`File ${filePath} was not found or did not have time to be generated...`)
+    );
+
+    return false;
+  }
+};
+
+const getS3LocationLink = async (out) => {
+  const response = await out.done();
+
+  let s3Location = response?.Location;
+
+  debug('Uploaded response.Location', s3Location);
+
+  if (!s3Location) {
+    // TODO: out: a fallback case  - remove after deeper testing
+    s3Location = out?.singleUploadResult?.Location;
+    debug('Uploaded singleUploadResult.Location', s3Location);
+
+    if (!s3Location) {
+      throw new Error("Problems getting the S3 artifact's link. Please check S3 permissions!");
+    }
+  }
+
+  return s3Location;
+};
+
 module.exports = {
   uploadFileByPath: memoize(uploadFileByPath),
   uploadFileAsBuffer: memoize(uploadFileAsBuffer),

package/lib/pipe/gitlab.js

@@ -23,7 +23,7 @@ class GitLabPipe {
     this.store = store;
     this.tests = [];
     // GitLab PAT looks like glpat-nKGdja3jsG4850sGksh7
-    this.token = params.GITLAB_PAT || this.ENV.GITLAB_PAT;
+    this.token = params.GITLAB_PAT || process.env.GITLAB_PAT || this.ENV.GITLAB_PAT;
     this.hiddenCommentData = `<!--- testomat.io report ${process.env.CI_JOB_NAME || ''} -->`;
 
     debug(