@testomatio/reporter 1.1.0-beta-3 → 1.1.0-beta.codeceptjs-before-suite-logs

package/Changelog.md

@@ -1,4 +1,8 @@
 <!-- pending release updates -->
+# 1.0.18
+
+* Fixed stack traces for CodeceptJS
+
 # 1.0.17
 
 Renamed `TESTOMATIO_STACK_FILTER` to `TESTOMATIO_STACK_IGNORE`

package/lib/adapter/codecept.js

@@ -78,8 +78,35 @@ function CodeceptReporter(config) {
     global.testomatioDataStore.steps = [];
   });
 
+  let hookSteps = [];
+  let suiteHookRunning = false;
+
+  event.dispatcher.on(event.suite.before, (suite) => {
+    suiteHookRunning = true;
+    hookSteps = [];
+    global.testomatioDataStore.steps = [];
+  });
+
+  event.dispatcher.on(event.test.before, () => {
+    suiteHookRunning = false;
+    global.testomatioDataStore.steps = []
+  });
+
+  event.dispatcher.on(event.hook.started, (suite) => {
+    // global.testomatioDataStore.steps = [];
+  });
+
+  event.dispatcher.on(event.hook.passed, (suite) => {
+    if (suiteHookRunning) hookSteps.push(...global.testomatioDataStore.steps);
+  });
+
+  event.dispatcher.on(event.hook.failed, (suite) => {
+    if (suiteHookRunning) hookSteps.push(...global.testomatioDataStore.steps);
+  });  
+
   event.dispatcher.on(event.test.started, test => {
     testTimeMap[test.id] = Date.now();
+
     if (global.testomatioDataStore) global.testomatioDataStore.currentlyRunningTestId = getIdFromTestTitle(test.title);
   });
 
@@ -89,7 +116,7 @@ function CodeceptReporter(config) {
     await Promise.all(reportTestPromises);
 
     if (upload.isArtifactsEnabled()) {
-      uploadAttachments(client, videos, '🎞️  Uploading', 'video');
+      uploadAttachments(client, videos, '🎞️ Uploading', 'video');
       uploadAttachments(client, traces, '📁 Uploading', 'trace');
     }
 
@@ -109,7 +136,7 @@ function CodeceptReporter(config) {
       suite_title: test.parent && test.parent.title,
       message: testObj.message,
       time: getDuration(test),
-      steps: global.testomatioDataStore.steps.join('\n') || null,
+      steps: [...hookSteps, ...global.testomatioDataStore.steps].join('\n') || null,
       test_id: testId,
     });
     // output.stop();
@@ -133,6 +160,7 @@ function CodeceptReporter(config) {
         ...stripExampleFromTitle(title),
         suite_title: suite.title,
         test_id: testId,
+        steps: hookSteps.join('\n') || null,
         error,
         time: 0,
       });
@@ -147,9 +175,6 @@ function CodeceptReporter(config) {
     failedTests.push(id || title);
     let testId = parseTest(tags);
     const testObj = getTestAndMessage(title);
-    if (error && error.stack && test.steps && test.steps.length) {
-      error.stack = test.steps[test.steps.length - 1].line();
-    }
 
     const files = [];
     if (artifacts.screenshot) files.push({ path: artifacts.screenshot, type: 'image/png' });
@@ -164,7 +189,7 @@ function CodeceptReporter(config) {
         message: testObj.message,
         time: getDuration(test),
         files,
-        steps: global.testomatioDataStore?.steps?.join('\n') || null,
+        steps: [...hookSteps, ...global.testomatioDataStore.steps].join('\n') || null,
       })
       .then(pipes => {
         testId = pipes.filter(p => p.pipe.includes('Testomatio'))[0]?.result?.data?.test_id;

package/lib/adapter/cypress-plugin/index.js

@@ -10,6 +10,7 @@ const testomatioReporter = on => {
   const client = new TestomatClient({ apiKey: process.env.TESTOMATIO });
 
   on('before:run', async (run) => {
+    // TODO: looks like client.env does not exist
     if (!client.env) {
       client.env = `${run.browser.displayName},${run.system.osName}`
     }
@@ -48,10 +49,16 @@ const testomatioReporter = on => {
         }
       }
 
-      const screenshots = results.screenshots
-        .filter(screenshot => screenshot.path.includes(title))
-        .filter(screenshot => screenshot.testAttemptIndex === lastAttemptIndex)
-        .map(screenshot => screenshot.path);
+      const screenshots = Array.isArray(results.screenshots)
+        ? results.screenshots
+            .filter(
+              screenshot =>
+                screenshot?.path &&
+                screenshot?.path.includes(title) &&
+                screenshot?.takenAt
+            )
+            .map(screenshot => screenshot.path)
+        : [];
 
       const files = [...videos, ...screenshots];
 

package/lib/bin/startTest.js

@@ -53,6 +53,12 @@ program
 
     let exitCode = 0;
 
+    if (!command.split) {
+      process.exitCode = 255;
+      console.log(APP_PREFIX, `No command provided. Use -c option to launch a test runner.`);
+      return;
+    }
+
     const client = new TestomatClient({ apiKey, title, parallel: true });
 
     if(filter) {
@@ -74,12 +80,6 @@ program
       }
     }
 
-    if (!command.split) {
-      process.exitCode = 255;
-      console.log(APP_PREFIX, `No command provided. Use -c option to launch a test runner.`);
-      return;
-    }
-
     const testCmds = command.split(' ');
     console.log(APP_PREFIX, `🚀 Running`, chalk.green(command));
 

package/lib/config.js

@@ -0,0 +1,24 @@
+// This file is used to read environment variables from .env file and process.env
+
+// ! uncommenting next line leads ro reading vars from .env file
+// require('dotenv').config();
+const debug = require('debug')('@testomatio/reporter:config');
+
+/* for possibility to use multiple env files (reading different paths)
+const dotenv = require('dotenv');
+const envFileVars = dotenv.config({ path: '.env' }).parsed; */
+
+// select only TESTOMATIO related variables (only to print them in debug)
+const testomatioEnvVars =
+  Object.keys(process.env)
+    .filter(key => key.startsWith('TESTOMATIO') || key.startsWith('S3_'))
+    .reduce((obj, key) => {
+      obj[key] = process.env[key];
+      return obj;
+    }, {}) || {};
+debug('TESTOMATIO variables:', testomatioEnvVars);
+
+// includes variables from .env file and process.env
+const config = process.env;
+
+module.exports = config;

package/lib/constants.js

@@ -20,10 +20,17 @@ const STATUS = {
   SKIPPED: 'skipped',
   FINISHED: 'finished',
 };
+// html pipe var
+const HTML_REPORT = {
+  FOLDER: "html-report",
+  REPORT_DEFAULT_NAME: "testomatio-report.html",
+  TEMPLATE_NAME: 'testomatio.hbs'
+};
 
 module.exports = {
   APP_PREFIX,
   TESTOMAT_TMP_STORAGE_DIR,
   CSV_HEADERS,
   STATUS,
+  HTML_REPORT
 }

package/lib/fileUploader.js

@@ -3,10 +3,16 @@ const { S3 } = require('@aws-sdk/client-s3');
 const { Upload } = require('@aws-sdk/lib-storage');
 
 const fs = require('fs');
+const util = require('util');
 const path = require('path');
+const promiseRetry = require('promise-retry');
+
+const readFile = util.promisify(fs.readFile);
+const stat = util.promisify(fs.stat);
 const chalk = require('chalk');
 const { randomUUID } = require('crypto');
 const memoize = require('lodash.memoize');
+
 const { APP_PREFIX } = require('./constants');
 
 const keys = [
@@ -93,51 +99,64 @@ const uploadUsingS3 = async (filePath, runId) => {
     Key = filePath.name;
   }
 
-  if (!fs.existsSync(filePath)) {
-    console.error(chalk.yellow(`Artifacts file ${filePath} does not exist. Skipping...`));
-    return;
-  }
-
   const {
-    TESTOMATIO_PRIVATE_ARTIFACTS, S3_BUCKET
+    TESTOMATIO_PRIVATE_ARTIFACTS,
+    S3_BUCKET
   } = getConfig();
 
-  debug('S3 config', getMaskedConfig());
-  debug('Uploading', filePath, 'to', S3_BUCKET);
-  
-const fileData = fs.readFileSync(filePath);
+  try {
+    debug('S3 config', getMaskedConfig());
+    debug('Started upload', filePath, 'to ', S3_BUCKET);
 
-  Key = `${runId}/${randomUUID()}-${Key || path.basename(filePath)}`;
-  const ACL = TESTOMATIO_PRIVATE_ARTIFACTS ? 'private' : 'public-read';
+    // Verification that the file was actually created: 20 attempts of 0.5 second => 10sec
+    const isFileExist = await checkFileExists(filePath, 20, 500);
 
-  const s3 = new S3(_getS3Config());
+    if (!isFileExist) {
+      console.error(chalk.yellow(`Artifacts file ${filePath} does not exist. Skipping...`));
+      return;
+    }
 
-  const params = {
-    Bucket: S3_BUCKET,
-    Key,
-    Body: fileData,
-    ContentType,
-    ACL,
-  };
+    debug('File: ', filePath, ' exists');
+    
+    const fileData = await readFile(filePath);
 
-  try {
+    Key = `${runId}/${randomUUID()}-${Key || path.basename(filePath)}`;
+
+    const ACL = TESTOMATIO_PRIVATE_ARTIFACTS ? 'private' : 'public-read';
+
+    if (!S3_BUCKET || !fileData) {
+      console.log(
+        APP_PREFIX,
+        chalk.bold.red(`Failed uploading '${Key}'. Please check S3 credentials`), getMaskedConfig());
+      return;
+    }
+
+    const s3 = new S3(_getS3Config());
+
+    const params = {
+      Bucket: S3_BUCKET,
+      Key,
+      Body: fileData,
+      ContentType,
+      ACL,
+    };
+  
     const out = new Upload({
       client: s3,
       params
     });
 
-    await out.done();
-    debug('Uploaded', out.singleUploadResult.Location)
-
-    return out.singleUploadResult.Location;
-  } catch (e) {
-    console.log(e);
-    console.log(APP_PREFIX, chalk.red(`Failed uploading '${Key}'. Please check S3 credentials`), getMaskedConfig());
-
+    return await getS3LocationLink(out);
+  } 
+  catch (e) {
+    debug('S3 file uploading error: ', e);
+   
     console.log(APP_PREFIX, `To ${chalk.bold('disable')} artifact uploads set: TESTOMATIO_DISABLE_ARTIFACTS=1`);
+
     if (!TESTOMATIO_PRIVATE_ARTIFACTS) {
       console.log(APP_PREFIX, `To enable ${chalk.bold('PRIVATE')} uploads set: TESTOMATIO_PRIVATE_ARTIFACTS=1`);
-    } else {
+    } 
+    else {
       console.log(
         APP_PREFIX,
         `To enable ${chalk.bold('PUBLIC')} uploads remove TESTOMATIO_PRIVATE_ARTIFACTS env variable`,
@@ -148,7 +167,6 @@ const fileData = fs.readFileSync(filePath);
 };
 
 const uploadUsingS3AsBuffer = async (buffer, fileName, runId) => {
-
   const {
     S3_REGION, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_ENDPOINT, TESTOMATIO_PRIVATE_ARTIFACTS, S3_BUCKET
   } = getConfig();
@@ -158,6 +176,18 @@ const uploadUsingS3AsBuffer = async (buffer, fileName, runId) => {
   const fileExtension = _getFileExtBase64(buffer.toString('base64'));
   const Key = `${runId}/${fileName}${fileExtension}`;
 
+  if (!S3_BUCKET || !buffer) {
+    console.log(APP_PREFIX, chalk.bold.red(`Failed uploading '${Key}'. Please check S3 credentials`), {
+      accessKeyId: S3_ACCESS_KEY_ID,
+      secretAccessKey: S3_SECRET_ACCESS_KEY ? '**** (hidden) ***' : '(empty)',
+      region: S3_REGION,
+      bucket: S3_BUCKET,
+      acl: ACL,
+      endpoint: S3_ENDPOINT,
+    });
+    return;
+  }
+
   const s3 = new S3(_getS3Config());
 
   try {
@@ -171,20 +201,14 @@ const uploadUsingS3AsBuffer = async (buffer, fileName, runId) => {
         ACL,
       }
     });
-    await out.done();
 
-    return out.singleUploadResult.Location;
-  } catch (e) {
-    console.log(APP_PREFIX, chalk.bold.red(`Failed uploading '${Key}'. Please check S3 credentials`), {
-      accessKeyId: S3_ACCESS_KEY_ID,
-      secretAccessKey: S3_SECRET_ACCESS_KEY ? '**** (hidden) ***' : '(empty)',
-      region: S3_REGION,
-      bucket: S3_BUCKET,
-      acl: ACL,
-      endpoint: S3_ENDPOINT,
-    });
+    return await getS3LocationLink(out);
+  } 
+  catch (e) {
+    debug('S3 buffer uploading error: ', e);
 
     console.log(APP_PREFIX, `To ${chalk.bold('disable')} artifact uploads set: TESTOMATIO_DISABLE_ARTIFACTS=1`);
+
     if (!TESTOMATIO_PRIVATE_ARTIFACTS) {
       console.log(APP_PREFIX, `To enable ${chalk.bold('PRIVATE')} uploads set: TESTOMATIO_PRIVATE_ARTIFACTS=1`);
     } else {
@@ -203,7 +227,9 @@ const uploadFileByPath = async (filePath, runId) => {
       return uploadUsingS3(filePath, runId);
     }
   } catch (e) {
-    console.error(chalk.red('Error occurred while uploading artifacts'), e);
+    debug(e);
+
+    console.error(chalk.red('Error occurred while uploading artifacts! '), e);
   }
 };
 
@@ -213,10 +239,61 @@ const uploadFileAsBuffer = async (buffer, fileName, runId) => {
       return uploadUsingS3AsBuffer(buffer, fileName, runId);
     }
   } catch (e) {
-    console.error(chalk.red('Error occurred while uploading artifacts'), e);
+    debug(e);
+
+    console.error(chalk.red('Error occurred while uploading artifacts! '), e);
   }
 };
 
+const checkFileExists = async (filePath, attempts = 5, intervalMs = 500) => {
+  const checkFile = async () => {
+    const fileStats = await stat(filePath);
+    if (fileStats.isFile()) {
+      return true;
+    }
+
+    throw new Error('File not found');
+  };
+
+  try {
+    await promiseRetry(
+      { 
+        retries: attempts,
+        minTimeout: intervalMs 
+      },
+      checkFile
+    );
+
+    return true;
+  } catch (err) {
+    console.error(
+      chalk.yellow(`File ${filePath} was not found or did not have time to be generated...`)
+    );
+
+    return false;
+  }
+};
+
+const getS3LocationLink = async (out) => {
+  const response = await out.done();
+
+  let s3Location = response?.Location;
+
+  debug('Uploaded response.Location', s3Location);
+
+  if (!s3Location) {
+    // TODO: out: a fallback case  - remove after deeper testing
+    s3Location = out?.singleUploadResult?.Location;
+    debug('Uploaded singleUploadResult.Location', s3Location);
+
+    if (!s3Location) {
+      throw new Error("Problems getting the S3 artifact's link. Please check S3 permissions!");
+    }
+  }
+
+  return s3Location;
+};
+
 module.exports = {
   uploadFileByPath: memoize(uploadFileByPath),
   uploadFileAsBuffer: memoize(uploadFileAsBuffer),

package/lib/pipe/gitlab.js

@@ -23,7 +23,7 @@ class GitLabPipe {
     this.store = store;
     this.tests = [];
     // GitLab PAT looks like glpat-nKGdja3jsG4850sGksh7
-    this.token = params.GITLAB_PAT || this.ENV.GITLAB_PAT;
+    this.token = params.GITLAB_PAT || process.env.GITLAB_PAT || this.ENV.GITLAB_PAT;
     this.hiddenCommentData = `<!--- testomat.io report ${process.env.CI_JOB_NAME || ''} -->`;
 
     debug(