@testomatio/reporter 1.6.0-beta-6-artifacts → 1.6.0-beta-artifacts

package/lib/adapter/mocha.js

@@ -24,7 +24,6 @@ function MochaReporter(runner, opts) {
   let passes = 0;
   let failures = 0;
   let skipped = 0;
-  // let artifactStore;
 
   const apiKey = opts?.reporterOptions?.apiKey || config.TESTOMATIO;
 
@@ -33,6 +32,7 @@ function MochaReporter(runner, opts) {
   runner.on(EVENT_RUN_BEGIN, () => {
     client.createRun();
 
+    // clear dir with artifacts/logs
     fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
   });
 

package/lib/adapter/playwright.js

@@ -104,6 +104,7 @@ class PlaywrightReporter {
     if (!this.client) return;
 
     await Promise.all(reportTestPromises);
+    await this.client.updateRunStatus(checkStatus(result.status));
 
     if (!this.uploads.length) return;
 
@@ -135,8 +136,6 @@ class PlaywrightReporter {
       );
     }
     await Promise.all(promises);
-
-    await this.client.updateRunStatus(checkStatus(result.status));
   }
 }
 

package/lib/adapter/webdriver.js

@@ -1,7 +1,9 @@
 // eslint-disable-next-line global-require, import/no-extraneous-dependencies
 const WDIOReporter = require('@wdio/reporter').default;
 const TestomatClient = require('../client');
-const { getTestomatIdFromTestTitle } = require('../utils/utils');
+const { getTestomatIdFromTestTitle, fileSystem } = require('../utils/utils');
+const { services } = require('../services');
+const { TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
 
 class WebdriverReporter extends WDIOReporter {
   constructor(options) {
@@ -15,6 +17,15 @@ class WebdriverReporter extends WDIOReporter {
     this._isSynchronising = false;
   }
 
+  onRunnerStart() {
+    // clear dir with artifacts/logs
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
+  }
+
+  onTestStart(test) {
+    services.setContext(test.fullTitle);
+  }
+
   get isSynchronised() {
     return this._isSynchronising === false;
   }
@@ -28,15 +39,18 @@ class WebdriverReporter extends WDIOReporter {
   }
 
   onTestEnd(test) {
+    test.suite = test.parent;
+    const logs = getTestLogs(test.fullTitle);
+    // TODO: FIX: artifacts for some reason leads to empty report on Testomat.io
+    // const artifacts = services.artifacts.get(test.fullTitle);
+    // const keyValues = services.keyValues.get(test.fullTitle);
+    test.logs = logs;
+    // test.artifacts = artifacts;
+    // test.meta = keyValues;
     this._addTestPromises.push(this.addTest(test));
   }
 
   // wdio-cucumber does not trigger onTestEnd hook, thus, using this one
-  /**
-   *
-   * @param {} scerario
-   * @returns
-   */
   onSuiteEnd(scerario) {
     if (scerario.type === 'scenario') {
       this._addTestPromises.push(this.addBddScenario(scerario));
@@ -56,7 +70,10 @@ class WebdriverReporter extends WDIOReporter {
       .map(el => Buffer.from(el.result.value, 'base64'));
 
     await this.client.addTestRun(state, {
+      // manuallyAttachedArtifacts: test.artifacts,
       error,
+      logs: test.logs,
+      // meta: test.meta,
       title,
       test_id: testId,
       time: duration,
@@ -98,4 +115,16 @@ class WebdriverReporter extends WDIOReporter {
   }
 }
 
+/**
+ *
+ * @param {*} fullTestTitle
+ * @returns string
+ */
+function getTestLogs(fullTestTitle) {
+  const logsArr = services.logger.getLogs(fullTestTitle);
+  // remove duplicates (for some reason, logs are duplicated several times)
+  const logs = logsArr ? Array.from(new Set(logsArr)).join('\n').trim() : '';
+  return logs;
+}
+
 module.exports = WebdriverReporter;

package/lib/bin/cli.js

@@ -185,7 +185,7 @@ program
       isBatchEnabled: false,
     });
 
-    let testruns = client.uploader.readUploadedFiles(process.env.TESTOMATIO_RUN);
+    let testruns = client.uploader.readUploadedFiles(runId);
     const numTotalArtifacts = testruns.length;
 
     debug('Found testruns:', testruns);

package/lib/pipe/testomatio.js

@@ -98,6 +98,7 @@ class TestomatioPipe {
     this.runId = params.runId || process.env.runId;
     this.createNewTests = params.createNewTests ?? !!process.env.TESTOMATIO_CREATE;
     this.hasUnmatchedTests = false;
+    this.requestFailures = 0;
 
     if (!isValidUrl(this.url.trim())) {
       this.isEnabled = false;
@@ -217,10 +218,11 @@ class TestomatioPipe {
       process.env.runId = this.runId;      
       debug('Run created', this.runId);
     } catch (err) {
+      const errorInfo = `${err?.response?.statusText} ${chalk.red(err?.status)} ${err.message}`;
       console.error(
         APP_PREFIX,
-        'Error creating Testomat.io report, please check if your API key is valid. Skipping report | ',
-        err?.response?.statusText || err?.status || err.message,
+        'Error creating Testomat.io report, please check if your API key is valid. Skipping report\n',
+        errorInfo || JSON.stringify(err),
       );
       printCreateIssue(err);
     }
@@ -229,37 +231,25 @@ class TestomatioPipe {
 
   /**
    * Decides whether to skip test reporting in case of too many request failures
-   * @param {TestData} testData
    * @returns {boolean}
    */
-  #cancelTestReportingInCaseOfTooManyReqFailures(testData) {
-    if (this.reportingCanceledDueToReqFailures) return true;
-
-    const retriesCountWithinTime = this.retriesTimestamps.filter(
-      timestamp => Date.now() - timestamp < REPORTER_REQUEST_RETRIES.withinTimeSeconds * 1000,
-    ).length;
-    debug(`${retriesCountWithinTime} failed requests within ${REPORTER_REQUEST_RETRIES.withinTimeSeconds}s`);
-
-    if (retriesCountWithinTime > REPORTER_REQUEST_RETRIES.maxTotalRetries) {
-      const errorMessage = chalk.yellow(
-        `${retriesCountWithinTime} requests were failed within ${REPORTER_REQUEST_RETRIES.withinTimeSeconds}s,\
- reporting for test "${testData.title}" to Testomat is skipped`,
-      );
-      console.warn(`${APP_PREFIX} ${errorMessage}`);
+  #cancelTestReportingInCaseOfTooManyReqFailures() {
+    if (!process.env.TESTOMATIO_MAX_REQUEST_FAILURES) return;
 
+    const cancelReporting = this.requestFailures >= parseInt(process.env.TESTOMATIO_MAX_REQUEST_FAILURES, 10);
+    if (cancelReporting) {
       this.reportingCanceledDueToReqFailures = true;
-      this.notReportedTestsCount++;
-
-      return true;
+      const errorMessage = 
+        `⚠️ ${process.env.TESTOMATIO_MAX_REQUEST_FAILURES} requests were failed, reporting to Testomat aborted.`;
+      console.warn(`${APP_PREFIX} ${chalk.yellow(errorMessage)}`);
     }
-
-    return false;
+    return cancelReporting;
   }
 
   #uploadSingleTest = async data => {
     if (!this.isEnabled) return;
     if (!this.runId) return;
-    if (this.#cancelTestReportingInCaseOfTooManyReqFailures(data)) return;
+    if (this.#cancelTestReportingInCaseOfTooManyReqFailures()) return;
 
     data.api_key = this.apiKey;
     data.create = this.createNewTests;
@@ -275,6 +265,8 @@ class TestomatioPipe {
     return this.axios
       .post(`/api/reporter/${this.runId}/testrun`, json, axiosAddTestrunRequestConfig)
       .catch(err => {
+        this.requestFailures++;
+        this.notReportedTestsCount++;
         if (err.response) {
           if (err.response.status >= 400) {
             const responseData = err.response.data || { message: '' };
@@ -308,6 +300,7 @@ class TestomatioPipe {
     this.batch.batchIndex++;
     if (!this.batch.isEnabled) return;
     if (!this.batch.tests.length) return;
+    if (this.#cancelTestReportingInCaseOfTooManyReqFailures()) return;
 
     // get tests from batch and clear batch
     const testsToSend = this.batch.tests.splice(0);
@@ -322,6 +315,8 @@ class TestomatioPipe {
         axiosAddTestrunRequestConfig,
       )
       .catch(err => {
+        this.requestFailures++;
+        this.notReportedTestsCount += testsToSend.length;
         if (err.response) {
           if (err.response.status >= 400) {
             const responseData = err.response.data || { message: '' };
@@ -399,6 +394,7 @@ class TestomatioPipe {
         await this.axios.put(`/api/reporter/${this.runId}`, {
           api_key: this.apiKey,
           status_event,
+          duration: params.duration,
           tests: params.tests,
         });
         if (this.runUrl) {

package/lib/reporter.js

@@ -4,7 +4,7 @@ const { services } = require('./services');
 
 const reporterFunctions = require('./reporter-functions');
 
-module.exports = {
+const testomat = {
   // TODO: deprecate in future; use log or testomat.log
   testomatioLogger: services.logger,
 
@@ -17,3 +17,14 @@ module.exports = {
   TestomatClient,
   TRConstants,
 };
+
+module.exports = testomat;
+
+module.exports.testomatioLogger = testomat;
+module.exports.artifact = reporterFunctions.artifact;
+module.exports.log = reporterFunctions.log;
+module.exports.logger = services.logger;
+module.exports.meta = reporterFunctions.keyValue;
+module.exports.step = reporterFunctions.step;
+module.exports.TestomatClient = TestomatClient;
+module.exports.TRConstants = TRConstants;

package/lib/uploader.js

@@ -79,7 +79,7 @@ class S3Uploader {
     this.storeEnabled = false;
   }
 
-  async uploadToS3(Body, Key) {
+  async #uploadToS3(Body, Key) {
     const { S3_BUCKET, TESTOMATIO_PRIVATE_ARTIFACTS } = this.getConfig();
     const ACL = TESTOMATIO_PRIVATE_ARTIFACTS ? 'private' : 'public-read';
 
@@ -94,7 +94,7 @@ class S3Uploader {
 
     debug('Uploading to S3:', Key);
 
-    const s3 = new S3(this.getS3Config());
+    const s3 = new S3(this.#getS3Config());
 
     try {
       const upload = new Upload({
@@ -118,8 +118,13 @@ class S3Uploader {
     }
   }
 
+  /**
+    * Returns an array of uploaded files list
+    * 
+    * @returns {{rid: string, file: string, uploaded: boolean}[]}
+   */
   readUploadedFiles(runId) {
-    const tempFilePath = this.#getUploadFilePath(runId);
+    const tempFilePath = this.#getFilePathWithUploadsList(runId);
 
     debug('Reading file', tempFilePath);
 
@@ -146,9 +151,9 @@ class S3Uploader {
     return lines.map(line => JSON.parse(line));
   }
 
-  #getUploadFilePath(runId, forceCreate = false) {
-    const tempFilePath = path.join(os.tmpdir(), `testomatio.run.${runId}.jsonl`);
-    if (!fs.existsSync(tempFilePath) || forceCreate) {
+  #getFilePathWithUploadsList(runId) {
+    const tempFilePath = path.join(os.tmpdir(), `testomatio.run.${runId}.json`);
+    if (!fs.existsSync(tempFilePath)) {
       debug('Creating artifacts file:', tempFilePath);
       fs.writeFileSync(tempFilePath, '');
     }
@@ -160,7 +165,11 @@ class S3Uploader {
 
     if (!filePath || !runId || !rid) return;
 
-    const tempFilePath = this.#getUploadFilePath(runId);
+    const tempFilePath = this.#getFilePathWithUploadsList(runId);
+
+    if (typeof filePath === 'object') {
+      filePath = filePath.path;
+    }
 
     if (typeof filePath === 'string' && !path.isAbsolute(filePath)) {
       filePath = path.join(process.cwd(), filePath);
@@ -168,14 +177,15 @@ class S3Uploader {
 
     const data = { rid, file: filePath, uploaded };
     const jsonLine = `${JSON.stringify(data)}\n`;
-
     fs.appendFileSync(tempFilePath, jsonLine);
   }
 
-  getskippedUpload() {
-    return this.skippedUploadsCount;
-  }
-
+  /**
+   * 
+   * @param {*} filePath 
+   * @param {*} pathInS3 contains runId, rid and filename
+   * @returns 
+   */
   async uploadFileByPath(filePath, pathInS3) {
     const [runId, rid] = pathInS3;
 
@@ -191,7 +201,7 @@ class S3Uploader {
 
     debug('Started upload', filePath, 'to', S3_BUCKET);
 
-    const isFileExist = await this.checkFileExists(filePath, 20, 500);
+    const isFileExist = await this.checkArtifactExistsInFileSystem(filePath, 20, 500);
 
     if (!isFileExist) {
       console.error(chalk.yellow(`Artifacts file ${filePath} does not exist. Skipping...`));
@@ -211,7 +221,7 @@ class S3Uploader {
     const fileStream = fs.createReadStream(filePath);
     const Key = pathInS3.join('/');
 
-    const link = await this.uploadToS3(fileStream, Key);
+    const link = await this.#uploadToS3(fileStream, Key);
 
     this.storeUploadedFile(filePath, runId, rid, !!link);
 
@@ -228,10 +238,10 @@ class S3Uploader {
       Key = `${Key}.${ext}`;
     }
 
-    return this.uploadToS3(buffer, Key);
+    return this.#uploadToS3(buffer, Key);
   }
 
-  async checkFileExists(filePath, attempts = 5, intervalMs = 500) {
+  async checkArtifactExistsInFileSystem(filePath, attempts = 5, intervalMs = 500) {
     return promiseRetry(
       async (retry, number) => {
         try {
@@ -286,7 +296,7 @@ class S3Uploader {
     );
   }
 
-  getS3Config() {
+  #getS3Config() {
     const { S3_REGION, S3_SESSION_TOKEN, S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY, S3_FORCE_PATH_STYLE, S3_ENDPOINT } =
       this.getConfig();
 

package/lib/utils/utils.js

@@ -321,6 +321,7 @@ const testRunnerHelper = {
 };
 
 function storeRunId(runId) {
+  if (!runId || runId === 'undefined') return;
   const filePath = path.join(os.tmpdir(), `testomatio.latest.run`);
   fs.writeFileSync(filePath, runId);
 }

package/lib/xmlReader.js

@@ -107,7 +107,7 @@ class XmlReader {
   }
 
   processJUnit(jsonSuite) {
-    const { testsuite, name, tests, failures, errors } = jsonSuite;
+    const { testsuite, name, tests, failures, errors, time } = jsonSuite;
 
     reduceOptions.preferClassname = this.stats.language === 'python';
     const resultTests = processTestSuite(testsuite);
@@ -115,12 +115,18 @@ class XmlReader {
     const hasFailures = resultTests.filter(t => t.status === 'failed').length > 0;
     const status = failures > 0 || errors > 0 || hasFailures ? 'failed' : 'passed';
 
+    if (time) {
+      if (!this.stats.duration) this.stats.duration = 0;
+      this.stats.duration += parseFloat(time);
+    }
+
     this.tests = this.tests.concat(resultTests);
 
     return {
       status,
       create_tests: true,
       name,
+      duration: parseFloat(time),
       tests_count: parseInt(tests, 10),
       passed_count: parseInt(tests, 10) - parseInt(failures, 10),
       failed_count: parseInt(failures, 10),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@testomatio/reporter",
-  "version": "1.6.0-beta-6-artifacts",
+  "version": "1.6.0-beta-artifacts",
   "description": "Testomatio Reporter Client",
   "main": "./lib/reporter.js",
   "typings": "typings/index.d.ts",