@testomatio/reporter 0.8.0-beta.30 → 0.8.0-beta.32

package/README.md

@@ -37,7 +37,7 @@ You can verify installed packages via `npm list` or `npm list -g`.
 
 ### CodeceptJS
 
-Make sure you load all your tests using [check-test](https://github.com/testomatio/check-tests#cli).
+Make sure you load all your tests using [check-tests](https://github.com/testomatio/check-tests#cli).
 
 Add plugin to [codecept conf](https://github.com/testomatio/reporter/blob/master/example/codecept/codecept.conf.js#L23):
 
@@ -94,7 +94,7 @@ TESTOMATIO={API_KEY} npx playwright test
 
 ### Mocha
 
-Load the test using using `check-test` if not done already. Get the test id from testomat account and add it to your mocha test like in this [example](https://github.com/testomatio/reporter/blob/master/example/mocha/test/index.test.js#L4).
+Load the test using using `check-tests` if not done already. Get the test id from testomat account and add it to your mocha test like in this [example](https://github.com/testomatio/reporter/blob/master/example/mocha/test/index.test.js#L4).
 
 Run the following command from you project folder:
 
@@ -104,7 +104,7 @@ mocha --reporter ./node_modules/testomat-reporter/lib/adapter/mocha.js --reporte
 
 ### Jest
 
-Load the test using using `check-test`. Add the test id to your tests like in this [example](https://github.com/testomatio/reporter/blob/master/example/jest/index.test.js#L1).
+Load the test using using `check-tests`. Add the test id to your tests like in this [example](https://github.com/testomatio/reporter/blob/master/example/jest/index.test.js#L1).
 
 Add the following line to [jest.config.js](https://github.com/testomatio/reporter/blob/master/example/jest/jest.config.js#L100):
 
@@ -126,7 +126,7 @@ TESTOMATIO={API_KEY} ./node_modules/.bin/cucumber-js --format ./node_modules/@te
 
 ### TestCafe
 
-Load the test using using `check-test`.
+Load the test using using `check-tests`.
 
 Run the following command from you project folder:
 
@@ -138,14 +138,16 @@ TESTOMATIO={API_KEY} npx testcafe chrome -r testomatio
 
 Run collection and specify `testomatio` as reporter:
 
-`TESTOMATIO={API_KEY} npx newman run {collection_name.json} -r testomatio`
+```bash
+TESTOMATIO={API_KEY} npx newman run {collection_name.json} -r testomatio
+```
 
-> _`check-test` not supported for newman for now, tests will be created on testomatio by default_
+> _`check-tests` not supported for newman for now, tests will be created on testomatio by default_
 
 
 ### Cypress
 
-Load the test using using `check-test`.
+Load the test using using `check-tests`.
 
 Register our `cypress-plugin` in `cypress/plugins/index.js`:
 
@@ -173,7 +175,7 @@ TESTOMATIO={API_KEY} npx cypress run
 
 ### Protractor
 
-Load the test using using `check-test`.
+Load the test using using `check-tests`.
 
 Add the following lines to [conf.js](https://github.com/angular/protractor/blob/5.4.1/example/conf.js):
 
@@ -195,7 +197,7 @@ TESTOMATIO={API_KEY} npx start-test-run -c 'npx protractor conf.js'
 
 ### WebdriverIO
 
-Load the test using using `check-test`.
+Load the test using using `check-tests`.
 
 Add the following lines to [wdio.conf.js](https://webdriver.io/docs/configurationfile/):
 
@@ -583,3 +585,8 @@ If you want to finish a run started by `--launch` use `--finish` option. `TESTOM
 ```bash
 TESTOMATIO={API_KEY} TESTOMATIO_RUN={RUN_ID} npx start-test-run --finish
 ```
+
+### Debug logs
+Pass `DEBUG` variable with module name e.g. `DEBUG=@testomatio/reporter:pipe:github`.
+(Module name could be taken directly from the required module code).
+To log all debug info pass `DEBUG=*`.

package/lib/ArtifactStorage.js

@@ -0,0 +1,131 @@
+const debug = require('debug')('@testomatio/reporter:storage');
+const { join, resolve } = require('path');
+const fs = require('fs');
+const os = require('os');
+const uuid = require('uuid');
+const { TESTOMAT_ARTIFACT_SUFFIX } = require('./constants');
+const { specificTestInfo } = require('./util');
+
+class ArtifactStorage {
+
+  constructor(params) {
+    this._tmpPrefix = "tsmt_reporter";
+    this.isFile = false;
+    this.isMemory = true;
+    this.storage = params?.toFile;
+
+    if (this.storage) {
+      this.isFile = true;
+      this.isMemory = false;
+      this.tmpDirFullpath = this.createTestomatTmpDir(this._tmpPrefix);
+      debug('SAVE to tmp folder mode enabled!');
+    }
+  }
+
+  static async storeToFile(tmpDirName, artifact, testSuffix = "test") {
+    const suffix = TESTOMAT_ARTIFACT_SUFFIX + uuid.v4() + testSuffix;
+    const dirpath = join(os.tmpdir(), tmpDirName);
+    const filepath = resolve(dirpath, `${suffix}.json`);
+
+    return fs.promises.appendFile(filepath, JSON.stringify(artifact));
+  }
+
+  static async artifact(artifact, context) {
+    // TODO: mode for different pre-hooks. As variant - "all-tests" || "only_fail"
+    // const mode = process.env.TESTOMAT_ARTIFACTS || "only_fail";
+
+    if (Array.isArray(global.testomatioArtifacts)) {
+      debug("Saving artifacts to global storage");
+
+      global.testomatioArtifacts.push(artifact);
+    }
+
+    if (global?.testomatioArtifacts === undefined) {
+      const tmpDirNames = ArtifactStorage.tmpTestomatDirNames();
+
+      const testSuffix = specificTestInfo(context.test);
+
+      if (Array.isArray(tmpDirNames) && tmpDirNames.length) {
+        debug("Saving artifacts to memory tmp folder");
+
+        return ArtifactStorage.storeToFile(tmpDirNames[tmpDirNames.length - 1], artifact, testSuffix);
+      }
+    }
+  }
+
+  async artifactByTestName(test) {
+    const list = [];
+
+    if (this.isFile && this.tmpDirFullpath) {
+      const files = fs.readdirSync(this.tmpDirFullpath);
+
+      for (const file of files) {
+        if (file.includes(test)) {
+          const buff = await fs.promises.readFile(`${this.tmpDirFullpath  }/${  file}`);
+
+          list.push(JSON.parse(buff.toString()));
+        }
+      }
+    }
+
+    return list;
+  }
+
+  async tmpContents() {
+    const contents = [];
+
+    if (this.isFile && this.tmpDirFullpath) {
+      const files = fs.readdirSync(this.tmpDirFullpath);
+
+      for (const file of files) {
+        const buff = await fs.promises.readFile(`${this.tmpDirFullpath  }/${  file}`);
+        const content = buff.toString();
+
+        contents.push(content);
+      }
+    }
+
+    return contents;
+  }
+
+  createTestomatTmpDir(clientPrefix) {
+    return fs.mkdtempSync(join(os.tmpdir(), clientPrefix));
+  }
+
+  clearTmpDirByName(name) {
+    const tmpDirPath = join(os.tmpdir(), name);
+
+    if (fs.existsSync(tmpDirPath)) {
+      fs.rmSync(tmpDirPath, { recursive: true });
+      debug(` Testomat tmpDir = ${tmpDirPath} was deleted successfully!`);
+
+      
+    }
+  }
+
+  static tmpTestomatDirNames() {
+    const subname = this._tmpPrefix || "tsmt_reporter";
+
+    return fs.readdirSync(os.tmpdir(), { withFileTypes: true })
+      .filter((item) => item.isDirectory())
+      .map((item) => item.name)
+      .filter((name) => name.includes(subname));
+  }
+
+  cleanup() {
+    if (this.isFile && this.tmpDirFullpath) {
+      const tmpDirNames = ArtifactStorage.tmpTestomatDirNames();
+
+      if (Array.isArray(tmpDirNames) && tmpDirNames.length) {
+        for (const name of tmpDirNames) {
+          this.clearTmpDirByName(name);
+        }
+      }
+    }
+    else {
+      debug("The tmp folder has not been created! Nothing to delete");
+    }
+  }
+}
+
+module.exports = ArtifactStorage;

package/lib/adapter/codecept.js

@@ -1,15 +1,15 @@
+const chalk = require('chalk');
+const TestomatClient = require('../client');
+const { STATUS, APP_PREFIX } = require('../constants');
+const upload = require('../fileUploader');
+const Output = require('../output');
+
 if (!global.codeceptjs) {
   // eslint-disable-next-line global-require, import/no-extraneous-dependencies
   global.codeceptjs = require('codeceptjs');
 }
 
 const { event, recorder, codecept } = global.codeceptjs;
-const chalk = require('chalk');
-const TestomatClient = require('../client');
-const { FAILED } = require('../constants');
-const TRConstants = require('../constants');
-const upload = require('../fileUploader');
-const Output = require('../output');
 
 let currentMetaStep = [];
 let error;
@@ -68,15 +68,16 @@ function CodeceptReporter(config) {
 
   event.dispatcher.on(event.all.result, async () => {
     if (videos.length && upload.isArtifactsEnabled()) {
-      console.log(TRConstants.APP_PREFIX, `🎞️  Uploading ${videos.length} videos...`);
+      console.log(APP_PREFIX, `🎞️  Uploading ${videos.length} videos...`);
 
       const promises = [];
       for (const video of videos) {
         const { testId, title, path, type } = video;
         const file = { path, type, title };
         promises.push(
-          client.addTestRun(testId, undefined, {
+          client.addTestRun(undefined, {
             ...stripExampleFromTitle(title),
+            test_id: testId,
             files: [file],
           }),
         );
@@ -84,7 +85,7 @@ function CodeceptReporter(config) {
       await Promise.all(promises);
     }
 
-    const status = failedTests.length === 0 ? TRConstants.PASSED : TRConstants.FAILED;
+    const status = failedTests.length === 0 ? STATUS.PASSED : STATUS.FAILED;
     client.updateRunStatus(status);
   });
 
@@ -95,12 +96,13 @@ function CodeceptReporter(config) {
     }
     const testId = parseTest(tags);
     const testObj = getTestAndMessage(title);
-    client.addTestRun(testId, TRConstants.PASSED, {
+    client.addTestRun(STATUS.PASSED, {
       ...stripExampleFromTitle(title),
       suite_title: test.parent && test.parent.title,
       message: testObj.message,
       time: getDuration(test),
       steps: output.text(),
+      test_id: testId,
     });
     output.stop();
   });
@@ -119,9 +121,10 @@ function CodeceptReporter(config) {
       failedTests.push(id || title);
       const testId = parseTest(tags);
 
-      client.addTestRun(testId, TRConstants.FAILED, {
+      client.addTestRun(STATUS.FAILED, {
         ...stripExampleFromTitle(title),
         suite_title: suite.title,
+        test_id: testId,
         error,
         time: 0,
       });
@@ -130,7 +133,7 @@ function CodeceptReporter(config) {
   });
 
   event.dispatcher.on(event.test.after, test => {
-    if (test.state && test.state !== FAILED) return;
+    if (test.state && test.state !== STATUS.FAILED) return;
     if (test.err) error = test.err;
     const { id, tags, title, artifacts } = test;
     failedTests.push(id || title);
@@ -149,8 +152,9 @@ function CodeceptReporter(config) {
       if (aid.startsWith('trace')) files.push({ testId: id, title, path: artifacts[aid], type: 'application/zip' }); 
     }
 
-    client.addTestRun(testId, TRConstants.FAILED, {
+    client.addTestRun(STATUS.FAILED, {
       ...stripExampleFromTitle(title),
+      test_id: testId,
       suite_title: test.parent && test.parent.title,
       error,
       message: testObj.message,
@@ -167,8 +171,9 @@ function CodeceptReporter(config) {
 
     const testId = parseTest(tags);
     const testObj = getTestAndMessage(title);
-    client.addTestRun(testId, TRConstants.SKIPPED, {
+    client.addTestRun(STATUS.SKIPPED, {
       ...stripExampleFromTitle(title),
+      test_id: testId,
       suite_title: test.parent && test.parent.title,
       message: testObj.message,
       time: getDuration(test),
@@ -207,12 +212,13 @@ function CodeceptReporter(config) {
     currentMetaStep = metaSteps;
     stepShift = 2 * shift;
 
-    let duration = new Date() - stepStart;
-    if (duration) {
-      duration = repeat(1) + chalk.grey(`(${duration}ms)`);
+    const durationMs = (+new Date() - (+stepStart));
+    let duration = '';
+    if (durationMs) {
+      duration = repeat(1) + chalk.grey(`(${durationMs}ms)`);
     }
 
-    if (step.status === TRConstants.FAILED) {
+    if (step.status === STATUS.FAILED) {
       output.push(repeat(stepShift) + chalk.red(step.toString()) + duration);
     } else {
       output.push(repeat(stepShift) + step.toString() + duration);

package/lib/adapter/cucumber/current.js

@@ -2,7 +2,8 @@
 const { Formatter, formatterHelpers } = require("@cucumber/cucumber");
 const chalk = require('chalk');
 const fs = require('fs');
-const { TestomatClient, TRConstants } = require('../../reporter');
+const { STATUS } = require('../../constants');
+const TestomatClient = require('../../client');
 
 const { GherkinDocumentParser, PickleParser } = formatterHelpers
 const {
@@ -19,7 +20,7 @@ class CucumberReporter extends Formatter {
     this.cases = [];
 
     this.client = new TestomatClient();
-    this.status = TRConstants.PASSED;
+    this.status = STATUS.PASSED;
 
   }
 
@@ -33,7 +34,8 @@ class CucumberReporter extends Formatter {
     const testCaseAttempt = this.eventDataCollector.getTestCaseAttempt(testCaseFinished.testCaseStartedId);
 
     let example;
-    let status = TRConstants.PASSED;
+
+    let status = STATUS.PASSED;
     let color = 'green';
     let message;
 
@@ -41,22 +43,23 @@ class CucumberReporter extends Formatter {
 
     if (testCaseAttempt.worstTestStepResult) {
       if (testCaseAttempt.worstTestStepResult.status === 'SKIPPED') {
-        status = TRConstants.SKIPPED;
+        status = STATUS.SKIPPED;
       }
       if (testCaseAttempt.worstTestStepResult.status === 'UNDEFINED') {
-        status = TRConstants.SKIPPED;
+        status = STATUS.SKIPPED;
         message = 'Undefined steps. Implement missing steps and rerun this scenario';
       }
       if (testCaseAttempt.worstTestStepResult.status === 'FAILED') {
         message = testCaseAttempt?.worstTestStepResult?.message;
-        status = TRConstants.FAILED;
+        status = STATUS.FAILED;
       }
       color = getStatusColor(testCaseAttempt.worstTestStepResult.status);
-      if (status !== TRConstants.PASSED) this.failures.push(testCaseAttempt);
+      if (status !== STATUS.PASSED) this.failures.push(testCaseAttempt);
     }
 
     if (testCaseAttempt.pickle.astNodeIds.length > 1) {
       example = getExample(testCaseAttempt);
+      // @ts-ignore
       testCaseAttempt.example = example;
     }
 
@@ -70,8 +73,8 @@ class CucumberReporter extends Formatter {
     if (message) cliMessage += chalk.gray(message.split('\n')[0]);
     console.log(cliMessage);
 
-    if (status !== TRConstants.PASSED && status !== TRConstants.SKIPPED) {
-      this.status = TRConstants.FAILED;
+    if (status !== STATUS.PASSED && status !== STATUS.SKIPPED) {
+      this.status = STATUS.FAILED;
     }
 
     const time = Object.values(testCaseAttempt.stepResults)
@@ -80,12 +83,13 @@ class CucumberReporter extends Formatter {
 
     if (!this.client) return;
 
-    this.client.addTestRun(testId, status, {
+    this.client.addTestRun(status, {
       // error: testCaseAttempt.worstTestStepResult.message,
       message,
       steps: getSteps(testCaseAttempt).map(s => s.toString()).join('\n').trim(),
       example: { ...example},
       title: scenario,
+      test_id: testId,
       time,
     });
   }
@@ -121,9 +125,9 @@ class CucumberReporter extends Formatter {
 
     if (!this.client) return;
 
-    this.client.updateRunStatus(testRunFinished.success ? TRConstants.PASSED : TRConstants.FAILED)
+    this.client.updateRunStatus(testRunFinished.success ? STATUS.PASSED : STATUS.FAILED)
   }
-};
+}
 
 function getSteps(tc) {
   const stepIds = Object.keys(tc.stepResults);

package/lib/adapter/cucumber/legacy.js

@@ -1,8 +1,9 @@
-// eslint-disable-next-line import/no-unresolved
+// eslint-disable-next-line global-require, import/no-extraneous-dependencies, import/no-unresolved
 const { Formatter } = require('cucumber');
 const chalk = require('chalk');
-const { TestomatClient, TRConstants } = require('../../reporter');
 const { parseTest } = require('../../util');
+const { STATUS } = require('../../constants');
+const TestomatClient = require('../../client');
 
 const createTestomatFormatter = apiKey => {
   if (!apiKey || apiKey === '') {
@@ -92,18 +93,18 @@ const createTestomatFormatter = apiKey => {
       if (!apiKey) return;
 
       this.client = new TestomatClient({ apiKey });
-      this.status = TRConstants.PASSED;
+      this.status = STATUS.PASSED.toString();
 
       options.eventBroadcaster.on('gherkin-document', addDocument);
-      options.eventBroadcaster.on('test-run-started', () => this.client.createRun());
+      options.eventBroadcaster.on('test-run-started', () => this?.client?.createRun());
       options.eventBroadcaster.on('test-case-finished', this.onTestCaseFinished.bind(this));
-      options.eventBroadcaster.on('test-run-finished', () => this.client.updateRunStatus(this.status));
+      options.eventBroadcaster.on('test-run-finished', () => this?.client?.updateRunStatus(this.status));
     }
 
     onTestCaseFinished(event) {
       const scenario = getScenario(event.sourceLocation);
       const testId = getTestId(scenario);
-      const status = event.result.status === 'undefined' ? TRConstants.SKIPPED : event.result.status;
+      const status = event.result.status === 'undefined' ? STATUS.SKIPPED : event.result.status;
 
       let example = getDataTableMap(scenario, event.sourceLocation);
       if (example) example = JSON.parse(example);
@@ -120,14 +121,17 @@ const createTestomatFormatter = apiKey => {
         message = 'Undefined steps. Implement missing steps and rerun this scenario';
       }
       console.log(cliMessage);
-      if (status !== TRConstants.PASSED && status !== TRConstants.SKIPPED) {
-        this.status = TRConstants.FAILED;
+      if (status !== STATUS.PASSED && status !== STATUS.SKIPPED) {
+        this.status = STATUS.FAILED;
       }
-      this.client.addTestRun(testId, status, {
-        error: event.result.exception,
+
+      this.client?.addTestRun(status, {
+        error: event.result.exception instanceof Error ? event.result.exception : undefined,
         message,
         example,
+        test_id: testId,
         title: getTitle(scenario),
+        suite_title: getTitle(getFeature(event.sourceLocation.uri)),
       });
     }
   };

package/lib/adapter/cypress-plugin/index.js

@@ -1,4 +1,4 @@
-const TRConstants = require('../../constants');
+const { STATUS } = require('../../constants');
 const { parseTest, parseSuite } = require('../../util');
 const TestomatClient = require('../../client');
 
@@ -57,16 +57,16 @@ const testomatioReporter = on => {
 
       let state;
       switch (test.state) {
-        case 'passed': state = TRConstants.PASSED; break;
-        case 'failed': state = TRConstants.FAILED; break;
+        case 'passed': state = STATUS.PASSED; break;
+        case 'failed': state = STATUS.FAILED; break;
         case 'skipped':
         case 'pending': 
         default:
-          state = TRConstants.SKIPPED;
+          state = STATUS.SKIPPED;
       }
 
-      addSpecTestsPromises.push(client.addTestRun(testId, state, {
-        title, time, example, error, files, suite_title: suiteTitle, suite_id: suiteId
+      addSpecTestsPromises.push(client.addTestRun(state, {
+        title, time, example, error, files, suite_title: suiteTitle, test_id: testId, suite_id: suiteId
       }));
     }
 
@@ -74,7 +74,7 @@ const testomatioReporter = on => {
   });
 
   on('after:run', async results => {
-    const status = results.totalFailed ? TRConstants.FAILED : TRConstants.PASSED;
+    const status = results.totalFailed ? STATUS.FAILED : STATUS.PASSED;
     await client.updateRunStatus(status);
   });
 };

package/lib/adapter/jasmine.js

@@ -1,6 +1,6 @@
 const TestomatClient = require('../client');
 const { parseTest, ansiRegExp } = require('../util');
-const { PASSED, FAILED } = require('../constants');
+const { STATUS } = require('../constants');
 
 class JasmineReporter {
   constructor(options) {
@@ -32,13 +32,14 @@ class JasmineReporter {
       errorMessage = `${errorMessage}Failure: ${result.failedExpectations[i].message}\n`;
       errorMessage = `${errorMessage}\n ${result.failedExpectations[i].stack}`;
     }
-    console.log(`${title} : ${PASSED}`);
+    console.log(`${title} : ${STATUS.PASSED}`);
     console.log(errorMessage);
     const testId = parseTest(title);
     errorMessage = errorMessage.replace(ansiRegExp(), '');
-    this.client.addTestRun(testId, status, {
+    this.client.addTestRun(status, {
       error: result.failedExpectations[0],
       message: errorMessage,
+      test_id: testId,
       title,
       time: this.getDuration(result),
     });
@@ -48,7 +49,7 @@ class JasmineReporter {
     if (!this.client) return;
 
     const { overallStatus } = suiteInfo;
-    const status = overallStatus === 'failed' ? FAILED : PASSED;
+    const status = overallStatus === 'failed' ? STATUS.FAILED : STATUS.PASSED;
 
     this.client.updateRunStatus(status).then(() => done);
   }

package/lib/adapter/jest.js

@@ -1,5 +1,5 @@
 const TestomatClient = require('../client');
-const TRConstants = require('../constants');
+const { STATUS } = require('../constants');
 const { parseTest, ansiRegExp } = require('../util');
 
 class JestReporter {
@@ -16,26 +16,22 @@ class JestReporter {
 
     const { testResults } = testResult;
     for (const result of testResults) {
-      let error = null;
-      let steps = null;
+      let error;
+      let steps;
       const { status, title, duration, failureMessages } = result;
       if (failureMessages[0]) {
         let errorMessage = failureMessages[0].replace(ansiRegExp(), '');
         errorMessage = errorMessage.split('\n')[0];
-        error = {
-          message: errorMessage,
-        };
+        error = new Error(errorMessage);
         steps = failureMessages[0];
       }
       const testId = parseTest(title);
       const deducedStatus = status === 'pending' ? 'skipped' : status;
-
-      const suite_title = result.ancestorTitles && result.ancestorTitles[result.ancestorTitles.length - 1];
       // In jest if test is not matched with test name pattern it is considered as skipped.
       // So adding a check if it is skipped for real or because of test pattern
       if (!this._globalConfig.testNamePattern || deducedStatus !== 'skipped') {
-        this.client.addTestRun(testId, deducedStatus, {
-          suite_title,
+        this.client.addTestRun(deducedStatus, {
+          test_id: testId,
           error,
           steps,
           title,
@@ -49,7 +45,7 @@ class JestReporter {
     if (!this.client) return;
 
     const { numFailedTests } = results;
-    const status = numFailedTests === 0 ? TRConstants.PASSED : TRConstants.FAILED;
+    const status = numFailedTests === 0 ? STATUS.PASSED : STATUS.FAILED;
     this.client.updateRunStatus(status);
   }
 }