@testomatio/reporter 1.4.5 → 1.5.0-beta-vitest

package/lib/adapter/codecept.js

@@ -129,8 +129,8 @@ function CodeceptReporter(config) {
     await Promise.all(reportTestPromises);
 
     if (upload.isArtifactsEnabled()) {
-      await uploadAttachments(client, videos, '🎞️ Uploading', 'video');
-      await uploadAttachments(client, traces, '📁 Uploading', 'trace');
+      uploadAttachments(client, videos, '🎞️  Uploading', 'video');
+      uploadAttachments(client, traces, '📁 Uploading', 'trace');
     }
 
     const status = failedTests.length === 0 ? STATUS.PASSED : STATUS.FAILED;
@@ -142,6 +142,7 @@ function CodeceptReporter(config) {
     if (id && failedTests.includes(id)) {
       failedTests = failedTests.filter(failed => id !== failed);
     }
+    const testId = getTestomatIdFromTestTitle(`${title} ${tags?.join(' ')}`);
     const testObj = getTestAndMessage(title);
 
     const logs = getTestLogs(test);
@@ -151,12 +152,11 @@ function CodeceptReporter(config) {
 
     client.addTestRun(STATUS.PASSED, {
       ...stripExampleFromTitle(title),
-      rid: id,
       suite_title: test.parent && test.parent.title,
       message: testObj.message,
       time: getDuration(test),
       steps: global.testomatioDataStore.steps.join('\n') || null,
-      test_id: getTestomatIdFromTestTitle(`${title} ${tags?.join(' ')}`),
+      test_id: testId,
       logs,
       manuallyAttachedArtifacts,
       meta: keyValues,
@@ -179,7 +179,6 @@ function CodeceptReporter(config) {
       const testId = getTestomatIdFromTestTitle(`${title} ${tags?.join(' ')}`);
 
       client.addTestRun(STATUS.FAILED, {
-        rid: id,
         ...stripExampleFromTitle(title),
         suite_title: suite.title,
         test_id: testId,
@@ -195,6 +194,7 @@ function CodeceptReporter(config) {
     if (test.err) error = test.err;
     const { id, tags, title, artifacts } = test;
     failedTests.push(id || title);
+    let testId = getTestomatIdFromTestTitle(`${title} ${tags?.join(' ')}`);
     const testObj = getTestAndMessage(title);
 
     const files = [];
@@ -206,10 +206,10 @@ function CodeceptReporter(config) {
     const keyValues = services.keyValues.get(test.fullTitle());
     services.setContext(null);
 
-    client.addTestRun(STATUS.FAILED, {
+    const reportTestPromise = client
+      .addTestRun(STATUS.FAILED, {
         ...stripExampleFromTitle(title),
-        rid: id,
-        test_id: getTestomatIdFromTestTitle(`${title} ${tags?.join(' ')}`),
+        test_id: testId,
         suite_title: test.parent && test.parent.title,
         error,
         message: testObj.message,
@@ -219,14 +219,19 @@ function CodeceptReporter(config) {
         logs,
         manuallyAttachedArtifacts,
         meta: keyValues,
-    });
+      })
+      .then(pipes => {
+        testId = pipes.filter(p => p.pipe.includes('Testomatio'))[0]?.result?.data?.test_id;
 
-    debug('artifacts', artifacts);
+        debug('artifacts', artifacts);
 
-    for (const aid in artifacts) {
-      if (aid.startsWith('video')) videos.push({ rid: id, title, path: artifacts[aid], type: 'video/webm' });
-      if (aid.startsWith('trace')) traces.push({ rid: id, title, path: artifacts[aid], type: 'application/zip' });
-    }
+        for (const aid in artifacts) {
+          if (aid.startsWith('video')) videos.push({ testId, title, path: artifacts[aid], type: 'video/webm' });
+          if (aid.startsWith('trace')) traces.push({ testId, title, path: artifacts[aid], type: 'application/zip' });
+        }
+      });
+
+    reportTestPromises.push(reportTestPromise);
 
     // output.stop();
   });
@@ -235,11 +240,11 @@ function CodeceptReporter(config) {
     const { id, tags, title } = test;
     if (failedTests.includes(id || title)) return;
 
+    const testId = getTestomatIdFromTestTitle(`${title} ${tags?.join(' ')}`);
     const testObj = getTestAndMessage(title);
     client.addTestRun(STATUS.SKIPPED, {
-      rid: id,
       ...stripExampleFromTitle(title),
-      test_id: getTestomatIdFromTestTitle(`${title} ${tags?.join(' ')}`),
+      test_id: testId,
       suite_title: test.parent && test.parent.title,
       message: testObj.message,
       time: getDuration(test),
@@ -304,22 +309,21 @@ function CodeceptReporter(config) {
 }
 
 async function uploadAttachments(client, attachments, messagePrefix, attachmentType) {
-  if (!attachments?.length) return;
-
-  console.log(APP_PREFIX, `Attachments: ${messagePrefix} ${attachments.length} ${attachmentType} ...`);
+  if (attachments.length > 0) {
+    console.log(APP_PREFIX, `Attachments: ${messagePrefix} ${attachments.length} ${attachmentType}/-s ...`);
 
-  const promises = attachments.map(async attachment => {
-    const { rid, title, path, type } = attachment;
-    const file = { path, type, title };
-    return client.addTestRun(undefined, {
-      ...stripExampleFromTitle(title),
-      rid,
-      files: [file],
+    const promises = attachments.map(async attachment => {
+      const { testId, title, path, type } = attachment;
+      const file = { path, type, title };
+      return client.addTestRun(undefined, {
+        ...stripExampleFromTitle(title),
+        test_id: testId,
+        files: [file],
+      });
     });
-  });
-
-  await Promise.all(promises);
 
+    await Promise.all(promises);
+  }
 }
 
 function getTestAndMessage(title) {

package/lib/adapter/playwright.js

@@ -2,7 +2,6 @@ const chalk = require('chalk');
 const crypto = require('crypto');
 const os = require('os');
 const path = require('path');
-const { v4: uuidv4 } = require('uuid');
 const fs = require('fs');
 const { APP_PREFIX, STATUS: Status, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
 const TestomatioClient = require('../client');
@@ -40,6 +39,8 @@ class PlaywrightReporter {
 
     const { title } = test;
 
+    let testId = getTestomatIdFromTestTitle(`${title} ${test.tags?.join(' ')}`);
+
     const { error, duration } = result;
 
     const suite_title = test.parent ? test.parent?.title : path.basename(test?.location?.file);
@@ -57,12 +58,10 @@ class PlaywrightReporter {
     const manuallyAttachedArtifacts = services.artifacts.get(fullTestTitle);
     const keyValues = services.keyValues.get(fullTestTitle);
 
-    const rid = test.id || test.testId || uuidv4();
-
-    const reportTestPromise = this.client.addTestRun(checkStatus(result.status), {
-        rid,
+    const reportTestPromise = this.client
+      .addTestRun(checkStatus(result.status), {
         error,
-        test_id: getTestomatIdFromTestTitle(`${title} ${test.tags?.join(' ')}`),
+        test_id: testId,
         suite_title,
         title,
         steps: steps.join('\n'),
@@ -71,16 +70,20 @@ class PlaywrightReporter {
         manuallyAttachedArtifacts,
         meta: keyValues,
         file: test.location?.file,
-      });
+      })
+      .then(pipes => {
+        testId = pipes?.filter(p => p.pipe.includes('Testomatio'))[0]?.result?.data?.test_id;
 
-    this.uploads.push({
-      rid,
-      title: test.title,
-      files: result.attachments.filter(a => a.body || a.path),
-      file: test.location?.file,
-    });
-    // remove empty uploads
-    this.uploads = this.uploads.filter(upload => upload.files.length);
+        this.uploads.push({
+          testId,
+          title,
+          suite_title,
+          files: result.attachments.filter(a => a.body || a.path),
+          file: test.location?.file,
+        });
+        // remove empty uploads
+        this.uploads = this.uploads.filter(upload => upload.files.length);
+      });
 
     reportTestPromises.push(reportTestPromise);
   }
@@ -112,7 +115,7 @@ class PlaywrightReporter {
       const promises = [];
 
       for (const upload of this.uploads) {
-        const { rid, file, title } = upload;
+        const { title, testId, suite_title } = upload;
 
         const files = upload.files.map(attachment => ({
           path: this.#getArtifactPath(attachment),
@@ -122,10 +125,11 @@ class PlaywrightReporter {
 
         promises.push(
           this.client.addTestRun(undefined, {
-            rid,
+            test_id: testId,
             title,
+            suite_title,
             files,
-            file,
+            file: upload.file,
           }),
         );
       }

package/lib/adapter/vitest.js

@@ -0,0 +1,182 @@
+const chalk = require('chalk');
+const { TestomatioClient } = require('../client');
+const { STATUS } = require('../constants');
+const { getTestomatIdFromTestTitle } = require('../utils/utils');
+const debug = require('debug')('@testomatio/reporter:adapter:vitest');
+
+/**
+ * @typedef {import('../../types').VitestTest} VitestTest
+ * @typedef {import('../../types').VitestTestFile} VitestTestFile
+ * @typedef {import('../../types').VitestSuite} VitestSuite
+ * @typedef {import('../../types').VitestTestLogs} VitestTestLogs
+ * @typedef {import('../../vitest.types').ErrorWithDiff} ErrorWithDiff
+ * @typedef {typeof import('../constants').STATUS} STATUS
+ * @typedef {import('../../types').TestData} TestData
+ */
+
+class VitestReporter {
+  constructor(config = {}) {
+    this.client = new TestomatioClient({ apiKey: config?.apiKey });
+    /**
+     * @type {(TestData & {status: string})[]} tests
+     */
+    this.tests = [];
+  }
+
+  // on run start
+  onInit() {
+    this.client.createRun();
+  }
+
+  /**
+   * @param {VitestTestFile[] | undefined} files // array with results;
+   * @param {unknown[] | undefined} errors // errors does not contain errors from tests; probably its testrunner errors
+   */
+  async onFinished(files, errors) {
+    if (!files || !files.length) console.info('No tests executed');
+
+    files.forEach(file => {
+      // task could be test or suite
+      file.tasks.forEach(taskOrSuite => {
+        if (taskOrSuite.type === 'test') {
+          const test = taskOrSuite;
+          this.tests.push(this.#getDataFromTest(test));
+        } else if (taskOrSuite.type === 'suite') {
+          const suite = taskOrSuite;
+          this.#processTasksOfSuite(suite);
+        } else {
+          throw new Error('Unprocessed case. Unknown task type');
+        }
+      });
+    });
+
+    debug(this.tests.length, 'tests collected');
+
+    // send tests to Testomat.io
+    for (const test of this.tests) {
+      await this.client.addTestRun(test.status, test);
+    }
+
+    console.log('finished');
+    if (errors.length) console.error('Vitest adapter errors:', errors);
+
+    await this.client.updateRunStatus(getRunStatusFromResults(files));
+  }
+
+  /* non-used listeners
+  onUserConsoleLog(log) {}
+  onPathsCollected(paths) {} // paths array to files with tests
+  onCollected(files) {} // files array with tests (but without results)
+  onTaskUpdate(packs) {} // some updates come here on afterAll block execution
+  onTestRemoved(trigger) {}
+  onWatcherStart(files, errors) {}
+  onWatcherRerun(files, trigger) {}
+  onServerRestart(reason) {}
+  onProcessTimeout() {}
+  */
+
+  /**
+   * Recursively gets all tasks from suite and pushes them to "tests" array
+   *
+   * @param {VitestSuite} suite
+   */
+  #processTasksOfSuite(suite) {
+    suite.tasks.forEach(taskOrSuite => {
+      if (taskOrSuite.type === 'test') {
+        const test = taskOrSuite;
+        this.tests.push(this.#getDataFromTest(test));
+      } else if (taskOrSuite.type === 'suite') {
+        const theSuite = taskOrSuite;
+        this.#processTasksOfSuite(theSuite);
+      } else {
+        throw new Error('Unprocessed case. Unknown task type');
+      }
+    });
+  }
+
+  /**
+   * Processes task and returns test data ready to be sent to Testomat.io
+   *
+   * @param {VitestTest} test
+   *
+   * @returns {TestData & {status: string}}
+   */
+  #getDataFromTest(test) {
+    return {
+      error: test.result?.errors ? test.result.errors[0] : undefined,
+      file: test.file.name,
+      logs: test.logs ? transformLogsToString(test.logs) : '',
+      meta: test.meta,
+      status: getTestStatus(test),
+      suite_title: test.suite.name || test.file?.name,
+      test_id: getTestomatIdFromTestTitle(test.name),
+      time: test.result?.duration || 0,
+      title: test.name,
+      // testomatio functions (artifacts, logs, steps, meta) are not supported
+    };
+  }
+}
+
+/**
+ * Returns run status based on test results
+ *
+ * @param {VitestTestFile[]} files
+ * @returns {'passed' | 'failed' | 'finished'}
+ */
+function getRunStatusFromResults(files) {
+  /**
+   * @type {'passed' | 'failed' | 'finished'}
+   */
+  let status = 'finished'; // default status (if no failed or passed tests)
+
+  files.forEach(file => {
+    // search for failed tests
+    file.tasks.forEach(taskOrSuite => {
+      if (taskOrSuite.result?.state === 'fail') {
+        status = 'failed'; // set status to failed if any test failed
+      }
+    });
+
+    // if there are no failed tests > search for passed tests
+    if (status !== 'failed') {
+      file.tasks.forEach(taskOrSuite => {
+        if (taskOrSuite.result?.state === 'pass') {
+          status = 'passed'; // set status to passed if any test passed (and there are no failed tests)
+        }
+      });
+    }
+  });
+
+  return status;
+}
+
+/**
+ * Returns test status in Testomat.io format
+ *
+ * @param {VitestTest} test
+ * @returns 'passed' | 'failed' | 'skipped'
+ */
+function getTestStatus(test) {
+  if (test.result?.state === 'fail') return STATUS.FAILED;
+  if (test.result?.state === 'pass') return STATUS.PASSED;
+  if (!test.result && test.mode === 'skip') return STATUS.SKIPPED;
+  console.error(chalk.red('Unprocessed case for defining test status. Contact dev team. Test:'), test);
+}
+
+/**
+ * @param {VitestTestLogs[]} logs
+ * @returns string
+ */
+function transformLogsToString(logs) {
+  if (!logs) return '';
+  let logsStr = '';
+  logs.forEach(log => {
+    if (log.type === 'stdout') logsStr += `${log.content}\n`;
+    if (log.type === 'stderr') logsStr += `${chalk.red(log.content)}\n`;
+  });
+  return logsStr;
+}
+
+module.exports.VitestReporter = VitestReporter;
+module.exports.default = VitestReporter;
+module.exports = VitestReporter;

package/lib/bin/reportXml.js

@@ -44,10 +44,15 @@ program
 
     let timeoutTimer;
     if (opts.timelimit) {
-      timeoutTimer = setTimeout(() => {
-        console.log(`⚠️  Reached timeout of ${opts.timelimit}s. Exiting... (Exit code is 0 to not fail the pipeline)`);
-        process.exit(0);
-      }, parseInt(opts.timelimit, 10) * 1000);
+      timeoutTimer = setTimeout(
+        () => {
+          console.log(
+            `⚠️  Reached timeout of ${opts.timelimit}s. Exiting... (Exit code is 0 to not fail the pipeline)`,
+          );
+          process.exit(0);
+        },
+        parseInt(opts.timelimit, 10) * 1000,
+      );
     }
 
     try {

package/lib/client.js

@@ -123,7 +123,7 @@ class Client {
     if (isTestShouldBeExculedFromReport(testData)) return [];
 
     if (status === STATUS.SKIPPED && process.env.TESTOMATIO_EXCLUDE_SKIPPED) {
-      debug('Skipping test from report', testData?.title)
+      debug('Skipping test from report', testData?.title);
       return []; // do not log skipped tests
     }
 
@@ -133,10 +133,12 @@ class Client {
         suite_title: 'Unknown suite',
       };
 
+    /**
+     * @type {TestData}
+     */
     const {
-      rid,
       error = null,
-      time = '',
+      time = 0,
       example = null,
       files = [],
       filesBuffers = [],
@@ -185,7 +187,6 @@ class Client {
     this.totalUploaded += artifacts.length;
 
     const data = {
-      rid,
       files,
       steps,
       status,
@@ -198,7 +199,7 @@ class Client {
       suite_id,
       test_id,
       message,
-      run_time: parseFloat(time),
+      run_time: typeof time === 'number' ? time : parseFloat(time),
       artifacts,
       meta,
     };
@@ -224,7 +225,8 @@ class Client {
   /**
    *
    * Updates the status of the current test run and finishes the run.
-   * @param {RunStatus} status - The status of the current test run. Must be one of "passed", "failed", or "finished"
+   * @param {'passed' | 'failed' | 'finished'} status - The status of the current test run.
+   * Must be one of "passed", "failed", or "finished"
    * @param {boolean} [isParallel] - Whether the current test run was executed in parallel with other tests.
    * @returns {Promise<any>} - A Promise that resolves when finishes the run.
    */
@@ -286,13 +288,22 @@ class Client {
     if (!message) message = error.message;
     if (error.inspect) message = error.inspect() || '';
 
-    let stack = `${message}\n`;
+    let stack = '';
+    if (error.name) stack += `${chalk.red(error.name)}`;
+    if (error.operator) stack += ` (${chalk.red(error.operator)})`;
+    // add new line if something was added to stack
+    if (stack) stack += ': ';
 
-    // diffs for mocha, cypress, codeceptjs style
-    if (error.actual && error.expected) {
+    stack += `${message}\n`;
+
+    if (error.diff) {
+      stack += error.diff;
+      stack += '\n\n';
+    } else if (error.actual && error.expected && error.actual !== error.expected) {
+      // diffs for mocha, cypress, codeceptjs style
       stack += `\n\n${chalk.bold.green('+ expected')} ${chalk.bold.red('- actual')}`;
-      stack += `\n${chalk.red(`- ${error.actual.toString().split('\n').join('\n- ')}`)}`;
       stack += `\n${chalk.green(`+ ${error.expected.toString().split('\n').join('\n+ ')}`)}`;
+      stack += `\n${chalk.red(`- ${error.actual.toString().split('\n').join('\n- ')}`)}`;
       stack += '\n\n';
     }
 
@@ -365,3 +376,4 @@ function isTestShouldBeExculedFromReport(testData) {
 }
 
 module.exports = Client;
+module.exports.TestomatioClient = Client;

package/lib/pipe/gitlab.js

@@ -79,13 +79,13 @@ class GitLabPipe {
     let summary = `${this.hiddenCommentData}
     
   | [![Testomat.io Report](${testomatLogoURL})](https://testomat.io)  | ${statusEmoji(
-      runParams.status,
-    )} ${runParams.status.toUpperCase()} ${statusEmoji(runParams.status)} |
+    runParams.status,
+  )} ${runParams.status.toUpperCase()} ${statusEmoji(runParams.status)} |
   | --- | --- |
   | Tests | ✔️  **${this.tests.length}** tests run  |
   | Summary | ${statusEmoji('failed')} **${failedCount}** failed; ${statusEmoji(
-      'passed',
-    )} **${passedCount}** passed; **${statusEmoji('skipped')}** ${skippedCount} skipped |
+    'passed',
+  )} **${passedCount}** passed; **${statusEmoji('skipped')}** ${skippedCount} skipped |
   | Duration | 🕐  **${humanizeDuration(
     parseInt(
       this.tests.reduce((a, t) => a + (t.run_time || 0), 0),

package/lib/pipe/testomatio.js

@@ -339,9 +339,6 @@ class TestomatioPipe {
   addTest(data) {
     if (!this.isEnabled) return;
     if (!this.runId) return;
-
-    // add test ID + run ID
-    data.rid = `${this.runId}-${data.rid}`;
     data.api_key = this.apiKey;
     data.create = this.createNewTests;
 

package/lib/utils/utils.js

@@ -248,7 +248,7 @@ const foundedTestLog = (app, tests) => {
 const humanize = text => {
   // if there are no spaces, decamelize
   if (!text.trim().includes(' ')) text = decamelize(text);
-  
+
   return text
     .replace(/_./g, match => ` ${match.charAt(1).toUpperCase()}`)
     .trim()

package/lib/xmlReader.js

@@ -460,7 +460,6 @@ function reduceTestCases(prev, item) {
       // SpecFlow config
       let { title, tags } = fetchProperties(item.type === 'ParameterizedMethod' ? item : testCaseItem);
       let example = null;
-      const suiteTitle = reduceOptions.preferClassname ? testCaseItem.classname : item.name || testCaseItem.classname;
 
       title ||= testCaseItem.name || testCaseItem.methodname || testCaseItem.classname;
       tags ||= [];
@@ -494,7 +493,7 @@ function reduceTestCases(prev, item) {
         run_time: parseFloat(testCaseItem.time || testCaseItem.duration) * 1000,
         status,
         title,
-        suite_title: suiteTitle,
+        suite_title: reduceOptions.preferClassname ? testCaseItem.classname : item.name || testCaseItem.classname,
       });
     });
   return prev;
@@ -528,4 +527,4 @@ function fetchProperties(item) {
     .filter(p => p.name === 'Category')
     .forEach(p => tags.push(p.value));
   return { title, tags };
-}
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@testomatio/reporter",
-  "version": "1.4.5",
+  "version": "1.5.0-beta-vitest",
   "description": "Testomatio Reporter Client",
   "main": "./lib/reporter.js",
   "typings": "typings/index.d.ts",
@@ -55,6 +55,7 @@
     "test:adapter:jasmine:example": "./tests/adapter/examples/jasmine/passReporterOpts.sh && jasmine './tests/adapter/examples/jasmine/index.test.js' --reporter=./../../../lib/adapter/jasmine.js",
     "test:adapter:codecept:example": "codeceptjs run --config='./tests/adapter/examples/codecept/codecept.conf.js'",
     "test:adapter:cucumber:example": "cd ./tests/adapter/examples/cucumber && npx cucumber-js",
+    "test:adapter:vitest:example": "vitest --config='./tests/adapter/examples/vitest/vitest.config.js'",
     "test:storage": "npx mocha tests-storage/artifact-storage.test.js && npx mocha tests-storage/data-storage.test.js && TESTOMATIO_INTERCEPT_CONSOLE_LOGS=true npx mocha tests-storage/logger.test.js && npx mocha tests-storage/logger-2.test.js && npx mocha tests-storage/reporter-functions.test.js"
   },
   "devDependencies": {