@testomatio/reporter 1.5.0-beta-vitest → 1.5.1-beta

package/lib/adapter/codecept.js

@@ -129,8 +129,8 @@ function CodeceptReporter(config) {
     await Promise.all(reportTestPromises);
 
     if (upload.isArtifactsEnabled()) {
-      uploadAttachments(client, videos, '🎞️  Uploading', 'video');
-      uploadAttachments(client, traces, '📁 Uploading', 'trace');
+      await uploadAttachments(client, videos, '🎞️ Uploading', 'video');
+      await uploadAttachments(client, traces, '📁 Uploading', 'trace');
     }
 
     const status = failedTests.length === 0 ? STATUS.PASSED : STATUS.FAILED;
@@ -142,7 +142,6 @@ function CodeceptReporter(config) {
     if (id && failedTests.includes(id)) {
       failedTests = failedTests.filter(failed => id !== failed);
     }
-    const testId = getTestomatIdFromTestTitle(`${title} ${tags?.join(' ')}`);
     const testObj = getTestAndMessage(title);
 
     const logs = getTestLogs(test);
@@ -152,11 +151,12 @@ function CodeceptReporter(config) {
 
     client.addTestRun(STATUS.PASSED, {
       ...stripExampleFromTitle(title),
+      rid: id,
       suite_title: test.parent && test.parent.title,
       message: testObj.message,
       time: getDuration(test),
       steps: global.testomatioDataStore.steps.join('\n') || null,
-      test_id: testId,
+      test_id: getTestomatIdFromTestTitle(`${title} ${tags?.join(' ')}`),
       logs,
       manuallyAttachedArtifacts,
       meta: keyValues,
@@ -179,6 +179,7 @@ function CodeceptReporter(config) {
       const testId = getTestomatIdFromTestTitle(`${title} ${tags?.join(' ')}`);
 
       client.addTestRun(STATUS.FAILED, {
+        rid: id,
         ...stripExampleFromTitle(title),
         suite_title: suite.title,
         test_id: testId,
@@ -194,7 +195,6 @@ function CodeceptReporter(config) {
     if (test.err) error = test.err;
     const { id, tags, title, artifacts } = test;
     failedTests.push(id || title);
-    let testId = getTestomatIdFromTestTitle(`${title} ${tags?.join(' ')}`);
     const testObj = getTestAndMessage(title);
 
     const files = [];
@@ -206,32 +206,27 @@ function CodeceptReporter(config) {
     const keyValues = services.keyValues.get(test.fullTitle());
     services.setContext(null);
 
-    const reportTestPromise = client
-      .addTestRun(STATUS.FAILED, {
-        ...stripExampleFromTitle(title),
-        test_id: testId,
-        suite_title: test.parent && test.parent.title,
-        error,
-        message: testObj.message,
-        time: getDuration(test),
-        files,
-        steps: global.testomatioDataStore?.steps?.join('\n') || null,
-        logs,
-        manuallyAttachedArtifacts,
-        meta: keyValues,
-      })
-      .then(pipes => {
-        testId = pipes.filter(p => p.pipe.includes('Testomatio'))[0]?.result?.data?.test_id;
-
-        debug('artifacts', artifacts);
-
-        for (const aid in artifacts) {
-          if (aid.startsWith('video')) videos.push({ testId, title, path: artifacts[aid], type: 'video/webm' });
-          if (aid.startsWith('trace')) traces.push({ testId, title, path: artifacts[aid], type: 'application/zip' });
-        }
-      });
+    client.addTestRun(STATUS.FAILED, {
+      ...stripExampleFromTitle(title),
+      rid: id,
+      test_id: getTestomatIdFromTestTitle(`${title} ${tags?.join(' ')}`),
+      suite_title: test.parent && test.parent.title,
+      error,
+      message: testObj.message,
+      time: getDuration(test),
+      files,
+      steps: global.testomatioDataStore?.steps?.join('\n') || null,
+      logs,
+      manuallyAttachedArtifacts,
+      meta: keyValues,
+    });
+
+    debug('artifacts', artifacts);
 
-    reportTestPromises.push(reportTestPromise);
+    for (const aid in artifacts) {
+      if (aid.startsWith('video')) videos.push({ rid: id, title, path: artifacts[aid], type: 'video/webm' });
+      if (aid.startsWith('trace')) traces.push({ rid: id, title, path: artifacts[aid], type: 'application/zip' });
+    }
 
     // output.stop();
   });
@@ -240,11 +235,11 @@ function CodeceptReporter(config) {
     const { id, tags, title } = test;
     if (failedTests.includes(id || title)) return;
 
-    const testId = getTestomatIdFromTestTitle(`${title} ${tags?.join(' ')}`);
     const testObj = getTestAndMessage(title);
     client.addTestRun(STATUS.SKIPPED, {
+      rid: id,
       ...stripExampleFromTitle(title),
-      test_id: testId,
+      test_id: getTestomatIdFromTestTitle(`${title} ${tags?.join(' ')}`),
       suite_title: test.parent && test.parent.title,
       message: testObj.message,
       time: getDuration(test),
@@ -309,21 +304,21 @@ function CodeceptReporter(config) {
 }
 
 async function uploadAttachments(client, attachments, messagePrefix, attachmentType) {
-  if (attachments.length > 0) {
-    console.log(APP_PREFIX, `Attachments: ${messagePrefix} ${attachments.length} ${attachmentType}/-s ...`);
+  if (!attachments?.length) return;
 
-    const promises = attachments.map(async attachment => {
-      const { testId, title, path, type } = attachment;
-      const file = { path, type, title };
-      return client.addTestRun(undefined, {
-        ...stripExampleFromTitle(title),
-        test_id: testId,
-        files: [file],
-      });
+  console.log(APP_PREFIX, `Attachments: ${messagePrefix} ${attachments.length} ${attachmentType} ...`);
+
+  const promises = attachments.map(async attachment => {
+    const { rid, title, path, type } = attachment;
+    const file = { path, type, title };
+    return client.addTestRun(undefined, {
+      ...stripExampleFromTitle(title),
+      rid,
+      files: [file],
     });
+  });
 
-    await Promise.all(promises);
-  }
+  await Promise.all(promises);
 }
 
 function getTestAndMessage(title) {

package/lib/adapter/playwright.js

@@ -2,6 +2,7 @@ const chalk = require('chalk');
 const crypto = require('crypto');
 const os = require('os');
 const path = require('path');
+const { v4: uuidv4 } = require('uuid');
 const fs = require('fs');
 const { APP_PREFIX, STATUS: Status, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
 const TestomatioClient = require('../client');
@@ -38,16 +39,15 @@ class PlaywrightReporter {
     if (!this.client) return;
 
     const { title } = test;
-
-    let testId = getTestomatIdFromTestTitle(`${title} ${test.tags?.join(' ')}`);
-
     const { error, duration } = result;
-
     const suite_title = test.parent ? test.parent?.title : path.basename(test?.location?.file);
 
     const steps = [];
     for (const step of result.steps) {
-      appendStep(step, steps);
+      const appendedStep = appendStep(step);
+      if (appendedStep) {
+        steps.push(appendedStep);
+      }
     }
 
     const fullTestTitle = getTestContextName(test);
@@ -57,33 +57,30 @@ class PlaywrightReporter {
     }
     const manuallyAttachedArtifacts = services.artifacts.get(fullTestTitle);
     const keyValues = services.keyValues.get(fullTestTitle);
+    const rid = test.id || test.testId || uuidv4();
+
+    const reportTestPromise = this.client.addTestRun(checkStatus(result.status), {
+      rid,
+      error,
+      test_id: getTestomatIdFromTestTitle(`${title} ${test.tags?.join(' ')}`),
+      suite_title,
+      title,
+      steps: steps.length ? steps : undefined,
+      time: duration,
+      logs,
+      manuallyAttachedArtifacts,
+      meta: keyValues,
+      file: test.location?.file,
+    });
 
-    const reportTestPromise = this.client
-      .addTestRun(checkStatus(result.status), {
-        error,
-        test_id: testId,
-        suite_title,
-        title,
-        steps: steps.join('\n'),
-        time: duration,
-        logs,
-        manuallyAttachedArtifacts,
-        meta: keyValues,
-        file: test.location?.file,
-      })
-      .then(pipes => {
-        testId = pipes?.filter(p => p.pipe.includes('Testomatio'))[0]?.result?.data?.test_id;
-
-        this.uploads.push({
-          testId,
-          title,
-          suite_title,
-          files: result.attachments.filter(a => a.body || a.path),
-          file: test.location?.file,
-        });
-        // remove empty uploads
-        this.uploads = this.uploads.filter(upload => upload.files.length);
-      });
+    this.uploads.push({
+      rid,
+      title: test.title,
+      files: result.attachments.filter(a => a.body || a.path),
+      file: test.location?.file,
+    });
+    // remove empty uploads
+    this.uploads = this.uploads.filter(upload => upload.files.length);
 
     reportTestPromises.push(reportTestPromise);
   }
@@ -115,7 +112,7 @@ class PlaywrightReporter {
       const promises = [];
 
       for (const upload of this.uploads) {
-        const { title, testId, suite_title } = upload;
+        const { rid, file, title } = upload;
 
         const files = upload.files.map(attachment => ({
           path: this.#getArtifactPath(attachment),
@@ -125,11 +122,10 @@ class PlaywrightReporter {
 
         promises.push(
           this.client.addTestRun(undefined, {
-            test_id: testId,
+            rid,
             title,
-            suite_title,
             files,
-            file: upload.file,
+            file,
           }),
         );
       }
@@ -150,18 +146,44 @@ function checkStatus(status) {
   );
 }
 
-function appendStep(step, steps = [], shift = 0) {
-  const prefix = ' '.repeat(shift);
-
-  if (step.error) {
-    steps.push(`${prefix}${chalk.red(step.title)} ${chalk.gray(`${step.duration}ms`)}`);
-  } else {
-    steps.push(`${prefix}${step.title} ${chalk.gray(`${step.duration}ms`)}`);
+function appendStep(step, shift = 0) {
+  let newCategory = step.category;
+  switch (newCategory) {
+    case 'test.step':
+      newCategory = 'user';
+      break;
+    case 'hook':
+      newCategory = 'hook';
+      break;
+    case 'attach':
+      return null; // Skip steps with category 'attach'
+    default:
+      newCategory = 'framework';
   }
 
+  const formattedSteps = [];
   for (const child of step.steps || []) {
-    appendStep(child, steps, shift + 2);
+    const appendedChild = appendStep(child, shift + 2);
+    if (appendedChild) {
+      formattedSteps.push(appendedChild);
+    }
+  }
+
+  const resultStep = {
+    category: newCategory,
+    title: step.title,
+    duration: step.duration,
+  };
+
+  if (formattedSteps.length) {
+    resultStep.steps = formattedSteps;
   }
+
+  if (step.error !== undefined) {
+    resultStep.error = step.error;
+  }
+
+  return resultStep;
 }
 
 function tmpFile(prefix = 'tmp.') {

package/lib/bin/reportXml.js

@@ -44,15 +44,10 @@ program
 
     let timeoutTimer;
     if (opts.timelimit) {
-      timeoutTimer = setTimeout(
-        () => {
-          console.log(
-            `⚠️  Reached timeout of ${opts.timelimit}s. Exiting... (Exit code is 0 to not fail the pipeline)`,
-          );
-          process.exit(0);
-        },
-        parseInt(opts.timelimit, 10) * 1000,
-      );
+      timeoutTimer = setTimeout(() => {
+        console.log(`⚠️  Reached timeout of ${opts.timelimit}s. Exiting... (Exit code is 0 to not fail the pipeline)`);
+        process.exit(0);
+      }, parseInt(opts.timelimit, 10) * 1000);
     }
 
     try {

package/lib/client.js

@@ -137,6 +137,7 @@ class Client {
      * @type {TestData}
      */
     const {
+      rid,
       error = null,
       time = 0,
       example = null,
@@ -187,6 +188,7 @@ class Client {
     this.totalUploaded += artifacts.length;
 
     const data = {
+      rid,
       files,
       steps,
       status,
@@ -225,7 +227,7 @@ class Client {
   /**
    *
    * Updates the status of the current test run and finishes the run.
-   * @param {'passed' | 'failed' | 'finished'} status - The status of the current test run.
+   * @param {'passed' | 'failed' | 'skipped' | 'finished'} status - The status of the current test run.
    * Must be one of "passed", "failed", or "finished"
    * @param {boolean} [isParallel] - Whether the current test run was executed in parallel with other tests.
    * @returns {Promise<any>} - A Promise that resolves when finishes the run.
@@ -274,9 +276,12 @@ class Client {
    */
   formatLogs({ error, steps, logs }) {
     error = error?.trim();
-    steps = steps?.trim();
     logs = logs?.trim();
 
+    if (Array.isArray(steps)) {
+      steps = steps.map(step => formatStep(step)).flat().join('\n');      
+    }
+
     let testLogs = '';
     if (steps) testLogs += `${chalk.bold.blue('################[ Steps ]################')}\n${steps}\n\n`;
     if (logs) testLogs += `${chalk.bold.gray('################[ Logs ]################')}\n${logs}\n\n`;
@@ -297,6 +302,7 @@ class Client {
     stack += `${message}\n`;
 
     if (error.diff) {
+      // diff for vitest
       stack += error.diff;
       stack += '\n\n';
     } else if (error.actual && error.expected && error.actual !== error.expected) {
@@ -339,6 +345,24 @@ function isNotInternalFrame(frame) {
   );
 }
 
+function formatStep(step, shift = 0) {
+  const prefix = ' '.repeat(shift);
+
+  const lines = [];
+
+  if (step.error) {
+    lines.push(`${prefix}${chalk.red(step.title)} ${chalk.gray(`${step.duration}ms`)}`);
+  } else {
+    lines.push(`${prefix}${step.title} ${chalk.gray(`${step.duration}ms`)}`);
+  }
+
+  for (const child of step.steps || []) {
+    lines.push(...formatStep(child, shift + 2));
+  } 
+  
+  return lines;
+}
+
 /**
  *
  * @param {TestData} testData

package/lib/pipe/gitlab.js

@@ -79,13 +79,13 @@ class GitLabPipe {
     let summary = `${this.hiddenCommentData}
     
   | [![Testomat.io Report](${testomatLogoURL})](https://testomat.io)  | ${statusEmoji(
-    runParams.status,
-  )} ${runParams.status.toUpperCase()} ${statusEmoji(runParams.status)} |
+      runParams.status,
+    )} ${runParams.status.toUpperCase()} ${statusEmoji(runParams.status)} |
   | --- | --- |
   | Tests | ✔️  **${this.tests.length}** tests run  |
   | Summary | ${statusEmoji('failed')} **${failedCount}** failed; ${statusEmoji(
-    'passed',
-  )} **${passedCount}** passed; **${statusEmoji('skipped')}** ${skippedCount} skipped |
+      'passed',
+    )} **${passedCount}** passed; **${statusEmoji('skipped')}** ${skippedCount} skipped |
   | Duration | 🕐  **${humanizeDuration(
     parseInt(
       this.tests.reduce((a, t) => a + (t.run_time || 0), 0),

package/lib/pipe/testomatio.js

@@ -42,6 +42,10 @@ class TestomatioPipe {
       return;
     }
     debug('Testomatio Pipe: Enabled');
+
+    const proxyUrl = process.env.HTTP_PROXY || process.env.HTTPS_PROXY;
+    const proxy = proxyUrl ? new URL(proxyUrl) : null;
+
     this.parallel = params.parallel;
     this.store = store || {};
     this.title = params.title || process.env.TESTOMATIO_TITLE;
@@ -53,6 +57,11 @@ class TestomatioPipe {
     this.axios = axios.create({
       baseURL: `${this.url.trim()}`,
       timeout: AXIOS_TIMEOUT,
+      proxy: proxy ? {
+        host: proxy.hostname,
+        port: proxy.port,
+        protocol: proxy.protocol,
+      } : false,
     });
 
     // Pass the axios instance to the retry function
@@ -339,6 +348,9 @@ class TestomatioPipe {
   addTest(data) {
     if (!this.isEnabled) return;
     if (!this.runId) return;
+
+    // add test ID + run ID
+    if (data.rid) data.rid = `${this.runId}-${data.rid}`;
     data.api_key = this.apiKey;
     data.create = this.createNewTests;
 

package/lib/xmlReader.js

@@ -2,6 +2,7 @@ const debug = require('debug')('@testomatio/reporter:xml');
 const path = require('path');
 const chalk = require('chalk');
 const fs = require('fs');
+const { randomUUID } = require('crypto');
 const { XMLParser } = require('fast-xml-parser');
 const { APP_PREFIX, STATUS } = require('./constants');
 const {
@@ -17,6 +18,8 @@ const pipesFactory = require('./pipe');
 const adapterFactory = require('./junit-adapter');
 const config = require('./config');
 
+const ridRunId = randomUUID();
+
 const TESTOMATIO_URL = process.env.TESTOMATIO_URL || 'https://app.testomat.io';
 const { TESTOMATIO_RUNGROUP_TITLE, TESTOMATIO_TITLE, TESTOMATIO_ENV, TESTOMATIO_RUN } = process.env;
 
@@ -457,16 +460,20 @@ function reduceTestCases(prev, item) {
       if (testCaseItem.error && testCaseItem.error['#text']) stack = testCaseItem.error['#text'];
       if (!message) message = stack.trim().split('\n')[0];
 
+      const isParametrized = item.type === 'ParameterizedMethod';
+      const preferClassname = reduceOptions.preferClassname || isParametrized;
+
       // SpecFlow config
-      let { title, tags } = fetchProperties(item.type === 'ParameterizedMethod' ? item : testCaseItem);
+      let { title, tags } = fetchProperties(isParametrized ? item : testCaseItem);
       let example = null;
+      const suiteTitle = preferClassname ? testCaseItem.classname : item.name || testCaseItem.classname;
 
       title ||= testCaseItem.name || testCaseItem.methodname || testCaseItem.classname;
       tags ||= [];
 
-      const exampleMatches = title.match(/\((.*?)\)/);
+      const exampleMatches = testCaseItem.name?.match(/\S\((.*?)\)/);
       if (exampleMatches) {
-        example = { ...exampleMatches[1].split(',').map(v => v.replace(/^['"]|['"]$/g, '')) };
+        example = { ...exampleMatches[1].split(',').map(v => v.trim().replace(/[^\w\s-]/g, '')) };
         title = title.replace(/\(.*?\)/, '').trim();
       }
 
@@ -480,12 +487,16 @@ function reduceTestCases(prev, item) {
       if ('failure' in testCaseItem || 'error' in testCaseItem) status = STATUS.FAILED;
       if ('skipped' in testCaseItem) status = STATUS.SKIPPED;
 
+      let rid = null;
+      if (testCaseItem.id) rid = `${ridRunId}-${testCaseItem.id}`;
+
       prev.push({
-        create: true,
+        rid,
         file,
         stack,
         example,
         tags,
+        create: true,
         test_id: testId,
         message,
         line: testCaseItem.lineno,
@@ -493,7 +504,7 @@ function reduceTestCases(prev, item) {
         run_time: parseFloat(testCaseItem.time || testCaseItem.duration) * 1000,
         status,
         title,
-        suite_title: reduceOptions.preferClassname ? testCaseItem.classname : item.name || testCaseItem.classname,
+        suite_title: suiteTitle,
       });
     });
   return prev;
@@ -509,9 +520,9 @@ function processTestSuite(testsuite) {
     suites = [testsuite];
   }
 
-  const res = suites.flat().reduce(reduceTestCases, []);
+  const subSuites = suites.filter(s => s['test-suite'] && !testsuite['test-case']);
 
-  return res;
+  return [...subSuites.map(s => processTestSuite(s['test-suite'])), ...suites.reduce(reduceTestCases, [])].flat();
 }
 
 function fetchProperties(item) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@testomatio/reporter",
-  "version": "1.5.0-beta-vitest",
+  "version": "1.5.1-beta",
   "description": "Testomatio Reporter Client",
   "main": "./lib/reporter.js",
   "typings": "typings/index.d.ts",
@@ -20,7 +20,7 @@
     "csv-writer": "^1.6.0",
     "debug": "^4.3.4",
     "dotenv": "^16.0.1",
-    "fast-xml-parser": "^4.0.8",
+    "fast-xml-parser": "^4.4.1",
     "file-url": "3.0.0",
     "glob": "^10.3",
     "handlebars": "^4.7.8",
@@ -63,7 +63,7 @@
     "@redocly/cli": "^1.0.0-beta.125",
     "@wdio/reporter": "^7.16.13",
     "chai": "^4.3.6",
-    "codeceptjs": "latest",
+    "codeceptjs": "^3.5.11",
     "cucumber": "^6.0.7",
     "eslint": "^8.7.0",
     "eslint-config-airbnb-base": "^15.0.0",
@@ -76,7 +76,7 @@
     "mock-http-server": "^1.4.5",
     "pino": "^8.15.0",
     "prettier": "^3.2.5",
-    "puppeteer": "^13.1.2"
+    "puppeteer": "^22.15.0"
   },
   "bin": {
     "report-xml": "./lib/bin/reportXml.js",