@testomatio/reporter 2.0.0-beta-esm → 2.0.0-beta.1-xml

package/src/adapter/nightwatch.js

@@ -0,0 +1,88 @@
+import TestomatClient from '../client.js';
+import { config } from '../config.js';
+import { STATUS } from '../constants.js';
+import { getTestomatIdFromTestTitle } from '../utils/utils.js';
+
+const apiKey = config.TESTOMATIO;
+const client = new TestomatClient({ apiKey });
+
+export default {
+  write: async (results, options, done) => {
+    await client.createRun();
+
+    const testFiles = results.modules;
+
+    for (const fileName in testFiles) {
+      // in nightwatch: object containing tests from a single file
+      const testModule = testFiles[fileName];
+
+      // passed and failed tests (tests with assertions)
+      const completedTests = testModule.completed;
+
+      // skipped tests (skipped by user or tests without assertions)
+      const skippedTests = testModule.skipped;
+
+      const tags = testModule.tags || [];
+
+      // if test file contains multiple suites, the last suite name is used as a name 🤷‍♂️
+      // no other places which contain suite name (even inside test object)
+      const suiteTitle = testModule.name;
+
+      for (const testTitle in completedTests) {
+        const test = completedTests[testTitle];
+        let status;
+        switch (test.status) {
+          case 'pass':
+            status = STATUS.PASSED;
+            break;
+          case 'fail':
+            status = STATUS.FAILED;
+            break;
+          // probably not required (because skipped tests are in separate array), but just in case
+          case 'skip':
+            status = STATUS.SKIPPED;
+            console.info('Skipped test is in completed tests array:', test, 'Not expected behavior.');
+            break;
+          default:
+            console.error('Test status processing error:', test.status);
+        }
+
+        const testId = getTestomatIdFromTestTitle(testTitle);
+
+        client.addTestRun(status, {
+          error: { name: test.assertions?.[0]?.name, message: test.assertions?.[0]?.message, stack: test.stackTrace },
+          file: testModule.modulePath?.replace(process.cwd(), ''),
+          message: test.assertions?.[0]?.message,
+          rid: `${testModule.uuid || ''}_${testTitle || ''}`,
+          stack: test.stackTrace,
+          suite_title: suiteTitle,
+          tags,
+          test_id: testId,
+          time: test.timeMs,
+          title: testTitle,
+        });
+      }
+
+      // just array with skipped tests titles, no any other info
+      for (const testTitle of skippedTests) {
+        client.addTestRun(STATUS.SKIPPED, {
+          suite_title: suiteTitle,
+          tags,
+          rid: `${testModule.uuid || ''}_${testTitle || ''}`,
+          title: testTitle,
+        });
+      }
+    }
+
+    /**
+     * @type {'passed' | 'failed' | 'finished'}
+     */
+    let runStatus = 'finished';
+    if (results.failed) runStatus = 'failed';
+    else if (results.passed) runStatus = 'passed';
+
+    await client.updateRunStatus(runStatus);
+
+    done();
+  },
+};

package/src/adapter/playwright.js

@@ -6,7 +6,6 @@ import { v4 as uuidv4 } from 'uuid';
 import fs from 'fs';
 import { APP_PREFIX, STATUS as Status, TESTOMAT_TMP_STORAGE_DIR } from '../constants.js';
 import TestomatioClient from '../client.js';
-import { upload } from '../fileUploader.js';
 import { getTestomatIdFromTestTitle, fileSystem } from '../utils/utils.js';
 import { services } from '../services/index.js';
 import { dataStorage } from '../data-storage.js';
@@ -57,11 +56,37 @@ class PlaywrightReporter {
       logs = `\n\n${pc.bold('Logs:')}\n${pc.red(result.stderr.join(''))}\n${result.stdout.join('')}`;
     }
     const manuallyAttachedArtifacts = services.artifacts.get(fullTestTitle);
-    const keyValues = services.keyValues.get(fullTestTitle);
+    const testMeta = services.keyValues.get(fullTestTitle);
     const rid = test.id || test.testId || uuidv4();
 
-    const reportTestPromise = this.client.addTestRun(checkStatus(result.status), {
-      rid,
+    /**
+     * @type {{
+     * browser?: string,
+     * dependencies: string[],
+     * isMobile?: boolean
+     * metadata: Record<string, any>,
+     * name: string,
+     * }}
+     */
+    const project = {
+      browser: test.parent.project().use.defaultBrowserType,
+      dependencies: test.parent.project().dependencies,
+      isMobile: test.parent.project().use.isMobile,
+      metadata: test.parent.project().metadata,
+      name: test.parent.project().name,
+    };
+
+    let status = result.status;
+    // process test.fail() annotation
+    if (test.expectedStatus === 'failed') {
+      // actual status = expected
+      if (result.status === 'failed') status = 'passed';
+      // actual status != expected
+      if (result.status === 'passed') status = 'failed';
+    }
+
+    const reportTestPromise = this.client.addTestRun(checkStatus(status), {
+      rid: `${rid}-${project.name}`,
       error,
       test_id: getTestomatIdFromTestTitle(`${title} ${test.tags?.join(' ')}`),
       suite_title,
@@ -70,12 +95,19 @@ class PlaywrightReporter {
       time: duration,
       logs,
       manuallyAttachedArtifacts,
-      meta: keyValues,
+      meta: {
+        browser: project.browser,
+        isMobile: project.isMobile,
+        project: project.name,
+        projectDependencies: project.dependencies?.length ? project.dependencies : null,
+        ...testMeta,
+        ...project.metadata, // metadata has any type (in playwright), but we will stringify it in client.js
+      },
       file: test.location?.file,
     });
 
     this.uploads.push({
-      rid,
+      rid: `${rid}-${project.name}`,
       title: test.title,
       files: result.attachments.filter(a => a.body || a.path),
       file: test.location?.file,
@@ -94,9 +126,13 @@ class PlaywrightReporter {
     }
 
     if (artifact.body) {
-      const fileName = tmpFile();
-      fs.writeFileSync(fileName, artifact.body);
-      return fileName;
+      let filePath = generateTmpFilepath(artifact.name);
+
+      const extension = artifact.contentType?.split('/')[1]?.replace('jpeg', 'jpg');
+      if (extension) filePath += `.${extension}`;
+
+      fs.writeFileSync(filePath, artifact.body);
+      return filePath;
     }
 
     return null;
@@ -107,20 +143,26 @@ class PlaywrightReporter {
 
     await Promise.all(reportTestPromises);
 
-    if (this.uploads.length && upload.isArtifactsEnabled()) {
-      console.log(APP_PREFIX, `🎞️  Uploading ${this.uploads.length} files...`);
+    if (this.uploads.length) {
+      if (this.client.uploader.isEnabled) console.log(APP_PREFIX, `🎞️  Uploading ${this.uploads.length} files...`);
 
       const promises = [];
 
-      for (const anUpload of this.uploads) {
-        const { rid, file, title } = anUpload;
+      // ? possible move to addTestRun (needs investigation if files are ready)
+      for (const upload of this.uploads) {
+        const { rid, file, title } = upload;
 
-        const files = anUpload.files.map(attachment => ({
+        const files = upload.files.map(attachment => ({
           path: this.#getArtifactPath(attachment),
           title,
           type: attachment.contentType,
         }));
 
+        if (!this.client.uploader.isEnabled) {
+          files.forEach(f => this.client.uploader.storeUploadedFile(f, this.client.runId, rid, false));
+          continue;
+        }
+
         promises.push(
           this.client.addTestRun(undefined, {
             rid,
@@ -190,9 +232,10 @@ function appendStep(step, shift = 0) {
   return resultStep;
 }
 
-function tmpFile(prefix = 'tmp.') {
+function generateTmpFilepath(filename = '') {
+  filename = filename || `tmp.${crypto.randomBytes(16).toString('hex')}`;
   const tmpdir = os.tmpdir();
-  return path.join(tmpdir, prefix + crypto.randomBytes(16).toString('hex'));
+  return path.join(tmpdir, filename);
 }
 
 /**
@@ -204,19 +247,4 @@ function getTestContextName(test) {
   return `${test._requireFile || ''}_${test.title}`;
 }
 
-function initPlaywrightForStorage() {
-  try {
-    // @ts-ignore-next-line
-    // eslint-disable-next-line import/no-extraneous-dependencies
-    const { test } = require('@playwright/test');
-    // eslint-disable-next-line no-empty-pattern
-    test.beforeEach(async ({}, testInfo) => {
-      global.testomatioTestTitle = `${testInfo.file || ''}_${testInfo.title}`;
-    });
-  } catch (e) {
-    // ignore
-  }
-}
-
 export default PlaywrightReporter;
-export { initPlaywrightForStorage };

package/src/adapter/vitest.js

@@ -7,13 +7,13 @@ import createDebugMessages from 'debug';
 const debug = createDebugMessages('@testomatio/reporter:adapter-jest');
 
 /**
- * @typedef {import('../../types').VitestTest} VitestTest
- * @typedef {import('../../types').VitestTestFile} VitestTestFile
- * @typedef {import('../../types').VitestSuite} VitestSuite
- * @typedef {import('../../types').VitestTestLogs} VitestTestLogs
- * @typedef {import('../../vitest.types').ErrorWithDiff} ErrorWithDiff
+ * @typedef {import('../../types/types.js').VitestTest} VitestTest
+ * @typedef {import('../../types/types.js').VitestTestFile} VitestTestFile
+ * @typedef {import('../../types/types.js').VitestSuite} VitestSuite
+ * @typedef {import('../../types/types.js').VitestTestLogs} VitestTestLogs
+ * @typedef {import('../../types/vitest.types.js').ErrorWithDiff} ErrorWithDiff
  * @typedef {typeof import('../constants.js').STATUS} STATUS
- * @typedef {import('../../types').TestData} TestData
+ * @typedef {import('../../types/types.js').TestData} TestData
  */
 
 class VitestReporter {

package/src/adapter/webdriver.js

@@ -1,8 +1,8 @@
-// eslint-disable-next-line
-import WDIOReporter, { RunnerStats } from '@wdio/reporter';
-
+import { default as WDIOReporter, RunnerStats } from '@wdio/reporter';
 import TestomatClient from '../client.js';
-import { getTestomatIdFromTestTitle } from '../utils/utils.js';
+import { getTestomatIdFromTestTitle, fileSystem } from '../utils/utils.js';
+import { services } from '../services/index.js';
+import { TESTOMAT_TMP_STORAGE_DIR } from '../constants.js';
 
 class WebdriverReporter extends WDIOReporter {
   constructor(options) {
@@ -23,8 +23,8 @@ class WebdriverReporter extends WDIOReporter {
   }
 
   /**
-   * 
-   * @param {RunnerStats} runData 
+   *
+   * @param {RunnerStats} runData
    */
   async onRunnerEnd(runData) {
     this._isSynchronising = true;
@@ -33,20 +33,35 @@ class WebdriverReporter extends WDIOReporter {
 
     this._isSynchronising = false;
 
-    // NOTE: new functionality; may break everything 
+    // NOTE: new functionality; may break everything
     // also this may require additional status mapping
     await this.client.updateRunStatus(runData.failures ? 'failed' : 'passed');
   }
 
+  onRunnerStart() {
+    // clear dir with artifacts/logs
+    //
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
+  }
+
+  onTestStart(test) {
+    services.setContext(test.fullTitle);
+  }
+
   onTestEnd(test) {
+    test.suite = test.parent;
+    const logs = getTestLogs(test.fullTitle);
+    // TODO: FIX: artifacts for some reason leads to empty report on Testomat.io
+    // const artifacts = services.artifacts.get(test.fullTitle);
+    // const keyValues = services.keyValues.get(test.fullTitle);
+    test.logs = logs;
+    // test.artifacts = artifacts;
+    // test.meta = keyValues;
+
     this._addTestPromises.push(this.addTest(test));
   }
 
   // wdio-cucumber does not trigger onTestEnd hook, thus, using this one
-  /**
-   *
-   * @returns
-   */
   onSuiteEnd(scerario) {
     if (scerario.type === 'scenario') {
       this._addTestPromises.push(this.addBddScenario(scerario));
@@ -66,7 +81,10 @@ class WebdriverReporter extends WDIOReporter {
       .map(el => Buffer.from(el.result.value, 'base64'));
 
     await this.client.addTestRun(state, {
+      manuallyAttachedArtifacts: test.artifacts,
       error,
+      logs: test.logs,
+      meta: test.meta,
       title,
       test_id: testId,
       time: duration,
@@ -75,7 +93,7 @@ class WebdriverReporter extends WDIOReporter {
   }
 
   /**
-   * @param {import('../../types').WebdriverIOScenario} scenario
+   * @param {import('../../types/types.js').WebdriverIOScenario} scenario
    */
   addBddScenario(scenario) {
     if (!this.client) return;
@@ -108,4 +126,16 @@ class WebdriverReporter extends WDIOReporter {
   }
 }
 
+/**
+ *
+ * @param {*} fullTestTitle
+ * @returns string
+ */
+function getTestLogs(fullTestTitle) {
+  const logsArr = services.logger.getLogs(fullTestTitle);
+  // remove duplicates (for some reason, logs are duplicated several times)
+  const logs = logsArr ? Array.from(new Set(logsArr)).join('\n').trim() : '';
+  return logs;
+}
+
 export default WebdriverReporter;

package/src/bin/cli.js

@@ -0,0 +1,303 @@
+#!/usr/bin/env node
+
+import { Command } from 'commander';
+import { spawn } from 'cross-spawn';
+import { glob } from 'glob';
+import createDebugMessages from 'debug';
+import TestomatClient from '../client.js';
+import XmlReader from '../xmlReader.js';
+import { APP_PREFIX, STATUS } from '../constants.js';
+import { getPackageVersion } from '../utils/utils.js';
+import { config } from '../config.js';
+import { readLatestRunId } from '../utils/utils.js';
+import pc from 'picocolors';
+import { filesize as prettyBytes } from 'filesize';
+import dotenv from 'dotenv';
+
+const debug = createDebugMessages('@testomatio/reporter:xml-cli');
+const version = getPackageVersion();
+console.log(pc.cyan(pc.bold(` 🤩 Testomat.io Reporter v${version}`)));
+const program = new Command();
+
+program
+  .version(version)
+  .option('--env-file <envfile>', 'Load environment variables from env file')
+  .hook('preAction', thisCommand => {
+    const opts = thisCommand.opts();
+    if (opts.envFile) {
+      dotenv.config({ path: opts.envFile });
+    } else {
+      dotenv.config();
+    }
+  });
+
+program
+  .command('start')
+  .description('Start a new run and return its ID')
+  .action(async () => {
+    console.log('Starting a new Run on Testomat.io...');
+    const apiKey = process.env['INPUT_TESTOMATIO-KEY'] || config.TESTOMATIO;
+    const client = new TestomatClient({ apiKey });
+
+    client.createRun().then(() => {
+      console.log(process.env.runId);
+      process.exit(0);
+    });
+  });
+
+program
+  .command('finish')
+  .description('Finish Run by its ID')
+  .action(async () => {
+    process.env.TESTOMATIO_RUN ||= readLatestRunId();
+
+    if (!process.env.TESTOMATIO_RUN) {
+      console.log('TESTOMATIO_RUN environment variable must be set or restored from a previous run.');
+      return process.exit(1);
+    }
+
+    console.log('Finishing Run on Testomat.io...');
+    const apiKey = process.env['INPUT_TESTOMATIO-KEY'] || config.TESTOMATIO;
+    const client = new TestomatClient({ apiKey });
+
+    // @ts-ignore
+    client.updateRunStatus(STATUS.FINISHED).then(() => {
+      console.log(pc.yellow(`Run ${process.env.TESTOMATIO_RUN} was finished`));
+      process.exit(0);
+    });
+  });
+
+program
+  .command('run')
+  .description('Run tests with the specified command')
+  .argument('<command>', 'Test runner command')
+  .option('--filter <filter>', 'Additional execution filter')
+  .action(async (command, opts) => {
+    const apiKey = process.env['INPUT_TESTOMATIO-KEY'] || config.TESTOMATIO;
+    const title = process.env.TESTOMATIO_TITLE;
+
+    if (!command || !command.split) {
+      console.log(APP_PREFIX, `No command provided. Use -c option to launch a test runner.`);
+      return process.exit(255);
+    }
+
+    const client = new TestomatClient({ apiKey, title, parallel: true });
+
+    if (opts.filter) {
+      const [pipe, ...optsArray] = opts.filter.split(':');
+      const pipeOptions = optsArray.join(':');
+
+      try {
+        const tests = await client.prepareRun({ pipe, pipeOptions });
+        if (tests && tests.length > 0) {
+          command += ` --grep (${tests.join('|')})`;
+        }
+      } catch (err) {
+        console.log(APP_PREFIX, err);
+      }
+    }
+
+    console.log(APP_PREFIX, `🚀 Running`, pc.green(command));
+
+    const runTests = () => {
+      const testCmds = command.split(' ');
+      const cmd = spawn(testCmds[0], testCmds.slice(1), { stdio: 'inherit' });
+
+      cmd.on('close', code => {
+        const emoji = code === 0 ? '🟢' : '🔴';
+        console.log(APP_PREFIX, emoji, `Runner exited with ${pc.bold(code)}`);
+        if (apiKey) {
+          const status = code === 0 ? 'passed' : 'failed';
+          client.updateRunStatus(status, true);
+        }
+        process.exit(code);
+      });
+    };
+
+    if (apiKey) {
+      client.createRun().then(runTests);
+    } else {
+      runTests();
+    }
+  });
+
+// program
+// .command('xml')
+// .description('Parse XML reports and upload to Testomat.io')
+// .argument('<pattern>', 'XML file pattern')
+// .option('-d, --dir <dir>', 'Project directory')
+// .option('--java-tests [java-path]', 'Load Java tests from path, by default: src/test/java')
+// .option('--lang <lang>', 'Language used (python, ruby, java)')
+// .option('--timelimit <time>', 'default time limit in seconds to kill a stuck process')
+// .action(async (pattern, opts) => {
+//   if (!pattern.endsWith('.xml')) {
+//     pattern += '.xml';
+//   }
+//   let { javaTests, lang } = opts;
+//   if (javaTests === true) javaTests = 'src/test/java';
+//   lang = lang?.toLowerCase();
+//   const runReader = new XmlReader({ javaTests, lang });
+//   const files = glob.sync(pattern, { cwd: opts.dir || process.cwd() });
+//   if (!files.length) {
+//     console.log(APP_PREFIX, `Report can't be created. No XML files found 😥`);
+//     process.exit(1);
+//   }
+
+program
+  .command('xml')
+  .description('Parse XML reports and upload to Testomat.io')
+  .argument('<pattern>', 'XML file pattern')
+  .option('-d, --dir <dir>', 'Project directory')
+  .option('--java-tests [java-path]', 'Load Java tests from path, by default: src/test/java')
+  .option('--lang <lang>', 'Language used (python, ruby, java)')
+  .option('--timelimit <time>', 'default time limit in seconds to kill a stuck process')
+  .action(async (pattern, opts) => {
+    if (!pattern.endsWith('.xml')) {
+      pattern += '.xml';
+    }
+    let { javaTests, lang } = opts;
+    if (javaTests === true) javaTests = 'src/test/java';
+    lang = lang?.toLowerCase();
+    const runReader = new XmlReader({ javaTests, lang });
+    const files = glob.sync(pattern, { cwd: opts.dir || process.cwd() });
+    if (!files.length) {
+      console.log(APP_PREFIX, `Report can't be created. No XML files found 😥`);
+      process.exit(1);
+    }
+
+    for (const file of files) {
+      console.log(APP_PREFIX, `Parsed ${file}`);
+      runReader.parse(file);
+    }
+
+    let timeoutTimer;
+    if (opts.timelimit) {
+      timeoutTimer = setTimeout(
+        () => {
+          console.log(
+            `⚠️  Reached timeout of ${opts.timelimit}s. Exiting... (Exit code is 0 to not fail the pipeline)`,
+          );
+          process.exit(0);
+        },
+        parseInt(opts.timelimit, 10) * 1000,
+      );
+    }
+
+    try {
+      await runReader.createRun();
+      await runReader.uploadData();
+    } catch (err) {
+      console.log(APP_PREFIX, 'Error updating status, skipping...', err);
+    }
+
+    if (timeoutTimer) clearTimeout(timeoutTimer);
+  });
+
+program
+  .command('upload-artifacts')
+  .description('Upload artifacts to Testomat.io')
+  .option('--force', 'Re-upload artifacts even if they were uploaded before')
+  .action(async opts => {
+    const apiKey = config.TESTOMATIO;
+
+    process.env.TESTOMATIO_DISABLE_ARTIFACTS = '';
+    const runId = process.env.TESTOMATIO_RUN || process.env.runId || readLatestRunId();
+
+    if (!runId) {
+      console.log('TESTOMATIO_RUN environment variable must be set or restored from a previous run.');
+      return process.exit(1);
+    }
+
+    const client = new TestomatClient({
+      apiKey,
+      runId,
+      isBatchEnabled: false,
+    });
+
+    let testruns = client.uploader.readUploadedFiles(runId);
+    const numTotalArtifacts = testruns.length;
+
+    debug('Found testruns:', testruns);
+
+    if (!opts.force) testruns = testruns.filter(tr => !tr.uploaded);
+
+    if (!testruns.length) {
+      console.log(APP_PREFIX, '🗄️ Total artifacts:', numTotalArtifacts);
+      if (numTotalArtifacts) {
+        console.log(APP_PREFIX, 'No new artifacts to upload');
+        console.log(APP_PREFIX, 'To re-upload artifacts run this command with --force flag');
+      }
+      process.exit(0);
+    }
+
+    const testrunsByRid = testruns.reduce((acc, { rid, file }) => {
+      if (!acc[rid]) {
+        acc[rid] = [];
+      }
+      if (!acc[rid].includes(file)) acc[rid].push(file);
+      return acc;
+    }, {});
+
+    await client.createRun();
+    client.uploader.checkEnabled();
+    client.uploader.disableLogStorage();
+
+    for (const rid in testrunsByRid) {
+      const files = testrunsByRid[rid];
+      await client.addTestRun(undefined, { rid, files });
+    }
+
+    console.log(APP_PREFIX, '🗄️', client.uploader.successfulUploads.length, 'artifacts 🟢uploaded');
+
+    if (client.uploader.successfulUploads.length) {
+      debug('\n', APP_PREFIX, `🗄️ ${client.uploader.successfulUploads.length} artifacts uploaded to S3 bucket`);
+      const uploadedArtifacts = client.uploader.successfulUploads.map(file => ({
+        relativePath: file.path.replace(process.cwd(), ''),
+        link: file.link,
+        sizePretty: prettyBytes(file.size, { round: 0 }).toString(),
+      }));
+
+      uploadedArtifacts.forEach(upload => {
+        debug(
+          `🟢Uploaded artifact`,
+          `${upload.relativePath},`,
+          'size:',
+          `${upload.sizePretty},`,
+          'link:',
+          `${upload.link}`,
+        );
+      });
+    }
+
+    const filesizeStrMaxLength = 7;
+
+    if (client.uploader.failedUploads.length) {
+      console.log(
+        '\n',
+        APP_PREFIX,
+        '🗄️',
+        client.uploader.failedUploads.length,
+        `artifacts 🔴${pc.bold('failed')} to upload`,
+      );
+
+      const failedUploads = client.uploader.failedUploads.map(({ path, size }) => ({
+        relativePath: path.replace(process.cwd(), ''),
+        sizePretty: prettyBytes(size, { round: 0 }).toString(),
+      }));
+
+      const pathPadding = Math.max(...failedUploads.map(upload => upload.relativePath.length)) + 1;
+      failedUploads.forEach(upload => {
+        console.log(
+          `  ${pc.gray('|')} 🔴 ${upload.relativePath.padEnd(pathPadding)} ${pc.gray(
+            `| ${upload.sizePretty.padStart(filesizeStrMaxLength)} |`,
+          )}`,
+        );
+      });
+    }
+  });
+
+program.parse(process.argv);
+
+if (!process.argv.slice(2).length) {
+  program.outputHelp();
+}