@testomatio/reporter 2.0.1-beta.5-timestamp → 2.0.1-beta.6

package/src/adapter/mocha.js

@@ -0,0 +1,156 @@
+import Mocha from 'mocha';
+import TestomatClient from '../client.js';
+import { STATUS, TESTOMAT_TMP_STORAGE_DIR } from '../constants.js';
+import { getTestomatIdFromTestTitle, fileSystem } from '../utils/utils.js';
+import { config } from '../config.js';
+import { services } from '../services/index.js';
+import pc from 'picocolors';
+
+const {
+  EVENT_RUN_BEGIN,
+  EVENT_RUN_END,
+  EVENT_TEST_FAIL,
+  EVENT_TEST_PASS,
+  EVENT_TEST_PENDING,
+  EVENT_SUITE_BEGIN,
+  EVENT_SUITE_END,
+  EVENT_TEST_BEGIN,
+  EVENT_TEST_END,
+} = Mocha.Runner.constants;
+
+function MochaReporter(runner, opts) {
+  Mocha.reporters.Base.call(this, runner);
+  let passes = 0;
+  let failures = 0;
+  let skipped = 0;
+  // let artifactStore;
+
+  const apiKey = opts?.reporterOptions?.apiKey || config.TESTOMATIO;
+
+  const client = new TestomatClient({ apiKey });
+
+  runner.on(EVENT_RUN_BEGIN, () => {
+    client.createRun();
+
+    // clear dir with artifacts/logs
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
+  });
+
+  runner.on(EVENT_SUITE_BEGIN, async suite => {
+    services.setContext(suite.fullTitle());
+  });
+
+  runner.on(EVENT_SUITE_END, async () => {
+    services.setContext(null);
+  });
+
+  runner.on(EVENT_TEST_BEGIN, async test => {
+    services.setContext(test.fullTitle());
+  });
+
+  runner.on(EVENT_TEST_END, async () => {
+    services.setContext(null);
+  });
+
+  runner.on(EVENT_TEST_PASS, async test => {
+    passes += 1;
+
+    console.log(pc.bold(pc.green('✔')), test.fullTitle());
+    const testId = getTestomatIdFromTestTitle(test.title);
+
+    const logs = getTestLogs(test);
+    const artifacts = services.artifacts.get(test.fullTitle());
+    const keyValues = services.keyValues.get(test.fullTitle());
+
+    client.addTestRun(STATUS.PASSED, {
+      test_id: testId,
+      suite_title: getSuiteTitle(test),
+      title: getTestName(test),
+      code: process.env.TESTOMATIO_UPDATE_CODE ? test.body.toString() : '',
+      file: getFile(test),
+      time: test.duration,
+      logs,
+      manuallyAttachedArtifacts: artifacts,
+      meta: keyValues,
+    });
+  });
+
+  runner.on(EVENT_TEST_PENDING, test => {
+    skipped += 1;
+    console.log('skip: %s', test.fullTitle());
+    const testId = getTestomatIdFromTestTitle(test.title);
+    client.addTestRun(STATUS.SKIPPED, {
+      title: getTestName(test),
+      suite_title: getSuiteTitle(test),
+      code: process.env.TESTOMATIO_UPDATE_CODE ? test.body.toString() : '',
+      file: getFile(test),
+      test_id: testId,
+      time: test.duration,
+    });
+  });
+
+  runner.on(EVENT_TEST_FAIL, async (test, err) => {
+    failures += 1;
+    console.log(pc.bold(pc.red('✖')), test.fullTitle(), pc.gray(err.message));
+    const testId = getTestomatIdFromTestTitle(test.title);
+
+    const logs = getTestLogs(test);
+
+    client.addTestRun(STATUS.FAILED, {
+      error: err,
+      suite_title: getSuiteTitle(test),
+      file: getFile(test),
+      test_id: testId,
+      title: getTestName(test),
+      code: process.env.TESTOMATIO_UPDATE_CODE ? test.body.toString() : '',
+      time: test.duration,
+      logs,
+    });
+  });
+
+  runner.on(EVENT_RUN_END, () => {
+    const status = failures === 0 ? STATUS.PASSED : STATUS.FAILED;
+    console.log(pc.bold(status), `${passes} passed, ${failures} failed, ${skipped} skipped`);
+    // @ts-ignore
+    client.updateRunStatus(status);
+  });
+}
+
+function getTestLogs(test) {
+  const suiteLogsArr = services.logger.getLogs(test.parent.fullTitle());
+  const suiteLogs = suiteLogsArr ? suiteLogsArr.join('\n').trim() : '';
+  const testLogsArr = services.logger.getLogs(test.fullTitle());
+  const testLogs = testLogsArr ? testLogsArr.join('\n').trim() : '';
+
+  let logs = '';
+  if (suiteLogs) {
+    logs += `${pc.bold('\t--- BeforeSuite ---')}\n${suiteLogs}`;
+  }
+  if (testLogs) {
+    logs += `\n${pc.bold('\t--- Test ---')}\n${testLogs}`;
+  }
+  return logs;
+}
+
+function getSuiteTitle(test, pathArr = []) {
+  if (test.parent.parent) getSuiteTitle(test.parent, pathArr);
+
+  pathArr.push(test.parent.title);
+
+  return pathArr.filter(t => !!t)[0];
+}
+
+function getFile(test) {
+  return test.parent.file?.replace(process.cwd(), '');
+}
+
+function getTestName(test) {
+  if (process.env.TESTOMATIO_CREATE === 'fulltitle') return test.fullTitle();
+  return test.title;
+}
+
+// To have this reporter "extend" a built-in reporter uncomment the following line:
+// @ts-ignore
+Mocha.utils.inherits(MochaReporter, Mocha.reporters.Spec);
+
+export default MochaReporter;

package/src/adapter/nightwatch.js

@@ -0,0 +1,88 @@
+import TestomatClient from '../client.js';
+import { config } from '../config.js';
+import { STATUS } from '../constants.js';
+import { getTestomatIdFromTestTitle } from '../utils/utils.js';
+
+const apiKey = config.TESTOMATIO;
+const client = new TestomatClient({ apiKey });
+
+export default {
+  write: async (results, options, done) => {
+    await client.createRun();
+
+    const testFiles = results.modules;
+
+    for (const fileName in testFiles) {
+      // in nightwatch: object containing tests from a single file
+      const testModule = testFiles[fileName];
+
+      // passed and failed tests (tests with assertions)
+      const completedTests = testModule.completed;
+
+      // skipped tests (skipped by user or tests without assertions)
+      const skippedTests = testModule.skipped;
+
+      const tags = testModule.tags || [];
+
+      // if test file contains multiple suites, the last suite name is used as a name 🤷‍♂️
+      // no other places which contain suite name (even inside test object)
+      const suiteTitle = testModule.name;
+
+      for (const testTitle in completedTests) {
+        const test = completedTests[testTitle];
+        let status;
+        switch (test.status) {
+          case 'pass':
+            status = STATUS.PASSED;
+            break;
+          case 'fail':
+            status = STATUS.FAILED;
+            break;
+          // probably not required (because skipped tests are in separate array), but just in case
+          case 'skip':
+            status = STATUS.SKIPPED;
+            console.info('Skipped test is in completed tests array:', test, 'Not expected behavior.');
+            break;
+          default:
+            console.error('Test status processing error:', test.status);
+        }
+
+        const testId = getTestomatIdFromTestTitle(testTitle);
+
+        client.addTestRun(status, {
+          error: { name: test.assertions?.[0]?.name, message: test.assertions?.[0]?.message, stack: test.stackTrace },
+          file: testModule.modulePath?.replace(process.cwd(), ''),
+          message: test.assertions?.[0]?.message,
+          rid: `${testModule.uuid || ''}_${testTitle || ''}`,
+          stack: test.stackTrace,
+          suite_title: suiteTitle,
+          tags,
+          test_id: testId,
+          time: test.timeMs,
+          title: testTitle,
+        });
+      }
+
+      // just array with skipped tests titles, no any other info
+      for (const testTitle of skippedTests) {
+        client.addTestRun(STATUS.SKIPPED, {
+          suite_title: suiteTitle,
+          tags,
+          rid: `${testModule.uuid || ''}_${testTitle || ''}`,
+          title: testTitle,
+        });
+      }
+    }
+
+    /**
+     * @type {'passed' | 'failed' | 'finished'}
+     */
+    let runStatus = 'finished';
+    if (results.failed) runStatus = 'failed';
+    else if (results.passed) runStatus = 'passed';
+
+    await client.updateRunStatus(runStatus);
+
+    done();
+  },
+};

package/src/adapter/playwright.js

@@ -0,0 +1,254 @@
+import pc from 'picocolors';
+import crypto from 'crypto';
+import os from 'os';
+import path from 'path';
+import { v4 as uuidv4 } from 'uuid';
+import fs from 'fs';
+import { APP_PREFIX, STATUS as Status, TESTOMAT_TMP_STORAGE_DIR } from '../constants.js';
+import TestomatioClient from '../client.js';
+import { getTestomatIdFromTestTitle, fileSystem } from '../utils/utils.js';
+import { services } from '../services/index.js';
+import { dataStorage } from '../data-storage.js';
+
+const reportTestPromises = [];
+
+class PlaywrightReporter {
+  constructor(config = {}) {
+    this.client = new TestomatioClient({ apiKey: config?.apiKey });
+
+    this.uploads = [];
+  }
+
+  onBegin(config, suite) {
+    // clean data storage
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
+    if (!this.client) return;
+    this.suite = suite;
+    this.config = config;
+    this.client.createRun();
+  }
+
+  onTestBegin(testInfo) {
+    const fullTestTitle = getTestContextName(testInfo);
+    dataStorage.setContext(fullTestTitle);
+  }
+
+  onTestEnd(test, result) {
+    // test.parent.project().__projectId
+
+    if (!this.client) return;
+
+    const { title } = test;
+    const { error, duration } = result;
+    const suite_title = test.parent ? test.parent?.title : path.basename(test?.location?.file);
+
+    const steps = [];
+    for (const step of result.steps) {
+      const appendedStep = appendStep(step);
+      if (appendedStep) {
+        steps.push(appendedStep);
+      }
+    }
+
+    const fullTestTitle = getTestContextName(test);
+    let logs = '';
+    if (result.stderr.length || result.stdout.length) {
+      logs = `\n\n${pc.bold('Logs:')}\n${pc.red(result.stderr.join(''))}\n${result.stdout.join('')}`;
+    }
+    const manuallyAttachedArtifacts = services.artifacts.get(fullTestTitle);
+    const testMeta = services.keyValues.get(fullTestTitle);
+    const rid = test.id || test.testId || uuidv4();
+
+    /**
+     * @type {{
+     * browser?: string,
+     * dependencies: string[],
+     * isMobile?: boolean
+     * metadata: Record<string, any>,
+     * name: string,
+     * }}
+     */
+    const project = {
+      browser: test.parent.project().use.defaultBrowserType,
+      dependencies: test.parent.project().dependencies,
+      isMobile: test.parent.project().use.isMobile,
+      metadata: test.parent.project().metadata,
+      name: test.parent.project().name,
+    };
+
+    let status = result.status;
+    // process test.fail() annotation
+    if (test.expectedStatus === 'failed') {
+      // actual status = expected
+      if (result.status === 'failed') status = 'passed';
+      // actual status != expected
+      if (result.status === 'passed') status = 'failed';
+    }
+
+    const reportTestPromise = this.client.addTestRun(checkStatus(status), {
+      rid: `${rid}-${project.name}`,
+      error,
+      test_id: getTestomatIdFromTestTitle(`${title} ${test.tags?.join(' ')}`),
+      suite_title,
+      title,
+      steps: steps.length ? steps : undefined,
+      time: duration,
+      logs,
+      manuallyAttachedArtifacts,
+      meta: {
+        browser: project.browser,
+        isMobile: project.isMobile,
+        project: project.name,
+        projectDependencies: project.dependencies?.length ? project.dependencies : null,
+        ...testMeta,
+        ...project.metadata, // metadata has any type (in playwright), but we will stringify it in client.js
+        ...test.annotations?.reduce((acc, annotation) => {
+          acc[annotation.type] = annotation.description;
+          return acc;
+        }, {}),
+      },
+      file: test.location?.file,
+    });
+
+    this.uploads.push({
+      rid: `${rid}-${project.name}`,
+      title: test.title,
+      files: result.attachments.filter(a => a.body || a.path),
+      file: test.location?.file,
+    });
+    // remove empty uploads
+    this.uploads = this.uploads.filter(anUpload => anUpload.files.length);
+
+    reportTestPromises.push(reportTestPromise);
+  }
+
+  #getArtifactPath(artifact) {
+    if (artifact.path) {
+      if (path.isAbsolute(artifact.path)) return artifact.path;
+
+      return path.join(this.config.outputDir || this.config.projects[0].outputDir, artifact.path);
+    }
+
+    if (artifact.body) {
+      let filePath = generateTmpFilepath(artifact.name);
+
+      const extension = artifact.contentType?.split('/')[1]?.replace('jpeg', 'jpg');
+      if (extension) filePath += `.${extension}`;
+
+      fs.writeFileSync(filePath, artifact.body);
+      return filePath;
+    }
+
+    return null;
+  }
+
+  async onEnd(result) {
+    if (!this.client) return;
+
+    await Promise.all(reportTestPromises);
+
+    if (this.uploads.length) {
+      if (this.client.uploader.isEnabled) console.log(APP_PREFIX, `🎞️  Uploading ${this.uploads.length} files...`);
+
+      const promises = [];
+
+      // ? possible move to addTestRun (needs investigation if files are ready)
+      for (const upload of this.uploads) {
+        const { rid, file, title } = upload;
+
+        const files = upload.files.map(attachment => ({
+          path: this.#getArtifactPath(attachment),
+          title,
+          type: attachment.contentType,
+        }));
+
+        if (!this.client.uploader.isEnabled) {
+          files.forEach(f => this.client.uploader.storeUploadedFile(f, this.client.runId, rid, false));
+          continue;
+        }
+
+        promises.push(
+          this.client.addTestRun(undefined, {
+            rid,
+            title,
+            files,
+            file,
+          }),
+        );
+      }
+      await Promise.all(promises);
+    }
+
+    await this.client.updateRunStatus(checkStatus(result.status));
+  }
+}
+
+function checkStatus(status) {
+  return (
+    {
+      skipped: Status.SKIPPED,
+      timedOut: Status.FAILED,
+      passed: Status.PASSED,
+    }[status] || Status.FAILED
+  );
+}
+
+function appendStep(step, shift = 0) {
+  // nesting too deep, ignore those steps
+  if (shift >= 10) return;
+
+  let newCategory = step.category;
+  switch (newCategory) {
+    case 'test.step':
+      newCategory = 'user';
+      break;
+    case 'hook':
+      newCategory = 'hook';
+      break;
+    case 'attach':
+      return null; // Skip steps with category 'attach'
+    default:
+      newCategory = 'framework';
+  }
+
+  const formattedSteps = [];
+  for (const child of step.steps || []) {
+    const appendedChild = appendStep(child, shift + 2);
+    if (appendedChild) {
+      formattedSteps.push(appendedChild);
+    }
+  }
+
+  const resultStep = {
+    category: newCategory,
+    title: step.title,
+    duration: step.duration,
+  };
+
+  if (formattedSteps.length) {
+    resultStep.steps = formattedSteps.filter(s => !!s);
+  }
+
+  if (step.error !== undefined) {
+    resultStep.error = step.error;
+  }
+
+  return resultStep;
+}
+
+function generateTmpFilepath(filename = '') {
+  filename = filename || `tmp.${crypto.randomBytes(16).toString('hex')}`;
+  const tmpdir = os.tmpdir();
+  return path.join(tmpdir, filename);
+}
+
+/**
+ * Returns filename + test title
+ * @param {*} test - testInfo object from Playwright
+ * @returns
+ */
+function getTestContextName(test) {
+  return `${test._requireFile || ''}_${test.title}`;
+}
+
+export default PlaywrightReporter;

package/src/adapter/vitest.js

@@ -0,0 +1,183 @@
+import pc from 'picocolors';
+import { Client as TestomatioClient } from '../client.js';
+import { STATUS } from '../constants.js';
+import { getTestomatIdFromTestTitle } from '../utils/utils.js';
+import createDebugMessages from 'debug';
+
+const debug = createDebugMessages('@testomatio/reporter:adapter-jest');
+
+/**
+ * @typedef {import('../../types/types.js').VitestTest} VitestTest
+ * @typedef {import('../../types/types.js').VitestTestFile} VitestTestFile
+ * @typedef {import('../../types/types.js').VitestSuite} VitestSuite
+ * @typedef {import('../../types/types.js').VitestTestLogs} VitestTestLogs
+ * @typedef {import('../../types/vitest.types.js').ErrorWithDiff} ErrorWithDiff
+ * @typedef {typeof import('../constants.js').STATUS} STATUS
+ * @typedef {import('../../types/types.js').TestData} TestData
+ */
+
+class VitestReporter {
+  constructor(config = {}) {
+    this.client = new TestomatioClient({ apiKey: config?.apiKey });
+    /**
+     * @type {(TestData & {status: string})[]} tests
+     */
+    this.tests = [];
+  }
+
+  // on run start
+  onInit() {
+    this.client.createRun();
+  }
+
+  /**
+   * @param {VitestTestFile[] | undefined} files // array with results;
+   * @param {unknown[] | undefined} errors // errors does not contain errors from tests; probably its testrunner errors
+   */
+  async onFinished(files, errors) {
+    if (!files || !files.length) console.info('No tests executed');
+
+    files.forEach(file => {
+      // task could be test or suite
+      file.tasks.forEach(taskOrSuite => {
+        if (taskOrSuite.type === 'test') {
+          const test = taskOrSuite;
+          this.tests.push(this.#getDataFromTest(test));
+        } else if (taskOrSuite.type === 'suite') {
+          const suite = taskOrSuite;
+          this.#processTasksOfSuite(suite);
+        } else {
+          throw new Error('Unprocessed case. Unknown task type');
+        }
+      });
+    });
+
+    debug(this.tests.length, 'tests collected');
+
+    // send tests to Testomat.io
+    for (const test of this.tests) {
+      await this.client.addTestRun(test.status, test);
+    }
+
+    console.log('finished');
+    if (errors.length) console.error('Vitest adapter errors:', errors);
+
+    await this.client.updateRunStatus(getRunStatusFromResults(files));
+  }
+
+  /* non-used listeners
+  onUserConsoleLog(log) {}
+  onPathsCollected(paths) {} // paths array to files with tests
+  onCollected(files) {} // files array with tests (but without results)
+  onTaskUpdate(packs) {} // some updates come here on afterAll block execution
+  onTestRemoved(trigger) {}
+  onWatcherStart(files, errors) {}
+  onWatcherRerun(files, trigger) {}
+  onServerRestart(reason) {}
+  onProcessTimeout() {}
+  */
+
+  /**
+   * Recursively gets all tasks from suite and pushes them to "tests" array
+   *
+   * @param {VitestSuite} suite
+   */
+  #processTasksOfSuite(suite) {
+    suite.tasks.forEach(taskOrSuite => {
+      if (taskOrSuite.type === 'test') {
+        const test = taskOrSuite;
+        this.tests.push(this.#getDataFromTest(test));
+      } else if (taskOrSuite.type === 'suite') {
+        const theSuite = taskOrSuite;
+        this.#processTasksOfSuite(theSuite);
+      } else {
+        throw new Error('Unprocessed case. Unknown task type');
+      }
+    });
+  }
+
+  /**
+   * Processes task and returns test data ready to be sent to Testomat.io
+   *
+   * @param {VitestTest} test
+   *
+   * @returns {TestData & {status: string}}
+   */
+  #getDataFromTest(test) {
+    return {
+      error: test.result?.errors ? test.result.errors[0] : undefined,
+      file: test.file.name,
+      logs: test.logs ? transformLogsToString(test.logs) : '',
+      meta: test.meta,
+      status: getTestStatus(test),
+      suite_title: test.suite.name || test.file?.name,
+      test_id: getTestomatIdFromTestTitle(test.name),
+      time: test.result?.duration || 0,
+      title: test.name,
+      // testomatio functions (artifacts, logs, steps, meta) are not supported
+    };
+  }
+}
+
+/**
+ * Returns run status based on test results
+ *
+ * @param {VitestTestFile[]} files
+ * @returns {'passed' | 'failed' | 'finished'}
+ */
+function getRunStatusFromResults(files) {
+  /**
+   * @type {'passed' | 'failed' | 'finished'}
+   */
+  let status = 'finished'; // default status (if no failed or passed tests)
+
+  files.forEach(file => {
+    // search for failed tests
+    file.tasks.forEach(taskOrSuite => {
+      if (taskOrSuite.result?.state === 'fail') {
+        status = 'failed'; // set status to failed if any test failed
+      }
+    });
+
+    // if there are no failed tests > search for passed tests
+    if (status !== 'failed') {
+      file.tasks.forEach(taskOrSuite => {
+        if (taskOrSuite.result?.state === 'pass') {
+          status = 'passed'; // set status to passed if any test passed (and there are no failed tests)
+        }
+      });
+    }
+  });
+
+  return status;
+}
+
+/**
+ * Returns test status in Testomat.io format
+ *
+ * @param {VitestTest} test
+ * @returns 'passed' | 'failed' | 'skipped'
+ */
+function getTestStatus(test) {
+  if (test.result?.state === 'fail') return STATUS.FAILED;
+  if (test.result?.state === 'pass') return STATUS.PASSED;
+  if (!test.result && test.mode === 'skip') return STATUS.SKIPPED;
+  console.error(pc.red('Unprocessed case for defining test status. Contact dev team. Test:'), test);
+}
+
+/**
+ * @param {VitestTestLogs[]} logs
+ * @returns string
+ */
+function transformLogsToString(logs) {
+  if (!logs) return '';
+  let logsStr = '';
+  logs.forEach(log => {
+    if (log.type === 'stdout') logsStr += `${log.content}\n`;
+    if (log.type === 'stderr') logsStr += `${pc.red(log.content)}\n`;
+  });
+  return logsStr;
+}
+
+export default VitestReporter;
+export { VitestReporter };