@testomatio/reporter 1.6.0-beta-2-artifacts → 2.0.0-beta-esm

package/src/adapter/mocha.js

@@ -0,0 +1,156 @@
+// eslint-disable-next-line global-require, import/no-extraneous-dependencies
+import Mocha from 'mocha';
+import TestomatClient from '../client.js';
+import { STATUS, TESTOMAT_TMP_STORAGE_DIR } from '../constants.js';
+import { getTestomatIdFromTestTitle, fileSystem } from '../utils/utils.js';
+import { config } from '../config.js';
+import { services } from '../services/index.js';
+import pc from 'picocolors';
+
+const {
+  EVENT_RUN_BEGIN,
+  EVENT_RUN_END,
+  EVENT_TEST_FAIL,
+  EVENT_TEST_PASS,
+  EVENT_TEST_PENDING,
+  EVENT_SUITE_BEGIN,
+  EVENT_SUITE_END,
+  EVENT_TEST_BEGIN,
+  EVENT_TEST_END,
+} = Mocha.Runner.constants;
+
+function MochaReporter(runner, opts) {
+  Mocha.reporters.Base.call(this, runner);
+  let passes = 0;
+  let failures = 0;
+  let skipped = 0;
+  // let artifactStore;
+
+  const apiKey = opts?.reporterOptions?.apiKey || config.TESTOMATIO;
+
+  const client = new TestomatClient({ apiKey });
+
+  runner.on(EVENT_RUN_BEGIN, () => {
+    client.createRun();
+
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
+  });
+
+  runner.on(EVENT_SUITE_BEGIN, async suite => {
+    services.setContext(suite.fullTitle());
+  });
+
+  runner.on(EVENT_SUITE_END, async () => {
+    services.setContext(null);
+  });
+
+  runner.on(EVENT_TEST_BEGIN, async test => {
+    services.setContext(test.fullTitle());
+  });
+
+  runner.on(EVENT_TEST_END, async () => {
+    services.setContext(null);
+  });
+
+  runner.on(EVENT_TEST_PASS, async test => {
+    passes += 1;
+
+    console.log(pc.bold(pc.green('✔')), test.fullTitle());
+    const testId = getTestomatIdFromTestTitle(test.title);
+
+    const logs = getTestLogs(test);
+    const artifacts = services.artifacts.get(test.fullTitle());
+    const keyValues = services.keyValues.get(test.fullTitle());
+
+    client.addTestRun(STATUS.PASSED, {
+      test_id: testId,
+      suite_title: getSuiteTitle(test),
+      title: getTestName(test),
+      code: test.body.toString(),
+      file: getFile(test),
+      time: test.duration,
+      logs,
+      manuallyAttachedArtifacts: artifacts,
+      meta: keyValues,
+    });
+  });
+
+  runner.on(EVENT_TEST_PENDING, test => {
+    skipped += 1;
+    console.log('skip: %s', test.fullTitle());
+    const testId = getTestomatIdFromTestTitle(test.title);
+    client.addTestRun(STATUS.SKIPPED, {
+      title: getTestName(test),
+      suite_title: getSuiteTitle(test),
+      code: test.body.toString(),
+      file: getFile(test),
+      test_id: testId,
+      time: test.duration,
+    });
+  });
+
+  runner.on(EVENT_TEST_FAIL, async (test, err) => {
+    failures += 1;
+    console.log(pc.bold(pc.red('✖')), test.fullTitle(), pc.gray(err.message));
+    const testId = getTestomatIdFromTestTitle(test.title);
+
+    const logs = getTestLogs(test);
+
+    client.addTestRun(STATUS.FAILED, {
+      error: err,
+      suite_title: getSuiteTitle(test),
+      file: getFile(test),
+      test_id: testId,
+      title: getTestName(test),
+      code: test.body.toString(),
+      time: test.duration,
+      logs,
+    });
+  });
+
+  runner.on(EVENT_RUN_END, () => {
+    const status = failures === 0 ? STATUS.PASSED : STATUS.FAILED;
+    console.log(pc.bold(status), `${passes} passed, ${failures} failed, ${skipped} skipped`);
+    // @ts-ignore
+    client.updateRunStatus(status);
+  });
+}
+
+function getTestLogs(test) {
+  const suiteLogsArr = services.logger.getLogs(test.parent.fullTitle());
+  const suiteLogs = suiteLogsArr ? suiteLogsArr.join('\n').trim() : '';
+  const testLogsArr = services.logger.getLogs(test.fullTitle());
+  const testLogs = testLogsArr ? testLogsArr.join('\n').trim() : '';
+
+  let logs = '';
+  if (suiteLogs) {
+    logs += `${pc.bold('\t--- BeforeSuite ---')}\n${suiteLogs}`;
+  }
+  if (testLogs) {
+    logs += `\n${pc.bold('\t--- Test ---')}\n${testLogs}`;
+  }
+  return logs;
+}
+
+function getSuiteTitle(test, pathArr = []) {
+  if (test.parent.parent) getSuiteTitle(test.parent, pathArr);
+
+  pathArr.push(test.parent.title);
+
+  return pathArr.filter(t => !!t)[0];
+}
+
+function getFile(test) {
+  return test.parent.file?.replace(process.cwd(), '');
+}
+
+function getTestName(test) {
+  if (process.env.TESTOMATIO_CREATE === 'fulltitle') return test.fullTitle();
+  return test.title;
+}
+
+// To have this reporter "extend" a built-in reporter uncomment the following line:
+// @ts-ignore
+Mocha.utils.inherits(MochaReporter, Mocha.reporters.Spec);
+
+export default MochaReporter;

package/src/adapter/playwright.js

@@ -0,0 +1,222 @@
+import pc from 'picocolors';
+import crypto from 'crypto';
+import os from 'os';
+import path from 'path';
+import { v4 as uuidv4 } from 'uuid';
+import fs from 'fs';
+import { APP_PREFIX, STATUS as Status, TESTOMAT_TMP_STORAGE_DIR } from '../constants.js';
+import TestomatioClient from '../client.js';
+import { upload } from '../fileUploader.js';
+import { getTestomatIdFromTestTitle, fileSystem } from '../utils/utils.js';
+import { services } from '../services/index.js';
+import { dataStorage } from '../data-storage.js';
+
+const reportTestPromises = [];
+
+class PlaywrightReporter {
+  constructor(config = {}) {
+    this.client = new TestomatioClient({ apiKey: config?.apiKey });
+
+    this.uploads = [];
+  }
+
+  onBegin(config, suite) {
+    // clean data storage
+    fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
+    if (!this.client) return;
+    this.suite = suite;
+    this.config = config;
+    this.client.createRun();
+  }
+
+  onTestBegin(testInfo) {
+    const fullTestTitle = getTestContextName(testInfo);
+    dataStorage.setContext(fullTestTitle);
+  }
+
+  onTestEnd(test, result) {
+    // test.parent.project().__projectId
+
+    if (!this.client) return;
+
+    const { title } = test;
+    const { error, duration } = result;
+    const suite_title = test.parent ? test.parent?.title : path.basename(test?.location?.file);
+
+    const steps = [];
+    for (const step of result.steps) {
+      const appendedStep = appendStep(step);
+      if (appendedStep) {
+        steps.push(appendedStep);
+      }
+    }
+
+    const fullTestTitle = getTestContextName(test);
+    let logs = '';
+    if (result.stderr.length || result.stdout.length) {
+      logs = `\n\n${pc.bold('Logs:')}\n${pc.red(result.stderr.join(''))}\n${result.stdout.join('')}`;
+    }
+    const manuallyAttachedArtifacts = services.artifacts.get(fullTestTitle);
+    const keyValues = services.keyValues.get(fullTestTitle);
+    const rid = test.id || test.testId || uuidv4();
+
+    const reportTestPromise = this.client.addTestRun(checkStatus(result.status), {
+      rid,
+      error,
+      test_id: getTestomatIdFromTestTitle(`${title} ${test.tags?.join(' ')}`),
+      suite_title,
+      title,
+      steps: steps.length ? steps : undefined,
+      time: duration,
+      logs,
+      manuallyAttachedArtifacts,
+      meta: keyValues,
+      file: test.location?.file,
+    });
+
+    this.uploads.push({
+      rid,
+      title: test.title,
+      files: result.attachments.filter(a => a.body || a.path),
+      file: test.location?.file,
+    });
+    // remove empty uploads
+    this.uploads = this.uploads.filter(anUpload => anUpload.files.length);
+
+    reportTestPromises.push(reportTestPromise);
+  }
+
+  #getArtifactPath(artifact) {
+    if (artifact.path) {
+      if (path.isAbsolute(artifact.path)) return artifact.path;
+
+      return path.join(this.config.outputDir || this.config.projects[0].outputDir, artifact.path);
+    }
+
+    if (artifact.body) {
+      const fileName = tmpFile();
+      fs.writeFileSync(fileName, artifact.body);
+      return fileName;
+    }
+
+    return null;
+  }
+
+  async onEnd(result) {
+    if (!this.client) return;
+
+    await Promise.all(reportTestPromises);
+
+    if (this.uploads.length && upload.isArtifactsEnabled()) {
+      console.log(APP_PREFIX, `🎞️  Uploading ${this.uploads.length} files...`);
+
+      const promises = [];
+
+      for (const anUpload of this.uploads) {
+        const { rid, file, title } = anUpload;
+
+        const files = anUpload.files.map(attachment => ({
+          path: this.#getArtifactPath(attachment),
+          title,
+          type: attachment.contentType,
+        }));
+
+        promises.push(
+          this.client.addTestRun(undefined, {
+            rid,
+            title,
+            files,
+            file,
+          }),
+        );
+      }
+      await Promise.all(promises);
+    }
+
+    await this.client.updateRunStatus(checkStatus(result.status));
+  }
+}
+
+function checkStatus(status) {
+  return (
+    {
+      skipped: Status.SKIPPED,
+      timedOut: Status.FAILED,
+      passed: Status.PASSED,
+    }[status] || Status.FAILED
+  );
+}
+
+function appendStep(step, shift = 0) {
+  // nesting too deep, ignore those steps
+  if (shift >= 10) return;
+
+  let newCategory = step.category;
+  switch (newCategory) {
+    case 'test.step':
+      newCategory = 'user';
+      break;
+    case 'hook':
+      newCategory = 'hook';
+      break;
+    case 'attach':
+      return null; // Skip steps with category 'attach'
+    default:
+      newCategory = 'framework';
+  }
+
+  const formattedSteps = [];
+  for (const child of step.steps || []) {
+    const appendedChild = appendStep(child, shift + 2);
+    if (appendedChild) {
+      formattedSteps.push(appendedChild);
+    }
+  }
+
+  const resultStep = {
+    category: newCategory,
+    title: step.title,
+    duration: step.duration,
+  };
+
+  if (formattedSteps.length) {
+    resultStep.steps = formattedSteps.filter(s => !!s);
+  }
+
+  if (step.error !== undefined) {
+    resultStep.error = step.error;
+  }
+
+  return resultStep;
+}
+
+function tmpFile(prefix = 'tmp.') {
+  const tmpdir = os.tmpdir();
+  return path.join(tmpdir, prefix + crypto.randomBytes(16).toString('hex'));
+}
+
+/**
+ * Returns filename + test title
+ * @param {*} test - testInfo object from Playwright
+ * @returns
+ */
+function getTestContextName(test) {
+  return `${test._requireFile || ''}_${test.title}`;
+}
+
+function initPlaywrightForStorage() {
+  try {
+    // @ts-ignore-next-line
+    // eslint-disable-next-line import/no-extraneous-dependencies
+    const { test } = require('@playwright/test');
+    // eslint-disable-next-line no-empty-pattern
+    test.beforeEach(async ({}, testInfo) => {
+      global.testomatioTestTitle = `${testInfo.file || ''}_${testInfo.title}`;
+    });
+  } catch (e) {
+    // ignore
+  }
+}
+
+export default PlaywrightReporter;
+export { initPlaywrightForStorage };

package/src/adapter/vitest.js

@@ -0,0 +1,183 @@
+import pc from 'picocolors';
+import { Client as TestomatioClient } from '../client.js';
+import { STATUS } from '../constants.js';
+import { getTestomatIdFromTestTitle } from '../utils/utils.js';
+import createDebugMessages from 'debug';
+
+const debug = createDebugMessages('@testomatio/reporter:adapter-jest');
+
+/**
+ * @typedef {import('../../types').VitestTest} VitestTest
+ * @typedef {import('../../types').VitestTestFile} VitestTestFile
+ * @typedef {import('../../types').VitestSuite} VitestSuite
+ * @typedef {import('../../types').VitestTestLogs} VitestTestLogs
+ * @typedef {import('../../vitest.types').ErrorWithDiff} ErrorWithDiff
+ * @typedef {typeof import('../constants.js').STATUS} STATUS
+ * @typedef {import('../../types').TestData} TestData
+ */
+
+class VitestReporter {
+  constructor(config = {}) {
+    this.client = new TestomatioClient({ apiKey: config?.apiKey });
+    /**
+     * @type {(TestData & {status: string})[]} tests
+     */
+    this.tests = [];
+  }
+
+  // on run start
+  onInit() {
+    this.client.createRun();
+  }
+
+  /**
+   * @param {VitestTestFile[] | undefined} files // array with results;
+   * @param {unknown[] | undefined} errors // errors does not contain errors from tests; probably its testrunner errors
+   */
+  async onFinished(files, errors) {
+    if (!files || !files.length) console.info('No tests executed');
+
+    files.forEach(file => {
+      // task could be test or suite
+      file.tasks.forEach(taskOrSuite => {
+        if (taskOrSuite.type === 'test') {
+          const test = taskOrSuite;
+          this.tests.push(this.#getDataFromTest(test));
+        } else if (taskOrSuite.type === 'suite') {
+          const suite = taskOrSuite;
+          this.#processTasksOfSuite(suite);
+        } else {
+          throw new Error('Unprocessed case. Unknown task type');
+        }
+      });
+    });
+
+    debug(this.tests.length, 'tests collected');
+
+    // send tests to Testomat.io
+    for (const test of this.tests) {
+      await this.client.addTestRun(test.status, test);
+    }
+
+    console.log('finished');
+    if (errors.length) console.error('Vitest adapter errors:', errors);
+
+    await this.client.updateRunStatus(getRunStatusFromResults(files));
+  }
+
+  /* non-used listeners
+  onUserConsoleLog(log) {}
+  onPathsCollected(paths) {} // paths array to files with tests
+  onCollected(files) {} // files array with tests (but without results)
+  onTaskUpdate(packs) {} // some updates come here on afterAll block execution
+  onTestRemoved(trigger) {}
+  onWatcherStart(files, errors) {}
+  onWatcherRerun(files, trigger) {}
+  onServerRestart(reason) {}
+  onProcessTimeout() {}
+  */
+
+  /**
+   * Recursively gets all tasks from suite and pushes them to "tests" array
+   *
+   * @param {VitestSuite} suite
+   */
+  #processTasksOfSuite(suite) {
+    suite.tasks.forEach(taskOrSuite => {
+      if (taskOrSuite.type === 'test') {
+        const test = taskOrSuite;
+        this.tests.push(this.#getDataFromTest(test));
+      } else if (taskOrSuite.type === 'suite') {
+        const theSuite = taskOrSuite;
+        this.#processTasksOfSuite(theSuite);
+      } else {
+        throw new Error('Unprocessed case. Unknown task type');
+      }
+    });
+  }
+
+  /**
+   * Processes task and returns test data ready to be sent to Testomat.io
+   *
+   * @param {VitestTest} test
+   *
+   * @returns {TestData & {status: string}}
+   */
+  #getDataFromTest(test) {
+    return {
+      error: test.result?.errors ? test.result.errors[0] : undefined,
+      file: test.file.name,
+      logs: test.logs ? transformLogsToString(test.logs) : '',
+      meta: test.meta,
+      status: getTestStatus(test),
+      suite_title: test.suite.name || test.file?.name,
+      test_id: getTestomatIdFromTestTitle(test.name),
+      time: test.result?.duration || 0,
+      title: test.name,
+      // testomatio functions (artifacts, logs, steps, meta) are not supported
+    };
+  }
+}
+
+/**
+ * Returns run status based on test results
+ *
+ * @param {VitestTestFile[]} files
+ * @returns {'passed' | 'failed' | 'finished'}
+ */
+function getRunStatusFromResults(files) {
+  /**
+   * @type {'passed' | 'failed' | 'finished'}
+   */
+  let status = 'finished'; // default status (if no failed or passed tests)
+
+  files.forEach(file => {
+    // search for failed tests
+    file.tasks.forEach(taskOrSuite => {
+      if (taskOrSuite.result?.state === 'fail') {
+        status = 'failed'; // set status to failed if any test failed
+      }
+    });
+
+    // if there are no failed tests > search for passed tests
+    if (status !== 'failed') {
+      file.tasks.forEach(taskOrSuite => {
+        if (taskOrSuite.result?.state === 'pass') {
+          status = 'passed'; // set status to passed if any test passed (and there are no failed tests)
+        }
+      });
+    }
+  });
+
+  return status;
+}
+
+/**
+ * Returns test status in Testomat.io format
+ *
+ * @param {VitestTest} test
+ * @returns 'passed' | 'failed' | 'skipped'
+ */
+function getTestStatus(test) {
+  if (test.result?.state === 'fail') return STATUS.FAILED;
+  if (test.result?.state === 'pass') return STATUS.PASSED;
+  if (!test.result && test.mode === 'skip') return STATUS.SKIPPED;
+  console.error(pc.red('Unprocessed case for defining test status. Contact dev team. Test:'), test);
+}
+
+/**
+ * @param {VitestTestLogs[]} logs
+ * @returns string
+ */
+function transformLogsToString(logs) {
+  if (!logs) return '';
+  let logsStr = '';
+  logs.forEach(log => {
+    if (log.type === 'stdout') logsStr += `${log.content}\n`;
+    if (log.type === 'stderr') logsStr += `${pc.red(log.content)}\n`;
+  });
+  return logsStr;
+}
+
+export default VitestReporter;
+export { VitestReporter };

package/src/adapter/webdriver.js

@@ -0,0 +1,111 @@
+// eslint-disable-next-line
+import WDIOReporter, { RunnerStats } from '@wdio/reporter';
+
+import TestomatClient from '../client.js';
+import { getTestomatIdFromTestTitle } from '../utils/utils.js';
+
+class WebdriverReporter extends WDIOReporter {
+  constructor(options) {
+    super(options);
+
+    this.client = new TestomatClient({ apiKey: options?.apiKey });
+    options = Object.assign(options, { stdout: true });
+
+    this._addTestPromises = [];
+
+    this._isSynchronising = false;
+    // NOTE: new functionality; may break everything
+    this.client.createRun();
+  }
+
+  get isSynchronised() {
+    return this._isSynchronising === false;
+  }
+
+  /**
+   * 
+   * @param {RunnerStats} runData 
+   */
+  async onRunnerEnd(runData) {
+    this._isSynchronising = true;
+
+    await Promise.all(this._addTestPromises);
+
+    this._isSynchronising = false;
+
+    // NOTE: new functionality; may break everything 
+    // also this may require additional status mapping
+    await this.client.updateRunStatus(runData.failures ? 'failed' : 'passed');
+  }
+
+  onTestEnd(test) {
+    this._addTestPromises.push(this.addTest(test));
+  }
+
+  // wdio-cucumber does not trigger onTestEnd hook, thus, using this one
+  /**
+   *
+   * @returns
+   */
+  onSuiteEnd(scerario) {
+    if (scerario.type === 'scenario') {
+      this._addTestPromises.push(this.addBddScenario(scerario));
+    }
+  }
+
+  async addTest(test) {
+    if (!this.client) return;
+
+    const { title, _duration: duration, state, error, output } = test;
+
+    const testId = getTestomatIdFromTestTitle(title);
+
+    const screenshotEndpoint = '/session/:sessionId/screenshot';
+    const screenshotsBuffers = output
+      .filter(el => el.endpoint === screenshotEndpoint && el.result && el.result.value)
+      .map(el => Buffer.from(el.result.value, 'base64'));
+
+    await this.client.addTestRun(state, {
+      error,
+      title,
+      test_id: testId,
+      time: duration,
+      filesBuffers: screenshotsBuffers,
+    });
+  }
+
+  /**
+   * @param {import('../../types').WebdriverIOScenario} scenario
+   */
+  addBddScenario(scenario) {
+    if (!this.client) return;
+
+    const { title, _duration: duration } = scenario;
+
+    const testId = getTestomatIdFromTestTitle(title || scenario.tags.map(tag => tag.name).join(' '));
+
+    let scenarioState = scenario.tests.every(test => test.state === 'passed') ? 'passed' : 'failed';
+    if (scenario.tests.every(test => test.state === 'skipped')) {
+      scenarioState = 'skipped';
+    }
+    const errors = scenario.tests
+      .filter(test => test.state === 'failed')
+      .map(test => test.error?.stack)
+      .filter(Boolean);
+    const error = errors.join('\n');
+
+    const tags = scenario.tags.map(tag => tag.name);
+
+    return this.client.addTestRun(scenarioState, {
+      error: error ? Error(error) : null,
+      title,
+      test_id: testId,
+      time: duration,
+      tags,
+      file: scenario.file,
+      // filesBuffers: screenshotsBuffers,
+    });
+  }
+}
+
+export default WebdriverReporter;