@testomatio/reporter 1.6.0-beta-1-artifacts → 2.0.0-beta-esm

package/src/bin/reportXml.js

@@ -0,0 +1,67 @@
+#!/usr/bin/env node
+import program from 'commander';
+import pc from 'picocolors';
+import glob from 'glob';
+import createDebugMessages from 'debug';
+import { APP_PREFIX } from '../constants.js';
+import XmlReader from '../xmlReader.js';
+import { version } from '../../package.json';
+
+const debug = createDebugMessages('@testomatio/reporter:xml-cli');
+console.log(pc.cyan(pc.bold(` 🤩 Testomat.io XML Reporter v${version}`)));
+
+program
+  .arguments('<pattern>')
+  .option('-d, --dir <dir>', 'Project directory')
+  .option('--java-tests [java-path]', 'Load Java tests from path, by default: src/test/java')
+  .option('--lang <lang>', 'Language used (python, ruby, java)')
+  .option('--timelimit <time>', 'default time limit in seconds to kill a stuck process')
+  .option('--env-file <envfile>', 'Load environment variables from env file')
+  .action(async (pattern, opts) => {
+    if (!pattern.endsWith('.xml')) {
+      pattern += '.xml';
+    }
+    let { javaTests, lang } = opts;
+    if (opts.envFile) {
+      console.log(APP_PREFIX, 'Loading env file:', opts.envFile);
+      debug('Loading env file: %s', opts.envFile);
+      require('dotenv').config({ path: opts.envFile }); // eslint-disable-line
+    }
+    if (javaTests === true) javaTests = 'src/test/java';
+    lang = lang?.toLowerCase();
+    const runReader = new XmlReader({ javaTests, lang });
+    const files = glob.sync(pattern, { cwd: opts.dir || process.cwd() });
+    if (!files.length) {
+      console.log(APP_PREFIX, `Report can't be created. No XML files found 😥`);
+      process.exitCode = 1;
+      return;
+    }
+
+    for (const file of files) {
+      console.log(APP_PREFIX, `Parsed ${file}`);
+      runReader.parse(file);
+    }
+
+    let timeoutTimer;
+    if (opts.timelimit) {
+      timeoutTimer = setTimeout(() => {
+        console.log(`⚠️  Reached timeout of ${opts.timelimit}s. Exiting... (Exit code is 0 to not fail the pipeline)`);
+        process.exit(0);
+      }, parseInt(opts.timelimit, 10) * 1000);
+    }
+
+    try {
+      await runReader.createRun();
+      await runReader.uploadData();
+    } catch (err) {
+      console.log(APP_PREFIX, 'Error updating status, skipping...', err);
+    }
+
+    if (timeoutTimer) clearTimeout(timeoutTimer);
+  });
+
+if (process.argv.length < 3) {
+  program.outputHelp();
+}
+
+program.parse(process.argv);

package/src/bin/startTest.js

@@ -0,0 +1,119 @@
+#!/usr/bin/env node
+import { spawn } from 'cross-spawn';
+import program from 'commander';
+import pc from 'picocolors';
+import TestomatClient from '../client.js';
+import { APP_PREFIX, STATUS } from '../constants.js';
+import { version } from '../../package.json';
+import {config} from '../config.js';
+
+console.log(pc.cyan(pc.bold(` 🤩 Testomat.io Reporter v${version}`)));
+
+program
+  .option('-c, --command <cmd>', 'Test runner command')
+  .option('--launch', 'Start a new run and return its ID')
+  .option('--finish', 'Finish Run by its ID')
+  .option('--env-file <envfile>', 'Load environment variables from env file')
+  .option('--filter <filter>', 'Additional execution filter')
+  .action(async opts => {
+    const { launch, finish, filter } = opts;
+    let { command } = opts;
+
+    if (opts.envFile) require('dotenv').config(opts.envFile); // eslint-disable-line
+
+    const apiKey = process.env['INPUT_TESTOMATIO-KEY'] || config.TESTOMATIO;
+    const title = process.env.TESTOMATIO_TITLE;
+
+    if (launch) {
+      console.log('Starting a new Run on Testomat.io...');
+      const client = new TestomatClient({ apiKey });
+
+      client.createRun().then(() => {
+        console.log(process.env.runId);
+        process.exit(0);
+      });
+      return;
+    }
+
+    if (finish) {
+      if (!process.env.TESTOMATIO_RUN) {
+        console.log('TESTOMATIO_RUN environment variable must be set.');
+        return process.exit(1);
+      }
+
+      console.log('Finishing Run on Testomat.io...');
+
+      const client = new TestomatClient({ apiKey });
+
+      // @ts-ignore
+      client.updateRunStatus(STATUS.FINISHED).then(() => {
+        console.log(pc.yellow(`Run ${process.env.TESTOMATIO_RUN} was finished`));
+        process.exit(0);
+      });
+      return;
+    }
+
+    let exitCode = 0;
+
+    if (!command.split) {
+      process.exitCode = 255;
+      console.log(APP_PREFIX, `No command provided. Use -c option to launch a test runner.`);
+      return;
+    }
+
+    const client = new TestomatClient({ apiKey, title, parallel: true });
+
+    if (filter) {
+      const [pipe, ...optsArray] = filter.split(':');
+      const pipeOptions = optsArray.join(':');
+
+      try {
+        const tests = await client.prepareRun({ pipe, pipeOptions });
+
+        if (!tests || tests.length === 0) {
+          return;
+        }
+
+        const grep = ` --grep (${tests.join('|')})`;
+        command += grep;
+      } catch (err) {
+        console.log(APP_PREFIX, err);
+      }
+    }
+
+    const testCmds = command.split(' ');
+    console.log(APP_PREFIX, `🚀 Running`, pc.green(command));
+
+    if (!apiKey) {
+      const cmd = spawn(testCmds[0], testCmds.slice(1), { stdio: 'inherit' });
+
+      cmd.on('close', code => {
+        console.log(APP_PREFIX, '⚠️ ', `Runner exited with ${pc.bold(code)}, report is ignored`);
+
+        if (code > exitCode) exitCode = code;
+        process.exitCode = exitCode;
+      });
+
+      return;
+    }
+
+    client.createRun().then(() => {
+      const cmd = spawn(testCmds[0], testCmds.slice(1), { stdio: 'inherit' });
+
+      cmd.on('close', code => {
+        const emoji = code === 0 ? '🟢' : '🔴';
+        console.log(APP_PREFIX, emoji, `Runner exited with ${pc.bold(code)}`);
+        const status = code === 0 ? 'passed' : 'failed';
+        client.updateRunStatus(status, true);
+
+        if (code > exitCode) exitCode = code;
+        process.exitCode = exitCode;
+      });
+    });
+  });
+
+if (process.argv.length <= 2) {
+  program.outputHelp();
+}
+
+program.parse(process.argv);

package/src/client.js

@@ -0,0 +1,423 @@
+import createDebugMessages from 'debug';
+import createCallsiteRecord from 'callsite-record';
+import { minimatch } from 'minimatch';
+import fs from 'fs';
+import pc from 'picocolors';
+import { randomUUID } from 'crypto';
+import {upload} from './fileUploader.js';
+import { APP_PREFIX, STATUS } from './constants.js';
+import { pipesFactory } from './pipe/index.js';
+import { glob } from 'glob';
+import path, { sep} from 'path';
+import { fileURLToPath } from 'node:url';
+
+const debug = createDebugMessages('@testomatio/reporter:client');
+
+// removed __dirname usage, because:
+// 1. replaced with ESM syntax (import.meta.url), but it throws an error on tsc compilation;
+// 2. got error "__dirname already defined" in compiles js code (cjs dir) 
+
+let listOfTestFilesToExcludeFromReport = null;
+
+/**
+ * @typedef {import('../types').TestData} TestData
+ * @typedef {import('../types').PipeResult} PipeResult
+ */
+
+class Client {
+  /**
+   * Create a Testomat client instance
+   * @returns
+   */
+  // eslint-disable-next-line 
+  constructor(params = {}) {
+    this.uuid = randomUUID();
+    this.queue = Promise.resolve();
+    this.totalUploaded = 0;
+    this.failedToUpload = 0;
+
+    // @ts-ignore this line will be removed in compiled code, because __dirname is defined in commonjs
+    const __dirname = path.dirname(fileURLToPath(import.meta.url));
+    const pathToPackageJSON = path.join(__dirname, '../package.json');
+    try {
+      this.version = JSON.parse(fs.readFileSync(pathToPackageJSON).toString()).version;
+      console.log(APP_PREFIX, `Testomatio Reporter v${this.version}`);
+    } catch (e) {
+      // do nothing
+    }
+    this.executionList = Promise.resolve();
+
+  }
+
+  /**
+   * Asynchronously prepares the execution list for running tests through various pipes.
+   * Each pipe in the client is checked for enablement,
+   * and if all pipes are disabled, the function returns a resolved Promise.
+   * Otherwise, it executes the `prepareRun` method for each enabled pipe and collects the results.
+   * The results are then filtered to remove any undefined values.
+   * If no valid results are found, the function returns undefined.
+   * Otherwise, it returns the first non-empty array from the filtered results.
+   *
+   * @param {Object} params - The options for preparing the test execution list.
+   * @param {string} params.pipe - Name of the executed pipe.
+   * @param {string} params.pipeOptions - Filter option.
+   * @returns {Promise<any>} - A Promise that resolves to an
+   * array containing the prepared execution list,
+   * or resolves to undefined if no valid results are found or if all pipes are disabled.
+   */
+  async prepareRun(params) {
+    const store = {};
+    this.pipes = await pipesFactory(params, store);
+    const { pipe, pipeOptions } = params;
+    // all pipes disabled, skipping
+    if (!this.pipes.some(p => p.isEnabled)) {
+      return Promise.resolve();
+    }
+
+    try {
+      const filterPipe = this.pipes.find(p => p.constructor.name.toLowerCase() === `${pipe.toLowerCase()}pipe`);
+
+      if (!filterPipe.isEnabled) {
+        // TODO:for the future for the another pipes
+        console.warn(
+          APP_PREFIX,
+          `At the moment processing is available only for the "testomatio" key. Example: "testomatio:tag-name=xxx"`,
+        );
+        return;
+      }
+
+      const results = await Promise.all(
+        this.pipes.map(async p => ({ pipe: p.toString(), result: await p.prepareRun(pipeOptions) })),
+      );
+
+      const result = results.filter(p => p.pipe.includes('Testomatio'))[0]?.result;
+
+      if (!result || result.length === 0) {
+        return;
+      }
+
+      debug('Execution tests list', result);
+
+      return result;
+    } catch (err) {
+      console.error(APP_PREFIX, err);
+    }
+  }
+
+  /**
+   * Used to create a new Test run
+   *
+   * @returns {Promise<any>} - resolves to Run id which should be used to update / add test
+   */
+  async createRun(params) {
+    if (!this.pipes || !this.pipes.length) this.pipes = await pipesFactory(params || {}, {});
+    debug('Creating run...');
+    // all pipes disabled, skipping
+    if (!this.pipes?.filter(p => p.isEnabled).length) return Promise.resolve();
+
+    this.queue = this.queue
+      .then(() => Promise.all(this.pipes.map(p => p.createRun())))
+      .catch(err => console.log(APP_PREFIX, err))
+      .then(() => undefined); // fixes return type
+    // debug('Run', this.queue);
+    return this.queue;
+  }
+
+  /**
+   * Updates test status and its data
+   *
+   * @param {string|undefined} status
+   * @param {TestData} [testData]
+   * @returns {Promise<PipeResult[]>}
+   */
+  async addTestRun(status, testData) {
+    // all pipes disabled, skipping
+    if (!this.pipes?.filter(p => p.isEnabled).length) return [];
+
+    if (isTestShouldBeExculedFromReport(testData)) return [];
+
+    if (status === STATUS.SKIPPED && process.env.TESTOMATIO_EXCLUDE_SKIPPED) {
+      debug('Skipping test from report', testData?.title);
+      return []; // do not log skipped tests
+    }
+
+    if (!testData)
+      testData = {
+        title: 'Unknown test',
+        suite_title: 'Unknown suite',
+      };
+
+    /**
+     * @type {TestData}
+     */
+    const {
+      rid,
+      error = null,
+      time = 0,
+      example = null,
+      files = [],
+      filesBuffers = [],
+      steps,
+      code = null,
+      title,
+      file,
+      suite_title,
+      suite_id,
+      test_id,
+      manuallyAttachedArtifacts,
+      meta,
+    } = testData;
+    let { message = '' } = testData;
+
+    let errorFormatted = '';
+    if (error) {
+      errorFormatted += this.formatError(error) || '';
+      message = error?.message;
+    }
+
+    // Attach logs
+    const fullLogs = this.formatLogs({ error: errorFormatted, steps, logs: testData.logs });
+
+    // add artifacts
+    if (manuallyAttachedArtifacts?.length) files.push(...manuallyAttachedArtifacts);
+
+    const uploadedFiles = [];
+
+    for (const f of files) {
+      uploadedFiles.push(upload.uploadFileByPath(f, this.uuid));
+    }
+
+    for (const [idx, buffer] of filesBuffers.entries()) {
+      const fileName = `${idx + 1}-${title.replace(/\s+/g, '-')}`;
+      uploadedFiles.push(upload.uploadFileAsBuffer(buffer, fileName, this.uuid));
+    }
+
+    const artifacts = (await Promise.all(uploadedFiles)).filter(n => !!n);
+
+    if (artifacts.length < uploadedFiles.length) {
+      const failedUploading = uploadedFiles.length - artifacts.length;
+      this.failedToUpload += failedUploading;
+    }
+
+    this.totalUploaded += artifacts.length;
+
+    const data = {
+      rid,
+      files,
+      steps,
+      status,
+      stack: fullLogs,
+      example,
+      file,
+      code,
+      title,
+      suite_title,
+      suite_id,
+      test_id,
+      message,
+      run_time: typeof time === 'number' ? time : parseFloat(time),
+      artifacts,
+      meta,
+    };
+
+    // debug('Adding test run...', data);
+
+    // @ts-ignore
+    this.queue = this.queue.then(() =>
+      Promise.all(
+        this.pipes.map(async pipe => {
+          try {
+            const result = await pipe.addTest(data);
+            return { pipe: pipe.toString(), result };
+          } catch (err) {
+            console.log(APP_PREFIX, pipe.toString(), err);
+          }
+        }),
+      ),
+    );
+
+    // @ts-ignore
+    return this.queue;
+  }
+
+  /**
+   *
+   * Updates the status of the current test run and finishes the run.
+   * @param {'passed' | 'failed' | 'skipped' | 'finished'} status - The status of the current test run.
+   * Must be one of "passed", "failed", or "finished"
+   * @param {boolean} [isParallel] - Whether the current test run was executed in parallel with other tests.
+   * @returns {Promise<any>} - A Promise that resolves when finishes the run.
+   */
+  updateRunStatus(status, isParallel = false) {
+    debug('Updating run status...');
+    // all pipes disabled, skipping
+    if (!this.pipes?.filter(p => p.isEnabled).length) return Promise.resolve();
+
+    const runParams = { status, parallel: isParallel };
+
+    this.queue = this.queue
+      .then(() => Promise.all(this.pipes.map(p => p.finishRun(runParams))))
+      .then(() => {
+        debug('TOTAL artifacts', this.totalUploaded);
+        if (this.totalUploaded && !upload.isArtifactsEnabled())
+          debug(`${this.totalUploaded} artifacts are not uploaded, because artifacts uploading is not enabled`);
+
+        if (this.totalUploaded && upload.isArtifactsEnabled()) {
+          console.log(
+            APP_PREFIX,
+            `🗄️ ${this.totalUploaded} artifacts ${
+              process.env.TESTOMATIO_PRIVATE_ARTIFACTS ? 'privately' : pc.bold('publicly')
+            } uploaded to S3 bucket`,
+          );
+
+          if (this.failedToUpload > 0) {
+            console.log(
+              APP_PREFIX,
+              pc.yellow(
+                `Some artifacts were not uploaded. ${this.failedToUpload} artifacts could not be uploaded.
+                Run tests with DEBUG="@testomatio/reporter:file-uploader" to see details"`,
+              ),
+            );
+          }
+        }
+      })
+      .catch(err => console.log(APP_PREFIX, err));
+
+    return this.queue;
+  }
+
+  /**
+   * Returns the formatted stack including the stack trace, steps, and logs.
+   * @returns {string}
+   */
+  formatLogs({ error, steps, logs }) {
+    error = error?.trim();
+    logs = logs?.trim();
+
+    if (Array.isArray(steps)) {
+      steps = steps
+        .map(step => formatStep(step))
+        .flat()
+        .join('\n');
+    }
+
+    let testLogs = '';
+    if (steps) testLogs += `${pc.bold(pc.blue('################[ Steps ]################'))}\n${steps}\n\n`;
+    if (logs) testLogs += `${pc.bold(pc.gray('################[ Logs ]################'))}\n${logs}\n\n`;
+    if (error) testLogs += `${pc.bold(pc.red('################[ Failure ]################'))}\n${error}`;
+    return testLogs;
+  }
+
+  formatError(error, message) {
+    if (!message) message = error.message;
+    if (error.inspect) message = error.inspect() || '';
+
+    let stack = '';
+    if (error.name) stack += `${pc.red(error.name)}`;
+    if (error.operator) stack += ` (${pc.red(error.operator)})`;
+    // add new line if something was added to stack
+    if (stack) stack += ': ';
+
+    stack += `${message}\n`;
+
+    if (error.diff) {
+      // diff for vitest
+      stack += error.diff;
+      stack += '\n\n';
+    } else if (error.actual && error.expected && error.actual !== error.expected) {
+      // diffs for mocha, cypress, codeceptjs style
+      stack += `\n\n${pc.bold(pc.green('+ expected'))} ${pc.bold(pc.red('- actual'))}`;
+      stack += `\n${pc.green(`+ ${error.expected.toString().split('\n').join('\n+ ')}`)}`;
+      stack += `\n${pc.red(`- ${error.actual.toString().split('\n').join('\n- ')}`)}`;
+      stack += '\n\n';
+    }
+
+    const customFilter = process.env.TESTOMATIO_STACK_IGNORE;
+
+    try {
+      let hasFrame = false;
+      const record = createCallsiteRecord({
+        forError: error,
+        isCallsiteFrame: frame => {
+          if (customFilter && minimatch(frame.fileName, customFilter)) return false;
+          if (hasFrame) return false;
+          if (isNotInternalFrame(frame)) hasFrame = true;
+          return hasFrame;
+        },
+      });
+      // @ts-ignore
+      if (record && !record.filename.startsWith('http')) {
+        stack += record.renderSync({ stackFilter: isNotInternalFrame });
+      }
+      return stack;
+    } catch (e) {
+      console.log(e);
+    }
+  }
+}
+
+function isNotInternalFrame(frame) {
+  return (
+    frame.getFileName() &&
+    frame.getFileName().includes(sep) &&
+    !frame.getFileName().includes('node_modules') &&
+    !frame.getFileName().includes('internal')
+  );
+}
+
+function formatStep(step, shift = 0) {
+  const prefix = ' '.repeat(shift);
+
+  const lines = [];
+
+  if (step.error) {
+    lines.push(`${prefix}${pc.red(step.title)} ${pc.gray(`${step.duration}ms`)}`);
+  } else {
+    lines.push(`${prefix}${step.title} ${pc.gray(`${step.duration}ms`)}`);
+  }
+
+  for (const child of step.steps || []) {
+    lines.push(...formatStep(child, shift + 2));
+  }
+
+  return lines;
+}
+
+/**
+ *
+ * @param {TestData} testData
+ * @returns boolean
+ */
+function isTestShouldBeExculedFromReport(testData) {
+  // const fileName = path.basename(test.location?.file || '');
+  const globExcludeFilesPattern = process.env.TESTOMATIO_EXCLUDE_FILES_FROM_REPORT_GLOB_PATTERN;
+  if (!globExcludeFilesPattern) return false;
+
+  if (!testData.file) {
+    debug('No "file" property found for test ', testData.title);
+    return false;
+  }
+
+  const excludeParretnsList = globExcludeFilesPattern.split(';');
+
+  // as scanning files is time consuming operation, just save the result in variable to avoid multiple scans
+  if (!listOfTestFilesToExcludeFromReport) {
+    // list of files with relative paths
+    listOfTestFilesToExcludeFromReport = glob.sync(excludeParretnsList, { ignore: '**/node_modules/**' });
+    debug('Tests from next files will not be reported:', listOfTestFilesToExcludeFromReport);
+  }
+
+  const testFileRelativePath = path.relative(process.cwd(), testData.file);
+
+  // no files found matching the exclusion pattern
+  if (!listOfTestFilesToExcludeFromReport.length) return false;
+
+  if (listOfTestFilesToExcludeFromReport.includes(testFileRelativePath)) {
+    debug(`Excluding test '${testData.title}' <${testFileRelativePath}> from reporting`);
+    return true;
+  }
+  return false;
+}
+
+export { Client };
+export default Client;

package/src/config.js

@@ -0,0 +1,30 @@
+// This file is used to read environment variables from .env file
+import createDebugMessages from 'debug';
+
+const debug = createDebugMessages('@testomatio/reporter:config');
+
+/* for possibility to use multiple env files (reading different paths)
+const envFileVars = dotenv.config({ path: '.env' }).parsed; */
+
+if (process.env.TESTOMATIO_API_KEY) {
+  process.env.TESTOMATIO = process.env.TESTOMATIO_API_KEY;
+}
+if (process.env.TESTOMATIO_TOKEN) {
+  process.env.TESTOMATIO = process.env.TESTOMATIO_TOKEN;
+}
+
+if (process.env.TESTOMATIO === 'undefined')
+  console.error('TESTOMATIO is "undefined". Something went wrong. Contact dev team.');
+
+// select only TESTOMATIO related variables (only to print them in debug)
+const testomatioEnvVars =
+  Object.keys(process.env)
+    .filter(key => key.startsWith('TESTOMATIO') || key.startsWith('S3_'))
+    .reduce((obj, key) => {
+      obj[key] = process.env[key];
+      return obj;
+    }, {}) || {};
+debug('TESTOMATIO variables:', testomatioEnvVars);
+
+// includes variables from .env file and process.env
+export const config = process.env;

package/src/constants.js

@@ -0,0 +1,49 @@
+import pc from 'picocolors';
+import os from 'os';
+import path from 'path';
+
+const APP_PREFIX = pc.gray('[TESTOMATIO]');
+const AXIOS_TIMEOUT = 20 * 1000; // sum = 20sec
+
+const TESTOMAT_TMP_STORAGE_DIR = path.join(os.tmpdir(), 'testomatio_tmp');
+
+const CSV_HEADERS = [
+  { id: 'suite_title', title: 'Suite_title' },
+  { id: 'title', title: 'Title' },
+  { id: 'status', title: 'Status' },
+  { id: 'message', title: 'Message' },
+  { id: 'stack', title: 'Stack' },
+];
+
+const STATUS = {
+  PASSED: 'passed',
+  FAILED: 'failed',
+  SKIPPED: 'skipped',
+  FINISHED: 'finished',
+};
+// html pipe var
+const HTML_REPORT = {
+  FOLDER: 'html-report',
+  REPORT_DEFAULT_NAME: 'testomatio-report.html',
+  TEMPLATE_NAME: 'testomatio.hbs',
+};
+
+const testomatLogoURL = 'https://avatars.githubusercontent.com/u/59105116?s=36&v=4';
+
+const REPORTER_REQUEST_RETRIES = {
+  retryTimeout: 5 * 1000, // sum = 5sec
+  retriesPerRequest: 2,
+  maxTotalRetries: Number(process.env.TESTOMATIO_MAX_REQUEST_FAILURES_COUNT) || 10,
+  withinTimeSeconds: Number(process.env.TESTOMATIO_MAX_REQUEST_RETRIES_WITHIN_TIME_SECONDS) || 60,
+};
+
+export {
+  APP_PREFIX,
+  TESTOMAT_TMP_STORAGE_DIR,
+  CSV_HEADERS,
+  STATUS,
+  HTML_REPORT,
+  AXIOS_TIMEOUT,
+  testomatLogoURL,
+  REPORTER_REQUEST_RETRIES,
+};