@testomatio/reporter 1.5.1-beta → 1.6.0-beta-1-artifacts

package/README.md

@@ -11,7 +11,7 @@ Testomat.io Reporter (this npm package) supports:
 - 🏄 Integarion with all popular [JavaScript/TypeScript frameworks](./docs/frameworks.md)
 - 🗄️ Screenshots, videos, traces [uploaded into S3 bucket](./docs/artifacts.md)
 - 🔎 [Stack traces](./docs/stacktrace.md) and error messages
-- 🐙 [GitHub](./docs/pipes/github.md) & [GitLab](./docs/pipes/gitlab.md) integration
+- 🐙 [GitHub](./docs/pipes/github.md), [GitLab](./docs/pipes/gitlab.md) & [Bitbucket](./docs/pipes/bitbucket.md) integration
 - 🚅 Realtime reports
 - 🗃️ Other test frameworks supported via [JUNit XML](./docs/junit.md)
 - 🚶‍♀️ Steps _(work in progress)_
@@ -128,6 +128,7 @@ Bring this reporter on CI and never lose test results again!
   - [Gitlab](./docs/pipes/gitlab.md)
   - [CSV](./docs/pipes/csv.md)
   - [HTML report](./docs/pipes/html.md)
+  - [Bitbucket](./docs/pipes/bitbucket.md)
 - 📓 [JUnit](./docs/junit.md)
 - 🗄️ [Artifacts](./docs/artifacts.md)
 - 🔂 [Workflows](./docs/workflows.md)

package/lib/adapter/codecept.js

@@ -2,7 +2,6 @@ const debug = require('debug')('@testomatio/reporter:adapter:codeceptjs');
 const chalk = require('chalk');
 const TestomatClient = require('../client');
 const { STATUS, APP_PREFIX, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
-const upload = require('../fileUploader');
 const { getTestomatIdFromTestTitle, fileSystem } = require('../utils/utils');
 const { services } = require('../services');
 
@@ -128,10 +127,8 @@ function CodeceptReporter(config) {
     // all tests were reported and we can upload videos
     await Promise.all(reportTestPromises);
 
-    if (upload.isArtifactsEnabled()) {
-      await uploadAttachments(client, videos, '🎞️ Uploading', 'video');
-      await uploadAttachments(client, traces, '📁 Uploading', 'trace');
-    }
+    await uploadAttachments(client, videos, '🎞️ Uploading', 'video');
+    await uploadAttachments(client, traces, '📁 Uploading', 'trace');
 
     const status = failedTests.length === 0 ? STATUS.PASSED : STATUS.FAILED;
     client.updateRunStatus(status);
@@ -304,13 +301,17 @@ function CodeceptReporter(config) {
 }
 
 async function uploadAttachments(client, attachments, messagePrefix, attachmentType) {
-  if (!attachments?.length) return;
+  if (!attachments?.length) return;  
 
-  console.log(APP_PREFIX, `Attachments: ${messagePrefix} ${attachments.length} ${attachmentType} ...`);
+  if (client.uploader.isEnabled) console.log(APP_PREFIX, `Attachments: ${messagePrefix} ${attachments.length} ${attachmentType} ...`);
 
   const promises = attachments.map(async attachment => {
     const { rid, title, path, type } = attachment;
     const file = { path, type, title };
+
+    // we are storing file if upload is disabled
+    if (!client.uploader.isEnabled) return client.uploader.storeUploadedFile(path, client.runId, rid, false);
+
     return client.addTestRun(undefined, {
       ...stripExampleFromTitle(title),
       rid,

package/lib/adapter/cucumber/current.js

@@ -32,16 +32,14 @@ class CucumberReporter extends Formatter {
 
     this.client = new TestomatClient({ apiKey: options.apiKey || config.TESTOMATIO });
     this.status = STATUS.PASSED;
+    this.client.createRun();
   }
 
   parseEnvelope(envelope) {
     if (envelope.testRunStarted) {
       fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
     }
-    if (envelope.testCaseStarted && this.client) {
-      this.client.createRun();
-      this.onTestCaseStarted(envelope.testCaseStarted);
-    }
+    if (envelope.testCaseStarted && this.client) this.onTestCaseStarted(envelope.testCaseStarted);
     if (envelope.testCaseFinished) this.onTestCaseFinished(envelope.testCaseFinished);
     if (envelope.testRunFinished) this.onTestRunFinished(envelope);
   }

package/lib/adapter/playwright.js

@@ -6,7 +6,6 @@ const { v4: uuidv4 } = require('uuid');
 const fs = require('fs');
 const { APP_PREFIX, STATUS: Status, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
 const TestomatioClient = require('../client');
-const { isArtifactsEnabled } = require('../fileUploader');
 const { getTestomatIdFromTestTitle, fileSystem } = require('../utils/utils');
 // const debug = require('debug')('@testomatio/reporter:adapter:playwright');
 const { services } = require('../services');
@@ -106,31 +105,37 @@ class PlaywrightReporter {
 
     await Promise.all(reportTestPromises);
 
-    if (this.uploads.length && isArtifactsEnabled()) {
-      console.log(APP_PREFIX, `🎞️  Uploading ${this.uploads.length} files...`);
+    if (!this.uploads.length) return;
+      
+    if (this.client.uploader.isEnabled) console.log(APP_PREFIX, `🎞️  Uploading ${this.uploads.length} files...`);
 
-      const promises = [];
+    const promises = [];
 
-      for (const upload of this.uploads) {
-        const { rid, file, title } = upload;
+    for (const upload of this.uploads) {
+      const { rid, file, title } = upload;
 
-        const files = upload.files.map(attachment => ({
-          path: this.#getArtifactPath(attachment),
-          title,
-          type: attachment.contentType,
-        }));
-
-        promises.push(
-          this.client.addTestRun(undefined, {
-            rid,
-            title,
-            files,
-            file,
-          }),
-        );
+      const files = upload.files.map(attachment => ({
+        path: this.#getArtifactPath(attachment),
+        title,
+        type: attachment.contentType,
+      }));
+
+      if (!this.client.uploader.isEnabled) {
+        files.forEach(f => this.client.uploader.storeUploadedFile(f, this.client.runId, rid, false));
+        continue;
       }
-      await Promise.all(promises);
+
+      promises.push(
+        this.client.addTestRun(undefined, {
+          rid,
+          title,
+          files,
+          file,
+        }),
+      );
     }
+    await Promise.all(promises);
+  
 
     await this.client.updateRunStatus(checkStatus(result.status));
   }
@@ -147,6 +152,9 @@ function checkStatus(status) {
 }
 
 function appendStep(step, shift = 0) {
+  // nesting too deep, ignore those steps
+  if (shift >= 10) return;
+
   let newCategory = step.category;
   switch (newCategory) {
     case 'test.step':
@@ -176,7 +184,7 @@ function appendStep(step, shift = 0) {
   };
 
   if (formattedSteps.length) {
-    resultStep.steps = formattedSteps;
+    resultStep.steps = formattedSteps.filter(s => !!s);
   }
 
   if (step.error !== undefined) {

package/lib/bin/cli.js

@@ -0,0 +1,220 @@
+#!/usr/bin/env node
+const { program } = require('commander');
+const spawn = require('cross-spawn');
+const chalk = require('chalk');
+const glob = require('glob');
+const debug = require('debug')('@testomatio/reporter:cli');
+const TestomatClient = require('../client');
+const XmlReader = require('../xmlReader');
+const { APP_PREFIX, STATUS } = require('../constants');
+const { version } = require('../../package.json');
+const config = require('../config');
+
+console.log(chalk.cyan.bold(` 🤩 Testomat.io Reporter v${version}`));
+
+program
+  .version(version)
+  .option('--env-file <envfile>', 'Load environment variables from env file')
+  .hook('preAction', (thisCommand) => {
+    const opts = thisCommand.opts();
+    if (opts.envFile) {
+      require('dotenv').config({ path: opts.envFile });
+    } else {
+      require('dotenv').config();
+    }
+  });
+
+program
+  .command('start')
+  .description('Start a new run and return its ID')
+  .action(async () => {
+    console.log('Starting a new Run on Testomat.io...');
+    const apiKey = process.env['INPUT_TESTOMATIO-KEY'] || config.TESTOMATIO;
+    const client = new TestomatClient({ apiKey });
+
+    client.createRun().then(() => {
+      console.log(process.env.runId);
+      process.exit(0);
+    });
+  });
+
+program
+  .command('finish')
+  .description('Finish Run by its ID')
+  .action(async () => {
+    if (!process.env.TESTOMATIO_RUN) {
+      console.log('TESTOMATIO_RUN environment variable must be set.');
+      return process.exit(1);
+    }
+
+    console.log('Finishing Run on Testomat.io...');
+    const apiKey = process.env['INPUT_TESTOMATIO-KEY'] || config.TESTOMATIO;
+    const client = new TestomatClient({ apiKey });
+
+    client.updateRunStatus(STATUS.FINISHED).then(() => {
+      console.log(chalk.yellow(`Run ${process.env.TESTOMATIO_RUN} was finished`));
+      process.exit(0);
+    });
+  });
+
+program
+  .command('run')
+  .description('Run tests with the specified command')
+  .argument('<command>', 'Test runner command')
+  .option('--filter <filter>', 'Additional execution filter')
+  .action(async (command, opts) => {
+    const apiKey = process.env['INPUT_TESTOMATIO-KEY'] || config.TESTOMATIO;
+    const title = process.env.TESTOMATIO_TITLE;
+
+    if (!command || !command.split) {
+      console.log(APP_PREFIX, `No command provided. Use -c option to launch a test runner.`);
+      return process.exit(255);
+    }
+
+    const client = new TestomatClient({ apiKey, title, parallel: true });
+
+    if (opts.filter) {
+      const [pipe, ...optsArray] = opts.filter.split(':');
+      const pipeOptions = optsArray.join(':');
+
+      try {
+        const tests = await client.prepareRun({ pipe, pipeOptions });
+        if (tests && tests.length > 0) {
+          command += ` --grep (${tests.join('|')})`;
+        }
+      } catch (err) {
+        console.log(APP_PREFIX, err);
+      }
+    }
+
+    console.log(APP_PREFIX, `🚀 Running`, chalk.green(command));
+
+    const runTests = () => {
+      const testCmds = command.split(' ');
+      const cmd = spawn(testCmds[0], testCmds.slice(1), { stdio: 'inherit' });
+
+      cmd.on('close', code => {
+        const emoji = code === 0 ? '🟢' : '🔴';
+        console.log(APP_PREFIX, emoji, `Runner exited with ${chalk.bold(code)}`);
+        if (apiKey) {
+          const status = code === 0 ? 'passed' : 'failed';
+          client.updateRunStatus(status, true);
+        }
+        process.exit(code);
+      });
+    };
+
+    if (apiKey) {
+      client.createRun().then(runTests);
+    } else {
+      runTests();
+    }
+  });
+
+program
+  .command('xml')
+  .description('Parse XML reports and upload to Testomat.io')
+  .argument('<pattern>', 'XML file pattern')
+  .option('-d, --dir <dir>', 'Project directory')
+  .option('--java-tests [java-path]', 'Load Java tests from path, by default: src/test/java')
+  .option('--lang <lang>', 'Language used (python, ruby, java)')
+  .option('--timelimit <time>', 'default time limit in seconds to kill a stuck process')
+  .action(async (pattern, opts) => {
+    if (!pattern.endsWith('.xml')) {
+      pattern += '.xml';
+    }
+    let { javaTests, lang } = opts;
+    if (javaTests === true) javaTests = 'src/test/java';
+    lang = lang?.toLowerCase();
+    const runReader = new XmlReader({ javaTests, lang });
+    const files = glob.sync(pattern, { cwd: opts.dir || process.cwd() });
+    if (!files.length) {
+      console.log(APP_PREFIX, `Report can't be created. No XML files found 😥`);
+      process.exit(1);
+    }
+
+    for (const file of files) {
+      console.log(APP_PREFIX, `Parsed ${file}`);
+      runReader.parse(file);
+    }
+
+    let timeoutTimer;
+    if (opts.timelimit) {
+      timeoutTimer = setTimeout(() => {
+        console.log(`⚠️  Reached timeout of ${opts.timelimit}s. Exiting... (Exit code is 0 to not fail the pipeline)`);
+        process.exit(0);
+      }, parseInt(opts.timelimit, 10) * 1000);
+    }
+
+    try {
+      await runReader.createRun();
+      await runReader.uploadData();
+    } catch (err) {
+      console.log(APP_PREFIX, 'Error updating status, skipping...', err);
+    }
+
+    if (timeoutTimer) clearTimeout(timeoutTimer);
+  });
+
+program
+  .command('upload-artifacts')
+  .description('Upload artifacts to Testomat.io')
+  .option('--force', 'Re-upload artifacts even if they were uploaded before')
+  .action(async (opts) => {
+    const apiKey = config.TESTOMATIO;
+    process.env.TESTOMATIO_DISABLE_ARTIFACTS = '';
+
+    const client = new TestomatClient({
+      apiKey,
+      isBatchEnabled: false,
+    });
+
+    if (!process.env.TESTOMATIO_RUN) {
+      console.log(APP_PREFIX, 'No run ID provided');
+      process.exit(1);
+    }
+
+    let testruns = client.uploader.readUploadedFiles(process.env.TESTOMATIO_RUN);
+    const numTotalArtifacts = testruns.length;
+
+    debug('Found testruns:', testruns);
+
+    if (!opts.force) testruns = testruns.filter(tr => !tr.uploaded);
+
+    if (!testruns.length) {
+      console.log(APP_PREFIX, 'Total artifacts:', numTotalArtifacts);
+      if (numTotalArtifacts) {
+        console.log(APP_PREFIX, 'No new artifacts to upload');
+        console.log(APP_PREFIX, 'To re-upload artifacts run this command with --force flag');
+      }
+      process.exit(0);
+    }
+
+    const testrunsByRid = testruns.reduce((acc, { rid, file }) => {
+      if (!acc[rid]) {
+        acc[rid] = [];
+      }
+      if (!acc[rid].includes(file)) acc[rid].push(file);
+      return acc;
+    }, {});
+
+    await client.createRun();
+    client.uploader.checkEnabled();
+    client.uploader.disbleLogStorage();
+
+    for (const rid in testrunsByRid) {
+      const files = testrunsByRid[rid];
+      await client.addTestRun(undefined, { rid, files });
+    }
+
+    console.log(APP_PREFIX, client.uploader.totalUploaded, 'artifacts uploaded');
+    if (client.uploader.failedUpload) {
+      console.log(APP_PREFIX, client.uploader.failedUpload, 'artifacts failed to upload');
+    }
+  });
+
+program.parse(process.argv);
+
+if (!process.argv.slice(2).length) {
+  program.outputHelp();
+}

package/lib/bin/startTest.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-const { spawn } = require('child_process');
+const spawn = require('cross-spawn');
 const program = require('commander');
 const chalk = require('chalk');
 const TestomatClient = require('../client');
@@ -45,7 +45,7 @@ program
 
       const client = new TestomatClient({ apiKey });
 
-      client.updateRunStatus(STATUS.FINISHED, true).then(() => {
+      client.updateRunStatus(STATUS.FINISHED).then(() => {
         console.log(chalk.yellow(`Run ${process.env.TESTOMATIO_RUN} was finished`));
         process.exit(0);
       });

package/lib/bin/uploadArtifacts.js

@@ -0,0 +1,91 @@
+#!/usr/bin/env node
+const program = require('commander');
+const chalk = require('chalk');
+const debug = require('debug')('@testomatio/reporter:upload-cli');
+const TestomatClient = require('../client');
+const { APP_PREFIX } = require('../constants');
+const { version } = require('../../package.json');
+const config = require('../config');
+
+console.log(chalk.cyan.bold(` 🤩 Testomat.io Reporter v${version}`));
+
+program
+  .option('--env-file <envfile>', 'Load environment variables from env file')
+  .option('--force', 'Re-upload artifacts even if they were uploaded before')
+  .action(async opts => {
+
+    if (opts.envFile) {
+      require('dotenv').config(opts.envFile); // eslint-disable-line
+    } else {
+      // try to load from env file
+      require('dotenv').config(); // eslint-disable-line
+    }
+
+    const apiKey = config.TESTOMATIO;
+
+    process.env.TESTOMATIO_DISABLE_ARTIFACTS = '';
+    // process.env.TESTOMATIO_ARTIFACTS_SIZE = null;    
+
+    const client = new TestomatClient({ 
+      apiKey,
+      isBatchEnabled: false,
+     });
+
+    if (!process.env.TESTOMATIO_RUN) {
+      console.log(APP_PREFIX, 'No run ID provided');
+      process.exit(1);
+    }
+
+    let testruns = client.uploader.readUploadedFiles(process.env.TESTOMATIO_RUN);
+
+    const numTotalArtifacts = testruns.length;
+
+    debug('Found testruns:', testruns);
+
+    if (!opts.force) testruns = testruns.filter(tr => !tr.uploaded);
+
+    if (!testruns.length) {
+      console.log(APP_PREFIX, 'Total artifacts:', numTotalArtifacts);
+      if (numTotalArtifacts) {
+        console.log(APP_PREFIX, 'No new artifacts to upload');
+        console.log(APP_PREFIX, 'To re-upload artifacts run this command with --force flag');
+      }
+      process.exit(0);
+    }
+
+    const testrunsByRid = testruns.reduce((acc, { rid, file }) => {
+      if (!acc[rid]) {
+        acc[rid] = [];
+      }
+      if (!acc[rid].includes(file)) acc[rid].push(file);
+      return acc;
+    }, {});
+
+    let numArtifacts = 0;
+
+    // we need to obtain S3 credentials
+    await client.createRun();
+
+    client.uploader.checkEnabled();
+    client.uploader.disbleLogStorage();
+
+    for (const rid in testrunsByRid) {
+      const files = testrunsByRid[rid];
+      numArtifacts += files.length;
+      await client.addTestRun(undefined, {
+        rid,
+        files,
+      });
+    }
+
+  console.log(APP_PREFIX, client.uploader.totalUploaded, 'artifacts uploaded');
+    if (client.uploader.failedUpload) {
+      console.log(APP_PREFIX, client.uploader.failedUpload, 'artifacts failed to upload');
+    }
+  });
+
+if (process.argv.length <= 1) {
+  program.outputHelp();
+}
+
+program.parse(process.argv);

package/lib/client.js

@@ -5,7 +5,7 @@ const { minimatch } = require('minimatch');
 const fs = require('fs');
 const chalk = require('chalk');
 const { randomUUID } = require('crypto');
-const upload = require('./fileUploader');
+const S3Uploader = require('./uploader');
 const { APP_PREFIX, STATUS } = require('./constants');
 const pipesFactory = require('./pipe');
 const { glob } = require('glob');
@@ -26,14 +26,14 @@ class Client {
    * @param {*} params
    */
   constructor(params = {}) {
-    const store = {};
-    this.uuid = randomUUID();
-    this.pipes = pipesFactory(params, store);
+    this.pipeStore = {};
+    this.runId = randomUUID(); // will be replaced by real run id
+    this.pipes = pipesFactory(params, this.pipeStore);
     this.queue = Promise.resolve();
-    this.totalUploaded = 0;
-    this.failedToUpload = 0;
     this.version = JSON.parse(fs.readFileSync(join(__dirname, '..', 'package.json')).toString()).version;
     this.executionList = Promise.resolve();
+    this.uploader = new S3Uploader();
+    this.uploader.checkEnabled();
 
     console.log(APP_PREFIX, `Testomatio Reporter v${this.version}`);
   }
@@ -104,7 +104,13 @@ class Client {
     this.queue = this.queue
       .then(() => Promise.all(this.pipes.map(p => p.createRun())))
       .catch(err => console.log(APP_PREFIX, err))
+      .then(() => {
+        const runId = this.pipeStore?.runId;
+        if (runId) this.runId = runId;
+        this.uploader.checkEnabled();
+      })
       .then(() => undefined); // fixes return type
+
     // debug('Run', this.queue);
     return this.queue;
   }
@@ -169,24 +175,28 @@ class Client {
 
     const uploadedFiles = [];
 
-    for (const f of files) {
-      uploadedFiles.push(upload.uploadFileByPath(f, this.uuid));
+    for (let f of files) {
+      if (typeof f === 'object') {
+        if (!f.path) continue;
+
+        f = f.path;
+      }
+
+      uploadedFiles.push(this.uploader.uploadFileByPath(f, [
+        this.runId,
+        rid,
+        path.basename(f)
+      ]));
+
     }
 
     for (const [idx, buffer] of filesBuffers.entries()) {
       const fileName = `${idx + 1}-${title.replace(/\s+/g, '-')}`;
-      uploadedFiles.push(upload.uploadFileAsBuffer(buffer, fileName, this.uuid));
+      uploadedFiles.push(this.uploader.uploadFileAsBuffer(buffer, [this.runId, rid, fileName]));
     }
 
     const artifacts = (await Promise.all(uploadedFiles)).filter(n => !!n);
 
-    if (artifacts.length < uploadedFiles.length) {
-      const failedUploading = uploadedFiles.length - artifacts.length;
-      this.failedToUpload += failedUploading;
-    }
-
-    this.totalUploaded += artifacts.length;
-
     const data = {
       rid,
       files,
@@ -242,28 +252,30 @@ class Client {
     this.queue = this.queue
       .then(() => Promise.all(this.pipes.map(p => p.finishRun(runParams))))
       .then(() => {
-        debug('TOTAL artifacts', this.totalUploaded);
-        if (this.totalUploaded && !upload.isArtifactsEnabled())
-          debug(`${this.totalUploaded} artifacts are not uploaded, because artifacts uploading is not enabled`);
+        debug('TOTAL artifacts', this.uploader.totalUploaded);        
+        debug(`${this.uploader.skippedUpload} artifacts skipped`);
 
-        if (this.totalUploaded && upload.isArtifactsEnabled()) {
+        if (this.uploader.totalUploaded && this.uploader.isEnabled) {
           console.log(
             APP_PREFIX,
-            `🗄️ ${this.totalUploaded} artifacts ${
+            `🗄️ ${this.uploader.totalUploaded} artifacts ${
               process.env.TESTOMATIO_PRIVATE_ARTIFACTS ? 'privately' : chalk.bold('publicly')
             } uploaded to S3 bucket`,
-          );
-
-          if (this.failedToUpload > 0) {
-            console.log(
-              APP_PREFIX,
-              chalk.yellow(
-                `Some artifacts were not uploaded. ${this.failedToUpload} artifacts could not be uploaded.
-                Run tests with DEBUG="@testomatio/reporter:file-uploader" to see details"`,
-              ),
-            );
-          }
+          );          
+        }
+
+        if (this.uploader.failedUpload) {
+          console.log(APP_PREFIX, `${this.uploader.failedUpload} artifacts failed to upload`);
+        }
+
+        if (this.uploader.isEnabled && this.uploader.skippedUpload) {
+          console.log(APP_PREFIX, `${chalk.bold(this.uploader.skippedUpload)} artifacts skipped to upload`);
         }
+
+        if (this.uploader.skippedUpload || this.uploader.failedUpload) {
+          console.log(APP_PREFIX, `Run "${chalk.magenta(`TESTOMATIO_RUN=${this.runId} npx upload-artifacts`)}" with valid S3 credentials to upload skipped & failed artifacts`);
+        }
+
       })
       .catch(err => console.log(APP_PREFIX, err));
 
@@ -279,7 +291,10 @@ class Client {
     logs = logs?.trim();
 
     if (Array.isArray(steps)) {
-      steps = steps.map(step => formatStep(step)).flat().join('\n');      
+      steps = steps
+        .map(step => formatStep(step))
+        .flat()
+        .join('\n');
     }
 
     let testLogs = '';
@@ -358,8 +373,8 @@ function formatStep(step, shift = 0) {
 
   for (const child of step.steps || []) {
     lines.push(...formatStep(child, shift + 2));
-  } 
-  
+  }
+
   return lines;
 }