@testomatio/reporter 1.1.0-beta.mocha-create.9 → 1.1.0-rc.1

package/README.md

@@ -153,3 +153,4 @@ To print all reporter logs of a specific pipe:
 ```
 DEBUG=@testomatio/reporter:pipe:github
 ```
+

package/lib/adapter/codecept.js

@@ -4,6 +4,7 @@ const TestomatClient = require('../client');
 const { STATUS, APP_PREFIX, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
 const upload = require('../fileUploader');
 const { parseTest: getIdFromTestTitle, fileSystem } = require('../utils/utils');
+const { services } = require('../services');
 
 if (!global.codeceptjs) {
   // eslint-disable-next-line global-require, import/no-extraneous-dependencies
@@ -51,21 +52,56 @@ function CodeceptReporter(config) {
 
   recorder.startUnlessRunning();
 
-  global.testomatioRunningEnvironment = 'codeceptjs';
-
   // Listening to events
   event.dispatcher.on(event.all.before, () => {
     // clear tmp dir
     fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
 
-    recorder.add('Creating new run', () => client.createRun());
+    // recorder.add('Creating new run', () => );
+    client.createRun();
     videos = [];
     traces = [];
 
     if (!global.testomatioDataStore) global.testomatioDataStore = {};
   });
 
-  event.dispatcher.on(event.test.before, () => {
+  let hookSteps = [];
+  let suiteHookRunning = false;
+
+  event.dispatcher.on(event.suite.before, suite => {
+    suiteHookRunning = true;
+    hookSteps = [];
+    global.testomatioDataStore.steps = [];
+
+    services.setContext(suite.fullTitle());
+  });
+
+  event.dispatcher.on(event.suite.after, () => {
+    services.setContext(null);
+  });
+
+  event.dispatcher.on(event.hook.started, () => {
+    // global.testomatioDataStore.steps = [];
+  });
+
+  event.dispatcher.on(event.hook.passed, () => {
+    if (suiteHookRunning) {
+      hookSteps.push(...global.testomatioDataStore.steps);
+      services.setContext(null);
+    }
+  });
+
+  event.dispatcher.on(event.hook.failed, () => {
+    if (suiteHookRunning) {
+      hookSteps.push(...global.testomatioDataStore.steps);
+      services.setContext(null);
+    }
+  });
+
+  event.dispatcher.on(event.test.before, test => {
+    suiteHookRunning = false;
+    global.testomatioDataStore.steps = [];
+
     recorder.add(() => {
       currentMetaStep = [];
       // output.reset();
@@ -76,11 +112,16 @@ function CodeceptReporter(config) {
     if (!global.testomatioDataStore) global.testomatioDataStore = {};
     // reset steps
     global.testomatioDataStore.steps = [];
+
+    services.setContext(test.fullTitle());
   });
 
   event.dispatcher.on(event.test.started, test => {
+    services.setContext(test.fullTitle());
+
     testTimeMap[test.id] = Date.now();
     if (global.testomatioDataStore) global.testomatioDataStore.currentlyRunningTestId = getIdFromTestTitle(test.title);
+    // start logging
   });
 
   event.dispatcher.on(event.all.result, async () => {
@@ -104,6 +145,12 @@ function CodeceptReporter(config) {
     }
     const testId = parseTest(tags);
     const testObj = getTestAndMessage(title);
+
+    const logs = getTestLogs(test);
+    const manuallyAttachedArtifacts = services.artifacts.get(test.fullTitle());
+    const keyValues = services.keyValues.get(test.fullTitle());
+    services.setContext(null);
+
     client.addTestRun(STATUS.PASSED, {
       ...stripExampleFromTitle(title),
       suite_title: test.parent && test.parent.title,
@@ -111,6 +158,9 @@ function CodeceptReporter(config) {
       time: getDuration(test),
       steps: global.testomatioDataStore.steps.join('\n') || null,
       test_id: testId,
+      logs,
+      manuallyAttachedArtifacts,
+      meta: keyValues,
     });
     // output.stop();
   });
@@ -152,6 +202,11 @@ function CodeceptReporter(config) {
     if (artifacts.screenshot) files.push({ path: artifacts.screenshot, type: 'image/png' });
     // todo: video must be uploaded later....
 
+    const logs = getTestLogs(test);
+    const manuallyAttachedArtifacts = services.artifacts.get(test.fullTitle());
+    const keyValues = services.keyValues.get(test.fullTitle());
+    services.setContext(null);
+
     const reportTestPromise = client
       .addTestRun(STATUS.FAILED, {
         ...stripExampleFromTitle(title),
@@ -162,6 +217,9 @@ function CodeceptReporter(config) {
         time: getDuration(test),
         files,
         steps: global.testomatioDataStore?.steps?.join('\n') || null,
+        logs,
+        manuallyAttachedArtifacts,
+        meta: keyValues,
       })
       .then(pipes => {
         testId = pipes.filter(p => p.pipe.includes('Testomatio'))[0]?.result?.data?.test_id;
@@ -230,7 +288,7 @@ function CodeceptReporter(config) {
     currentMetaStep = metaSteps;
     stepShift = 2 * shift;
 
-    const durationMs = +new Date() - (+stepStart);
+    const durationMs = +new Date() - +stepStart;
     let duration = '';
     if (durationMs) {
       duration = repeat(1) + chalk.grey(`(${durationMs}ms)`);
@@ -255,7 +313,7 @@ async function uploadAttachments(client, attachments, messagePrefix, attachmentT
   if (attachments.length > 0) {
     console.log(APP_PREFIX, `Attachments: ${messagePrefix} ${attachments.length} ${attachmentType}/-s ...`);
 
-    const promises = attachments.map(async (attachment) => {
+    const promises = attachments.map(async attachment => {
       const { testId, title, path, type } = attachment;
       const file = { path, type, title };
       return client.addTestRun(undefined, {
@@ -303,4 +361,21 @@ function repeat(num) {
   return ''.padStart(num, ' ');
 }
 
+// TODO: think about moving to some common utils
+function getTestLogs(test) {
+  const suiteLogsArr = services.logger.getLogs(test.parent.fullTitle());
+  const suiteLogs = suiteLogsArr ? suiteLogsArr.join('\n').trim() : '';
+  const testLogsArr = services.logger.getLogs(test.fullTitle());
+  const testLogs = testLogsArr ? testLogsArr.join('\n').trim() : '';
+
+  let logs = '';
+  if (suiteLogs) {
+    logs += `${chalk.bold('\t--- BeforeSuite ---')}\n${suiteLogs}`;
+  }
+  if (testLogs) {
+    logs += `\n${chalk.bold('\t--- Test ---')}\n${testLogs}`;
+  }
+  return logs;
+}
+
 module.exports = CodeceptReporter;

package/lib/adapter/cucumber/current.js

@@ -4,8 +4,9 @@ const chalk = require('chalk');
 const fs = require('fs');
 const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../../constants');
 const TestomatClient = require('../../client');
-const logger = require('../../storages/logger');
 const { parseTest, fileSystem } = require('../../utils/utils');
+const { TESTOMATIO } = require('../../config');
+const { services } = require('../../services');
 
 const { GherkinDocumentParser, PickleParser } = formatterHelpers;
 const { getGherkinScenarioLocationMap, getGherkinStepMap } = GherkinDocumentParser;
@@ -29,10 +30,8 @@ class CucumberReporter extends Formatter {
     this.failures = [];
     this.cases = [];
 
-    this.client = new TestomatClient({ apiKey: options.apiKey || process.env.TESTOMATIO });
+    this.client = new TestomatClient({ apiKey: options.apiKey || TESTOMATIO });
     this.status = STATUS.PASSED;
-
-    global.testomatioRunningEnvironment = 'cucumber:current';
   }
 
   parseEnvelope(envelope) {
@@ -49,9 +48,10 @@ class CucumberReporter extends Formatter {
 
   onTestCaseStarted(testCaseStarted) {
     const testCaseAttempt = this.eventDataCollector.getTestCaseAttempt(testCaseStarted.id);
-    const testId = getTestId(testCaseAttempt.pickle);
     if (!global.testomatioDataStore) global.testomatioDataStore = {};
-    global.testomatioDataStore.currentlyRunningTestId = testId;
+
+    const testTitle = testCaseAttempt.pickle.name + testCaseAttempt.pickle.uri;
+    services.setContext(testTitle);
   }
 
   onTestCaseFinished(testCaseFinished) {
@@ -109,7 +109,10 @@ class CucumberReporter extends Formatter {
 
     if (!this.client) return;
 
-    const logs = logger.getLogs(testId);
+    const testTitle = testCaseAttempt.pickle.name + testCaseAttempt.pickle.uri;
+    const logs = services.logger.getLogs(testTitle).join('\n');
+    const artifacts = services.artifacts.get(testTitle);
+    const keyValues = services.keyValues.get(testTitle);
 
     this.client.addTestRun(status, {
       // error: testCaseAttempt.worstTestStepResult.message,
@@ -119,11 +122,15 @@ class CucumberReporter extends Formatter {
         .join('\n')
         .trim(),
       example: { ...example },
-      stack: logs,
+      logs,
+      manuallyAttachedArtifacts: artifacts,
+      meta: keyValues,
       title: scenario,
       test_id: testId,
       time,
     });
+
+    services.setContext(null);
   }
 
   onTestRunFinished(envelope) {

package/lib/adapter/cucumber/legacy.js

@@ -4,6 +4,7 @@ const chalk = require('chalk');
 const { parseTest, fileSystem } = require('../../utils/utils');
 const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../../constants');
 const TestomatClient = require('../../client');
+const { TESTOMATIO } = require('../../config');
 
 const createTestomatFormatter = apiKey => {
   if (!apiKey || apiKey === '') {
@@ -151,4 +152,4 @@ const createTestomatFormatter = apiKey => {
   };
 };
 
-module.exports = createTestomatFormatter(process.env.TESTOMATIO);
+module.exports = createTestomatFormatter(TESTOMATIO);

package/lib/adapter/cypress-plugin/index.js

@@ -1,18 +1,19 @@
 const { STATUS } = require('../../constants');
 const { parseTest, parseSuite } = require('../../utils/utils');
 const TestomatClient = require('../../client');
+const { TESTOMATIO } = require('../../config');
 
 const testomatioReporter = on => {
-  if (!process.env.TESTOMATIO) {
+  if (!TESTOMATIO) {
     console.log('TESTOMATIO key is empty, ignoring reports');
-    return
+    return;
   }
-  const client = new TestomatClient({ apiKey: process.env.TESTOMATIO });
+  const client = new TestomatClient({ apiKey: TESTOMATIO });
 
-  on('before:run', async (run) => {
+  on('before:run', async run => {
     // TODO: looks like client.env does not exist
     if (!client.env) {
-      client.env = `${run.browser.displayName},${run.system.osName}`
+      client.env = `${run.browser.displayName},${run.system.osName}`;
     }
     await client.createRun();
   });
@@ -26,8 +27,9 @@ const testomatioReporter = on => {
       const lastAttemptIndex = test.attempts.length - 1;
       const latestAttempt = test.attempts[lastAttemptIndex];
 
-      const time = latestAttempt.duration;
-      const error = latestAttempt.error;
+      // latestAttempt.duration && latestAttempt.error were available in adapters version up to 13 JFYI
+      const time = latestAttempt.duration || latestAttempt.wallClockDuration || test.duration;
+      let error = latestAttempt.error;
 
       let title = test.title.pop();
       const examples = title.match(/\(example (#\d+)\)/);
@@ -40,23 +42,28 @@ const testomatioReporter = on => {
       const testId = parseTest(title);
       const suiteId = parseSuite(suiteTitle);
 
-      if (error) {
-        error.inspect = function() { // eslint-disable-line func-names
-          if (this && this.codeFrame) {
-            return this.codeFrame.frame;
-          }
-          return '';
-        }
+      if (!error && test.displayError) {
+        error = { message: test.displayError };
+        error.inspect = function () {
+          // eslint-disable-line func-names
+          return this.message;
+        };
       }
 
+      const formattedError = error
+        ? {
+            message: error.message,
+            inspect:
+              error.inspect ||
+              function () {
+                return this.message;
+              },
+          }
+        : '';
+
       const screenshots = Array.isArray(results.screenshots)
         ? results.screenshots
-            .filter(
-              screenshot =>
-                screenshot?.path &&
-                screenshot?.path.includes(title) &&
-                screenshot?.takenAt
-            )
+            .filter(screenshot => screenshot?.path && screenshot?.path.includes(title) && screenshot?.takenAt)
             .map(screenshot => screenshot.path)
         : [];
 
@@ -64,17 +71,30 @@ const testomatioReporter = on => {
 
       let state;
       switch (test.state) {
-        case 'passed': state = STATUS.PASSED; break;
-        case 'failed': state = STATUS.FAILED; break;
+        case 'passed':
+          state = STATUS.PASSED;
+          break;
+        case 'failed':
+          state = STATUS.FAILED;
+          break;
         case 'skipped':
-        case 'pending': 
+        case 'pending':
         default:
           state = STATUS.SKIPPED;
       }
 
-      addSpecTestsPromises.push(client.addTestRun(state, {
-        title, time, example, error, files, suite_title: suiteTitle, test_id: testId, suite_id: suiteId
-      }));
+      addSpecTestsPromises.push(
+        client.addTestRun(state, {
+          title,
+          time,
+          example,
+          error: formattedError,
+          files,
+          suite_title: suiteTitle,
+          test_id: testId,
+          suite_id: suiteId,
+        }),
+      );
     }
 
     await Promise.all(addSpecTestsPromises);

package/lib/adapter/jest.js

@@ -1,6 +1,9 @@
+const chalk = require('chalk');
 const TestomatClient = require('../client');
 const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
 const { parseTest, ansiRegExp, fileSystem } = require('../utils/utils');
+const { services } = require('../services');
+const debug = require('debug')('@testomatio/reporter:adapter-jest');
 
 class JestReporter {
   constructor(globalConfig, options) {
@@ -16,6 +19,19 @@ class JestReporter {
     fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
   }
 
+  // start of test file (including beforeAll)
+  onTestStart(testFile) {
+    debug('Start running test file:', testFile.path);
+    services.setContext(testFile.path);
+  }
+
+  // start of the test (including beforeEach)
+  onTestCaseStart(test, testCase) {
+    debug('Start running test:', testCase.fullName);
+    services.setContext(testCase.fullName);
+  }
+
+  // end of test file! (there is also onTestCaseResult listener)
   onTestResult(test, testResult) {
     if (!this.client) return;
 
@@ -39,6 +55,10 @@ class JestReporter {
         suite_title = result.fullName.replace(result.title, '');
       }
 
+      const logs = getTestLogs(result);
+      const artifacts = services.artifacts.get(testResult.fullName);
+      const keyValues = services.keyValues.get(testResult.fullName);
+
       const deducedStatus = status === 'pending' ? 'skipped' : status;
       // In jest if test is not matched with test name pattern it is considered as skipped.
       // So adding a check if it is skipped for real or because of test pattern
@@ -50,6 +70,9 @@ class JestReporter {
           steps,
           title,
           time: duration,
+          logs,
+          manuallyAttachedArtifacts: artifacts,
+          meta: keyValues,
         });
       }
     }
@@ -64,4 +87,20 @@ class JestReporter {
   }
 }
 
+function getTestLogs(testResult) {
+  const suiteLogsArr = services.logger.getLogs(testResult.testFilePath);
+  const suiteLogs = suiteLogsArr ? suiteLogsArr.join('\n').trim() : '';
+  const testLogsArr = services.logger.getLogs(testResult.fullName);
+  const testLogs = testLogsArr ? testLogsArr.join('\n').trim() : '';
+
+  let logs = '';
+  if (suiteLogs) {
+    logs += `${chalk.bold('\t--- Suite ---')}\n${suiteLogs}`;
+  }
+  if (testLogs) {
+    logs += `\n${chalk.bold('\t--- Test ---')}\n${testLogs}`;
+  }
+  return logs;
+}
+
 module.exports = JestReporter;

package/lib/adapter/mocha.js

@@ -4,9 +4,20 @@ const chalk = require('chalk');
 const TestomatClient = require('../client');
 const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
 const { parseTest, fileSystem } = require('../utils/utils');
-
-const { EVENT_RUN_BEGIN, EVENT_RUN_END, EVENT_TEST_FAIL, EVENT_TEST_PASS, EVENT_TEST_PENDING } =
-  Mocha.Runner.constants;
+const { TESTOMATIO } = require('../config');
+const { services } = require('../services');
+
+const {
+  EVENT_RUN_BEGIN,
+  EVENT_RUN_END,
+  EVENT_TEST_FAIL,
+  EVENT_TEST_PASS,
+  EVENT_TEST_PENDING,
+  EVENT_SUITE_BEGIN,
+  EVENT_SUITE_END,
+  EVENT_TEST_BEGIN,
+  EVENT_TEST_END,
+} = Mocha.Runner.constants;
 
 function MochaReporter(runner, opts) {
   Mocha.reporters.Base.call(this, runner);
@@ -15,7 +26,7 @@ function MochaReporter(runner, opts) {
   let skipped = 0;
   // let artifactStore;
 
-  const apiKey = opts?.reporterOptions?.apiKey || process.env.TESTOMATIO;
+  const apiKey = opts?.reporterOptions?.apiKey || TESTOMATIO;
 
   const client = new TestomatClient({ apiKey });
 
@@ -25,12 +36,32 @@ function MochaReporter(runner, opts) {
     fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
   });
 
+  runner.on(EVENT_SUITE_BEGIN, async suite => {
+    services.setContext(suite.fullTitle());
+  });
+
+  runner.on(EVENT_SUITE_END, async () => {
+    services.setContext(null);
+  });
+
+  runner.on(EVENT_TEST_BEGIN, async test => {
+    services.setContext(test.fullTitle());
+  });
+
+  runner.on(EVENT_TEST_END, async () => {
+    services.setContext(null);
+  });
+
   runner.on(EVENT_TEST_PASS, async test => {
     passes += 1;
   
     console.log(chalk.bold.green('✔'), test.fullTitle());
     const testId = parseTest(test.title);
 
+    const logs = getTestLogs(test);
+    const artifacts = services.artifacts.get(test.fullTitle());
+    const keyValues = services.keyValues.get(test.fullTitle());
+
     client.addTestRun(STATUS.PASSED, {
       test_id: testId,
       suite_title: getSuiteTitle(test),
@@ -38,6 +69,9 @@ function MochaReporter(runner, opts) {
       code: test.body.toString(),
       file: getFile(test),
       time: test.duration,
+      logs,
+      manuallyAttachedArtifacts: artifacts,
+      meta: keyValues,
     });
   });
 
@@ -60,6 +94,8 @@ function MochaReporter(runner, opts) {
     console.log(chalk.bold.red('✖'), test.fullTitle(), chalk.gray(err.message));
     const testId = parseTest(test.title);
 
+    const logs = getTestLogs(test);
+
     client.addTestRun(STATUS.FAILED, {
       error: err,
       suite_title: getSuiteTitle(test),
@@ -68,6 +104,7 @@ function MochaReporter(runner, opts) {
       title: getTestName(test),
       code: test.body.toString(),
       time: test.duration,
+      logs,
     });
   });
 
@@ -78,13 +115,29 @@ function MochaReporter(runner, opts) {
   });
 }
 
+function getTestLogs(test) {
+
+  const suiteLogsArr = services.logger.getLogs(test.parent.fullTitle());
+  const suiteLogs = suiteLogsArr ? suiteLogsArr.join('\n').trim() : '';
+  const testLogsArr = services.logger.getLogs(test.fullTitle());
+  const testLogs = testLogsArr ? testLogsArr.join('\n').trim() : '';
+
+  let logs = '';
+  if (suiteLogs) {
+    logs += `${chalk.bold('\t--- BeforeSuite ---')}\n${suiteLogs}`;
+  }
+  if (testLogs) {
+    logs += `\n${chalk.bold('\t--- Test ---')}\n${testLogs}`;
+  }
+  return logs;
+}
+
 // To have this reporter "extend" a built-in reporter uncomment the following line:
 Mocha.utils.inherits(MochaReporter, Mocha.reporters.Spec);
 
 module.exports = MochaReporter;
 
 function getSuiteTitle(test, pathArr = []) {
-  let root = pathArr.length === 0;
   
   if (test.parent.parent) getSuiteTitle(test.parent, pathArr);    
 

package/lib/adapter/playwright.js

@@ -25,18 +25,13 @@ class PlaywrightReporter {
     this.client.createRun();
   }
 
-  /*
-  onTestBegin(test) {
-    // does not work; value is not to the storage context
-    global.testTitle = test.title;
-  } */
-
   onTestEnd(test, result) {
     if (!this.client) return;
 
-    let testId = parseTest(test.title);
-
     const { title } = test;
+
+    let testId = parseTest(title);
+
     const { error, duration } = result;
 
     const suite_title = test.parent ? test.parent.title : null;
@@ -59,7 +54,8 @@ class PlaywrightReporter {
         title,
         steps: steps.join('\n'),
         time: duration,
-        stack: logs,
+        logs,
+        // TODO: attach artifacts
       })
       .then(pipes => {
         testId = pipes?.filter(p => p.pipe.includes('Testomatio'))[0]?.result?.data?.test_id;
@@ -143,4 +139,20 @@ function tmpFile(prefix = 'tmp.') {
   return path.join(tmpdir, prefix + crypto.randomBytes(16).toString('hex'));
 }
 
+function initPlaywrightForStorage() {
+  try {
+    // @ts-ignore-next-line
+    // eslint-disable-next-line import/no-unresolved
+    const { test } = require('@playwright/test');
+    // eslint-disable-next-line no-empty-pattern
+    test.beforeEach(async ({}, testInfo) => {
+      const fullTestTitle = `${testInfo.file}_${testInfo.title}`;
+      global.testomatioTestTitle = fullTestTitle;
+    });
+  } catch (e) {
+    // ignore
+  }
+}
+
 module.exports = PlaywrightReporter;
+module.exports.initPlaywrightForStorage = initPlaywrightForStorage;

package/lib/bin/startTest.js

@@ -5,6 +5,7 @@ const chalk = require('chalk');
 const TestomatClient = require('../client');
 const { APP_PREFIX, STATUS } = require('../constants');
 const { version } = require('../../package.json');
+const { TESTOMATIO } = require('../config');
 
 console.log(chalk.cyan.bold(` 🤩 Testomat.io Reporter v${version}`));
 
@@ -20,7 +21,7 @@ program
 
     if (opts.envFile) require('dotenv').config(opts.envFile); // eslint-disable-line
 
-    const apiKey = process.env['INPUT_TESTOMATIO-KEY'] || process.env.TESTOMATIO;
+    const apiKey = process.env['INPUT_TESTOMATIO-KEY'] || TESTOMATIO;
     const title = process.env.TESTOMATIO_TITLE;
 
     if (launch) {