@testomatio/reporter 2.1.0-beta-nightwatch → 2.1.0-beta.1-codeceptjs

package/src/adapter/codecept.js

@@ -4,7 +4,7 @@ import TestomatClient from '../client.js';
 import { STATUS, APP_PREFIX, TESTOMAT_TMP_STORAGE_DIR } from '../constants.js';
 import { getTestomatIdFromTestTitle, fileSystem } from '../utils/utils.js';
 import { services } from '../services/index.js';
-// eslint-disable-next-line
+import { dataStorage } from '../data-storage.js';
 import codeceptjs from 'codeceptjs';
 
 const debug = createDebugMessages('@testomatio/reporter:adapter:codeceptjs');
@@ -15,19 +15,15 @@ if (!global.codeceptjs) {
 }
 
 // @ts-ignore
-const { event, recorder, codecept } = global.codeceptjs;
+const { event, recorder, codecept, output } = global.codeceptjs;
 
-let currentMetaStep = [];
-let error;
-let stepShift = 0;
+const [, MAJOR_VERSION, MINOR_VERSION] = codecept.version().match(/(\d+)\.(\d+)/).map(Number);
 
-// const output = new Output({
-//   filterFn: stack => !stack.includes('codeceptjs/lib/output'), // output from codeceptjs
-// });
-
-let stepStart = new Date();
-
-const MAJOR_VERSION = parseInt(codecept.version().match(/\d/)[0], 10);
+// Constants for hook execution order
+const HOOK_EXECUTION_ORDER = {
+  PRE_TEST: ['BeforeSuiteHook', 'BeforeHook'],
+  POST_TEST: ['AfterHook', 'AfterSuiteHook']
+};
 
 const DATA_REGEXP = /[|\s]+?(\{".*\}|\[.*\])/;
 
@@ -35,8 +31,12 @@ if (MAJOR_VERSION < 3) {
   console.log('🔴 This reporter works with CodeceptJS 3+, please update your tests');
 }
 
+if (MAJOR_VERSION === 3 && MINOR_VERSION < 7) {
+  console.log('🔴 CodeceptJS 3.7+ is supported, please upgrade CodeceptJS or use 1.6 version of `@testomatio/reporter`');
+}
+
 function CodeceptReporter(config) {
-  let failedTests = [];
+  const failedTests = [];
   let videos = [];
   let traces = [];
   const reportTestPromises = [];
@@ -44,92 +44,104 @@ function CodeceptReporter(config) {
   const testTimeMap = {};
   const { apiKey } = config;
 
-  const getDuration = test => {
-    if (!test.uid) return 0;
-    if (testTimeMap[test.uid]) {
-      return Date.now() - testTimeMap[test.uid];
-    }
+  const client = new TestomatClient({ apiKey });
 
-    return 0;
+  // Store original output methods for fallback
+  const originalOutput = {
+    debug: output.debug,
+    log: output.log,
+    step: output.step,
+    say: output.say,
   };
 
-  const client = new TestomatClient({ apiKey });
+  output.debug = function(msg) {
+    originalOutput.debug(msg);
+    dataStorage.putData('log', repeat(this.stepShift) + pc.cyan(msg.toString()));
+  };
+
+  output.say = function(message, color = 'cyan') {
+    originalOutput.say(message, color);
+    const sayMsg = repeat(this.stepShift) + `  ${pc.bold(pc[color](message))}`;
+    dataStorage.putData('log', sayMsg);
+  };
+
+  output.log = function(msg) {
+    originalOutput.log(msg);
+    dataStorage.putData('log', repeat(this.stepShift) + pc.gray(msg));
+  };
 
   recorder.startUnlessRunning();
 
+  const hookSteps = new Map();
+  let currentHook = null;
+
+  event.dispatcher.on(event.workers.before, () => {
+    recorder.add('Creating new run', async () => {
+      await client.createRun();
+      process.env.TESTOMATIO_RUN = client.runId;
+      process.env.TESTOMATIO_PROCEED = 'true';
+      debug('Run ID:', client.runId);
+    });
+  });
+
+  event.dispatcher.on(event.workers.after, () => {
+    client.updateRunStatus('finished');
+  });
+
   // Listening to events
   event.dispatcher.on(event.all.before, () => {
     // clear tmp dir
-    fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
+    // fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
 
     // recorder.add('Creating new run', () => );
-    client.createRun();
+    recorder.add('Creating new run', () => {
+      return client.createRun();
+    });
     videos = [];
     traces = [];
 
     if (!global.testomatioDataStore) global.testomatioDataStore = {};
   });
 
-  let hookSteps = [];
-  let suiteHookRunning = false;
-
-  event.dispatcher.on(event.suite.before, suite => {
-    suiteHookRunning = true;
-    hookSteps = [];
-    global.testomatioDataStore.steps = [];
-
-    services.setContext(suite.fullTitle());
+  // Hook event listeners
+  event.dispatcher.on(event.hook.started, (hook) => {
+    output.stepShift = 2;
+    currentHook = hook.name;
+    let title = hook.hookName;
+    if (hook.suite) title += ' ' + hook.suite.fullTitle();
+    if (hook.test) title += ' ' + hook.test.fullTitle();
+    if (hook.ctx.currentTest) title += ' ' + hook.ctx.currentTest.fullTitle();
+
+    services.setContext(title);
+    hookSteps.set(hook.name, []);
   });
 
-  event.dispatcher.on(event.suite.after, () => {
+  event.dispatcher.on(event.hook.finished, () => {
+    currentHook = null;
+    output.stepShift = 2;
     services.setContext(null);
   });
 
-  event.dispatcher.on(event.hook.started, () => {
-    // global.testomatioDataStore.steps = [];
-  });
 
-  event.dispatcher.on(event.hook.passed, () => {
-    if (suiteHookRunning) {
-      hookSteps.push(...global.testomatioDataStore.steps);
-      services.setContext(null);
-    }
+  event.dispatcher.on(event.suite.before, suite => {
+    dataStorage.setContext(suite.fullTitle());
   });
 
-  event.dispatcher.on(event.hook.failed, () => {
-    if (suiteHookRunning) {
-      hookSteps.push(...global.testomatioDataStore.steps);
-      services.setContext(null);
-    }
+  event.dispatcher.on(event.suite.after, () => {
+    services.setContext(null);
   });
 
   event.dispatcher.on(event.test.before, test => {
-    suiteHookRunning = false;
-    global.testomatioDataStore.steps = [];
-
-    recorder.add(() => {
-      currentMetaStep = [];
-      // output.reset();
-      // output.start();
-      stepShift = 0;
-    });
-
-    if (!global.testomatioDataStore) global.testomatioDataStore = {};
-    // reset steps
-    global.testomatioDataStore.steps = [];
-
+    initializeTestDataStore();
     services.setContext(test.fullTitle());
   });
 
   event.dispatcher.on(event.test.started, test => {
     services.setContext(test.fullTitle());
-
-    testTimeMap[test.id] = Date.now();
-    if (!test.uid) return;
     testTimeMap[test.uid] = Date.now();
   });
 
-  event.dispatcher.on(event.all.result, async () => {
+  event.dispatcher.on(event.all.result, async (result) => {
     debug('waiting for all tests to be reported');
     // all tests were reported and we can upload videos
     await Promise.all(reportTestPromises);
@@ -137,174 +149,50 @@ function CodeceptReporter(config) {
     await uploadAttachments(client, videos, '🎞️ Uploading', 'video');
     await uploadAttachments(client, traces, '📁 Uploading', 'trace');
 
-    const status = failedTests.length === 0 ? STATUS.PASSED : STATUS.FAILED;
-    // @ts-ignore
-    client.updateRunStatus(status);
-  });
-
-  event.dispatcher.on(event.test.passed, test => {
-    const { uid, tags, title } = test;
-    if (uid && failedTests.includes(uid)) {
-      failedTests = failedTests.filter(failed => uid !== failed);
-    }
-    const testObj = getTestAndMessage(title);
-
-    const logs = getTestLogs(test);
-    const manuallyAttachedArtifacts = services.artifacts.get(test.fullTitle());
-    const keyValues = services.keyValues.get(test.fullTitle());
-    services.setContext(null);
-
-    client.addTestRun(STATUS.PASSED, {
-      ...stripExampleFromTitle(title),
-      rid: uid,
-      suite_title: test.parent && test.parent.title,
-      message: testObj.message,
-      time: getDuration(test),
-      steps: global.testomatioDataStore.steps.join('\n') || null,
-      test_id: getTestomatIdFromTestTitle(`${title} ${tags?.join(' ')}`),
-      logs,
-      manuallyAttachedArtifacts,
-      meta: keyValues,
-    });
-    // output.stop();
-  });
-
-  event.dispatcher.on(event.test.failed, (test, err) => {
-    error = err;
-  });
-
-  event.dispatcher.on(event.hook.failed, (suite, err) => {
-    error = err;
-
-    if (!suite) return;
-    if (!suite.tests) return;
-    for (const test of suite.tests) {
-      const { uid, tags, title } = test;
-      failedTests.push(uid || title);
-      const testId = getTestomatIdFromTestTitle(`${title} ${tags?.join(' ')}`);
-
-      client.addTestRun(STATUS.FAILED, {
-        rid: uid,
-        ...stripExampleFromTitle(title),
-        suite_title: suite.title,
-        test_id: testId,
-        error,
-        time: 0,
-      });
-    }
-    // output.stop();
+    client.updateRunStatus('finished');
   });
 
   event.dispatcher.on(event.test.after, test => {
-    if (test.state && test.state !== STATUS.FAILED) return;
-    if (test.err) error = test.err;
-    const { uid, tags, title, artifacts } = test;
+    const { uid, tags, title, artifacts } = test.simplify();
+    const error = test.err || null;
     failedTests.push(uid || title);
     const testObj = getTestAndMessage(title);
-
-    const files = [];
-    if (artifacts.screenshot) files.push({ path: artifacts.screenshot, type: 'image/png' });
-    // todo: video must be uploaded later....
-
+    const files = buildArtifactFiles(artifacts);
     const logs = getTestLogs(test);
     const manuallyAttachedArtifacts = services.artifacts.get(test.fullTitle());
     const keyValues = services.keyValues.get(test.fullTitle());
+    const stepHierarchy = buildUnifiedStepHierarchy(test.steps, hookSteps);
+    const labels = services.labels.get(test.fullTitle());
+
     services.setContext(null);
 
-    client.addTestRun(STATUS.FAILED, {
+    client.addTestRun(test.state, {
       ...stripExampleFromTitle(title),
       rid: uid,
       test_id: getTestomatIdFromTestTitle(`${title} ${tags?.join(' ')}`),
-      suite_title: test.parent && test.parent.title,
+      suite_title: test.parent && stripTagsFromTitle(stripExampleFromTitle(test.parent.title).title),
       error,
       message: testObj.message,
-      time: getDuration(test),
+      time: test.duration,
       files,
-      steps: global.testomatioDataStore?.steps?.join('\n') || null,
+      steps: stepHierarchy, // Array of step objects per API schema
       logs,
+      labels,
       manuallyAttachedArtifacts,
-      meta: keyValues,
+      meta: { ...keyValues, ...test.meta },
     });
 
-    debug('artifacts', artifacts);
-
-    for (const aid in artifacts) {
-      if (aid.startsWith('video')) videos.push({ rid: uid, title, path: artifacts[aid], type: 'video/webm' });
-      if (aid.startsWith('trace')) traces.push({ rid: uid, title, path: artifacts[aid], type: 'application/zip' });
-    }
-
-    // output.stop();
-  });
-
-  event.dispatcher.on(event.test.skipped, test => {
-    const { uid, tags, title } = test;
-    if (failedTests.includes(uid || title)) return;
-
-    const testObj = getTestAndMessage(title);
-    client.addTestRun(STATUS.SKIPPED, {
-      rid: uid,
-      ...stripExampleFromTitle(title),
-      test_id: getTestomatIdFromTestTitle(`${title} ${tags?.join(' ')}`),
-      suite_title: test.parent && test.parent.title,
-      message: testObj.message,
-      time: getDuration(test),
-    });
-    // output.stop();
+    processArtifactsForUpload(artifacts, uid, title, videos, traces);
   });
 
   event.dispatcher.on(event.step.started, step => {
-    stepShift = 0;
-    step.started = true;
-    stepStart = new Date();
+    const stepText = `${repeat(output.stepShift)} ${step.toCliStyled ? step.toCliStyled() : step.toString()}`;
+    dataStorage.putData('log', stepText);
   });
 
   event.dispatcher.on(event.step.finished, step => {
-    if (!step.started) return;
-    let processingStep = step;
-    const metaSteps = [];
-    while (processingStep.metaStep) {
-      metaSteps.unshift(processingStep.metaStep);
-      processingStep = processingStep.metaStep;
-    }
-    const shift = metaSteps.length;
-
-    for (let i = 0; i < Math.max(currentMetaStep.length, metaSteps.length); i++) {
-      if (currentMetaStep[i] !== metaSteps[i]) {
-        stepShift = 2 * i;
-        // eslint-disable-next-line no-continue
-        if (!metaSteps[i]) continue;
-        if (metaSteps[i].isBDD()) {
-          // output.push(repeat(stepShift) + pc.bold(metaSteps[i].toString()) + metaSteps[i].comment);
-          global.testomatioDataStore?.steps?.push(
-            repeat(stepShift) + pc.bold(metaSteps[i].toString()) + metaSteps[i].comment,
-          );
-        } else {
-          // output.push(repeat(stepShift) + pc.green.bold(metaSteps[i].toString()));
-          global.testomatioDataStore?.steps?.push(repeat(stepShift) + pc.green(pc.bold(metaSteps[i].toString())));
-        }
-      }
-    }
-    currentMetaStep = metaSteps;
-    stepShift = 2 * shift;
-
-    const durationMs = +new Date() - +stepStart;
-    let duration = '';
-    if (durationMs) {
-      duration = repeat(1) + pc.gray(`(${durationMs}ms)`);
-    }
-
-    if (step.status === STATUS.FAILED) {
-      // output.push(repeat(stepShift) + pc.red(step.toString()) + duration);
-      global.testomatioDataStore?.steps?.push(repeat(stepShift) + pc.red(step.toString()) + duration);
-    } else {
-      // output.push(repeat(stepShift) + step.toString() + duration);
-      global.testomatioDataStore?.steps?.push(repeat(stepShift) + step.toString() + duration);
-    }
-  });
-
-  event.dispatcher.on(event.step.comment, step => {
-    // output.push(pc.cyan.bold(step.toString()));
-    global.testomatioDataStore?.steps?.push(pc.cyan(pc.bold(step.toString())));
+    processMetaStepsForDisplay(step);
+    captureHookStep(step, currentHook, hookSteps);
   });
 }
 
@@ -344,32 +232,250 @@ function stripExampleFromTitle(title) {
   const res = title.match(DATA_REGEXP);
   if (!res) return { title, example: null };
 
-  const example = JSON.parse(res[1]);
-  title = title.replace(DATA_REGEXP, '').trim();
+  try {
+    const example = JSON.parse(res[1]);
+    title = title.replace(DATA_REGEXP, '').trim();
+    return { title, example };
+  } catch (e) {
+    // If JSON parsing fails, return title without example
+    debug('Failed to parse example JSON:', res[1], e.message);
+    return { title: title.replace(DATA_REGEXP, '').trim(), example: null };
+  }
+}
 
-  return { title, example };
+function stripTagsFromTitle(title) {
+  // Remove @tags from the end of titles (e.g., "Hooks Test Suite @hooks" -> "Hooks Test Suite")
+  return title.replace(/\s+@[\w-]+\s*$/, '').trim();
 }
 
 function repeat(num) {
   return ''.padStart(num, ' ');
 }
 
+// Helper functions for cleaner event handling
+function initializeTestDataStore() {
+  if (!global.testomatioDataStore) global.testomatioDataStore = {};
+  global.testomatioDataStore.steps = [];
+}
+
+function buildArtifactFiles(artifacts) {
+  const files = [];
+  if (artifacts.screenshot) {
+    files.push({ path: artifacts.screenshot, type: 'image/png' });
+  }
+  return files;
+}
+
+function processArtifactsForUpload(artifacts, uid, title, videos, traces) {
+  for (const aid in artifacts) {
+    if (aid.startsWith('video')) {
+      videos.push({ rid: uid, title, path: artifacts[aid], type: 'video/webm' });
+    }
+    if (aid.startsWith('trace')) {
+      traces.push({ rid: uid, title, path: artifacts[aid], type: 'application/zip' });
+    }
+  }
+}
+
+function processMetaStepsForDisplay(step) {
+  const metaSteps = [];
+  let processingStep = step;
+
+  while (processingStep.metaStep) {
+    metaSteps.unshift(processingStep.metaStep);
+    processingStep = processingStep.metaStep;
+  }
+}
+
+function captureHookStep(step, currentHook, hookSteps) {
+  if (!currentHook) return;
+
+  const startTime = step.startTime;
+  const endTime = step.endTime;
+
+  const hookStepsArray = hookSteps.get(currentHook) || [];
+  hookStepsArray.push({
+    name: step.name,
+    actor: step.actor,
+    args: step.args,
+    status: step.status,
+    startTime,
+    endTime,
+    helperMethod: step.helperMethod
+  });
+  hookSteps.set(currentHook, hookStepsArray);
+}
+
 // TODO: think about moving to some common utils
 function getTestLogs(test) {
-  const suiteLogsArr = services.logger.getLogs(test.parent.fullTitle());
-  const suiteLogs = suiteLogsArr ? suiteLogsArr.join('\n').trim() : '';
-  const testLogsArr = services.logger.getLogs(test.fullTitle());
+  // Contexts for each log section
+  const suiteTitle = test.parent.fullTitle();
+  const testTitle = test.fullTitle();
+  const beforeSuiteLogsArr = services.logger.getLogs(`BeforeSuite ${suiteTitle}`);
+  const beforeLogsArr = services.logger.getLogs(`Before ${testTitle}`);
+  const testLogsArr = services.logger.getLogs(testTitle);
+  const afterLogsArr = services.logger.getLogs(`After ${testTitle}`);
+  const afterSuiteLogsArr = services.logger.getLogs(`AfterSuite ${suiteTitle}`);
+
+  const beforeSuiteLogs = beforeSuiteLogsArr ? beforeSuiteLogsArr.join('\n').trim() : '';
+  const beforeLogs = beforeLogsArr ? beforeLogsArr.join('\n').trim() : '';
   const testLogs = testLogsArr ? testLogsArr.join('\n').trim() : '';
+  const afterLogs = afterLogsArr ? afterLogsArr.join('\n').trim() : '';
+  const afterSuiteLogs = afterSuiteLogsArr ? afterSuiteLogsArr.join('\n').trim() : '';
 
   let logs = '';
-  if (suiteLogs) {
-    logs += `${pc.bold('\t--- BeforeSuite ---')}\n${suiteLogs}`;
+  if (beforeSuiteLogs) {
+    logs += `${pc.bold('--- BeforeSuite ---')}\n${beforeSuiteLogs}`;
+  }
+  if (beforeLogs) {
+    logs += `\n${pc.bold('--- Before ---')}\n${beforeLogs}`;
   }
   if (testLogs) {
-    logs += `\n${pc.bold('\t--- Test ---')}\n${testLogs}`;
+    logs += `\n${pc.bold('--- Test ---')}\n${testLogs}`;
+  }
+  if (afterLogs) {
+    logs += `\n${pc.bold('--- After ---')}\n${afterLogs}`;
+  }
+  if (afterSuiteLogs) {
+    logs += `\n${pc.bold('--- AfterSuite ---')}\n${afterSuiteLogs}`;
   }
   return logs;
 }
 
+// Build step hierarchy using CodeceptJS built-in methods
+function buildUnifiedStepHierarchy(steps, hookSteps) {
+  const hierarchy = [];
+
+  // Add pre-test hooks
+  addHooksToHierarchy(hierarchy, hookSteps, HOOK_EXECUTION_ORDER.PRE_TEST);
+
+  // Process test steps if they exist
+  if (steps && steps.length > 0) {
+    processTestSteps(steps, hierarchy);
+  }
+
+  // Add post-test hooks
+  addHooksToHierarchy(hierarchy, hookSteps, HOOK_EXECUTION_ORDER.POST_TEST);
+
+  return hierarchy;
+}
+
+function addHooksToHierarchy(hierarchy, hookSteps, hookNames) {
+  for (const hookName of hookNames) {
+    if (hookSteps.has(hookName)) {
+      const hookSection = createHookSection(hookName, hookSteps.get(hookName));
+      if (hookSection) hierarchy.push(hookSection);
+    }
+  }
+}
+
+function processTestSteps(steps, hierarchy) {
+  const sectionMap = new Map();
+
+  for (const step of steps) {
+    const formattedStep = formatCodeceptStep(step);
+    if (!formattedStep) continue;
+
+    if (step.metaStep) {
+      // Step belongs to a section (meta step)
+      const sectionKey = step.metaStep;
+      let sectionStep = sectionMap.get(sectionKey);
+
+      if (!sectionStep) {
+        sectionStep = createSectionStep(step.metaStep);
+        sectionMap.set(sectionKey, sectionStep);
+        hierarchy.push(sectionStep);
+      }
+
+      sectionStep.steps.push(formattedStep);
+      sectionStep.duration += formattedStep.duration || 0;
+    } else {
+      // Regular step
+      hierarchy.push(formattedStep);
+    }
+  }
+}
+
+
+function createSectionStep(metaStep) {
+  return {
+    category: 'user',
+    title: metaStep.toString(), // Use built-in toString method
+    duration: metaStep.duration || 0, // Use built-in duration
+    steps: []
+  };
+}
+
+function createHookSection(hookName, steps) {
+  if (!steps || steps.length === 0) return null;
+
+  const hookSection = {
+    category: 'hook',
+    title: formatHookName(hookName),
+    duration: 0,
+    steps: []
+  };
+
+  for (const step of steps) {
+    const formattedStep = formatHookStep(step);
+    if (formattedStep) {
+      hookSection.steps.push(formattedStep);
+      hookSection.duration += formattedStep.duration || 0;
+    }
+  }
+
+  return hookSection.steps.length > 0 ? hookSection : null;
+}
+
+function formatHookName(hookName) {
+  return hookName.replace(/Hook$/, '');
+}
+
+
+// Format CodeceptJS step using its built-in methods
+function formatCodeceptStep(step) {
+  if (!step) return null;
+
+  const category = step.constructor.name === 'HelperStep' ? 'framework' : 'user';
+  const title = step.toString(); // Use built-in toString
+  const duration = step.duration || 0; // Use built-in duration
+
+  const formattedStep = {
+    category,
+    title,
+    duration
+  };
+
+  // Add error if step failed
+  if (step.status === 'failed' && step.err) {
+    formattedStep.error = {
+      message: step.err.message || 'Step failed',
+      stack: step.err.stack || ''
+    };
+  }
+
+  return formattedStep;
+}
+
+function formatHookStep(step) {
+  if (!step) return null;
+
+  // For hook steps, construct title from available properties
+  let title = step.name;
+  if (step.actor && step.name) {
+    title = `${step.actor}.${step.name}`;
+    if (step.args && step.args.length > 0) {
+      const argsStr = step.args.map(arg => JSON.stringify(arg)).join(', ');
+      title += `(${argsStr})`;
+    }
+  }
+
+  return {
+    category: 'hook',
+    title,
+    duration: step.duration || 0
+  };
+}
+
 export { CodeceptReporter };
 export default CodeceptReporter;

package/src/adapter/cypress-plugin/index.js

@@ -44,7 +44,6 @@ const testomatioReporter = on => {
 
       if (!error && test.displayError) {
         error = { message: test.displayError };
-        // eslint-disable-next-line
         error.inspect = function () {
           return this.message;
         };
@@ -56,7 +55,6 @@ const testomatioReporter = on => {
             name: error.name,
             inspect:
               error.inspect ||
-              // eslint-disable-next-line
               function () {
                 return this.message;
               },

package/src/adapter/mocha.js

@@ -1,4 +1,3 @@
-// eslint-disable-next-line global-require, import/no-extraneous-dependencies
 import Mocha from 'mocha';
 import TestomatClient from '../client.js';
 import { STATUS, TESTOMAT_TMP_STORAGE_DIR } from '../constants.js';

package/src/adapter/nightwatch.js

@@ -1,6 +1,6 @@
 import TestomatClient from '../client.js';
 import { config } from '../config.js';
-import { STATUS } from '../constants';
+import { STATUS } from '../constants.js';
 import { getTestomatIdFromTestTitle } from '../utils/utils.js';
 
 const apiKey = config.TESTOMATIO;