@testomatio/reporter 2.7.0 → 2.7.2-beta.1

package/README.md

@@ -63,7 +63,7 @@ yarn add @testomatio/reporter --dev
 ### 1️⃣ Attach Reporter to the Test Runner
 
 |                                                 |                                               |                                                           |
-|-------------------------------------------------|-----------------------------------------------|-----------------------------------------------------------|
+| ----------------------------------------------- | --------------------------------------------- | --------------------------------------------------------- |
 | [Playwright](./docs/frameworks.md#playwright)   | [CodeceptJS](./docs/frameworks.md#codeceptjs) | [Cypress](./docs/frameworks.md#cypress)                   |
 | [Jest](./docs/frameworks.md#jest)               | [Mocha](./docs/frameworks.md#mocha)           | [WebDriverIO](./docs/frameworks.md#webdriverio)           |
 | [TestCafe](./docs/frameworks.md#testcafe)       | [Detox](./docs/frameworks.md#detox)           | [Codeception](https://github.com/testomatio/php-reporter) |
@@ -135,6 +135,7 @@ Bring this reporter on CI and never lose test results again!
 - 🔂 [Workflows](./docs/workflows.md)
 - 🖊️ [Logger](./docs/logger.md)
 - 🪲 [Debug File Format](./docs/debug-file-format.md)
+- 📊 [Control output logs](./docs/log-level.md)
 
 ## Development
 

package/lib/adapter/codecept.js

@@ -11,7 +11,9 @@ const constants_js_1 = require("../constants.js");
 const utils_js_1 = require("../utils/utils.js");
 const index_js_1 = require("../services/index.js");
 const data_storage_js_1 = require("../data-storage.js");
+const step_formatter_js_1 = require("./utils/step-formatter.js");
 const codeceptjs_1 = __importDefault(require("codeceptjs"));
+const log_js_1 = require("../utils/log.js");
 const debug = (0, debug_1.default)('@testomatio/reporter:adapter:codeceptjs');
 // @ts-ignore
 if (!global.codeceptjs) {
@@ -43,6 +45,7 @@ function CodeceptReporter(config) {
     let videos = [];
     let traces = [];
     const reportTestPromises = [];
+    let isRunFinalized = false;
     const testTimeMap = {};
     const { apiKey } = config;
     const client = new client_js_1.default({ apiKey });
@@ -70,6 +73,17 @@ function CodeceptReporter(config) {
     recorder.startUnlessRunning();
     const hookSteps = new Map();
     let currentHook = null;
+    const finalizeRun = async (origin) => {
+        if (isRunFinalized)
+            return;
+        isRunFinalized = true;
+        debug(`finalizing run from ${origin}`);
+        debug('waiting for all tests to be reported');
+        await Promise.allSettled(reportTestPromises);
+        await uploadAttachments(client, videos, '🎞️ Uploading', 'video');
+        await uploadAttachments(client, traces, '📁 Uploading', 'trace');
+        await client.updateRunStatus('finished');
+    };
     event.dispatcher.on(event.workers.before, () => {
         recorder.add('Creating new run', async () => {
             await client.createRun();
@@ -79,7 +93,9 @@ function CodeceptReporter(config) {
         });
     });
     event.dispatcher.on(event.workers.after, () => {
-        client.updateRunStatus('finished');
+        recorder.add('Finishing run', async () => {
+            await finalizeRun('workers.after');
+        });
     });
     // Listening to events
     event.dispatcher.on(event.all.before, () => {
@@ -91,6 +107,8 @@ function CodeceptReporter(config) {
         });
         videos = [];
         traces = [];
+        isRunFinalized = false;
+        reportTestPromises.length = 0;
         if (!global.testomatioDataStore)
             global.testomatioDataStore = {};
     });
@@ -122,7 +140,7 @@ function CodeceptReporter(config) {
             return;
         const error = hook?.ctx?.currentTest?.err;
         for (const test of suite.tests) {
-            client.addTestRun('failed', {
+            const reportTestPromise = client.addTestRun('failed', {
                 ...stripExampleFromTitle(test.title),
                 rid: test.uid,
                 test_id: (0, utils_js_1.getTestomatIdFromTestTitle)(test.title),
@@ -130,6 +148,7 @@ function CodeceptReporter(config) {
                 error,
                 time: hook?.runnable?.duration,
             });
+            reportTestPromises.push(reportTestPromise);
         }
     });
     event.dispatcher.on(event.suite.before, suite => {
@@ -146,13 +165,10 @@ function CodeceptReporter(config) {
         index_js_1.services.setContext(test.fullTitle());
         testTimeMap[test.uid] = Date.now();
     });
-    event.dispatcher.on(event.all.result, async (result) => {
-        debug('waiting for all tests to be reported');
-        // all tests were reported and we can upload videos
-        await Promise.all(reportTestPromises);
-        await uploadAttachments(client, videos, '🎞️ Uploading', 'video');
-        await uploadAttachments(client, traces, '📁 Uploading', 'trace');
-        client.updateRunStatus('finished');
+    event.dispatcher.on(event.all.after, () => {
+        recorder.add('Finishing run', async () => {
+            await finalizeRun('all.after');
+        });
     });
     event.dispatcher.on(event.test.after, test => {
         const { uid, tags, title, artifacts } = test.simplify();
@@ -163,10 +179,12 @@ function CodeceptReporter(config) {
         const logs = getTestLogs(test);
         const manuallyAttachedArtifacts = index_js_1.services.artifacts.get(test.fullTitle());
         const keyValues = index_js_1.services.keyValues.get(test.fullTitle());
-        const stepHierarchy = buildUnifiedStepHierarchy(test.steps, hookSteps);
         const links = index_js_1.services.links.get(test.fullTitle());
+        const screenshotOnFailPath = artifacts.screenshot || null;
+        // Build step hierarchy with screenshot from screenshotOnFail
+        const stepHierarchy = buildUnifiedStepHierarchy(test.steps, hookSteps, screenshotOnFailPath);
         index_js_1.services.setContext(null);
-        client.addTestRun(test.state, {
+        const reportTestPromise = client.addTestRun(test.state, {
             ...stripExampleFromTitle(title),
             rid: uid,
             test_id: (0, utils_js_1.getTestomatIdFromTestTitle)(`${title} ${tags?.join(' ')}`),
@@ -181,6 +199,7 @@ function CodeceptReporter(config) {
             manuallyAttachedArtifacts,
             meta: { ...keyValues, ...test.meta },
         });
+        reportTestPromises.push(reportTestPromise);
         processArtifactsForUpload(artifacts, uid, title, videos, traces);
     });
     event.dispatcher.on(event.step.started, step => {
@@ -196,7 +215,7 @@ async function uploadAttachments(client, attachments, messagePrefix, attachmentT
     if (!attachments?.length)
         return;
     if (client.uploader.isEnabled) {
-        console.log(constants_js_1.APP_PREFIX, `Attachments: ${messagePrefix} ${attachments.length} ${attachmentType} ...`);
+        log_js_1.log.info(`Attachments: ${messagePrefix} ${attachments.length} ${attachmentType} ...`);
     }
     const promises = attachments.map(async (attachment) => {
         const { rid, title, path, type } = attachment;
@@ -322,13 +341,13 @@ function getTestLogs(test) {
     return logs;
 }
 // Build step hierarchy using CodeceptJS built-in methods
-function buildUnifiedStepHierarchy(steps, hookSteps) {
+function buildUnifiedStepHierarchy(steps, hookSteps, screenshotOnFailPath = null) {
     const hierarchy = [];
     // Add pre-test hooks
     addHooksToHierarchy(hierarchy, hookSteps, HOOK_EXECUTION_ORDER.PRE_TEST);
     // Process test steps if they exist
     if (steps && steps.length > 0) {
-        processTestSteps(steps, hierarchy);
+        processTestSteps(steps, hierarchy, screenshotOnFailPath);
     }
     // Add post-test hooks
     addHooksToHierarchy(hierarchy, hookSteps, HOOK_EXECUTION_ORDER.POST_TEST);
@@ -343,10 +362,16 @@ function addHooksToHierarchy(hierarchy, hookSteps, hookNames) {
         }
     }
 }
-function processTestSteps(steps, hierarchy) {
+function processTestSteps(steps, hierarchy, screenshotOnFailPath = null) {
     const sectionMap = new Map();
+    let screenshotAttached = false;
     for (const step of steps) {
-        const formattedStep = formatCodeceptStep(step);
+        let stepScreenshotPath = null;
+        if (screenshotOnFailPath && !screenshotAttached && step.status === 'failed') {
+            stepScreenshotPath = screenshotOnFailPath;
+            screenshotAttached = true;
+        }
+        const formattedStep = formatCodeceptStep(step, stepScreenshotPath);
         if (!formattedStep)
             continue;
         if (step.metaStep) {
@@ -397,24 +422,38 @@ function formatHookName(hookName) {
     return hookName.replace(/Hook$/, '');
 }
 // Format CodeceptJS step using its built-in methods
-function formatCodeceptStep(step) {
+function formatCodeceptStep(step, screenshotOnFailPath = null) {
     if (!step)
         return null;
     const category = step.constructor.name === 'HelperStep' ? 'framework' : 'user';
-    const title = (0, utils_js_1.truncate)(step); // Use built-in toString
-    const duration = step.duration || 0; // Use built-in duration
-    const formattedStep = {
+    const title = (0, utils_js_1.truncate)(String(step));
+    const duration = step.duration || 0;
+    const formattedStep = (0, step_formatter_js_1.formatStep)({
         category,
         title,
         duration,
-    };
+    });
+    // Add status
+    (0, step_formatter_js_1.addStatusToStep)(formattedStep, step.status, step.err);
     // Add error if step failed
     if (step.status === 'failed' && step.err) {
         formattedStep.error = {
-            message: step.err.message || 'Step failed',
-            stack: step.err.stack || '',
+            message: (0, utils_js_1.truncate)(String(step.err.message || 'Step failed'), 250),
+            stack: (0, utils_js_1.truncate)(String(step.err.stack || ''), 250),
         };
     }
+    // Add artifacts
+    if (step.artifacts && constants_js_1.SCREENSHOTS_ON_STEPS) {
+        (0, step_formatter_js_1.addArtifactsToStep)(formattedStep, step.artifacts);
+    }
+    // Add screenshot from screenshotOnFail plugin
+    if (screenshotOnFailPath && constants_js_1.SCREENSHOTS_ON_STEPS) {
+        (0, step_formatter_js_1.addArtifactPathToStep)(formattedStep, screenshotOnFailPath);
+    }
+    // Add log if present
+    if (step.log) {
+        formattedStep.log = (0, utils_js_1.truncate)(String(step.log), 250);
+    }
     return formattedStep;
 }
 function formatHookStep(step) {
@@ -425,16 +464,32 @@ function formatHookStep(step) {
     if (step.actor && step.name) {
         title = `${step.actor} ${step.name}`;
         if (step.args && step.args.length > 0) {
-            const argsStr = step.args.map(arg => (0, utils_js_1.truncate)(JSON.stringify(arg))).join(', ');
+            const argsStr = step.args.map(arg => (0, utils_js_1.truncate)(JSON.stringify(arg), 250)).join(', ');
             title += ` ${argsStr}`;
         }
     }
     title = (0, utils_js_1.truncate)(title);
-    return {
+    const formattedStep = (0, step_formatter_js_1.formatStep)({
         category: 'hook',
         title,
         duration: step.duration || 0,
-    };
+    });
+    (0, step_formatter_js_1.addStatusToStep)(formattedStep, step.status, step.err);
+    if (step.status === 'failed' && step.err) {
+        formattedStep.error = {
+            message: (0, utils_js_1.truncate)(String(step.err.message || 'Hook failed'), 250),
+            stack: (0, utils_js_1.truncate)(String(step.err.stack || ''), 250),
+        };
+    }
+    // Add artifacts
+    if (step.artifacts && constants_js_1.SCREENSHOTS_ON_STEPS) {
+        (0, step_formatter_js_1.addArtifactsToStep)(formattedStep, step.artifacts);
+    }
+    // Add log if present
+    if (step.log) {
+        formattedStep.log = (0, utils_js_1.truncate)(String(step.log), 250);
+    }
+    return formattedStep;
 }
 module.exports = CodeceptReporter;
 

package/lib/adapter/playwright.d.ts

@@ -7,7 +7,7 @@ declare class PlaywrightReporter {
     suite: any;
     config: any;
     onTestBegin(testInfo: any): void;
-    onTestEnd(test: any, result: any): void;
+    onTestEnd(test: any, result: any): Promise<void>;
     onEnd(result: any): Promise<void>;
     #private;
 }

package/lib/adapter/playwright.js

@@ -19,6 +19,8 @@ const constants_js_2 = require("../utils/constants.js");
 const picocolors_1 = __importDefault(require("picocolors"));
 const playwright_js_1 = require("./utils/playwright.js");
 Object.defineProperty(exports, "fetchLinksFromLogs", { enumerable: true, get: function () { return playwright_js_1.fetchLinksFromLogs; } });
+const step_formatter_js_1 = require("./utils/step-formatter.js");
+const log_js_1 = require("../utils/log.js");
 const reportTestPromises = [];
 class PlaywrightReporter {
     constructor(config = {}) {
@@ -38,7 +40,7 @@ class PlaywrightReporter {
         const fullTestTitle = getTestContextName(testInfo);
         data_storage_js_1.dataStorage.setContext(fullTestTitle);
     }
-    onTestEnd(test, result) {
+    async onTestEnd(test, result) {
         // test.parent.project().__projectId
         if (!this.client)
             return;
@@ -53,13 +55,24 @@ class PlaywrightReporter {
         }))
             .filter(f => f.path);
         const suite_title = test.parent ? test.parent?.title : path_1.default.basename(test?.location?.file);
-        const steps = [];
-        for (const step of result.steps) {
-            const appendedStep = appendStep(step);
-            if (appendedStep) {
-                steps.push(appendedStep);
-            }
-        }
+        const rid = test.id || test.testId || (0, uuid_1.v4)();
+        /**
+         * @type {{
+         * browser?: string,
+         * dependencies: string[],
+         * isMobile?: boolean
+         * metadata: Record<string, any>,
+         * name: string,
+         * }}
+         */
+        const project = {
+            browser: test.parent.project().use.defaultBrowserType,
+            dependencies: test.parent.project().dependencies,
+            isMobile: test.parent.project().use.isMobile,
+            metadata: test.parent.project().metadata,
+            name: test.parent.project().name,
+        };
+        const steps = result.steps.map(step => appendStep(step, 0)).filter(step => step !== null);
         // Extract and normalize tags
         const tags = extractTags(test);
         const fullTestTitle = getTestContextName(test);
@@ -81,23 +94,6 @@ class PlaywrightReporter {
         */
         const manuallyAttachedArtifacts = index_js_1.services.artifacts.get(fullTestTitle);
         const testMeta = index_js_1.services.keyValues.get(fullTestTitle);
-        const rid = test.id || test.testId || (0, uuid_1.v4)();
-        /**
-         * @type {{
-         * browser?: string,
-         * dependencies: string[],
-         * isMobile?: boolean
-         * metadata: Record<string, any>,
-         * name: string,
-         * }}
-         */
-        const project = {
-            browser: test.parent.project().use.defaultBrowserType,
-            dependencies: test.parent.project().dependencies,
-            isMobile: test.parent.project().use.isMobile,
-            metadata: test.parent.project().metadata,
-            name: test.parent.project().name,
-        };
         let status = result.status;
         // process test.fail() annotation
         if (test.expectedStatus === 'failed') {
@@ -171,7 +167,7 @@ class PlaywrightReporter {
         await Promise.all(reportTestPromises);
         if (this.uploads.length) {
             if (this.client.uploader.isEnabled)
-                console.log(constants_js_1.APP_PREFIX, `🎞️  Uploading ${this.uploads.length} files...`);
+                log_js_1.log.info(`🎞️ Uploading ${this.uploads.length} files...`);
             const promises = [];
             // ? possible move to addTestRun (needs investigation if files are ready)
             for (const upload of this.uploads) {
@@ -221,6 +217,38 @@ function appendStep(step, shift = 0) {
         default:
             newCategory = 'framework';
     }
+    const resultStep = (0, step_formatter_js_1.formatStep)({
+        category: newCategory,
+        title: step.title,
+        duration: step.duration,
+    });
+    // Add status based on error
+    (0, step_formatter_js_1.addStatusToStep)(resultStep, step.error ? 'failed' : 'passed', step.error);
+    // Add error if present
+    if (step.error !== undefined) {
+        if (typeof step.error === 'object') {
+            resultStep.error = {
+                message: (0, utils_js_1.truncate)(String(step.error.message), 250),
+                stack: (0, utils_js_1.truncate)(String(step.error.stack || ''), 250),
+            };
+        }
+        else {
+            resultStep.error = (0, utils_js_1.truncate)(String(step.error), 250);
+        }
+    }
+    // Add log if present
+    if (step.log) {
+        resultStep.log = (0, utils_js_1.truncate)(String(step.log), 250);
+    }
+    // Add artifacts from attachments
+    if (step.attachments && step.attachments.length > 0 && constants_js_1.SCREENSHOTS_ON_STEPS) {
+        const screenshotAttachment = step.attachments.find(att => att.contentType === 'image/png' && att.name === 'screenshot');
+        if (screenshotAttachment && screenshotAttachment.path) {
+            const artifacts = { screenshot: screenshotAttachment.path };
+            (0, step_formatter_js_1.addArtifactsToStep)(resultStep, artifacts);
+        }
+    }
+    // Process nested steps
     const formattedSteps = [];
     for (const child of step.steps || []) {
         const appendedChild = appendStep(child, shift + 2);
@@ -228,17 +256,9 @@ function appendStep(step, shift = 0) {
             formattedSteps.push(appendedChild);
         }
     }
-    const resultStep = {
-        category: newCategory,
-        title: step.title,
-        duration: step.duration,
-    };
     if (formattedSteps.length) {
         resultStep.steps = formattedSteps.filter(s => !!s);
     }
-    if (step.error !== undefined) {
-        resultStep.error = step.error;
-    }
     return resultStep;
 }
 function generateTmpFilepath(filename = '') {

package/lib/adapter/utils/step-formatter.d.ts

@@ -0,0 +1,134 @@
+/**
+ * Generates a short unique filename from screenshot path
+ * If original filename is too long, uses hash-based name
+ *
+ * @param {string} screenshotPath - Path to screenshot file
+ * @returns {string} Short filename (max 80 chars)
+ */
+export function generateShortFilename(screenshotPath: string): string;
+/**
+ * Formats a step object according to Testomat.io Step Schema
+ *
+ * This function transforms a raw step object from test frameworks (CodeceptJS, Playwright, etc.)
+ * into a standardized format compatible with Testomat.io API. It ensures all text fields are
+ * truncated to 250 characters as defined in testomat-api-definition.yml.
+ *
+ * Processed fields:
+ * - category: step type (framework, user, hook) - defaults to 'user'
+ * - title: step name/description, truncated to 250 chars
+ * - duration: step execution time in seconds
+ * - log: optional log output, truncated to 250 chars
+ * - artifacts: optional array of artifact URLs (screenshots), each truncated to 250 chars
+ * - error: error details (message + stack) if step failed, each truncated to 250 chars
+ * - steps: recursively formats nested steps
+ *
+ * Schema reference: testomat-api-definition.yml (Step object)
+ *
+ * @param {Object} step - Raw step object from test framework
+ * @param {string} [step.category] - Step category: 'user', 'framework', or 'hook'
+ * @param {string} [step.title] - Step title/name
+ * @param {number} [step.duration] - Step duration in seconds
+ * @param {string} [step.log] - Log output for this step
+ * @param {string[]} [step.artifacts] - Array of artifact URLs (screenshots)
+ * @param {string|Object} [step.error] - Error details - can be string or object with message/stack
+ * @param {Object[]} [step.steps] - Array of nested child steps
+ * @returns {Object} Formatted step object matching Testomat.io Step Schema with:
+ * category, title, duration, and optional log, artifacts, error, and steps fields
+ *
+ * @example
+ * const rawStep = {
+ *   category: 'user',
+ *   title: 'I click on button',
+ *   duration: 1.5,
+ *   error: { message: 'Element not found', stack: 'at test.js:10:5' }
+ * };
+ * const formatted = formatStep(rawStep);
+ * // Returns: { category: 'user', title: 'I click on button', duration: 1.5, error: {...} }
+ */
+export function formatStep(step: {
+    category?: string;
+    title?: string;
+    duration?: number;
+    log?: string;
+    artifacts?: string[];
+    error?: string | any;
+    steps?: any[];
+}): any;
+/**
+ * Adds status field to step
+ *
+ * Normalizes step status from test frameworks to Testomat.io standard format.
+ * Maps framework-specific statuses ('success', 'failed', 'passed') to Testomat.io
+ * standard values ('passed', 'failed').
+ *
+ * Status mapping:
+ * - 'success' → 'passed'
+ * - 'passed' → 'passed'
+ * - 'failed' → 'failed'
+ * - Any other value → 'passed' (default)
+ *
+ * If step already has a status, it won't be overwritten. If error is provided
+ * and step doesn't have status, it will be set to 'failed'.
+ *
+ * Schema reference: testomat-api-definition.yml (Step.status enum)
+ *
+ * @param {Object} step - Step object to add status to (modified in place)
+ * @param {string} [step.status] - Existing status (won't be overwritten if present)
+ * @param {string} status - Status from test framework: 'success', 'failed', or 'passed'
+ * @param {Error|Object|null} err - Error object if step failed
+ * @returns {Object} The same step object with added status field
+ *
+ * @example
+ * const step = { title: 'Click button' };
+ * addStatusToStep(step, 'success', null);
+ * // step.status === 'passed'
+ *
+ * @example
+ * const step2 = { title: 'Find element' };
+ * addStatusToStep(step2, 'failed', new Error('Not found'));
+ * // step2.status === 'failed'
+ */
+export function addStatusToStep(step: {
+    status?: string;
+}, status: string, err: Error | any | null): any;
+/**
+ * Adds screenshot to step as artifacts array
+ *
+ * Extracts screenshot path from artifacts and adds it to the step's artifacts array.
+ * The actual upload will happen in the client's addTestRun method.
+ *
+ * Artifact format supports:
+ * - Array format: [{ screenshot: '/path/to/screenshot.png' }]
+ * - Object format: { screenshot: '/path/to/screenshot.png' }
+ *
+ * Screenshot path can be specified as:
+ * - Object with path property: { screenshot: { path: '/path/to/file.png' } }
+ * - Object with screenshot property: { screenshot: { screenshot: '/path/to/file.png' } }
+ * - Direct string path: { screenshot: '/path/to/file.png' }
+ *
+ * @param {Object} step - Step object to add artifacts to (modified in place)
+ * @param {string[]} [step.artifacts] - Existing artifacts array (won't be overwritten if present)
+ * @param {Object|Object[]|null} artifacts - Artifacts from test framework
+ * @returns {Object} The same step object with artifacts array added
+ *
+ * @example
+ * const step = { title: 'Click button' };
+ * const artifacts = { screenshot: '/tmp/screenshot.png' };
+ * addArtifactsToStep(step, artifacts);
+ * // step.artifacts === ['/tmp/screenshot.png']
+ */
+export function addArtifactsToStep(step: {
+    artifacts?: string[];
+}, artifacts: any | any[] | null): any;
+/**
+ * Appends one artifact path to a step.
+ *
+ * Unlike addArtifactsToStep, this helper accepts a direct path (or URL-like string)
+ * and does not check file existence, so callers can attach fallback artifacts
+ * collected from logs or async trace outputs.
+ *
+ * @param {Object} step - Step object to update (modified in place)
+ * @param {string} artifactPath - Artifact path to append
+ * @returns {Object} The same step object with updated artifacts
+ */
+export function addArtifactPathToStep(step: any, artifactPath: string): any;