@testomatio/reporter 2.7.3 → 2.7.4

package/README.md

@@ -58,6 +58,13 @@ pnpm install @testomatio/reporter --save-dev
 yarn add @testomatio/reporter --dev
 ```
 
+For Yarn 4 (Berry), use CLI wrapper package:
+
+```
+yarn add testomatio-reporter-cli --dev
+npx testomatio-reporter <command> [options]
+```
+
 ## Getting Started
 
 ### 1️⃣ Attach Reporter to the Test Runner

package/lib/adapter/vitest.d.ts

@@ -18,15 +18,22 @@ export type TestData = import("../../types/types.js").TestData;
 export class VitestReporter {
     constructor(config?: {});
     client: TestomatioClient;
-    /**
-     * @type {(TestData & {status: string})[]} tests
-     */
+    /** @type {(TestData & {status: string, _reportKey?: string | null})[]} tests */
     tests: (TestData & {
         status: string;
+        _reportKey?: string | null;
     })[];
     _finalized: boolean;
     _finalizing: boolean;
+    _runStartedAtMs: number;
+    _runStartedAtMicros: number;
+    _reportedTestKeys: Set<any>;
+    _liveQueue: Promise<void>;
     onInit(): void;
+    /**
+     * Vitest 3/4 callback fired when test run starts.
+     */
+    onTestRunStart(): void;
     /**
      * @param {VitestTestFile[] | undefined} files // array with results;
      * @param {unknown[] | undefined} errors // errors does not contain errors from tests; probably its testrunner errors
@@ -39,6 +46,18 @@ export class VitestReporter {
      * @param {unknown[] | undefined} errors
      */
     onTestRunEnd(testModules: Array<unknown> | undefined, errors: unknown[] | undefined): Promise<void>;
+    /**
+     * Vitest 4 callback fired when single test case is finished.
+     *
+     * @param {unknown} testCase
+     */
+    onTestCaseResult(testCase: unknown): Promise<void>;
+    /**
+     * Vitest 3 fallback callback with task updates.
+     *
+     * @param {unknown[] | undefined} packs
+     */
+    onTaskUpdate(packs: unknown[] | undefined): Promise<void>;
     #private;
 }
 import { Client as TestomatioClient } from '../client.js';

package/lib/adapter/vitest.js

@@ -22,19 +22,34 @@ const debug = (0, debug_1.default)('@testomatio/reporter:adapter-jest');
 class VitestReporter {
     constructor(config = {}) {
         this.client = new client_js_1.Client({ apiKey: config?.apiKey });
-        /**
-         * @type {(TestData & {status: string})[]} tests
-         */
+        /** @type {(TestData & {status: string, _reportKey?: string | null})[]} tests */
         this.tests = [];
         this._finalized = false;
         this._finalizing = false;
+        this._runStartedAtMs = null;
+        this._runStartedAtMicros = null;
+        this._reportedTestKeys = new Set();
+        this._liveQueue = Promise.resolve();
     }
     // on run start
     onInit() {
+        const now = Date.now();
         this._finalized = false;
         this._finalizing = false;
+        this._runStartedAtMs = now;
+        this._runStartedAtMicros = now * 1000;
+        this._reportedTestKeys = new Set();
+        this._liveQueue = Promise.resolve();
         this.client.createRun();
     }
+    /**
+     * Vitest 3/4 callback fired when test run starts.
+     */
+    onTestRunStart() {
+        const now = Date.now();
+        this._runStartedAtMs = now;
+        this._runStartedAtMicros = now * 1000;
+    }
     /**
      * @param {VitestTestFile[] | undefined} files // array with results;
      * @param {unknown[] | undefined} errors // errors does not contain errors from tests; probably its testrunner errors
@@ -68,12 +83,19 @@ class VitestReporter {
             debug(this.tests.length, 'tests collected');
             // send tests to Testomat.io
             for (const test of this.tests) {
+                if (test._reportKey && this._reportedTestKeys.has(test._reportKey))
+                    continue;
+                if (test._reportKey)
+                    this._reportedTestKeys.add(test._reportKey);
                 await this.client.addTestRun(test.status, test);
             }
+            await this._liveQueue;
             console.log('finished');
             if (errors.length)
                 console.error('Vitest adapter errors:', errors);
-            await this.client.updateRunStatus(getRunStatusFromResults(files));
+            const startedAtMs = this._runStartedAtMs || getEarliestTestStartMs(files) || Date.now();
+            const duration = Math.max(0, (Date.now() - startedAtMs) / 1000);
+            await this.client.updateRunStatus(getRunStatusFromResults(files), { duration });
             this._finalized = true;
         }
         finally {
@@ -92,6 +114,28 @@ class VitestReporter {
             .filter(Boolean);
         await this.onFinished(files, errors);
     }
+    /**
+     * Vitest 4 callback fired when single test case is finished.
+     *
+     * @param {unknown} testCase
+     */
+    async onTestCaseResult(testCase) {
+        await this.#reportLive(testCase);
+    }
+    /**
+     * Vitest 3 fallback callback with task updates.
+     *
+     * @param {unknown[] | undefined} packs
+     */
+    async onTaskUpdate(packs) {
+        if (!Array.isArray(packs) || !packs.length)
+            return;
+        for (const pack of packs) {
+            const test = getTestFromTaskUpdatePack(pack);
+            if (test)
+                await this.#reportLive(test);
+        }
+    }
     /* non-used listeners
     onUserConsoleLog(log) {}
     onPathsCollected(paths) {} // paths array to files with tests
@@ -126,25 +170,50 @@ class VitestReporter {
     /**
      * Processes task and returns test data ready to be sent to Testomat.io
      *
-     * @param {VitestTest} test
+     * @param {any} test
      *
-     * @returns {TestData & {status: 'passed' | 'failed' | 'skipped'}}
+     * @returns {TestData & {status: 'passed' | 'failed' | 'skipped', _reportKey?: string | null}}
      */
     #getDataFromTest(test) {
+        const normalized = normalizeVitestTest(test);
+        const reportKey = getReportKey(test, normalized);
+        const startMicros = typeof normalized.startTime === 'number'
+            ? Math.floor(normalized.startTime * 1000)
+            : this._runStartedAtMicros || undefined;
         return {
-            error: test.result?.errors ? test.result.errors[0] : undefined,
-            file: test.file?.name || test.file?.filepath || '',
-            logs: test.logs ? transformLogsToString(test.logs) : '',
-            meta: test.meta,
+            _reportKey: reportKey,
+            error: normalized.error,
+            file: normalized.file,
+            logs: normalized.logs,
+            meta: normalized.meta,
             // @ts-ignore - STATUS values are string literals but type system sees them as string
-            status: getTestStatus(test),
-            suite_title: test.suite?.name || test.file?.name || test.file?.filepath,
-            test_id: (0, utils_js_1.getTestomatIdFromTestTitle)(test.name),
-            time: test.result?.duration || 0,
-            title: test.name,
+            status: getTestStatus(normalized.state, normalized.mode),
+            suite_title: normalized.suiteTitle,
+            test_id: (0, utils_js_1.getTestomatIdFromTestTitle)(normalized.name),
+            time: normalized.duration,
+            timestamp: startMicros,
+            title: normalized.name,
             // testomatio functions (artifacts, logs, steps, meta) are not supported
         };
     }
+    /**
+     * @param {unknown} testCase
+     */
+    async #reportLive(testCase) {
+        if (this._finalized || this._finalizing)
+            return;
+        const normalized = normalizeVitestTest(testCase);
+        if (!isLiveReportableState(normalized.state, normalized.mode))
+            return;
+        const data = this.#getDataFromTest(testCase);
+        if (!data._reportKey || this._reportedTestKeys.has(data._reportKey))
+            return;
+        this._reportedTestKeys.add(data._reportKey);
+        this._liveQueue = this._liveQueue
+            .then(() => this.client.addTestRun(data.status, data))
+            .catch(() => undefined);
+        await this._liveQueue;
+    }
 }
 exports.VitestReporter = VitestReporter;
 /**
@@ -159,16 +228,15 @@ function getRunStatusFromResults(files) {
      */
     let status = 'finished'; // default status (if no failed or passed tests)
     files.forEach(file => {
-        // search for failed tests
-        file.tasks.forEach(taskOrSuite => {
-            if (taskOrSuite.result?.state === 'fail') {
+        getTasks(file).forEach(taskOrSuite => {
+            if (isFailedState(taskOrSuite?.result?.state)) {
                 status = 'failed'; // set status to failed if any test failed
             }
         });
         // if there are no failed tests > search for passed tests
         if (status !== 'failed') {
-            file.tasks.forEach(taskOrSuite => {
-                if (taskOrSuite.result?.state === 'pass') {
+            getTasks(file).forEach(taskOrSuite => {
+                if (isPassedState(taskOrSuite?.result?.state)) {
                     status = 'passed'; // set status to passed if any test passed (and there are no failed tests)
                 }
             });
@@ -179,17 +247,18 @@ function getRunStatusFromResults(files) {
 /**
  * Returns test status in Testomat.io format
  *
- * @param {VitestTest} test
+ * @param {string | undefined} state
+ * @param {string | undefined} mode
  * @returns 'passed' | 'failed' | 'skipped'
  */
-function getTestStatus(test) {
-    if (test.result?.state === 'fail')
+function getTestStatus(state, mode) {
+    if (isFailedState(state))
         return constants_js_1.STATUS.FAILED;
-    if (test.result?.state === 'pass')
+    if (isPassedState(state))
         return constants_js_1.STATUS.PASSED;
-    if (test.result?.state === 'skip' || (!test.result && test.mode === 'skip'))
+    if (isSkippedState(state) || (!state && mode === 'skip'))
         return constants_js_1.STATUS.SKIPPED;
-    console.error(picocolors_1.default.red('Unprocessed case for defining test status. Contact dev team. Test:'), test);
+    console.error(picocolors_1.default.red('Unprocessed case for defining test status. Contact dev team. State:'), state);
     return constants_js_1.STATUS.SKIPPED;
 }
 /**
@@ -221,10 +290,165 @@ function getTasks(node) {
         return node.tasks;
     if (Array.isArray(node.children))
         return node.children;
+    if (node.children && typeof node.children[Symbol.iterator] === 'function')
+        return Array.from(node.children);
     if (node.task)
         return [node.task];
     return [];
 }
+/**
+ * @param {string | undefined} state
+ * @returns {boolean}
+ */
+function isFailedState(state) {
+    return state === 'fail' || state === 'failed';
+}
+/**
+ * @param {string | undefined} state
+ * @returns {boolean}
+ */
+function isPassedState(state) {
+    return state === 'pass' || state === 'passed';
+}
+/**
+ * @param {string | undefined} state
+ * @returns {boolean}
+ */
+function isSkippedState(state) {
+    return state === 'skip' || state === 'skipped' || state === 'todo';
+}
+/**
+ * Accept only completed test states for live upload to avoid reporting
+ * intermediate task updates as skipped.
+ *
+ * @param {string | undefined} state
+ * @param {string | undefined} mode
+ * @returns {boolean}
+ */
+function isLiveReportableState(state, mode) {
+    if (isFailedState(state) || isPassedState(state) || isSkippedState(state))
+        return true;
+    if (!state && mode === 'skip')
+        return true;
+    return false;
+}
+/**
+ * @param {VitestTestFile[] | undefined} files
+ * @returns {number | null}
+ */
+function getEarliestTestStartMs(files) {
+    let earliest = null;
+    const walk = node => {
+        if (!node)
+            return;
+        const startTime = node?.result?.startTime;
+        if (typeof startTime === 'number' && !Number.isNaN(startTime)) {
+            if (earliest == null || startTime < earliest)
+                earliest = startTime;
+        }
+        getTasks(node).forEach(walk);
+    };
+    (files || []).forEach(walk);
+    return earliest;
+}
+/**
+ * @param {any} test
+ * @returns {{
+ *  name: string,
+ *  state: string | undefined,
+ *  mode: string | undefined,
+ *  duration: number,
+ *  startTime: number | undefined,
+ *  error: any,
+ *  file: string,
+ *  suiteTitle: string,
+ *  logs: string,
+ *  meta: any
+ * }}
+ */
+function normalizeVitestTest(test) {
+    if (test && typeof test.result === 'function') {
+        const result = test.result();
+        const diagnostic = typeof test.diagnostic === 'function' ? test.diagnostic() : undefined;
+        const state = result?.state;
+        const duration = diagnostic?.duration || 0;
+        const startTime = diagnostic?.startTime;
+        const error = Array.isArray(result?.errors) ? result.errors[0] : undefined;
+        const file = test.module?.relativeModuleId ||
+            test.module?.moduleId ||
+            test.task?.file?.name ||
+            test.task?.file?.filepath ||
+            '';
+        const suiteTitle = (test.parent?.type === 'suite' ? test.parent?.name : null) ||
+            test.task?.suite?.name ||
+            test.task?.file?.name ||
+            file;
+        return {
+            name: test.name || test.task?.name || '',
+            state,
+            mode: test.options?.mode || test.task?.mode,
+            duration,
+            startTime,
+            error,
+            file,
+            suiteTitle,
+            logs: '',
+            meta: typeof test.meta === 'function' ? test.meta() : {},
+        };
+    }
+    return {
+        name: test?.name || '',
+        state: test?.result?.state,
+        mode: test?.mode,
+        duration: test?.result?.duration || 0,
+        startTime: test?.result?.startTime,
+        error: test?.result?.errors ? test.result.errors[0] : undefined,
+        file: test?.file?.name || test?.file?.filepath || '',
+        suiteTitle: test?.suite?.name || test?.file?.name || test?.file?.filepath || '',
+        logs: test?.logs ? transformLogsToString(test.logs) : '',
+        meta: test?.meta,
+    };
+}
+/**
+ * @param {any} test
+ * @param {{file: string, suiteTitle: string, name: string, startTime?: number}} normalized
+ * @returns {string | null}
+ */
+function getReportKey(test, normalized) {
+    if (test?.id)
+        return String(test.id);
+    if (test?.task?.id)
+        return String(test.task.id);
+    if (!normalized?.name)
+        return null;
+    const loc = test?.location || test?.task?.location;
+    const locationKey = loc ? `${loc.line || ''}:${loc.column || ''}` : '';
+    const startKey = typeof normalized.startTime === 'number' && !Number.isNaN(normalized.startTime) ? String(normalized.startTime) : '';
+    return `${normalized.file}::${normalized.suiteTitle}::${normalized.name}::${locationKey}::${startKey}`;
+}
+/**
+ * Vitest can pass task updates as tuples. Try to extract a test-like object.
+ *
+ * @param {unknown} pack
+ * @returns {any | null}
+ */
+function getTestFromTaskUpdatePack(pack) {
+    if (!pack)
+        return null;
+    if (Array.isArray(pack)) {
+        if (pack[2]?.type === 'test')
+            return pack[2];
+        if (pack[1]?.type === 'test')
+            return pack[1];
+        if (pack[0]?.type === 'test')
+            return pack[0];
+        return null;
+    }
+    const objectPack = /** @type {any} */ (pack);
+    if (typeof objectPack === 'object' && objectPack?.type === 'test')
+        return objectPack;
+    return null;
+}
 module.exports = VitestReporter;
 
 module.exports.VitestReporter = VitestReporter;

package/lib/client.d.ts

@@ -65,9 +65,10 @@ export class Client {
      *
      * Updates the status of the current test run and finishes the run.
      * @param {'passed' | 'failed' | 'skipped' | 'finished'} status - The status of the current test run.
+     * @param {Partial<import('../types/types.js').RunData>} [params] - Additional run params (e.g. duration).
      * Must be one of "passed", "failed", or "finished"
      * @returns {Promise<any>} - A Promise that resolves when finishes the run.
      */
-    updateRunStatus(status: "passed" | "failed" | "skipped" | "finished"): Promise<any>;
+    updateRunStatus(status: "passed" | "failed" | "skipped" | "finished", params?: Partial<import("../types/types.js").RunData>): Promise<any>;
 }
 import { S3Uploader } from './uploader.js';

package/lib/client.js

@@ -308,17 +308,18 @@ class Client {
      *
      * Updates the status of the current test run and finishes the run.
      * @param {'passed' | 'failed' | 'skipped' | 'finished'} status - The status of the current test run.
+     * @param {Partial<import('../types/types.js').RunData>} [params] - Additional run params (e.g. duration).
      * Must be one of "passed", "failed", or "finished"
      * @returns {Promise<any>} - A Promise that resolves when finishes the run.
      */
-    async updateRunStatus(status) {
+    async updateRunStatus(status, params = {}) {
         this.pipes ||= await (0, index_js_1.pipesFactory)(this.paramsForPipesFactory || {}, this.pipeStore);
         this.runId ||= (0, utils_js_1.readLatestRunId)();
         debug('Updating run status...');
         // all pipes disabled, skipping
         if (!this.pipes?.filter(p => p.isEnabled).length)
             return Promise.resolve();
-        const runParams = { status };
+        const runParams = { ...params, status };
         this.queue = this.queue
             .then(() => Promise.all(this.pipes.map(p => p.finishRun(runParams))))
             .then(() => {

package/lib/pipe/bitbucket.js

@@ -101,6 +101,14 @@ class BitbucketPipe {
     async finishRun(runParams) {
         if (!this.isEnabled)
             return;
+        if (!this.ENV.BITBUCKET_PR_ID) {
+            log_js_1.log.warn(picocolors_1.default.yellow('Bitbucket'), 'Skipping PR comment: BITBUCKET_PR_ID is not set. Run this pipe in a Bitbucket pull-requests pipeline.');
+            return;
+        }
+        if (!this.ENV.BITBUCKET_WORKSPACE || !this.ENV.BITBUCKET_REPO_SLUG) {
+            log_js_1.log.warn(picocolors_1.default.yellow('Bitbucket'), 'Skipping PR comment: BITBUCKET_WORKSPACE or BITBUCKET_REPO_SLUG is missing.');
+            return;
+        }
         if (runParams.tests)
             runParams.tests.forEach(t => this.addTest(t));
         // Clean up the logs from ANSI codes
@@ -194,9 +202,16 @@ class BitbucketPipe {
             log_js_1.log.info(picocolors_1.default.yellow('Bitbucket'), `Report created: ${picocolors_1.default.magenta(commentURL)}`);
         }
         catch (err) {
+            const isForbiddenError = `${err}`.includes('Forbidden') || `${err}`.includes('403');
+            const scopeHint = isForbiddenError
+                ? '\nHint: use a token that can write PR comments '
+                    + '(recommended: Repository Access Token with Pull requests: Write '
+                    + 'and Repository: Read) and run inside a pull-requests pipeline '
+                    + 'where BITBUCKET_PR_ID is available.'
+                : '';
             console.error(constants_js_1.APP_PREFIX, picocolors_1.default.yellow('Bitbucket'), `Couldn't create Bitbucket report\n${err}.
       Request URL: ${commentsRequestURL}
-      Request data: ${body}`);
+      Request data: ${body}${scopeHint}`);
         }
     }
     async sync() {

package/lib/pipe/html.js

@@ -936,6 +936,16 @@ function normalizeArtifacts(test) {
     return allArtifacts
         .map(artifact => {
         if (typeof artifact === 'string') {
+            if (/^https?:\/\//i.test(artifact)) {
+                const base = path_1.default.basename(new URL(artifact).pathname) || artifact;
+                return {
+                    name: base,
+                    title: base,
+                    path: artifact,
+                    fsPath: null,
+                    relativePath: artifact,
+                };
+            }
             const abs = path_1.default.isAbsolute(artifact) ? artifact : path_1.default.resolve(process.cwd(), artifact);
             const href = artifact.startsWith('file://') ? artifact : (0, file_url_1.default)(abs, { resolve: true });
             const base = path_1.default.basename(abs);
@@ -950,9 +960,14 @@ function normalizeArtifacts(test) {
         if (artifact?.path) {
             const raw = String(artifact.path);
             const isFileUrl = raw.startsWith('file://');
-            const abs = isFileUrl ? null : path_1.default.isAbsolute(raw) ? raw : path_1.default.resolve(process.cwd(), raw);
-            const href = isFileUrl ? raw : (0, file_url_1.default)(abs, { resolve: true });
-            const base = abs ? path_1.default.basename(abs) : artifact.name || artifact.title || 'attachment';
+            const isHttpUrl = /^https?:\/\//i.test(raw);
+            const abs = isFileUrl || isHttpUrl ? null : path_1.default.isAbsolute(raw) ? raw : path_1.default.resolve(process.cwd(), raw);
+            const href = isFileUrl || isHttpUrl ? raw : (0, file_url_1.default)(abs, { resolve: true });
+            const base = abs
+                ? path_1.default.basename(abs)
+                : isHttpUrl
+                    ? path_1.default.basename(new URL(raw).pathname) || artifact.name || artifact.title || 'attachment'
+                    : artifact.name || artifact.title || 'attachment';
             return {
                 ...artifact,
                 name: artifact.name || artifact.title || base,