@testomatio/reporter 2.20.0 → 2.21.0

package/lib/utils/detectRunner.js

@@ -0,0 +1,149 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isPlaceholderCommand = isPlaceholderCommand;
+exports.detectRunner = detectRunner;
+exports.runnerHint = runnerHint;
+exports.buildFilterWarning = buildFilterWarning;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const picocolors_1 = __importDefault(require("picocolors"));
+const LAUNCHER_PREFIXES = ['npx', 'npm', 'yarn', 'pnpm', 'bun', 'node', 'bunx'];
+const PLACEHOLDER_PHRASE_RE = /^<?-*(?:run|your|test)\s+(?:[\w-]+\s+)?(?:tests?|command|runner)>?$/i;
+const RUNNERS = [
+    {
+        name: 'Playwright',
+        configs: ['playwright.config.js', 'playwright.config.ts', 'playwright.config.mjs', 'playwright.config.cjs'],
+        dependencies: ['@playwright/test'],
+        command: 'npx playwright test',
+    },
+    {
+        name: 'CodeceptJS',
+        configs: ['codecept.conf.js', 'codecept.conf.ts'],
+        dependencies: ['codeceptjs'],
+        command: 'npx codeceptjs run',
+    },
+    {
+        name: 'WebdriverIO',
+        configs: ['wdio.conf.js', 'wdio.conf.ts'],
+        dependencies: ['@wdio/cli'],
+        dependencyPrefix: '@wdio/',
+        command: 'npx wdio run ./wdio.conf.js',
+    },
+    {
+        name: 'Cypress',
+        configs: ['cypress.config.js', 'cypress.config.ts'],
+        dependencies: ['cypress'],
+        command: 'npx cypress run',
+    },
+    {
+        name: 'Jest',
+        configs: ['jest.config.js', 'jest.config.ts', 'jest.config.mjs', 'jest.config.cjs', 'jest.config.json'],
+        dependencies: ['jest'],
+        command: 'npx jest',
+    },
+    {
+        name: 'Vitest',
+        configs: ['vitest.config.js', 'vitest.config.ts', 'vitest.config.mjs'],
+        dependencies: ['vitest'],
+        command: 'npx vitest run',
+    },
+    {
+        name: 'Mocha',
+        configs: ['.mocharc.js', '.mocharc.cjs', '.mocharc.json', '.mocharc.yml', '.mocharc.yaml'],
+        dependencies: ['mocha'],
+        command: 'npx mocha',
+    },
+    {
+        name: 'Cucumber',
+        configs: ['cucumber.js', 'cucumber.cjs'],
+        dependencies: ['@cucumber/cucumber'],
+        command: 'npx cucumber-js',
+    },
+];
+/**
+ * @param {string} command
+ * @returns {boolean}
+ */
+function isPlaceholderCommand(command) {
+    const cmd = (command || '').trim();
+    if (!cmd)
+        return false;
+    if (cmd.includes('<') || cmd.includes('>'))
+        return true;
+    const firstToken = cmd.split(/\s+/)[0].toLowerCase();
+    if (LAUNCHER_PREFIXES.includes(firstToken))
+        return false;
+    return PLACEHOLDER_PHRASE_RE.test(cmd);
+}
+/**
+ * @param {string} cwd
+ * @returns {Record<string, string>}
+ */
+function readPackageDependencies(cwd) {
+    try {
+        const pkg = JSON.parse(fs_1.default.readFileSync(path_1.default.join(cwd, 'package.json'), 'utf8'));
+        return { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
+    }
+    catch {
+        return {};
+    }
+}
+/**
+ * @param {string} [cwd]
+ * @returns {{ name: string, command: string } | null}
+ */
+function detectRunner(cwd = process.cwd()) {
+    const dependencies = readPackageDependencies(cwd);
+    for (const runner of RUNNERS) {
+        const config = runner.configs.find(file => fs_1.default.existsSync(path_1.default.join(cwd, file)));
+        const hasDependency = runner.dependencies?.some(dep => dependencies[dep])
+            || (runner.dependencyPrefix && Object.keys(dependencies).some(dep => dep.startsWith(runner.dependencyPrefix)));
+        if (!config && !hasDependency)
+            continue;
+        let command = runner.command;
+        if (runner.name === 'WebdriverIO') {
+            command = `npx wdio run ./${config || 'wdio.conf.js'}`;
+        }
+        return { name: runner.name, command };
+    }
+    return null;
+}
+/**
+ * @param {string} [cwd]
+ * @returns {string}
+ */
+function runnerHint(cwd = process.cwd()) {
+    const runner = detectRunner(cwd);
+    if (runner) {
+        return `💡 Looks like this project uses ${runner.name}. Try: ${picocolors_1.default.green(runner.command)}`;
+    }
+    return `💡 Replace the placeholder with your test runner command, e.g. ${picocolors_1.default.green('npx playwright test')}`;
+}
+/**
+ * @param {string} command
+ * @returns {string[]}
+ */
+function buildFilterWarning(command) {
+    return [
+        '⚠️  The runner exited with a non-zero code while a Testomat.io filter was applied.',
+        'Your command was modified to run only tests matching Testomat IDs:',
+        `  ${picocolors_1.default.bold(command)}`,
+        'This usually means no local tests matched the filter:',
+        '  - tests are not annotated with the matching @Txxxxxx IDs in their titles',
+        '  - the filter selected tests from another repository or branch',
+        '  - the test runner does not support the appended --grep / --testNamePattern option',
+        '💡 Hint: run the printed command without the appended filter option to see all your tests,',
+        'then annotate them with Testomat IDs.',
+    ];
+}
+
+module.exports.isPlaceholderCommand = isPlaceholderCommand;
+
+module.exports.detectRunner = detectRunner;
+
+module.exports.runnerHint = runnerHint;
+
+module.exports.buildFilterWarning = buildFilterWarning;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@testomatio/reporter",
-  "version": "2.20.0",
+  "version": "2.21.0",
   "description": "Testomatio Reporter Client",
   "engines": {
     "node": ">=22"
@@ -102,7 +102,7 @@
     "jasmine": "^5.2.0",
     "jest": "^27.4.7",
     "jsdom": "^22.1.0",
-    "mocha": "^9.2.0",
+    "mocha": "^12.0.3",
     "mock-http-server": "^1.4.5",
     "pino": "^8.15.0",
     "prettier": "^3.2.5",

package/src/adapter/codecept.js

@@ -20,7 +20,7 @@ if (!global.codeceptjs) {
 }
 
 // @ts-ignore
-const { event, recorder, codecept, output } = global.codeceptjs;
+const { event, recorder, codecept, output, container } = global.codeceptjs;
 
 const [, MAJOR_VERSION, MINOR_VERSION] = codecept
   .version()
@@ -59,6 +59,7 @@ function CodeceptReporter(config) {
   let isRunFinalized = false;
 
   const testTimeMap = {};
+  const failedUrls = new Map();
   const clientConfig = buildCodeceptClientConfig(config);
   const client = new TestomatClient(clientConfig);
 
@@ -243,6 +244,22 @@ function CodeceptReporter(config) {
     }, true);
   });
 
+  event.dispatcher.on(event.test.failed, (test, _err, hookName) => {
+    // no browser in suite hooks
+    if (hookName === 'BeforeSuite' || hookName === 'AfterSuite') return;
+    const helper = Object.values(container.helpers()).find(h => typeof h.grabCurrentUrl === 'function');
+    if (!helper) return;
+
+    recorder.add('url of failed test', async () => {
+      try {
+        const url = await helper.grabCurrentUrl();
+        if (url) failedUrls.set(test.uid, url);
+      } catch (err) {
+        debug('Failed to grab url of failed test', err?.message || err);
+      }
+    }, true);
+  });
+
   event.dispatcher.on(event.test.skipped, test => {
     const { uid, tags, title } = test.simplify();
 
@@ -274,6 +291,9 @@ function CodeceptReporter(config) {
     const keyValues = services.keyValues.get(test.fullTitle());
     const links = services.links.get(test.fullTitle());
     const screenshotOnFailPath = artifacts.screenshot || null;
+    const meta = { ...keyValues, ...test.meta };
+    if (test.state === STATUS.FAILED && failedUrls.has(uid)) meta.failedUrl = failedUrls.get(uid);
+    failedUrls.delete(uid);
 
     // Build step hierarchy with screenshot from screenshotOnFail
     const stepHierarchy = buildUnifiedStepHierarchy(
@@ -297,7 +317,7 @@ function CodeceptReporter(config) {
       logs,
       links,
       manuallyAttachedArtifacts,
-      meta: { ...keyValues, ...test.meta },
+      meta,
     });
     reportTestPromises.push(reportTestPromise);
 

package/src/adapter/e2e.js

@@ -94,6 +94,8 @@ export function toTestData(result, groups, artifactsRoot) {
   if (result.repeat) meta.repeat = result.repeat;
   if (attempts.length > 1) meta.retryCount = attempts.length - 1;
   if (result.status === 'flaky') meta.isFlaky = true;
+  const status = testStatus(result.status);
+  if (status === STATUS.FAILED && final?.failure?.url) meta.failedUrl = final.failure.url;
 
   const filesById = new Map();
   for (const artifact of final?.artifacts || []) {
@@ -128,7 +130,7 @@ export function toTestData(result, groups, artifactsRoot) {
     meta,
   };
 
-  return { status: testStatus(result.status), data };
+  return { status, data };
 }
 
 /** Attempts of a result; a serial member keeps its part of the group's attempts. */
@@ -140,6 +142,7 @@ function attemptsOf(result, groups) {
     return {
       status: member?.status || attempt.status,
       error: member?.error || attempt.error,
+      failure: member?.failure || attempt.failure,
       steps: member?.steps || [],
       appLog: member?.appLog || [],
       skip: member?.skip,

package/src/adapter/playwright.js

@@ -110,7 +110,11 @@ class PlaywrightReporter {
       if (result.status === 'passed') status = 'failed';
     }
 
-    const reportTestPromise = this.client.addTestRun(checkStatus(status), {
+    const testStatus = checkStatus(status);
+    let failedUrl;
+    if (testStatus === Status.FAILED) failedUrl = lastNavigatedUrl(result.steps, test.parent.project().use.baseURL);
+
+    const reportTestPromise = this.client.addTestRun(testStatus, {
       rid: `${rid}-${project.name}`,
       error,
       test_id: getTestomatIdFromTestTitle(`${title} ${tags.join(' ')}`),
@@ -128,6 +132,7 @@ class PlaywrightReporter {
         isMobile: project.isMobile,
         project: project.name,
         projectDependencies: project.dependencies?.length ? project.dependencies : null,
+        failedUrl,
         ...testMeta,
         ...meta,
         ...project.metadata, // metadata has any type (in playwright), but we will stringify it in client.js
@@ -244,6 +249,31 @@ function isScreenshotArtifact(attachment) {
   return attachment?.contentType === 'image/png' && attachment?.name === 'screenshot';
 }
 
+/**
+ * Last url the test navigated to; the reporter has no access to the page itself.
+ * @returns {string|undefined}
+ */
+function lastNavigatedUrl(steps, baseURL) {
+  const url = findLastNavigation(steps);
+  if (!url) return;
+  try {
+    return new URL(url, baseURL).href;
+  } catch {
+    return url;
+  }
+}
+
+function findLastNavigation(steps = []) {
+  let url;
+  for (const step of steps) {
+    // 'Navigate to "url"' since Playwright 1.53, 'page.goto(url)' before
+    const match = step.title.match(/^Navigate to "(.+)"$/) || step.title.match(/^page\.goto\((.+)\)$/);
+    if (step.category === 'pw:api' && match) url = match[1];
+    url = findLastNavigation(step.steps) || url;
+  }
+  return url;
+}
+
 function appendStep(step, shift = 0) {
   // nesting too deep, ignore those steps
   if (shift >= 10) return;

package/src/adapter/vitest.js

@@ -398,7 +398,10 @@ function normalizeVitestTest(test) {
       file,
       suiteTitle,
       logs: '',
-      meta: typeof test.meta === 'function' ? test.meta() : {},
+      meta: {
+        ...(test.task?.meta || {}),
+        ...(typeof test.meta === 'function' ? test.meta() : test.meta || {}),
+      },
     };
   }
 

package/src/adapter/webdriver.js

@@ -4,6 +4,9 @@ import { getTestomatIdFromTestTitle, fileSystem } from '../utils/utils.js';
 import { services } from '../services/index.js';
 import { TESTOMAT_TMP_STORAGE_DIR } from '../constants.js';
 import { stringToMD5Hash } from '../data-storage.js';
+import createDebugMessages from 'debug';
+
+const debug = createDebugMessages('@testomatio/reporter:adapter:webdriver');
 
 global.TESTOMATIO_ADAPTER = 'webdriverio';
 
@@ -122,7 +125,8 @@ class WebdriverReporter extends WDIOReporter {
   async addTest(test) {
     if (!this.client) return;
 
-    const { title, _duration: duration, state, error, output, links, artifacts, meta, logs } = test;
+    const { title, _duration: duration, state, error, output, links, artifacts, logs } = test;
+    let { meta } = test;
 
     const testId = getTestomatIdFromTestTitle(title);
 
@@ -133,6 +137,11 @@ class WebdriverReporter extends WDIOReporter {
 
     const rid = stringToMD5Hash(test.fullTitle);
 
+    if (state === 'failed') {
+      const failedUrl = (await grabCurrentUrl()) || lastNavigatedUrl(output);
+      if (failedUrl) meta = { ...meta, failedUrl };
+    }
+
     await this.client.addTestRun(state, {
       rid,
       manuallyAttachedArtifacts: test.artifacts,
@@ -150,7 +159,7 @@ class WebdriverReporter extends WDIOReporter {
   /**
    * @param {import('../../types/types.js').WebdriverIOScenario} scenario
    */
-  addBddScenario(scenario) {
+  async addBddScenario(scenario) {
     if (!this.client) return;
 
     const { title, _duration: duration } = scenario;
@@ -169,6 +178,12 @@ class WebdriverReporter extends WDIOReporter {
 
     const tags = scenario.tags.map(tag => tag.name);
 
+    let meta;
+    if (scenarioState === 'failed') {
+      const failedUrl = await grabCurrentUrl();
+      if (failedUrl) meta = { failedUrl };
+    }
+
     return this.client.addTestRun(scenarioState, {
       error: error ? Error(error) : null,
       title,
@@ -176,11 +191,38 @@ class WebdriverReporter extends WDIOReporter {
       time: duration,
       tags,
       file: scenario.file,
+      meta,
       // filesBuffers: screenshotsBuffers,
     });
   }
 }
 
+/**
+ * Current url of the browser in this worker; reporters run in the worker process.
+ * @returns {Promise<string|undefined>}
+ */
+async function grabCurrentUrl() {
+  // @ts-ignore
+  const browser = globalThis.browser;
+  if (typeof browser?.getUrl !== 'function') return;
+  try {
+    const url = await browser.getUrl();
+    // multiremote returns a url per browser
+    if (typeof url === 'string') return url;
+  } catch (err) {
+    debug('Failed to grab url of failed test', err?.message || err);
+  }
+}
+
+/**
+ * Last url opened by the test, taken from its command log.
+ * @returns {string|undefined}
+ */
+function lastNavigatedUrl(output = []) {
+  const navigations = output.filter(el => el.endpoint === '/session/:sessionId/url' && el.body?.url);
+  return navigations.at(-1)?.body.url;
+}
+
 /**
  *
  * @param {*} fullTestTitle