@testomatio/reporter 2.19.0 → 2.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/adapter/codecept.js +26 -2
- package/lib/adapter/cucumber/current.js +1 -0
- package/lib/adapter/cypress-plugin/index.js +1 -0
- package/lib/adapter/e2e.d.ts +49 -0
- package/lib/adapter/e2e.js +254 -0
- package/lib/adapter/jasmine.js +1 -0
- package/lib/adapter/jest.js +1 -0
- package/lib/adapter/mocha.d.ts +4 -1
- package/lib/adapter/mocha.js +123 -122
- package/lib/adapter/nightwatch.js +1 -0
- package/lib/adapter/playwright.js +33 -1
- package/lib/adapter/vitest.js +5 -1
- package/lib/adapter/webdriver.js +45 -2
- package/lib/allureReader.d.ts +4 -1
- package/lib/allureReader.js +2 -1
- package/lib/bin/cli.js +212 -19
- package/lib/client.d.ts +1 -0
- package/lib/client.js +12 -3
- package/lib/data-storage.js +4 -3
- package/lib/fetch/report.d.ts +31 -0
- package/lib/fetch/report.js +81 -0
- package/lib/fetch/testruns.d.ts +133 -0
- package/lib/fetch/testruns.js +286 -0
- package/lib/pipe/markdown.d.ts +1 -0
- package/lib/pipe/markdown.js +7 -0
- package/lib/pipe/testomatio.js +17 -1
- package/lib/replay.js +1 -0
- package/lib/reporter-functions.js +39 -0
- package/lib/uploader.d.ts +17 -0
- package/lib/uploader.js +75 -1
- package/lib/utils/detectRunner.d.ts +23 -0
- package/lib/utils/detectRunner.js +149 -0
- package/lib/utils/pipe_utils.d.ts +16 -3
- package/lib/utils/pipe_utils.js +45 -4
- package/lib/xmlReader.js +2 -0
- package/package.json +8 -3
- package/src/adapter/codecept.js +24 -2
- package/src/adapter/cucumber/current.js +2 -0
- package/src/adapter/cypress-plugin/index.js +2 -0
- package/src/adapter/e2e.js +248 -0
- package/src/adapter/jasmine.js +2 -0
- package/src/adapter/jest.js +2 -0
- package/src/adapter/mocha.js +146 -145
- package/src/adapter/nightwatch.js +2 -0
- package/src/adapter/playwright.js +33 -1
- package/src/adapter/vitest.js +6 -1
- package/src/adapter/webdriver.js +46 -2
- package/src/allureReader.js +2 -1
- package/src/bin/cli.js +246 -18
- package/src/client.js +11 -3
- package/src/data-storage.js +4 -3
- package/src/fetch/report.js +86 -0
- package/src/fetch/testruns.js +283 -0
- package/src/pipe/markdown.js +7 -0
- package/src/pipe/testomatio.js +27 -1
- package/src/replay.js +1 -0
- package/src/reporter-functions.js +41 -0
- package/src/uploader.js +81 -1
- package/src/utils/detectRunner.js +151 -0
- package/src/utils/pipe_utils.js +45 -4
- package/src/xmlReader.js +3 -0
- package/types/types.d.ts +1 -0
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.isPlaceholderCommand = isPlaceholderCommand;
|
|
7
|
+
exports.detectRunner = detectRunner;
|
|
8
|
+
exports.runnerHint = runnerHint;
|
|
9
|
+
exports.buildFilterWarning = buildFilterWarning;
|
|
10
|
+
const fs_1 = __importDefault(require("fs"));
|
|
11
|
+
const path_1 = __importDefault(require("path"));
|
|
12
|
+
const picocolors_1 = __importDefault(require("picocolors"));
|
|
13
|
+
const LAUNCHER_PREFIXES = ['npx', 'npm', 'yarn', 'pnpm', 'bun', 'node', 'bunx'];
|
|
14
|
+
const PLACEHOLDER_PHRASE_RE = /^<?-*(?:run|your|test)\s+(?:[\w-]+\s+)?(?:tests?|command|runner)>?$/i;
|
|
15
|
+
const RUNNERS = [
|
|
16
|
+
{
|
|
17
|
+
name: 'Playwright',
|
|
18
|
+
configs: ['playwright.config.js', 'playwright.config.ts', 'playwright.config.mjs', 'playwright.config.cjs'],
|
|
19
|
+
dependencies: ['@playwright/test'],
|
|
20
|
+
command: 'npx playwright test',
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
name: 'CodeceptJS',
|
|
24
|
+
configs: ['codecept.conf.js', 'codecept.conf.ts'],
|
|
25
|
+
dependencies: ['codeceptjs'],
|
|
26
|
+
command: 'npx codeceptjs run',
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
name: 'WebdriverIO',
|
|
30
|
+
configs: ['wdio.conf.js', 'wdio.conf.ts'],
|
|
31
|
+
dependencies: ['@wdio/cli'],
|
|
32
|
+
dependencyPrefix: '@wdio/',
|
|
33
|
+
command: 'npx wdio run ./wdio.conf.js',
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
name: 'Cypress',
|
|
37
|
+
configs: ['cypress.config.js', 'cypress.config.ts'],
|
|
38
|
+
dependencies: ['cypress'],
|
|
39
|
+
command: 'npx cypress run',
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
name: 'Jest',
|
|
43
|
+
configs: ['jest.config.js', 'jest.config.ts', 'jest.config.mjs', 'jest.config.cjs', 'jest.config.json'],
|
|
44
|
+
dependencies: ['jest'],
|
|
45
|
+
command: 'npx jest',
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
name: 'Vitest',
|
|
49
|
+
configs: ['vitest.config.js', 'vitest.config.ts', 'vitest.config.mjs'],
|
|
50
|
+
dependencies: ['vitest'],
|
|
51
|
+
command: 'npx vitest run',
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
name: 'Mocha',
|
|
55
|
+
configs: ['.mocharc.js', '.mocharc.cjs', '.mocharc.json', '.mocharc.yml', '.mocharc.yaml'],
|
|
56
|
+
dependencies: ['mocha'],
|
|
57
|
+
command: 'npx mocha',
|
|
58
|
+
},
|
|
59
|
+
{
|
|
60
|
+
name: 'Cucumber',
|
|
61
|
+
configs: ['cucumber.js', 'cucumber.cjs'],
|
|
62
|
+
dependencies: ['@cucumber/cucumber'],
|
|
63
|
+
command: 'npx cucumber-js',
|
|
64
|
+
},
|
|
65
|
+
];
|
|
66
|
+
/**
|
|
67
|
+
* @param {string} command
|
|
68
|
+
* @returns {boolean}
|
|
69
|
+
*/
|
|
70
|
+
function isPlaceholderCommand(command) {
|
|
71
|
+
const cmd = (command || '').trim();
|
|
72
|
+
if (!cmd)
|
|
73
|
+
return false;
|
|
74
|
+
if (cmd.includes('<') || cmd.includes('>'))
|
|
75
|
+
return true;
|
|
76
|
+
const firstToken = cmd.split(/\s+/)[0].toLowerCase();
|
|
77
|
+
if (LAUNCHER_PREFIXES.includes(firstToken))
|
|
78
|
+
return false;
|
|
79
|
+
return PLACEHOLDER_PHRASE_RE.test(cmd);
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* @param {string} cwd
|
|
83
|
+
* @returns {Record<string, string>}
|
|
84
|
+
*/
|
|
85
|
+
function readPackageDependencies(cwd) {
|
|
86
|
+
try {
|
|
87
|
+
const pkg = JSON.parse(fs_1.default.readFileSync(path_1.default.join(cwd, 'package.json'), 'utf8'));
|
|
88
|
+
return { ...(pkg.dependencies || {}), ...(pkg.devDependencies || {}) };
|
|
89
|
+
}
|
|
90
|
+
catch {
|
|
91
|
+
return {};
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* @param {string} [cwd]
|
|
96
|
+
* @returns {{ name: string, command: string } | null}
|
|
97
|
+
*/
|
|
98
|
+
function detectRunner(cwd = process.cwd()) {
|
|
99
|
+
const dependencies = readPackageDependencies(cwd);
|
|
100
|
+
for (const runner of RUNNERS) {
|
|
101
|
+
const config = runner.configs.find(file => fs_1.default.existsSync(path_1.default.join(cwd, file)));
|
|
102
|
+
const hasDependency = runner.dependencies?.some(dep => dependencies[dep])
|
|
103
|
+
|| (runner.dependencyPrefix && Object.keys(dependencies).some(dep => dep.startsWith(runner.dependencyPrefix)));
|
|
104
|
+
if (!config && !hasDependency)
|
|
105
|
+
continue;
|
|
106
|
+
let command = runner.command;
|
|
107
|
+
if (runner.name === 'WebdriverIO') {
|
|
108
|
+
command = `npx wdio run ./${config || 'wdio.conf.js'}`;
|
|
109
|
+
}
|
|
110
|
+
return { name: runner.name, command };
|
|
111
|
+
}
|
|
112
|
+
return null;
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* @param {string} [cwd]
|
|
116
|
+
* @returns {string}
|
|
117
|
+
*/
|
|
118
|
+
function runnerHint(cwd = process.cwd()) {
|
|
119
|
+
const runner = detectRunner(cwd);
|
|
120
|
+
if (runner) {
|
|
121
|
+
return `💡 Looks like this project uses ${runner.name}. Try: ${picocolors_1.default.green(runner.command)}`;
|
|
122
|
+
}
|
|
123
|
+
return `💡 Replace the placeholder with your test runner command, e.g. ${picocolors_1.default.green('npx playwright test')}`;
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* @param {string} command
|
|
127
|
+
* @returns {string[]}
|
|
128
|
+
*/
|
|
129
|
+
function buildFilterWarning(command) {
|
|
130
|
+
return [
|
|
131
|
+
'⚠️ The runner exited with a non-zero code while a Testomat.io filter was applied.',
|
|
132
|
+
'Your command was modified to run only tests matching Testomat IDs:',
|
|
133
|
+
` ${picocolors_1.default.bold(command)}`,
|
|
134
|
+
'This usually means no local tests matched the filter:',
|
|
135
|
+
' - tests are not annotated with the matching @Txxxxxx IDs in their titles',
|
|
136
|
+
' - the filter selected tests from another repository or branch',
|
|
137
|
+
' - the test runner does not support the appended --grep / --testNamePattern option',
|
|
138
|
+
'💡 Hint: run the printed command without the appended filter option to see all your tests,',
|
|
139
|
+
'then annotate them with Testomat IDs.',
|
|
140
|
+
];
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
module.exports.isPlaceholderCommand = isPlaceholderCommand;
|
|
144
|
+
|
|
145
|
+
module.exports.detectRunner = detectRunner;
|
|
146
|
+
|
|
147
|
+
module.exports.runnerHint = runnerHint;
|
|
148
|
+
|
|
149
|
+
module.exports.buildFilterWarning = buildFilterWarning;
|
|
@@ -139,16 +139,29 @@ export function formatRunOutput(store: {
|
|
|
139
139
|
}, format?: string): string;
|
|
140
140
|
/**
|
|
141
141
|
* Format the runs fetched via the `fetch` command for machine-readable output.
|
|
142
|
-
* `json` prints the full array of run objects; `id` prints one run id per line;
|
|
142
|
+
* `json` prints the full array of run objects (a single object for one run); `id` prints one run id per line;
|
|
143
143
|
* any other format prints a human-readable multi-line summary per run.
|
|
144
144
|
*
|
|
145
|
-
* @param {{data?: Array}} body - Parsed response body from GET /api/v2/:project/runs.
|
|
145
|
+
* @param {{data?: Array|Object}} body - Parsed response body from GET /api/v2/:project/runs[/:id].
|
|
146
146
|
* @param {string} [format] - Value of the CLI `--format` option.
|
|
147
147
|
* @returns {string}
|
|
148
148
|
*/
|
|
149
149
|
export function formatFetchRunsOutput(body: {
|
|
150
|
-
data?: any[];
|
|
150
|
+
data?: any[] | any;
|
|
151
151
|
}, format?: string): string;
|
|
152
|
+
/**
|
|
153
|
+
* Parse a Testomat.io app URL of a run (/projects/:project/runs/:id)
|
|
154
|
+
* or of a runs page (/projects/:project/runs?filterParam=...) into API v2 request parts.
|
|
155
|
+
*
|
|
156
|
+
* @param {string} pageUrl
|
|
157
|
+
* @returns {{baseUrl: string, project: string, runId?: string, params: string[][]}|null}
|
|
158
|
+
*/
|
|
159
|
+
export function parseRunsUrl(pageUrl: string): {
|
|
160
|
+
baseUrl: string;
|
|
161
|
+
project: string;
|
|
162
|
+
runId?: string;
|
|
163
|
+
params: string[][];
|
|
164
|
+
} | null;
|
|
152
165
|
/**
|
|
153
166
|
* Calculate the approximate size of data in bytes (JSON stringified, UTF-8 encoded length).
|
|
154
167
|
* @param {Object} data - Data to measure
|
package/lib/utils/pipe_utils.js
CHANGED
|
@@ -18,6 +18,7 @@ exports.parsePipeOptions = parsePipeOptions;
|
|
|
18
18
|
exports.formatFilterListIds = formatFilterListIds;
|
|
19
19
|
exports.formatRunOutput = formatRunOutput;
|
|
20
20
|
exports.formatFetchRunsOutput = formatFetchRunsOutput;
|
|
21
|
+
exports.parseRunsUrl = parseRunsUrl;
|
|
21
22
|
exports.getObjectSize = getObjectSize;
|
|
22
23
|
exports.splitTestsIntoChunks = splitTestsIntoChunks;
|
|
23
24
|
const humanize_duration_1 = __importDefault(require("humanize-duration"));
|
|
@@ -362,17 +363,18 @@ function formatRunOutput(store, format) {
|
|
|
362
363
|
}
|
|
363
364
|
/**
|
|
364
365
|
* Format the runs fetched via the `fetch` command for machine-readable output.
|
|
365
|
-
* `json` prints the full array of run objects; `id` prints one run id per line;
|
|
366
|
+
* `json` prints the full array of run objects (a single object for one run); `id` prints one run id per line;
|
|
366
367
|
* any other format prints a human-readable multi-line summary per run.
|
|
367
368
|
*
|
|
368
|
-
* @param {{data?: Array}} body - Parsed response body from GET /api/v2/:project/runs.
|
|
369
|
+
* @param {{data?: Array|Object}} body - Parsed response body from GET /api/v2/:project/runs[/:id].
|
|
369
370
|
* @param {string} [format] - Value of the CLI `--format` option.
|
|
370
371
|
* @returns {string}
|
|
371
372
|
*/
|
|
372
373
|
function formatFetchRunsOutput(body, format) {
|
|
373
|
-
const
|
|
374
|
+
const data = body?.data || [];
|
|
375
|
+
const runs = [data].flat();
|
|
374
376
|
if (format === 'json')
|
|
375
|
-
return JSON.stringify(
|
|
377
|
+
return JSON.stringify(data, null, 2);
|
|
376
378
|
if (format === 'id')
|
|
377
379
|
return runs.map(run => run.id).filter(Boolean).join('\n');
|
|
378
380
|
return runs
|
|
@@ -380,6 +382,8 @@ function formatFetchRunsOutput(body, format) {
|
|
|
380
382
|
const lines = [`* ID: ${run.id}`];
|
|
381
383
|
if (run.title)
|
|
382
384
|
lines.push(` title: ${run.title}`);
|
|
385
|
+
if (run.rungroup_id)
|
|
386
|
+
lines.push(` rungroup: ${run.rungroup_id}`);
|
|
383
387
|
if (run.launched_at)
|
|
384
388
|
lines.push(` started at: ${run.launched_at}`);
|
|
385
389
|
if (run.finished_at)
|
|
@@ -400,6 +404,41 @@ function formatFetchRunsOutput(body, format) {
|
|
|
400
404
|
})
|
|
401
405
|
.join('\n');
|
|
402
406
|
}
|
|
407
|
+
// filterParam keys of the runs page which are API params on their own; the rest go under filter[...]
|
|
408
|
+
const RUNS_PAGE_PARAMS = ['search', 'tql', 'groupId'];
|
|
409
|
+
/**
|
|
410
|
+
* Parse a Testomat.io app URL of a run (/projects/:project/runs/:id)
|
|
411
|
+
* or of a runs page (/projects/:project/runs?filterParam=...) into API v2 request parts.
|
|
412
|
+
*
|
|
413
|
+
* @param {string} pageUrl
|
|
414
|
+
* @returns {{baseUrl: string, project: string, runId?: string, params: string[][]}|null}
|
|
415
|
+
*/
|
|
416
|
+
function parseRunsUrl(pageUrl) {
|
|
417
|
+
let url;
|
|
418
|
+
try {
|
|
419
|
+
url = new URL(pageUrl);
|
|
420
|
+
}
|
|
421
|
+
catch (e) {
|
|
422
|
+
return null;
|
|
423
|
+
}
|
|
424
|
+
const match = url.pathname.match(/\/projects\/([^/]+)\/runs(?:\/([^/]+))?\/?$/);
|
|
425
|
+
if (!match)
|
|
426
|
+
return null;
|
|
427
|
+
const [, project, runId] = match;
|
|
428
|
+
const params = [];
|
|
429
|
+
for (const [key, value] of new URLSearchParams(url.searchParams.get('filterParam') || '')) {
|
|
430
|
+
// "groups" only switches the page to the grouped view
|
|
431
|
+
if (!value || key === 'groups')
|
|
432
|
+
continue;
|
|
433
|
+
if (RUNS_PAGE_PARAMS.includes(key)) {
|
|
434
|
+
params.push([key, value]);
|
|
435
|
+
continue;
|
|
436
|
+
}
|
|
437
|
+
const [name, ...nested] = key.split('[');
|
|
438
|
+
params.push([`filter[${name}]${nested.map(part => `[${part}`).join('')}`, value]);
|
|
439
|
+
}
|
|
440
|
+
return { baseUrl: url.origin, project, runId, params };
|
|
441
|
+
}
|
|
403
442
|
|
|
404
443
|
module.exports.updateFilterType = updateFilterType;
|
|
405
444
|
|
|
@@ -431,6 +470,8 @@ module.exports.formatRunOutput = formatRunOutput;
|
|
|
431
470
|
|
|
432
471
|
module.exports.formatFetchRunsOutput = formatFetchRunsOutput;
|
|
433
472
|
|
|
473
|
+
module.exports.parseRunsUrl = parseRunsUrl;
|
|
474
|
+
|
|
434
475
|
module.exports.getObjectSize = getObjectSize;
|
|
435
476
|
|
|
436
477
|
module.exports.splitTestsIntoChunks = splitTestsIntoChunks;
|
package/lib/xmlReader.js
CHANGED
|
@@ -432,6 +432,8 @@ class XmlReader {
|
|
|
432
432
|
}
|
|
433
433
|
this.adapter.formatTest(t);
|
|
434
434
|
t.title = (0, utils_js_1.humanize)(t.title);
|
|
435
|
+
// meta must always include reporter version and adapter used
|
|
436
|
+
t.meta = { ...t.meta, reporterVersion: this.version, adapter: 'xml' };
|
|
435
437
|
});
|
|
436
438
|
}
|
|
437
439
|
formatErrors() {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@testomatio/reporter",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.21.0",
|
|
4
4
|
"description": "Testomatio Reporter Client",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22"
|
|
@@ -65,7 +65,8 @@
|
|
|
65
65
|
"test:playwright": "mocha tests/adapter/playwright.test.js",
|
|
66
66
|
"test:codecept": "mocha tests/adapter/codecept.test.js tests/adapter/codecept_comprehensive.test.js tests/adapter/codecept_steps_sections.test.js",
|
|
67
67
|
"test:vitest": "mocha tests/adapter/vitest.test.js",
|
|
68
|
-
"test:
|
|
68
|
+
"test:e2e": "mocha tests/adapter/e2e.test.js",
|
|
69
|
+
"test:frameworks": "npm run test:playwright && npm run test:codecept && npm run test:vitest && npm run test:e2e",
|
|
69
70
|
"test:all": "npm run test && npm run test:frameworks",
|
|
70
71
|
"test:adapters": "mocha tests/adapter/*.test.js",
|
|
71
72
|
"test:codecept:bug948": "mocha tests/adapter/codecept_aftersuite_failure.test.js",
|
|
@@ -101,7 +102,7 @@
|
|
|
101
102
|
"jasmine": "^5.2.0",
|
|
102
103
|
"jest": "^27.4.7",
|
|
103
104
|
"jsdom": "^22.1.0",
|
|
104
|
-
"mocha": "^
|
|
105
|
+
"mocha": "^12.0.3",
|
|
105
106
|
"mock-http-server": "^1.4.5",
|
|
106
107
|
"pino": "^8.15.0",
|
|
107
108
|
"prettier": "^3.2.5",
|
|
@@ -129,6 +130,10 @@
|
|
|
129
130
|
"./lib/adapter/cypress-plugin": "./lib/adapter/cypress-plugin/index.js",
|
|
130
131
|
"./cypress-plugin": "./lib/adapter/cypress-plugin/index.js",
|
|
131
132
|
"./cypress": "./lib/adapter/cypress-plugin/index.js",
|
|
133
|
+
"./e2e": {
|
|
134
|
+
"import": "./src/adapter/e2e.js",
|
|
135
|
+
"require": "./lib/adapter/e2e.js"
|
|
136
|
+
},
|
|
132
137
|
"./lib/adapter/jasmine.js": "./lib/adapter/jasmine.js",
|
|
133
138
|
"./jasmine": "./lib/adapter/jasmine.js",
|
|
134
139
|
"./lib/adapter/jest.js": "./lib/adapter/jest.js",
|
package/src/adapter/codecept.js
CHANGED
|
@@ -11,6 +11,8 @@ import codeceptjs from 'codeceptjs';
|
|
|
11
11
|
import { log } from '../utils/log.js';
|
|
12
12
|
|
|
13
13
|
const debug = createDebugMessages('@testomatio/reporter:adapter:codeceptjs');
|
|
14
|
+
|
|
15
|
+
global.TESTOMATIO_ADAPTER = 'codeceptjs';
|
|
14
16
|
// @ts-ignore
|
|
15
17
|
if (!global.codeceptjs) {
|
|
16
18
|
// @ts-ignore
|
|
@@ -18,7 +20,7 @@ if (!global.codeceptjs) {
|
|
|
18
20
|
}
|
|
19
21
|
|
|
20
22
|
// @ts-ignore
|
|
21
|
-
const { event, recorder, codecept, output } = global.codeceptjs;
|
|
23
|
+
const { event, recorder, codecept, output, container } = global.codeceptjs;
|
|
22
24
|
|
|
23
25
|
const [, MAJOR_VERSION, MINOR_VERSION] = codecept
|
|
24
26
|
.version()
|
|
@@ -57,6 +59,7 @@ function CodeceptReporter(config) {
|
|
|
57
59
|
let isRunFinalized = false;
|
|
58
60
|
|
|
59
61
|
const testTimeMap = {};
|
|
62
|
+
const failedUrls = new Map();
|
|
60
63
|
const clientConfig = buildCodeceptClientConfig(config);
|
|
61
64
|
const client = new TestomatClient(clientConfig);
|
|
62
65
|
|
|
@@ -241,6 +244,22 @@ function CodeceptReporter(config) {
|
|
|
241
244
|
}, true);
|
|
242
245
|
});
|
|
243
246
|
|
|
247
|
+
event.dispatcher.on(event.test.failed, (test, _err, hookName) => {
|
|
248
|
+
// no browser in suite hooks
|
|
249
|
+
if (hookName === 'BeforeSuite' || hookName === 'AfterSuite') return;
|
|
250
|
+
const helper = Object.values(container.helpers()).find(h => typeof h.grabCurrentUrl === 'function');
|
|
251
|
+
if (!helper) return;
|
|
252
|
+
|
|
253
|
+
recorder.add('url of failed test', async () => {
|
|
254
|
+
try {
|
|
255
|
+
const url = await helper.grabCurrentUrl();
|
|
256
|
+
if (url) failedUrls.set(test.uid, url);
|
|
257
|
+
} catch (err) {
|
|
258
|
+
debug('Failed to grab url of failed test', err?.message || err);
|
|
259
|
+
}
|
|
260
|
+
}, true);
|
|
261
|
+
});
|
|
262
|
+
|
|
244
263
|
event.dispatcher.on(event.test.skipped, test => {
|
|
245
264
|
const { uid, tags, title } = test.simplify();
|
|
246
265
|
|
|
@@ -272,6 +291,9 @@ function CodeceptReporter(config) {
|
|
|
272
291
|
const keyValues = services.keyValues.get(test.fullTitle());
|
|
273
292
|
const links = services.links.get(test.fullTitle());
|
|
274
293
|
const screenshotOnFailPath = artifacts.screenshot || null;
|
|
294
|
+
const meta = { ...keyValues, ...test.meta };
|
|
295
|
+
if (test.state === STATUS.FAILED && failedUrls.has(uid)) meta.failedUrl = failedUrls.get(uid);
|
|
296
|
+
failedUrls.delete(uid);
|
|
275
297
|
|
|
276
298
|
// Build step hierarchy with screenshot from screenshotOnFail
|
|
277
299
|
const stepHierarchy = buildUnifiedStepHierarchy(
|
|
@@ -295,7 +317,7 @@ function CodeceptReporter(config) {
|
|
|
295
317
|
logs,
|
|
296
318
|
links,
|
|
297
319
|
manuallyAttachedArtifacts,
|
|
298
|
-
meta
|
|
320
|
+
meta,
|
|
299
321
|
});
|
|
300
322
|
reportTestPromises.push(reportTestPromise);
|
|
301
323
|
|
|
@@ -8,6 +8,8 @@ import { getTestomatIdFromTestTitle, fileSystem } from '../../utils/utils.js';
|
|
|
8
8
|
import { config } from '../../config.js';
|
|
9
9
|
import { services } from '../../services/index.js';
|
|
10
10
|
|
|
11
|
+
global.TESTOMATIO_ADAPTER = 'cucumber';
|
|
12
|
+
|
|
11
13
|
const { GherkinDocumentParser, PickleParser } = formatterHelpers;
|
|
12
14
|
const { getGherkinScenarioLocationMap, getGherkinStepMap } = GherkinDocumentParser;
|
|
13
15
|
const { getPickleStepMap } = PickleParser;
|
|
@@ -3,6 +3,8 @@ import { getTestomatIdFromTestTitle, parseSuite } from '../../utils/utils.js';
|
|
|
3
3
|
import TestomatClient from '../../client.js';
|
|
4
4
|
import { config } from '../../config.js';
|
|
5
5
|
|
|
6
|
+
global.TESTOMATIO_ADAPTER = 'cypress';
|
|
7
|
+
|
|
6
8
|
const testomatioReporter = on => {
|
|
7
9
|
if (!config.TESTOMATIO) {
|
|
8
10
|
console.log('TESTOMATIO key is empty, ignoring reports');
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
import path from 'path';
|
|
2
|
+
import createDebugMessages from 'debug';
|
|
3
|
+
import TestomatioClient from '../client.js';
|
|
4
|
+
import { STATUS, SCREENSHOTS_ON_STEPS } from '../constants.js';
|
|
5
|
+
import { getTestomatIdFromTestTitle, truncate } from '../utils/utils.js';
|
|
6
|
+
import { formatStep, addStatusToStep, addArtifactsToStep } from './utils/step-formatter.js';
|
|
7
|
+
import { log } from '../utils/log.js';
|
|
8
|
+
|
|
9
|
+
const debug = createDebugMessages('@testomatio/reporter:adapter-e2e');
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Reporter for the e2e framework (https://github.com/tester-army/e2e).
|
|
13
|
+
* Add it to `reporters` in `e2e.config.ts`: `reporters: ['list', testomatio()]`.
|
|
14
|
+
*
|
|
15
|
+
* The config is loaded by every worker, so the client is created only inside
|
|
16
|
+
* handlers, which run in the runner process.
|
|
17
|
+
*
|
|
18
|
+
* @param {{ apiKey?: string }} [options]
|
|
19
|
+
*/
|
|
20
|
+
export function testomatio(options = {}) {
|
|
21
|
+
let client = null;
|
|
22
|
+
let runPromise = null;
|
|
23
|
+
|
|
24
|
+
function getClient() {
|
|
25
|
+
if (!client) client = new TestomatioClient({ apiKey: options.apiKey });
|
|
26
|
+
return client;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function startRun() {
|
|
30
|
+
if (!runPromise) {
|
|
31
|
+
runPromise = getClient()
|
|
32
|
+
.createRun()
|
|
33
|
+
.catch(err => log.error('Failed to create Testomat.io run:', err?.message || err));
|
|
34
|
+
}
|
|
35
|
+
return runPromise;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
return {
|
|
39
|
+
name: 'testomatio',
|
|
40
|
+
|
|
41
|
+
onEvent(event) {
|
|
42
|
+
try {
|
|
43
|
+
if (event?.type === 'run-started') startRun();
|
|
44
|
+
} catch (err) {
|
|
45
|
+
debug('onEvent failed', err);
|
|
46
|
+
}
|
|
47
|
+
},
|
|
48
|
+
|
|
49
|
+
async onRunFinished(run, signal) {
|
|
50
|
+
const reporter = getClient();
|
|
51
|
+
await startRun();
|
|
52
|
+
|
|
53
|
+
const { report, artifactsRoot } = run;
|
|
54
|
+
const groups = new Map((report.run.serialGroups || []).map(group => [group.id, group]));
|
|
55
|
+
|
|
56
|
+
const promises = [];
|
|
57
|
+
for (const result of report.run.results) {
|
|
58
|
+
if (signal?.aborted) break;
|
|
59
|
+
// results left out by `--last-failed` belong to the previous run
|
|
60
|
+
if (result.selected === false) continue;
|
|
61
|
+
const { status, data } = toTestData(result, groups, artifactsRoot);
|
|
62
|
+
promises.push(reporter.addTestRun(status, data));
|
|
63
|
+
}
|
|
64
|
+
await Promise.all(promises);
|
|
65
|
+
|
|
66
|
+
const duration = (Date.parse(report.run.finishedAt) - Date.parse(report.run.startedAt)) / 1000;
|
|
67
|
+
await reporter.updateRunStatus(runStatus(report.run.status), { duration });
|
|
68
|
+
|
|
69
|
+
const runUrl = reporter.pipeStore?.runUrl;
|
|
70
|
+
if (runUrl) return [{ label: 'Testomat.io', text: runUrl }];
|
|
71
|
+
},
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Converts one report-1 result into Testomat.io test data.
|
|
77
|
+
* Only the final attempt is reported; earlier attempts are counted as retries.
|
|
78
|
+
*/
|
|
79
|
+
export function toTestData(result, groups, artifactsRoot) {
|
|
80
|
+
const attempts = attemptsOf(result, groups);
|
|
81
|
+
const final = attempts.at(-1);
|
|
82
|
+
const titlePath = [...result.titlePath];
|
|
83
|
+
const title = titlePath.pop();
|
|
84
|
+
|
|
85
|
+
let suiteTitle = titlePath.join(' › ');
|
|
86
|
+
if (!suiteTitle) suiteTitle = path.basename(result.file);
|
|
87
|
+
|
|
88
|
+
const meta = {
|
|
89
|
+
target: result.targetId,
|
|
90
|
+
platform: result.platform,
|
|
91
|
+
browser: final?.environment?.browser,
|
|
92
|
+
};
|
|
93
|
+
if (result.agent && result.agent !== 'default') meta.agent = result.agent;
|
|
94
|
+
if (result.repeat) meta.repeat = result.repeat;
|
|
95
|
+
if (attempts.length > 1) meta.retryCount = attempts.length - 1;
|
|
96
|
+
if (result.status === 'flaky') meta.isFlaky = true;
|
|
97
|
+
const status = testStatus(result.status);
|
|
98
|
+
if (status === STATUS.FAILED && final?.failure?.url) meta.failedUrl = final.failure.url;
|
|
99
|
+
|
|
100
|
+
const filesById = new Map();
|
|
101
|
+
for (const artifact of final?.artifacts || []) {
|
|
102
|
+
if (artifact.path) {
|
|
103
|
+
filesById.set(artifact.id, {
|
|
104
|
+
path: path.join(artifactsRoot, artifact.path),
|
|
105
|
+
type: artifact.mediaType,
|
|
106
|
+
title: artifact.kind,
|
|
107
|
+
});
|
|
108
|
+
} else if (artifact.url) {
|
|
109
|
+
// hosted engines keep videos remotely
|
|
110
|
+
meta[artifact.kind] = artifact.url;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
const files = [...filesById.values()];
|
|
114
|
+
|
|
115
|
+
const steps = (final?.steps || []).map(step => toStep(step, filesById));
|
|
116
|
+
|
|
117
|
+
const data = {
|
|
118
|
+
rid: result.id,
|
|
119
|
+
title,
|
|
120
|
+
suite_title: suiteTitle,
|
|
121
|
+
file: result.file,
|
|
122
|
+
test_id: getTestomatIdFromTestTitle(title),
|
|
123
|
+
tags: result.tags || [],
|
|
124
|
+
time: final?.durationMs || 0,
|
|
125
|
+
timestamp: timestampOf(final),
|
|
126
|
+
error: toError(final?.error, final?.secondaryErrors),
|
|
127
|
+
steps: steps.length ? steps : undefined,
|
|
128
|
+
logs: logsOf(result, final),
|
|
129
|
+
files: files.length ? files : undefined,
|
|
130
|
+
meta,
|
|
131
|
+
};
|
|
132
|
+
|
|
133
|
+
return { status, data };
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** Attempts of a result; a serial member keeps its part of the group's attempts. */
|
|
137
|
+
function attemptsOf(result, groups) {
|
|
138
|
+
if (!result.serialGroupId) return result.attempts || [];
|
|
139
|
+
const groupAttempts = groups.get(result.serialGroupId)?.attempts || [];
|
|
140
|
+
return groupAttempts.map(attempt => {
|
|
141
|
+
const member = attempt.members?.find(candidate => candidate.testId === result.testId);
|
|
142
|
+
return {
|
|
143
|
+
status: member?.status || attempt.status,
|
|
144
|
+
error: member?.error || attempt.error,
|
|
145
|
+
failure: member?.failure || attempt.failure,
|
|
146
|
+
steps: member?.steps || [],
|
|
147
|
+
appLog: member?.appLog || [],
|
|
148
|
+
skip: member?.skip,
|
|
149
|
+
durationMs: member?.durationMs,
|
|
150
|
+
startedAt: member?.startedAt || attempt.startedAt,
|
|
151
|
+
artifacts: attempt.artifacts || [],
|
|
152
|
+
secondaryErrors: [...(member?.secondaryErrors || []), ...(attempt.secondaryErrors || [])],
|
|
153
|
+
environment: attempt.environment,
|
|
154
|
+
};
|
|
155
|
+
});
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function timestampOf(attempt) {
|
|
159
|
+
if (!attempt?.startedAt) return undefined;
|
|
160
|
+
return Date.parse(attempt.startedAt);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function testStatus(status) {
|
|
164
|
+
if (status === 'passed' || status === 'flaky') return STATUS.PASSED;
|
|
165
|
+
if (status === 'skipped') return STATUS.SKIPPED;
|
|
166
|
+
// failed, timed-out, interrupted: the test reached no passing verdict
|
|
167
|
+
return STATUS.FAILED;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
function runStatus(status) {
|
|
171
|
+
if (status === 'passed') return STATUS.PASSED;
|
|
172
|
+
return STATUS.FAILED;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function toError(error, secondaryErrors = []) {
|
|
176
|
+
if (!error) return undefined;
|
|
177
|
+
/** @type {Error & { expected?: string, actual?: string }} */
|
|
178
|
+
const err = new Error(error.message);
|
|
179
|
+
err.name = error.code;
|
|
180
|
+
err.stack = `${error.code}: ${error.message}\n${error.stack || sourceFrame(error.source)}`;
|
|
181
|
+
if (error.details?.expected && error.details?.observed) {
|
|
182
|
+
err.expected = error.details.expected;
|
|
183
|
+
err.actual = error.details.observed;
|
|
184
|
+
}
|
|
185
|
+
for (const secondary of secondaryErrors) {
|
|
186
|
+
err.stack += `\n\nAlso failed (${secondary.phase || secondary.category}) ${secondary.code}: ${secondary.message}`;
|
|
187
|
+
}
|
|
188
|
+
return err;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function sourceFrame(source) {
|
|
192
|
+
if (!source) return '';
|
|
193
|
+
return ` at ${source.file}:${source.line}:${source.column}`;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
function toStep(step, filesById) {
|
|
197
|
+
let category = 'user';
|
|
198
|
+
if (step.phase === 'beforeEach' || step.phase === 'afterEach') category = 'hook';
|
|
199
|
+
|
|
200
|
+
let title = step.api;
|
|
201
|
+
if (step.label) title = `${step.api} ${step.label}`;
|
|
202
|
+
|
|
203
|
+
const formatted = formatStep({
|
|
204
|
+
category,
|
|
205
|
+
title,
|
|
206
|
+
duration: step.durationMs,
|
|
207
|
+
log: stepLog(step),
|
|
208
|
+
});
|
|
209
|
+
|
|
210
|
+
let status = 'failed';
|
|
211
|
+
if (step.status === 'passed') status = 'passed';
|
|
212
|
+
addStatusToStep(formatted, status, step.error);
|
|
213
|
+
|
|
214
|
+
if (step.error) {
|
|
215
|
+
formatted.error = { message: truncate(`${step.error.code}: ${step.error.message}`, 250) };
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
if (SCREENSHOTS_ON_STEPS) {
|
|
219
|
+
const screenshot = (step.artifacts || []).map(id => filesById.get(id)).find(file => file?.type === 'image/png');
|
|
220
|
+
if (screenshot) addArtifactsToStep(formatted, { screenshot: screenshot.path });
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
return formatted;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/** Agent reasoning and non-binary step statuses, which a passed/failed step would lose. */
|
|
227
|
+
function stepLog(step) {
|
|
228
|
+
const lines = [];
|
|
229
|
+
if (step.status !== 'passed' && step.status !== 'failed') lines.push(`status: ${step.status}`);
|
|
230
|
+
if (step.explanation) lines.push(step.explanation);
|
|
231
|
+
for (const event of step.events || []) {
|
|
232
|
+
if (event.reasoning) lines.push(event.reasoning);
|
|
233
|
+
}
|
|
234
|
+
return lines.join('\n');
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function logsOf(result, final) {
|
|
238
|
+
const lines = [];
|
|
239
|
+
const skip = final?.skip || result.skip;
|
|
240
|
+
if (skip) lines.push(`Skipped (${skip.cause}): ${skip.reason}`);
|
|
241
|
+
if (final?.appLog?.length) {
|
|
242
|
+
lines.push('App log:');
|
|
243
|
+
for (const line of final.appLog) lines.push(`[${line.source}] ${line.level}: ${line.text}`);
|
|
244
|
+
}
|
|
245
|
+
return lines.join('\n');
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
export default testomatio;
|
package/src/adapter/jasmine.js
CHANGED
|
@@ -2,6 +2,8 @@ import TestomatClient from '../client.js';
|
|
|
2
2
|
import { getTestomatIdFromTestTitle, ansiRegExp } from '../utils/utils.js';
|
|
3
3
|
import { STATUS } from '../constants.js';
|
|
4
4
|
|
|
5
|
+
global.TESTOMATIO_ADAPTER = 'jasmine';
|
|
6
|
+
|
|
5
7
|
class JasmineReporter {
|
|
6
8
|
constructor(options) {
|
|
7
9
|
this.testTimeMap = {};
|
package/src/adapter/jest.js
CHANGED
|
@@ -8,6 +8,8 @@ import createDebugMessages from 'debug';
|
|
|
8
8
|
|
|
9
9
|
const debug = createDebugMessages('@testomatio/reporter:adapter-jest');
|
|
10
10
|
|
|
11
|
+
global.TESTOMATIO_ADAPTER = 'jest';
|
|
12
|
+
|
|
11
13
|
export class JestReporter {
|
|
12
14
|
constructor(globalConfig, options) {
|
|
13
15
|
this._globalConfig = globalConfig;
|