@testomatio/reporter 1.4.1-beta-1 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +60 -57
- package/lib/adapter/codecept.js +86 -27
- package/lib/adapter/cucumber/current.js +17 -10
- package/lib/adapter/cucumber/legacy.js +5 -4
- package/lib/adapter/cucumber.js +2 -2
- package/lib/adapter/cypress-plugin/index.js +53 -26
- package/lib/adapter/jasmine.js +2 -2
- package/lib/adapter/jest.js +48 -10
- package/lib/adapter/mocha.js +90 -10
- package/lib/adapter/playwright.js +68 -18
- package/lib/adapter/webdriver.js +47 -2
- package/lib/bin/reportXml.js +21 -16
- package/lib/bin/startTest.js +12 -12
- package/lib/client.js +112 -52
- package/lib/config.js +34 -0
- package/lib/constants.js +28 -1
- package/lib/data-storage.js +203 -0
- package/lib/fileUploader.js +66 -53
- package/lib/junit-adapter/adapter.js +0 -2
- package/lib/junit-adapter/csharp.js +3 -4
- package/lib/junit-adapter/index.js +3 -3
- package/lib/junit-adapter/java.js +11 -11
- package/lib/junit-adapter/javascript.js +1 -2
- package/lib/junit-adapter/python.js +12 -14
- package/lib/junit-adapter/ruby.js +1 -2
- package/lib/pipe/csv.js +35 -32
- package/lib/pipe/github.js +4 -3
- package/lib/pipe/gitlab.js +17 -10
- package/lib/pipe/html.js +361 -0
- package/lib/pipe/index.js +3 -1
- package/lib/pipe/testomatio.js +230 -51
- package/lib/reporter-functions.js +12 -9
- package/lib/reporter.js +8 -12
- package/lib/services/artifacts.js +57 -0
- package/lib/services/index.js +13 -0
- package/lib/{storages/key-value-storage.js → services/key-values.js} +19 -19
- package/lib/{storages → services}/logger.js +58 -37
- package/lib/template/emptyData.svg +23 -0
- package/lib/template/testomatio.hbs +1421 -0
- package/lib/utils/pipe_utils.js +73 -79
- package/lib/utils/utils.js +53 -40
- package/lib/xmlReader.js +113 -105
- package/package.json +13 -7
- package/lib/storages/artifact-storage.js +0 -70
- package/lib/storages/data-storage.js +0 -307
package/README.md
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
# Testomatio Reporter
|
|
2
2
|
|
|
3
|
-
|
|
4
3
|
👋 Hey, do you need some test reporting?
|
|
5
4
|
|
|
6
5
|
Testomat.io Reporter is a library that integrates with popular **JavaScript and TypeScript** test frameworks to provide a common interface for test reporting. By default, Testomat.io Reporter works with our reporting cloud service [Testomat.io App](https://testomat.io), however it is not locked to it. Reporter can be used as a standalone tool.
|
|
@@ -9,18 +8,17 @@ Testomat.io Reporter is a library that integrates with popular **JavaScript and
|
|
|
9
8
|
|
|
10
9
|
Testomat.io Reporter (this npm package) supports:
|
|
11
10
|
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
11
|
+
- 🏄 Integarion with all popular [JavaScript/TypeScript frameworks](./docs/frameworks.md)
|
|
12
|
+
- 🗄️ Screenshots, videos, traces [uploaded into S3 bucket](./docs/artifacts.md)
|
|
13
|
+
- 🔎 [Stack traces](./docs/stacktrace.md) and error messages
|
|
14
|
+
- 🐙 [GitHub](./docs/pipes/github.md) & [GitLab](./docs/pipes/gitlab.md) integration
|
|
15
|
+
- 🚅 Realtime reports
|
|
16
|
+
- 🗃️ Other test frameworks supported via [JUNit XML](./docs/junit.md)
|
|
17
|
+
- 🚶♀️ Steps _(work in progress)_
|
|
18
|
+
- 📄 [Logger](./docs/logger.md) _(work in progress, supports Jest for now)_
|
|
19
|
+
- ☁️ Custom properties and metadata _(work in progress)_
|
|
20
|
+
- 💯 Free & open-source.
|
|
21
|
+
- 📊 Public and private Run reports on cloud via [Testomat.io App](https://testomat.io) 👇
|
|
24
22
|
|
|
25
23
|

|
|
26
24
|
|
|
@@ -29,12 +27,12 @@ Testomat.io Reporter (this npm package) supports:
|
|
|
29
27
|
Testomat.io Reporter provides common API to store and organize test reports.
|
|
30
28
|
It can receive test result data from any [test framework](./docs/frameworks.md) and send it to different services via [pipes](./docs/pipes.md).
|
|
31
29
|
|
|
32
|
-
| 🌊 Input
|
|
33
|
-
|
|
34
|
-
| Playwright
|
|
35
|
-
| Cypress
|
|
36
|
-
| Jest
|
|
37
|
-
| ...
|
|
30
|
+
| 🌊 Input | 📊 Output |
|
|
31
|
+
| ---------- | -------------------------------------------- |
|
|
32
|
+
| Playwright | Report to GitHub |
|
|
33
|
+
| Cypress | Report to GitLab |
|
|
34
|
+
| Jest | Report to [Testomat.io](https://testomat.io) |
|
|
35
|
+
| ... | ... your custom report |
|
|
38
36
|
|
|
39
37
|
If you use multiple test frameworks and you need to use one customizable reporter, check Testomat.io Reporter, as you can adjust it once and attach it to all your projects.
|
|
40
38
|
|
|
@@ -46,7 +44,6 @@ Artifacts like screenshots, videos, traces, are **uploaded to your own cloud sto
|
|
|
46
44
|
|
|
47
45
|
To enable Testomat.io Reporter install `@testomatio/reporter` package
|
|
48
46
|
|
|
49
|
-
|
|
50
47
|
Use one of your favorite package managers:
|
|
51
48
|
|
|
52
49
|
```
|
|
@@ -65,31 +62,22 @@ yarn add @testomatio/reporter --dev
|
|
|
65
62
|
|
|
66
63
|
### 1️⃣ Attach Reporter to the Test Runner
|
|
67
64
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
* #### [Detox](./docs/frameworks.md#Detox)
|
|
76
|
-
* #### [Codeception](https://github.com/testomatio/php-reporter)
|
|
77
|
-
* #### [Newman (Postman)](./docs/frameworks.md#Newman)
|
|
78
|
-
* #### [JUnit](./docs/junit.md#junit)
|
|
79
|
-
* #### [NUnit](./docs/junit.md#nunit)
|
|
80
|
-
* #### [PyTest](./docs/junit.md#pytest)
|
|
81
|
-
* #### [PHPUnit](./docs/junit.md#phpunit)
|
|
82
|
-
* #### [Protractor](./docs/frameworks.md#protractor)
|
|
83
|
-
|
|
84
|
-
or any [other via JUnit](./docs/junit.md) report....
|
|
65
|
+
| | | |
|
|
66
|
+
| ----------------------------------------------- | --------------------------------------------- | --------------------------------------------------------- |
|
|
67
|
+
| [Playwright](./docs/frameworks.md#playwright) | [CodeceptJS](./docs/frameworks.md#CodeceptJS) | [Cypress](./docs/frameworks.md#Cypress) |
|
|
68
|
+
| [Jest](./docs/frameworks.md#Jest) | [Mocha](./docs/frameworks.md#Mocha) | [WebDriverIO](./docs/frameworks.md#WebDriverIO) |
|
|
69
|
+
| [TestCafe](./docs/frameworks.md#TestCafe) | [Detox](./docs/frameworks.md#Detox) | [Codeception](https://github.com/testomatio/php-reporter) |
|
|
70
|
+
| [Newman (Postman)](./docs/frameworks.md#Newman) | [JUnit](./docs/junit.md#junit) | [NUnit](./docs/junit.md#nunit) |
|
|
71
|
+
| [PyTest](./docs/junit.md#pytest) | [PHPUnit](./docs/junit.md#phpunit) | [Protractor](./docs/frameworks.md#protractor) |
|
|
85
72
|
|
|
86
|
-
|
|
73
|
+
or **any [other via JUnit](./docs/junit.md)** report....
|
|
87
74
|
|
|
88
|
-
|
|
89
|
-
* [Create brief summary report for GitHub Pull Request](./docs/pipes/github.md) 👇
|
|
90
|
-
* [Create brief summary report for GitLab Merge Request](./docs/pipes/gitlab.md).
|
|
91
|
-
* [Configure other pipes](./docs/pipes/md) for other ways to process test results output.
|
|
75
|
+
### 2️⃣ Configure Reports
|
|
92
76
|
|
|
77
|
+
- [Create report on Testomat.io](./docs/pipes/testomatio.md).
|
|
78
|
+
- [Create brief summary report for GitHub Pull Request](./docs/pipes/github.md) 👇
|
|
79
|
+
- [Create brief summary report for GitLab Merge Request](./docs/pipes/gitlab.md).
|
|
80
|
+
- [Configure other pipes](./docs/pipes/md) for other ways to process test results output.
|
|
93
81
|
|
|
94
82
|

|
|
95
83
|
|
|
@@ -97,14 +85,29 @@ GitHub report published as a comment to Pull Request:
|
|
|
97
85
|
|
|
98
86
|
### 3️⃣ Enable Artifacts Storage
|
|
99
87
|
|
|
100
|
-
1. Create bucket on AWS, Google Cloud, or any other cloud storage provider supporting S3 protocol.
|
|
88
|
+
1. Create bucket on AWS, Google Cloud, or any other cloud storage provider supporting S3 protocol.
|
|
101
89
|
2. [Pass S3 credentials](./docs/artifacts.md) to reporter to enable artifacts uploading.
|
|
102
90
|
|
|
103
91
|
### 4️⃣ Use Logger
|
|
104
92
|
|
|
105
93
|
Intercept your logger messages or log anything with our [Logger](./docs/logger.md) (_work in progress_).
|
|
106
94
|
|
|
107
|
-
### 5️⃣
|
|
95
|
+
### 5️⃣ TESTOMATIO HTML Report
|
|
96
|
+
|
|
97
|
+
Testomatio now features a custom reporting tool that allows you to generate a standalone HTML/CSS report for easy visualization of your test runs. This mode provides a quick way to gain a clear and visually appealing overview of your test execution.
|
|
98
|
+
|
|
99
|
+
With our reporter, you can:
|
|
100
|
+
|
|
101
|
+
- Easily analyze the results of your test runs in a convenient HTML format.
|
|
102
|
+
|
|
103
|
+
* Visualize data on successful and failed tests, including statistics and error details.
|
|
104
|
+
* Quickly share reports with your team members or stakeholders.
|
|
105
|
+
|
|
106
|
+

|
|
107
|
+
|
|
108
|
+
Learn more about generating HTML reports [here](./docs/pipes/html.md)
|
|
109
|
+
|
|
110
|
+
### 6️⃣ Add to CI Pipeline
|
|
108
111
|
|
|
109
112
|
After you tested reporter locally add it to your CI pipeline.
|
|
110
113
|
|
|
@@ -116,24 +119,23 @@ After you tested reporter locally add it to your CI pipeline.
|
|
|
116
119
|
|
|
117
120
|
Bring this reporter on CI and never lose test results again!
|
|
118
121
|
|
|
119
|
-
|
|
120
122
|
## Documentation
|
|
121
123
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
124
|
+
- 🛠️ [Frameworks](./docs/frameworks.md)
|
|
125
|
+
- ⛲ [Pipes](./docs/pipes.md)
|
|
126
|
+
- [Testomat.io](./docs/pipes/testomatio.md)
|
|
127
|
+
- [GitHub](./docs/pipes/github.md)
|
|
128
|
+
- [Gitlab](./docs/pipes/gitlab.md)
|
|
129
|
+
- [CSV](./docs/pipes/csv.md)
|
|
130
|
+
- [HTML report](./docs/pipes/html.md)
|
|
131
|
+
- 📓 [JUnit](./docs/junit.md)
|
|
132
|
+
- 🗄️ [Artifacts](./docs/artifacts.md)
|
|
133
|
+
- 🔂 [Workflows](./docs/workflows.md)
|
|
134
|
+
- 🖊️ [Logger](./docs/logger.md)
|
|
132
135
|
|
|
133
136
|
## Development
|
|
134
137
|
|
|
135
|
-
|
|
136
|
-
### REST API
|
|
138
|
+
### REST API
|
|
137
139
|
|
|
138
140
|
Testomat.io App uses REST API to collect data from the reporter.
|
|
139
141
|
|
|
@@ -148,6 +150,7 @@ To print all reporter logs:
|
|
|
148
150
|
```
|
|
149
151
|
DEBUG=@testomatio/reporter:*
|
|
150
152
|
```
|
|
153
|
+
|
|
151
154
|
To print all reporter logs of a specific pipe:
|
|
152
155
|
|
|
153
156
|
```
|
package/lib/adapter/codecept.js
CHANGED
|
@@ -3,7 +3,8 @@ const chalk = require('chalk');
|
|
|
3
3
|
const TestomatClient = require('../client');
|
|
4
4
|
const { STATUS, APP_PREFIX, TESTOMAT_TMP_STORAGE_DIR } = require('../constants');
|
|
5
5
|
const upload = require('../fileUploader');
|
|
6
|
-
const {
|
|
6
|
+
const { getTestomatIdFromTestTitle, fileSystem } = require('../utils/utils');
|
|
7
|
+
const { services } = require('../services');
|
|
7
8
|
|
|
8
9
|
if (!global.codeceptjs) {
|
|
9
10
|
// eslint-disable-next-line global-require, import/no-extraneous-dependencies
|
|
@@ -51,21 +52,56 @@ function CodeceptReporter(config) {
|
|
|
51
52
|
|
|
52
53
|
recorder.startUnlessRunning();
|
|
53
54
|
|
|
54
|
-
global.testomatioRunningEnvironment = 'codeceptjs';
|
|
55
|
-
|
|
56
55
|
// Listening to events
|
|
57
56
|
event.dispatcher.on(event.all.before, () => {
|
|
58
57
|
// clear tmp dir
|
|
59
58
|
fileSystem.clearDir(TESTOMAT_TMP_STORAGE_DIR);
|
|
60
59
|
|
|
61
|
-
recorder.add('Creating new run', () =>
|
|
60
|
+
// recorder.add('Creating new run', () => );
|
|
61
|
+
client.createRun();
|
|
62
62
|
videos = [];
|
|
63
63
|
traces = [];
|
|
64
64
|
|
|
65
65
|
if (!global.testomatioDataStore) global.testomatioDataStore = {};
|
|
66
66
|
});
|
|
67
67
|
|
|
68
|
-
|
|
68
|
+
let hookSteps = [];
|
|
69
|
+
let suiteHookRunning = false;
|
|
70
|
+
|
|
71
|
+
event.dispatcher.on(event.suite.before, suite => {
|
|
72
|
+
suiteHookRunning = true;
|
|
73
|
+
hookSteps = [];
|
|
74
|
+
global.testomatioDataStore.steps = [];
|
|
75
|
+
|
|
76
|
+
services.setContext(suite.fullTitle());
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
event.dispatcher.on(event.suite.after, () => {
|
|
80
|
+
services.setContext(null);
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
event.dispatcher.on(event.hook.started, () => {
|
|
84
|
+
// global.testomatioDataStore.steps = [];
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
event.dispatcher.on(event.hook.passed, () => {
|
|
88
|
+
if (suiteHookRunning) {
|
|
89
|
+
hookSteps.push(...global.testomatioDataStore.steps);
|
|
90
|
+
services.setContext(null);
|
|
91
|
+
}
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
event.dispatcher.on(event.hook.failed, () => {
|
|
95
|
+
if (suiteHookRunning) {
|
|
96
|
+
hookSteps.push(...global.testomatioDataStore.steps);
|
|
97
|
+
services.setContext(null);
|
|
98
|
+
}
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
event.dispatcher.on(event.test.before, test => {
|
|
102
|
+
suiteHookRunning = false;
|
|
103
|
+
global.testomatioDataStore.steps = [];
|
|
104
|
+
|
|
69
105
|
recorder.add(() => {
|
|
70
106
|
currentMetaStep = [];
|
|
71
107
|
// output.reset();
|
|
@@ -76,11 +112,15 @@ function CodeceptReporter(config) {
|
|
|
76
112
|
if (!global.testomatioDataStore) global.testomatioDataStore = {};
|
|
77
113
|
// reset steps
|
|
78
114
|
global.testomatioDataStore.steps = [];
|
|
115
|
+
|
|
116
|
+
services.setContext(test.fullTitle());
|
|
79
117
|
});
|
|
80
118
|
|
|
81
119
|
event.dispatcher.on(event.test.started, test => {
|
|
120
|
+
services.setContext(test.fullTitle());
|
|
121
|
+
|
|
82
122
|
testTimeMap[test.id] = Date.now();
|
|
83
|
-
|
|
123
|
+
// start logging
|
|
84
124
|
});
|
|
85
125
|
|
|
86
126
|
event.dispatcher.on(event.all.result, async () => {
|
|
@@ -102,8 +142,14 @@ function CodeceptReporter(config) {
|
|
|
102
142
|
if (id && failedTests.includes(id)) {
|
|
103
143
|
failedTests = failedTests.filter(failed => id !== failed);
|
|
104
144
|
}
|
|
105
|
-
const testId =
|
|
145
|
+
const testId = getTestomatIdFromTestTitle(`${title} ${tags?.join(' ')}`);
|
|
106
146
|
const testObj = getTestAndMessage(title);
|
|
147
|
+
|
|
148
|
+
const logs = getTestLogs(test);
|
|
149
|
+
const manuallyAttachedArtifacts = services.artifacts.get(test.fullTitle());
|
|
150
|
+
const keyValues = services.keyValues.get(test.fullTitle());
|
|
151
|
+
services.setContext(null);
|
|
152
|
+
|
|
107
153
|
client.addTestRun(STATUS.PASSED, {
|
|
108
154
|
...stripExampleFromTitle(title),
|
|
109
155
|
suite_title: test.parent && test.parent.title,
|
|
@@ -111,6 +157,9 @@ function CodeceptReporter(config) {
|
|
|
111
157
|
time: getDuration(test),
|
|
112
158
|
steps: global.testomatioDataStore.steps.join('\n') || null,
|
|
113
159
|
test_id: testId,
|
|
160
|
+
logs,
|
|
161
|
+
manuallyAttachedArtifacts,
|
|
162
|
+
meta: keyValues,
|
|
114
163
|
});
|
|
115
164
|
// output.stop();
|
|
116
165
|
});
|
|
@@ -127,7 +176,7 @@ function CodeceptReporter(config) {
|
|
|
127
176
|
for (const test of suite.tests) {
|
|
128
177
|
const { id, tags, title } = test;
|
|
129
178
|
failedTests.push(id || title);
|
|
130
|
-
const testId =
|
|
179
|
+
const testId = getTestomatIdFromTestTitle(`${title} ${tags?.join(' ')}`);
|
|
131
180
|
|
|
132
181
|
client.addTestRun(STATUS.FAILED, {
|
|
133
182
|
...stripExampleFromTitle(title),
|
|
@@ -145,16 +194,18 @@ function CodeceptReporter(config) {
|
|
|
145
194
|
if (test.err) error = test.err;
|
|
146
195
|
const { id, tags, title, artifacts } = test;
|
|
147
196
|
failedTests.push(id || title);
|
|
148
|
-
let testId =
|
|
197
|
+
let testId = getTestomatIdFromTestTitle(`${title} ${tags?.join(' ')}`);
|
|
149
198
|
const testObj = getTestAndMessage(title);
|
|
150
|
-
if (error && error.stack && test.steps && test.steps.length) {
|
|
151
|
-
error.stack = test.steps[test.steps.length - 1].line();
|
|
152
|
-
}
|
|
153
199
|
|
|
154
200
|
const files = [];
|
|
155
201
|
if (artifacts.screenshot) files.push({ path: artifacts.screenshot, type: 'image/png' });
|
|
156
202
|
// todo: video must be uploaded later....
|
|
157
203
|
|
|
204
|
+
const logs = getTestLogs(test);
|
|
205
|
+
const manuallyAttachedArtifacts = services.artifacts.get(test.fullTitle());
|
|
206
|
+
const keyValues = services.keyValues.get(test.fullTitle());
|
|
207
|
+
services.setContext(null);
|
|
208
|
+
|
|
158
209
|
const reportTestPromise = client
|
|
159
210
|
.addTestRun(STATUS.FAILED, {
|
|
160
211
|
...stripExampleFromTitle(title),
|
|
@@ -165,6 +216,9 @@ function CodeceptReporter(config) {
|
|
|
165
216
|
time: getDuration(test),
|
|
166
217
|
files,
|
|
167
218
|
steps: global.testomatioDataStore?.steps?.join('\n') || null,
|
|
219
|
+
logs,
|
|
220
|
+
manuallyAttachedArtifacts,
|
|
221
|
+
meta: keyValues,
|
|
168
222
|
})
|
|
169
223
|
.then(pipes => {
|
|
170
224
|
testId = pipes.filter(p => p.pipe.includes('Testomatio'))[0]?.result?.data?.test_id;
|
|
@@ -186,7 +240,7 @@ function CodeceptReporter(config) {
|
|
|
186
240
|
const { id, tags, title } = test;
|
|
187
241
|
if (failedTests.includes(id || title)) return;
|
|
188
242
|
|
|
189
|
-
const testId =
|
|
243
|
+
const testId = getTestomatIdFromTestTitle(`${title} ${tags?.join(' ')}`);
|
|
190
244
|
const testObj = getTestAndMessage(title);
|
|
191
245
|
client.addTestRun(STATUS.SKIPPED, {
|
|
192
246
|
...stripExampleFromTitle(title),
|
|
@@ -233,7 +287,7 @@ function CodeceptReporter(config) {
|
|
|
233
287
|
currentMetaStep = metaSteps;
|
|
234
288
|
stepShift = 2 * shift;
|
|
235
289
|
|
|
236
|
-
const durationMs = +new Date() -
|
|
290
|
+
const durationMs = +new Date() - +stepStart;
|
|
237
291
|
let duration = '';
|
|
238
292
|
if (durationMs) {
|
|
239
293
|
duration = repeat(1) + chalk.grey(`(${durationMs}ms)`);
|
|
@@ -258,7 +312,7 @@ async function uploadAttachments(client, attachments, messagePrefix, attachmentT
|
|
|
258
312
|
if (attachments.length > 0) {
|
|
259
313
|
console.log(APP_PREFIX, `Attachments: ${messagePrefix} ${attachments.length} ${attachmentType}/-s ...`);
|
|
260
314
|
|
|
261
|
-
const promises = attachments.map(async
|
|
315
|
+
const promises = attachments.map(async attachment => {
|
|
262
316
|
const { testId, title, path, type } = attachment;
|
|
263
317
|
const file = { path, type, title };
|
|
264
318
|
return client.addTestRun(undefined, {
|
|
@@ -272,18 +326,6 @@ async function uploadAttachments(client, attachments, messagePrefix, attachmentT
|
|
|
272
326
|
}
|
|
273
327
|
}
|
|
274
328
|
|
|
275
|
-
function parseTest(tags) {
|
|
276
|
-
if (tags) {
|
|
277
|
-
for (const tag of tags) {
|
|
278
|
-
if (tag.startsWith('@T')) {
|
|
279
|
-
return tag.substring(2, tag.length);
|
|
280
|
-
}
|
|
281
|
-
}
|
|
282
|
-
}
|
|
283
|
-
|
|
284
|
-
return null;
|
|
285
|
-
}
|
|
286
|
-
|
|
287
329
|
function getTestAndMessage(title) {
|
|
288
330
|
const testObj = { message: '' };
|
|
289
331
|
const testArr = title.split(/\s(\|\s\{.*?\})/);
|
|
@@ -306,4 +348,21 @@ function repeat(num) {
|
|
|
306
348
|
return ''.padStart(num, ' ');
|
|
307
349
|
}
|
|
308
350
|
|
|
351
|
+
// TODO: think about moving to some common utils
|
|
352
|
+
function getTestLogs(test) {
|
|
353
|
+
const suiteLogsArr = services.logger.getLogs(test.parent.fullTitle());
|
|
354
|
+
const suiteLogs = suiteLogsArr ? suiteLogsArr.join('\n').trim() : '';
|
|
355
|
+
const testLogsArr = services.logger.getLogs(test.fullTitle());
|
|
356
|
+
const testLogs = testLogsArr ? testLogsArr.join('\n').trim() : '';
|
|
357
|
+
|
|
358
|
+
let logs = '';
|
|
359
|
+
if (suiteLogs) {
|
|
360
|
+
logs += `${chalk.bold('\t--- BeforeSuite ---')}\n${suiteLogs}`;
|
|
361
|
+
}
|
|
362
|
+
if (testLogs) {
|
|
363
|
+
logs += `\n${chalk.bold('\t--- Test ---')}\n${testLogs}`;
|
|
364
|
+
}
|
|
365
|
+
return logs;
|
|
366
|
+
}
|
|
367
|
+
|
|
309
368
|
module.exports = CodeceptReporter;
|
|
@@ -4,8 +4,9 @@ const chalk = require('chalk');
|
|
|
4
4
|
const fs = require('fs');
|
|
5
5
|
const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../../constants');
|
|
6
6
|
const TestomatClient = require('../../client');
|
|
7
|
-
const
|
|
8
|
-
const
|
|
7
|
+
const { getTestomatIdFromTestTitle, fileSystem } = require('../../utils/utils');
|
|
8
|
+
const config = require('../../config');
|
|
9
|
+
const { services } = require('../../services');
|
|
9
10
|
|
|
10
11
|
const { GherkinDocumentParser, PickleParser } = formatterHelpers;
|
|
11
12
|
const { getGherkinScenarioLocationMap, getGherkinStepMap } = GherkinDocumentParser;
|
|
@@ -14,7 +15,7 @@ const { getPickleStepMap } = PickleParser;
|
|
|
14
15
|
function getTestId(scenario) {
|
|
15
16
|
if (scenario) {
|
|
16
17
|
for (const tag of scenario.tags) {
|
|
17
|
-
const testId =
|
|
18
|
+
const testId = getTestomatIdFromTestTitle(tag.name);
|
|
18
19
|
if (testId) return testId;
|
|
19
20
|
}
|
|
20
21
|
}
|
|
@@ -29,10 +30,8 @@ class CucumberReporter extends Formatter {
|
|
|
29
30
|
this.failures = [];
|
|
30
31
|
this.cases = [];
|
|
31
32
|
|
|
32
|
-
this.client = new TestomatClient({ apiKey: options.apiKey ||
|
|
33
|
+
this.client = new TestomatClient({ apiKey: options.apiKey || config.TESTOMATIO });
|
|
33
34
|
this.status = STATUS.PASSED;
|
|
34
|
-
|
|
35
|
-
global.testomatioRunningEnvironment = 'cucumber:current';
|
|
36
35
|
}
|
|
37
36
|
|
|
38
37
|
parseEnvelope(envelope) {
|
|
@@ -49,9 +48,10 @@ class CucumberReporter extends Formatter {
|
|
|
49
48
|
|
|
50
49
|
onTestCaseStarted(testCaseStarted) {
|
|
51
50
|
const testCaseAttempt = this.eventDataCollector.getTestCaseAttempt(testCaseStarted.id);
|
|
52
|
-
const testId = getTestId(testCaseAttempt.pickle);
|
|
53
51
|
if (!global.testomatioDataStore) global.testomatioDataStore = {};
|
|
54
|
-
|
|
52
|
+
|
|
53
|
+
const testTitle = testCaseAttempt.pickle.name + testCaseAttempt.pickle.uri;
|
|
54
|
+
services.setContext(testTitle);
|
|
55
55
|
}
|
|
56
56
|
|
|
57
57
|
onTestCaseFinished(testCaseFinished) {
|
|
@@ -109,7 +109,10 @@ class CucumberReporter extends Formatter {
|
|
|
109
109
|
|
|
110
110
|
if (!this.client) return;
|
|
111
111
|
|
|
112
|
-
const
|
|
112
|
+
const testTitle = testCaseAttempt.pickle.name + testCaseAttempt.pickle.uri;
|
|
113
|
+
const logs = services.logger.getLogs(testTitle).join('\n');
|
|
114
|
+
const artifacts = services.artifacts.get(testTitle);
|
|
115
|
+
const keyValues = services.keyValues.get(testTitle);
|
|
113
116
|
|
|
114
117
|
this.client.addTestRun(status, {
|
|
115
118
|
// error: testCaseAttempt.worstTestStepResult.message,
|
|
@@ -119,11 +122,15 @@ class CucumberReporter extends Formatter {
|
|
|
119
122
|
.join('\n')
|
|
120
123
|
.trim(),
|
|
121
124
|
example: { ...example },
|
|
122
|
-
|
|
125
|
+
logs,
|
|
126
|
+
manuallyAttachedArtifacts: artifacts,
|
|
127
|
+
meta: keyValues,
|
|
123
128
|
title: scenario,
|
|
124
129
|
test_id: testId,
|
|
125
130
|
time,
|
|
126
131
|
});
|
|
132
|
+
|
|
133
|
+
services.setContext(null);
|
|
127
134
|
}
|
|
128
135
|
|
|
129
136
|
onTestRunFinished(envelope) {
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
// eslint-disable-next-line global-require, import/no-extraneous-dependencies, import/no-unresolved
|
|
2
2
|
const { Formatter } = require('cucumber');
|
|
3
3
|
const chalk = require('chalk');
|
|
4
|
-
const {
|
|
4
|
+
const { getTestomatIdFromTestTitle, fileSystem } = require('../../utils/utils');
|
|
5
5
|
const { STATUS, TESTOMAT_TMP_STORAGE_DIR } = require('../../constants');
|
|
6
6
|
const TestomatClient = require('../../client');
|
|
7
|
+
const config = require('../../config');
|
|
7
8
|
|
|
8
9
|
const createTestomatFormatter = apiKey => {
|
|
9
10
|
if (!apiKey || apiKey === '') {
|
|
@@ -78,7 +79,7 @@ const createTestomatFormatter = apiKey => {
|
|
|
78
79
|
const getTestId = scenario => {
|
|
79
80
|
if (scenario) {
|
|
80
81
|
for (const tag of scenario.tags) {
|
|
81
|
-
const testId =
|
|
82
|
+
const testId = getTestomatIdFromTestTitle(tag.name);
|
|
82
83
|
if (testId) return testId;
|
|
83
84
|
}
|
|
84
85
|
}
|
|
@@ -110,7 +111,7 @@ const createTestomatFormatter = apiKey => {
|
|
|
110
111
|
onTestCaseStarted(event) {
|
|
111
112
|
const scenario = getScenario(event.sourceLocation);
|
|
112
113
|
const testId = getTestId(scenario);
|
|
113
|
-
|
|
114
|
+
|
|
114
115
|
if (!global.testomatioDataStore) global.testomatioDataStore = {};
|
|
115
116
|
global.testomatioDataStore.currentlyRunningTestId = testId;
|
|
116
117
|
}
|
|
@@ -151,4 +152,4 @@ const createTestomatFormatter = apiKey => {
|
|
|
151
152
|
};
|
|
152
153
|
};
|
|
153
154
|
|
|
154
|
-
module.exports = createTestomatFormatter(
|
|
155
|
+
module.exports = createTestomatFormatter(config.TESTOMATIO);
|
package/lib/adapter/cucumber.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
try {
|
|
2
2
|
// eslint-disable-next-line import/no-unresolved
|
|
3
|
-
require.resolve(
|
|
3
|
+
require.resolve('@cucumber/cucumber');
|
|
4
4
|
// eslint-disable-next-line global-require, import/no-extraneous-dependencies
|
|
5
5
|
module.exports = require('./cucumber/current');
|
|
6
6
|
} catch (_e) {
|
|
@@ -11,6 +11,6 @@ try {
|
|
|
11
11
|
module.exports = require('./cucumber/legacy');
|
|
12
12
|
} catch (_err) {
|
|
13
13
|
console.error('Cucumber packages: "@cucumber/cucumber" or "cucumber" were not detected. Report won\'t be sent');
|
|
14
|
-
module.exports = {}
|
|
14
|
+
module.exports = {};
|
|
15
15
|
}
|
|
16
16
|
}
|
|
@@ -1,17 +1,19 @@
|
|
|
1
1
|
const { STATUS } = require('../../constants');
|
|
2
|
-
const {
|
|
2
|
+
const { getTestomatIdFromTestTitle, parseSuite } = require('../../utils/utils');
|
|
3
3
|
const TestomatClient = require('../../client');
|
|
4
|
+
const config = require('../../config');
|
|
4
5
|
|
|
5
6
|
const testomatioReporter = on => {
|
|
6
|
-
if (!
|
|
7
|
+
if (!config.TESTOMATIO) {
|
|
7
8
|
console.log('TESTOMATIO key is empty, ignoring reports');
|
|
8
|
-
return
|
|
9
|
+
return;
|
|
9
10
|
}
|
|
10
|
-
const client = new TestomatClient({ apiKey:
|
|
11
|
+
const client = new TestomatClient({ apiKey: config.TESTOMATIO });
|
|
11
12
|
|
|
12
|
-
on('before:run', async
|
|
13
|
+
on('before:run', async run => {
|
|
14
|
+
// TODO: looks like client.env does not exist
|
|
13
15
|
if (!client.env) {
|
|
14
|
-
client.env = `${run.browser.displayName},${run.system.osName}
|
|
16
|
+
client.env = `${run.browser.displayName},${run.system.osName}`;
|
|
15
17
|
}
|
|
16
18
|
await client.createRun();
|
|
17
19
|
});
|
|
@@ -25,8 +27,9 @@ const testomatioReporter = on => {
|
|
|
25
27
|
const lastAttemptIndex = test.attempts.length - 1;
|
|
26
28
|
const latestAttempt = test.attempts[lastAttemptIndex];
|
|
27
29
|
|
|
28
|
-
|
|
29
|
-
const
|
|
30
|
+
// latestAttempt.duration && latestAttempt.error were available in adapters version up to 13 JFYI
|
|
31
|
+
const time = latestAttempt.duration || latestAttempt.wallClockDuration || test.duration;
|
|
32
|
+
let error = latestAttempt.error;
|
|
30
33
|
|
|
31
34
|
let title = test.title.pop();
|
|
32
35
|
const examples = title.match(/\(example (#\d+)\)/);
|
|
@@ -36,38 +39,62 @@ const testomatioReporter = on => {
|
|
|
36
39
|
|
|
37
40
|
const suiteTitle = test.title.pop();
|
|
38
41
|
|
|
39
|
-
const testId =
|
|
42
|
+
const testId = getTestomatIdFromTestTitle(title);
|
|
40
43
|
const suiteId = parseSuite(suiteTitle);
|
|
41
44
|
|
|
42
|
-
if (error) {
|
|
43
|
-
error
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
}
|
|
45
|
+
if (!error && test.displayError) {
|
|
46
|
+
error = { message: test.displayError };
|
|
47
|
+
error.inspect = function () {
|
|
48
|
+
// eslint-disable-line func-names
|
|
49
|
+
return this.message;
|
|
50
|
+
};
|
|
49
51
|
}
|
|
50
52
|
|
|
51
|
-
const
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
53
|
+
const formattedError = error
|
|
54
|
+
? {
|
|
55
|
+
message: error.message,
|
|
56
|
+
inspect:
|
|
57
|
+
error.inspect ||
|
|
58
|
+
function () {
|
|
59
|
+
return this.message;
|
|
60
|
+
},
|
|
61
|
+
}
|
|
62
|
+
: '';
|
|
63
|
+
|
|
64
|
+
const screenshots = Array.isArray(results.screenshots)
|
|
65
|
+
? results.screenshots
|
|
66
|
+
.filter(screenshot => screenshot?.path && screenshot?.path.includes(title) && screenshot?.takenAt)
|
|
67
|
+
.map(screenshot => screenshot.path)
|
|
68
|
+
: [];
|
|
55
69
|
|
|
56
70
|
const files = [...videos, ...screenshots];
|
|
57
71
|
|
|
58
72
|
let state;
|
|
59
73
|
switch (test.state) {
|
|
60
|
-
case 'passed':
|
|
61
|
-
|
|
74
|
+
case 'passed':
|
|
75
|
+
state = STATUS.PASSED;
|
|
76
|
+
break;
|
|
77
|
+
case 'failed':
|
|
78
|
+
state = STATUS.FAILED;
|
|
79
|
+
break;
|
|
62
80
|
case 'skipped':
|
|
63
|
-
case 'pending':
|
|
81
|
+
case 'pending':
|
|
64
82
|
default:
|
|
65
83
|
state = STATUS.SKIPPED;
|
|
66
84
|
}
|
|
67
85
|
|
|
68
|
-
addSpecTestsPromises.push(
|
|
69
|
-
|
|
70
|
-
|
|
86
|
+
addSpecTestsPromises.push(
|
|
87
|
+
client.addTestRun(state, {
|
|
88
|
+
title,
|
|
89
|
+
time,
|
|
90
|
+
example,
|
|
91
|
+
error: formattedError,
|
|
92
|
+
files,
|
|
93
|
+
suite_title: suiteTitle,
|
|
94
|
+
test_id: testId,
|
|
95
|
+
suite_id: suiteId,
|
|
96
|
+
}),
|
|
97
|
+
);
|
|
71
98
|
}
|
|
72
99
|
|
|
73
100
|
await Promise.all(addSpecTestsPromises);
|
package/lib/adapter/jasmine.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
const TestomatClient = require('../client');
|
|
2
|
-
const {
|
|
2
|
+
const { getTestomatIdFromTestTitle, ansiRegExp } = require('../utils/utils');
|
|
3
3
|
const { STATUS } = require('../constants');
|
|
4
4
|
|
|
5
5
|
class JasmineReporter {
|
|
@@ -34,7 +34,7 @@ class JasmineReporter {
|
|
|
34
34
|
}
|
|
35
35
|
console.log(`${title} : ${STATUS.PASSED}`);
|
|
36
36
|
console.log(errorMessage);
|
|
37
|
-
const testId =
|
|
37
|
+
const testId = getTestomatIdFromTestTitle(title);
|
|
38
38
|
errorMessage = errorMessage.replace(ansiRegExp(), '');
|
|
39
39
|
this.client.addTestRun(status, {
|
|
40
40
|
error: result.failedExpectations[0],
|