@zohodesk/testinglibrary 0.0.83-n20-experimental → 0.0.84-n20-experimental
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/core/playwright/builtInFixtures/context.js +27 -7
- package/build/core/playwright/constants/failureTagConstants.js +5 -2
- package/build/core/playwright/constants/logCollectorConstants.js +1 -4
- package/build/core/playwright/helpers/failureTagClassifier.js +176 -76
- package/build/core/playwright/helpers/logCollector.js +10 -4
- package/build/core/playwright/helpers/runtimeLogPath.js +61 -0
- package/build/core/playwright/helpers/runtimeLogReader.js +25 -16
- package/build/core/playwright/patterns/casePatterns.js +11 -0
- package/build/core/playwright/patterns/index.js +7 -0
- package/build/core/playwright/patterns/toolPatterns.js +1 -3
- package/build/core/playwright/setup/qc-custom-reporter.js +15 -3
- package/build/test/core/playwright/helpers/__tests__/failureTagClassifier.test.js +274 -28
- package/build/test/core/playwright/helpers/__tests__/logCollector.test.js +26 -12
- package/build/test/core/playwright/helpers/__tests__/runtimeLogReader.test.js +85 -25
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
- package/unit_reports/unit-report.html +1 -1
|
@@ -1,16 +1,24 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
|
+
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
|
|
3
4
|
Object.defineProperty(exports, "__esModule", {
|
|
4
5
|
value: true
|
|
5
6
|
});
|
|
6
7
|
exports.default = void 0;
|
|
8
|
+
var _path = _interopRequireDefault(require("path"));
|
|
7
9
|
var _readConfigFile = require("../readConfigFile");
|
|
8
|
-
var _logCollectorConstants = require("../constants/logCollectorConstants");
|
|
9
10
|
var _logCollector = require("../helpers/logCollector");
|
|
11
|
+
var _failureTagClassifier = require("../helpers/failureTagClassifier");
|
|
12
|
+
var _runtimeLogPath = require("../helpers/runtimeLogPath");
|
|
10
13
|
const {
|
|
11
14
|
testSetup,
|
|
15
|
+
uatDirectory,
|
|
12
16
|
browserLogs = {}
|
|
13
17
|
} = (0, _readConfigFile.generateConfigFromFile)();
|
|
18
|
+
|
|
19
|
+
// Keep runtime logs OUTSIDE reportPath (HTML reporter wipes it on onEnd) and
|
|
20
|
+
// outside test-results (Playwright outputDir is cleaned each run).
|
|
21
|
+
const LOGS_DIR = _path.default.join(uatDirectory, 'runtime-logs');
|
|
14
22
|
async function performDefaultContextSteps({
|
|
15
23
|
context
|
|
16
24
|
}) {
|
|
@@ -38,13 +46,25 @@ var _default = exports.default = {
|
|
|
38
46
|
collector.detach();
|
|
39
47
|
const failureOnly = browserLogs.mode === 'failure-only';
|
|
40
48
|
const isFailure = testInfo.status && testInfo.status !== testInfo.expectedStatus;
|
|
41
|
-
const
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
49
|
+
const shouldFlush = !failureOnly || isFailure;
|
|
50
|
+
if (shouldFlush) {
|
|
51
|
+
const base = (0, _runtimeLogPath.resolveRuntimeLogBaseName)({
|
|
52
|
+
workerIndex: testInfo.workerIndex,
|
|
53
|
+
retry: testInfo.retry,
|
|
54
|
+
title: testInfo.title,
|
|
55
|
+
titlePath: testInfo.titlePath
|
|
47
56
|
});
|
|
57
|
+
collector.flushToFile(LOGS_DIR, base);
|
|
58
|
+
}
|
|
59
|
+
if (isFailure) {
|
|
60
|
+
const metadata = (0, _failureTagClassifier.classifyFailureDetailsFromEvents)(collector.events);
|
|
61
|
+
const attachment = (0, _failureTagClassifier.buildFailureTagMetadataAttachment)(metadata);
|
|
62
|
+
if (attachment) {
|
|
63
|
+
await testInfo.attach(attachment.name, {
|
|
64
|
+
contentType: attachment.contentType,
|
|
65
|
+
body: attachment.body
|
|
66
|
+
});
|
|
67
|
+
}
|
|
48
68
|
}
|
|
49
69
|
}
|
|
50
70
|
}
|
|
@@ -3,8 +3,9 @@
|
|
|
3
3
|
Object.defineProperty(exports, "__esModule", {
|
|
4
4
|
value: true
|
|
5
5
|
});
|
|
6
|
-
exports.FAILURE_TAG_SUFFIX = exports.FAILURE_TAGS = void 0;
|
|
6
|
+
exports.FAILURE_TAG_SUFFIX = exports.FAILURE_TAG_METADATA_SCHEMA_VERSION = exports.FAILURE_TAG_METADATA_ATTACHMENT = exports.FAILURE_TAGS = void 0;
|
|
7
7
|
const FAILURE_TAGS = exports.FAILURE_TAGS = {
|
|
8
|
+
CASE: 'CASE',
|
|
8
9
|
CODE: 'CODE',
|
|
9
10
|
ENVIRONMENT: 'ENVIRONMENT',
|
|
10
11
|
FLAKY: 'FLAKY',
|
|
@@ -13,4 +14,6 @@ const FAILURE_TAGS = exports.FAILURE_TAGS = {
|
|
|
13
14
|
|
|
14
15
|
// Suffix appended to a category when merging into the existing spec.tags array,
|
|
15
16
|
// e.g. "CODE" -> "CODE_failure". The flaky category is emitted as-is ("FLAKY").
|
|
16
|
-
const FAILURE_TAG_SUFFIX = exports.FAILURE_TAG_SUFFIX = '_failure';
|
|
17
|
+
const FAILURE_TAG_SUFFIX = exports.FAILURE_TAG_SUFFIX = '_failure';
|
|
18
|
+
const FAILURE_TAG_METADATA_ATTACHMENT = exports.FAILURE_TAG_METADATA_ATTACHMENT = 'failure-tag-metadata';
|
|
19
|
+
const FAILURE_TAG_METADATA_SCHEMA_VERSION = exports.FAILURE_TAG_METADATA_SCHEMA_VERSION = 1;
|
|
@@ -3,12 +3,9 @@
|
|
|
3
3
|
Object.defineProperty(exports, "__esModule", {
|
|
4
4
|
value: true
|
|
5
5
|
});
|
|
6
|
-
exports.
|
|
6
|
+
exports.IGNORE_URL_PATTERNS = exports.IGNORE_MESSAGE_PATTERNS = void 0;
|
|
7
7
|
// Ignore patterns for the runtime-log LogCollector. Matching events are dropped here so it is never recorded.
|
|
8
8
|
|
|
9
|
-
const RUNTIME_LOG_EVENTS_ATTACHMENT = exports.RUNTIME_LOG_EVENTS_ATTACHMENT = 'runtime-log-events';
|
|
10
|
-
const RUNTIME_LOG_EVENTS_CONTENT_TYPE = exports.RUNTIME_LOG_EVENTS_CONTENT_TYPE = 'text/plain';
|
|
11
|
-
|
|
12
9
|
// Matched against console-error text, pageerror messages and requestfailed
|
|
13
10
|
|
|
14
11
|
const IGNORE_MESSAGE_PATTERNS = exports.IGNORE_MESSAGE_PATTERNS = [/Access is denied for this document/i, /enableDarkTheme is not a function/i, /enableZohoSearchAskZia is not a function/i, /enableSMSBridge is not a function/i, /Unexpected token '<'[\s\S]*not valid JSON/i, /blocked by CORS policy/i, /ERR_BLOCKED_BY_ORB/i];
|
|
@@ -1,25 +1,73 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
|
+
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
|
|
3
4
|
Object.defineProperty(exports, "__esModule", {
|
|
4
5
|
value: true
|
|
5
6
|
});
|
|
7
|
+
exports.buildFailureTagMetadataAttachment = buildFailureTagMetadataAttachment;
|
|
8
|
+
exports.classifyFailureDetails = classifyFailureDetails;
|
|
9
|
+
exports.classifyFailureDetailsFromEvents = classifyFailureDetailsFromEvents;
|
|
10
|
+
exports.classifyFailureDetailsFromResult = classifyFailureDetailsFromResult;
|
|
6
11
|
exports.classifyFailureTags = classifyFailureTags;
|
|
7
12
|
exports.mergeFailureTags = mergeFailureTags;
|
|
13
|
+
exports.upsertFailureTagMetadataAttachment = upsertFailureTagMetadataAttachment;
|
|
14
|
+
var _path = _interopRequireDefault(require("path"));
|
|
8
15
|
var _failureTagConstants = require("../constants/failureTagConstants");
|
|
9
16
|
var _patterns = require("../patterns");
|
|
17
|
+
var _readConfigFile = require("../readConfigFile");
|
|
10
18
|
var _runtimeLogReader = require("./runtimeLogReader");
|
|
11
19
|
// A failing test is tagged purely on presence: if its runtime log contains an
|
|
12
20
|
// environment pattern it gets `ENVIRONMENT`, if it contains a code pattern it
|
|
13
21
|
// gets `CODE`, and if its stdout/stderr contains a tool pattern it gets `TOOL`
|
|
14
|
-
// (any may apply). `
|
|
22
|
+
// or a case pattern it gets `CASE` (any may apply). A `CASE` match also adds
|
|
23
|
+
// `CODE` for now via case-implied metadata. `FLAKY` comes from the Playwright
|
|
24
|
+
// retry outcome, not the log.
|
|
15
25
|
|
|
16
26
|
const FAILING_RESULT_STATUSES = new Set(['failed', 'timedOut', 'interrupted']);
|
|
17
|
-
// Playwright TestCase.outcome() value for a failing (non-flaky) test.
|
|
18
27
|
const UNEXPECTED_OUTCOME = 'unexpected';
|
|
19
28
|
const FLAKY_OUTCOME = 'flaky';
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
29
|
+
const RUNTIME_LOG_SOURCE = 'runtime-log';
|
|
30
|
+
const MAX_MATCH_TEXT_LENGTH = 500;
|
|
31
|
+
const TAG_ORDER = [_failureTagConstants.FAILURE_TAGS.FLAKY, _failureTagConstants.FAILURE_TAGS.ENVIRONMENT, _failureTagConstants.FAILURE_TAGS.CASE, _failureTagConstants.FAILURE_TAGS.CODE, _failureTagConstants.FAILURE_TAGS.TOOL];
|
|
32
|
+
const RUNTIME_LOG_RULES = [{
|
|
33
|
+
tag: _failureTagConstants.FAILURE_TAGS.ENVIRONMENT,
|
|
34
|
+
patterns: _patterns.ENVIRONMENT_PATTERNS
|
|
35
|
+
}, {
|
|
36
|
+
tag: _failureTagConstants.FAILURE_TAGS.CODE,
|
|
37
|
+
patterns: _patterns.CODE_PATTERNS
|
|
38
|
+
}];
|
|
39
|
+
const STDIO_RULES = [{
|
|
40
|
+
tag: _failureTagConstants.FAILURE_TAGS.TOOL,
|
|
41
|
+
patterns: _patterns.TOOL_PATTERNS
|
|
42
|
+
}, {
|
|
43
|
+
tag: _failureTagConstants.FAILURE_TAGS.CASE,
|
|
44
|
+
patterns: _patterns.CASE_PATTERNS,
|
|
45
|
+
impliesCode: true
|
|
46
|
+
}];
|
|
47
|
+
function resolveLogsDir(logsDir) {
|
|
48
|
+
if (logsDir) {
|
|
49
|
+
return logsDir;
|
|
50
|
+
}
|
|
51
|
+
const {
|
|
52
|
+
uatDirectory
|
|
53
|
+
} = (0, _readConfigFile.generateConfigFromFile)();
|
|
54
|
+
return _path.default.join(uatDirectory, 'runtime-logs');
|
|
55
|
+
}
|
|
56
|
+
function patternLabel(pattern) {
|
|
57
|
+
return pattern.source;
|
|
58
|
+
}
|
|
59
|
+
function testPattern(pattern, text) {
|
|
60
|
+
if (pattern.global || pattern.sticky) {
|
|
61
|
+
pattern.lastIndex = 0;
|
|
62
|
+
}
|
|
63
|
+
return pattern.test(text);
|
|
64
|
+
}
|
|
65
|
+
function truncateText(text) {
|
|
66
|
+
if (!text || text.length <= MAX_MATCH_TEXT_LENGTH) {
|
|
67
|
+
return text || '';
|
|
68
|
+
}
|
|
69
|
+
return `${text.slice(0, MAX_MATCH_TEXT_LENGTH)}...`;
|
|
70
|
+
}
|
|
23
71
|
function eventToText(event) {
|
|
24
72
|
var _event$location;
|
|
25
73
|
switch (event.kind) {
|
|
@@ -35,23 +83,22 @@ function eventToText(event) {
|
|
|
35
83
|
return JSON.stringify(event);
|
|
36
84
|
}
|
|
37
85
|
}
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
function matchesAny(events, patterns) {
|
|
41
|
-
return events.some(event => {
|
|
86
|
+
function findEventMatch(events, patterns, tag, source) {
|
|
87
|
+
for (const event of events) {
|
|
42
88
|
const text = eventToText(event);
|
|
43
|
-
|
|
44
|
-
if (pattern
|
|
45
|
-
|
|
89
|
+
for (const pattern of patterns) {
|
|
90
|
+
if (testPattern(pattern, text)) {
|
|
91
|
+
return {
|
|
92
|
+
tag,
|
|
93
|
+
pattern: patternLabel(pattern),
|
|
94
|
+
source,
|
|
95
|
+
text: truncateText(text)
|
|
96
|
+
};
|
|
46
97
|
}
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return null;
|
|
50
101
|
}
|
|
51
|
-
|
|
52
|
-
// Flattens a result's stdout/stderr chunks into a single string. Each chunk is
|
|
53
|
-
// either a plain string, { text } or { buffer } (base64) as produced by the
|
|
54
|
-
// reporter's stdioEntry.
|
|
55
102
|
function stdioToText(entries = []) {
|
|
56
103
|
return entries.map(entry => {
|
|
57
104
|
if (typeof entry === 'string') {
|
|
@@ -73,41 +120,85 @@ function stdioToText(entries = []) {
|
|
|
73
120
|
return '';
|
|
74
121
|
}).join('\n');
|
|
75
122
|
}
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
// and code come from the browser runtime log; tool comes from stdout/stderr.
|
|
79
|
-
function classifyResult(result) {
|
|
80
|
-
const events = (0, _runtimeLogReader.readRuntimeLogEventsFromResult)(result);
|
|
81
|
-
const stdio = `${stdioToText(result.stdout)}\n${stdioToText(result.stderr)}`;
|
|
82
|
-
return {
|
|
83
|
-
environment: matchesAny(events, _patterns.ENVIRONMENT_PATTERNS),
|
|
84
|
-
code: matchesAny(events, _patterns.CODE_PATTERNS),
|
|
85
|
-
tool: _patterns.TOOL_PATTERNS.some(pattern => {
|
|
86
|
-
if (pattern.global || pattern.sticky) {
|
|
87
|
-
pattern.lastIndex = 0;
|
|
88
|
-
}
|
|
89
|
-
return pattern.test(stdio);
|
|
90
|
-
})
|
|
91
|
-
};
|
|
123
|
+
function buildStdioText(result = {}) {
|
|
124
|
+
return `${stdioToText(result.stdout)}\n${stdioToText(result.stderr)}`;
|
|
92
125
|
}
|
|
93
|
-
function
|
|
94
|
-
const
|
|
95
|
-
|
|
96
|
-
|
|
126
|
+
function findStdioMatch(stdio, patterns, tag) {
|
|
127
|
+
for (const pattern of patterns) {
|
|
128
|
+
if (testPattern(pattern, stdio)) {
|
|
129
|
+
return {
|
|
130
|
+
tag,
|
|
131
|
+
pattern: patternLabel(pattern),
|
|
132
|
+
source: 'stdio',
|
|
133
|
+
text: truncateText(stdio)
|
|
134
|
+
};
|
|
135
|
+
}
|
|
97
136
|
}
|
|
98
|
-
return
|
|
137
|
+
return null;
|
|
99
138
|
}
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
139
|
+
function collectRuntimeLogMatches(events, matches) {
|
|
140
|
+
for (const rule of RUNTIME_LOG_RULES) {
|
|
141
|
+
const match = findEventMatch(events, rule.patterns, rule.tag, RUNTIME_LOG_SOURCE);
|
|
142
|
+
if (match) {
|
|
143
|
+
matches.push(match);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
function collectStdioMatches(stdio, matches) {
|
|
148
|
+
for (const rule of STDIO_RULES) {
|
|
149
|
+
const match = findStdioMatch(stdio, rule.patterns, rule.tag);
|
|
150
|
+
if (!match) {
|
|
151
|
+
continue;
|
|
152
|
+
}
|
|
153
|
+
matches.push(match);
|
|
154
|
+
if (rule.impliesCode) {
|
|
155
|
+
implyCodeFromCase(matches, match);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
function implyCodeFromCase(matches, caseMatch) {
|
|
160
|
+
if (matches.some(match => match.tag === _failureTagConstants.FAILURE_TAGS.CODE)) {
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
163
|
+
matches.push({
|
|
164
|
+
tag: _failureTagConstants.FAILURE_TAGS.CODE,
|
|
165
|
+
pattern: null,
|
|
166
|
+
source: 'case-implied',
|
|
167
|
+
text: caseMatch.text
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
function classifyFailureDetailsFromEvents(events = []) {
|
|
171
|
+
const matches = [];
|
|
172
|
+
collectRuntimeLogMatches(events, matches);
|
|
173
|
+
return buildFailureTagMetadata(matches);
|
|
174
|
+
}
|
|
175
|
+
function classifyFailureDetailsFromResult(result, testTitle, logsDir) {
|
|
176
|
+
const resolvedLogsDir = resolveLogsDir(logsDir);
|
|
177
|
+
const events = (0, _runtimeLogReader.readRuntimeLogEventsForResult)(result, testTitle, resolvedLogsDir);
|
|
178
|
+
const matches = [];
|
|
179
|
+
collectRuntimeLogMatches(events, matches);
|
|
180
|
+
collectStdioMatches(buildStdioText(result), matches);
|
|
181
|
+
return buildFailureTagMetadata(matches);
|
|
182
|
+
}
|
|
183
|
+
function buildFailureTagMetadata(matches = []) {
|
|
184
|
+
const tags = TAG_ORDER.filter(tag => matches.some(match => match.tag === tag));
|
|
185
|
+
return {
|
|
186
|
+
schemaVersion: _failureTagConstants.FAILURE_TAG_METADATA_SCHEMA_VERSION,
|
|
187
|
+
tags,
|
|
188
|
+
matches
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
function classifyFailureDetails(testDetails = [], testTitle = 'test', logsDir) {
|
|
192
|
+
const resolvedLogsDir = resolveLogsDir(logsDir);
|
|
193
|
+
const matches = [];
|
|
108
194
|
for (const detail of testDetails) {
|
|
109
195
|
if (detail.status === FLAKY_OUTCOME) {
|
|
110
|
-
|
|
196
|
+
matches.push({
|
|
197
|
+
tag: _failureTagConstants.FAILURE_TAGS.FLAKY,
|
|
198
|
+
pattern: null,
|
|
199
|
+
source: 'retry-outcome',
|
|
200
|
+
text: null
|
|
201
|
+
});
|
|
111
202
|
continue;
|
|
112
203
|
}
|
|
113
204
|
if (detail.status !== UNEXPECTED_OUTCOME) {
|
|
@@ -117,43 +208,33 @@ function classifyFailureTags(testDetails = []) {
|
|
|
117
208
|
if (!failingResult) {
|
|
118
209
|
continue;
|
|
119
210
|
}
|
|
120
|
-
const
|
|
121
|
-
|
|
122
|
-
code = code || matched.code;
|
|
123
|
-
tool = tool || matched.tool;
|
|
211
|
+
const resultDetails = classifyFailureDetailsFromResult(failingResult, testTitle, resolvedLogsDir);
|
|
212
|
+
matches.push(...resultDetails.matches);
|
|
124
213
|
}
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
214
|
+
const uniqueMatches = [];
|
|
215
|
+
for (const match of matches) {
|
|
216
|
+
if (!uniqueMatches.some(existing => existing.tag === match.tag)) {
|
|
217
|
+
uniqueMatches.push(match);
|
|
218
|
+
}
|
|
130
219
|
}
|
|
131
|
-
|
|
132
|
-
|
|
220
|
+
return buildFailureTagMetadata(uniqueMatches);
|
|
221
|
+
}
|
|
222
|
+
function classifyFailureTags(testDetails = [], testTitle = 'test', logsDir) {
|
|
223
|
+
return classifyFailureDetails(testDetails, testTitle, logsDir).tags;
|
|
224
|
+
}
|
|
225
|
+
function pickFailingResult(results = []) {
|
|
226
|
+
const failing = results.filter(result => FAILING_RESULT_STATUSES.has(result.status));
|
|
227
|
+
if (failing.length === 0) {
|
|
228
|
+
return null;
|
|
133
229
|
}
|
|
134
|
-
|
|
135
|
-
// Stable order: flaky, environment, code, tool.
|
|
136
|
-
const ordered = [];
|
|
137
|
-
if (tags.has(_failureTagConstants.FAILURE_TAGS.FLAKY)) ordered.push(_failureTagConstants.FAILURE_TAGS.FLAKY);
|
|
138
|
-
if (tags.has(_failureTagConstants.FAILURE_TAGS.ENVIRONMENT)) ordered.push(_failureTagConstants.FAILURE_TAGS.ENVIRONMENT);
|
|
139
|
-
if (tags.has(_failureTagConstants.FAILURE_TAGS.CODE)) ordered.push(_failureTagConstants.FAILURE_TAGS.CODE);
|
|
140
|
-
if (tags.has(_failureTagConstants.FAILURE_TAGS.TOOL)) ordered.push(_failureTagConstants.FAILURE_TAGS.TOOL);
|
|
141
|
-
return ordered;
|
|
230
|
+
return failing[failing.length - 1];
|
|
142
231
|
}
|
|
143
|
-
|
|
144
|
-
// Formats a bare category into its emitted tag form (e.g. "CODE" -> "CODE_failure").
|
|
145
|
-
// The flaky category is a passed-on-retry outcome, not a failure, so it is emitted
|
|
146
|
-
// as "FLAKY" without a suffix.
|
|
147
232
|
function formatFailureTag(tag) {
|
|
148
233
|
if (tag === _failureTagConstants.FAILURE_TAGS.FLAKY) {
|
|
149
234
|
return tag;
|
|
150
235
|
}
|
|
151
236
|
return `${tag}${_failureTagConstants.FAILURE_TAG_SUFFIX}`;
|
|
152
237
|
}
|
|
153
|
-
|
|
154
|
-
// Merges derived failure tags into an existing tags array. Failure tags are
|
|
155
|
-
// placed first, followed by the existing tags, without removing or duplicating
|
|
156
|
-
// entries.
|
|
157
238
|
function mergeFailureTags(existingTags = [], failureTags = []) {
|
|
158
239
|
const formatted = failureTags.map(formatFailureTag);
|
|
159
240
|
const merged = [...formatted];
|
|
@@ -163,4 +244,23 @@ function mergeFailureTags(existingTags = [], failureTags = []) {
|
|
|
163
244
|
}
|
|
164
245
|
}
|
|
165
246
|
return merged;
|
|
247
|
+
}
|
|
248
|
+
function buildFailureTagMetadataAttachment(details) {
|
|
249
|
+
var _details$tags;
|
|
250
|
+
if (!(details !== null && details !== void 0 && (_details$tags = details.tags) !== null && _details$tags !== void 0 && _details$tags.length)) {
|
|
251
|
+
return null;
|
|
252
|
+
}
|
|
253
|
+
return {
|
|
254
|
+
name: _failureTagConstants.FAILURE_TAG_METADATA_ATTACHMENT,
|
|
255
|
+
contentType: 'application/json',
|
|
256
|
+
body: Buffer.from(JSON.stringify(details, null, 2))
|
|
257
|
+
};
|
|
258
|
+
}
|
|
259
|
+
function upsertFailureTagMetadataAttachment(attachments = [], details) {
|
|
260
|
+
const metadataAttachment = buildFailureTagMetadataAttachment(details);
|
|
261
|
+
if (!metadataAttachment) {
|
|
262
|
+
return attachments;
|
|
263
|
+
}
|
|
264
|
+
const withoutMetadata = attachments.filter(item => item.name !== _failureTagConstants.FAILURE_TAG_METADATA_ATTACHMENT);
|
|
265
|
+
return [...withoutMetadata, metadataAttachment];
|
|
166
266
|
}
|
|
@@ -1,13 +1,16 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
|
+
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
|
|
3
4
|
Object.defineProperty(exports, "__esModule", {
|
|
4
5
|
value: true
|
|
5
6
|
});
|
|
6
7
|
exports.LogCollector = void 0;
|
|
8
|
+
var _path = _interopRequireDefault(require("path"));
|
|
9
|
+
var _fileUtils = require("../../../utils/fileUtils");
|
|
7
10
|
var _logCollectorConstants = require("../constants/logCollectorConstants");
|
|
8
11
|
// Per-context Playwright runtime event collector — network failures,
|
|
9
|
-
// console errors, page errors and crashes. Buffered in-memory and
|
|
10
|
-
//
|
|
12
|
+
// console errors, page errors and crashes. Buffered in-memory and flushed
|
|
13
|
+
// to the project runtime-logs folder per test.
|
|
11
14
|
|
|
12
15
|
const DEFAULT_CONFIG = {
|
|
13
16
|
maxEvents: 2000,
|
|
@@ -128,11 +131,14 @@ class LogCollector {
|
|
|
128
131
|
}
|
|
129
132
|
this._bound.length = 0;
|
|
130
133
|
}
|
|
131
|
-
|
|
134
|
+
flushToFile(dir, baseName) {
|
|
132
135
|
if (this.events.length === 0) {
|
|
133
136
|
return null;
|
|
134
137
|
}
|
|
135
|
-
|
|
138
|
+
const file = _path.default.join(dir, `${baseName}.jsonl`);
|
|
139
|
+
const body = this.events.map(e => JSON.stringify(e)).join('\n') + '\n';
|
|
140
|
+
(0, _fileUtils.writeFileContents)(file, body);
|
|
141
|
+
return file;
|
|
136
142
|
}
|
|
137
143
|
}
|
|
138
144
|
exports.LogCollector = LogCollector;
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
|
|
4
|
+
Object.defineProperty(exports, "__esModule", {
|
|
5
|
+
value: true
|
|
6
|
+
});
|
|
7
|
+
exports.resolveRuntimeLogBaseName = resolveRuntimeLogBaseName;
|
|
8
|
+
exports.resolveRuntimeLogFilePath = resolveRuntimeLogFilePath;
|
|
9
|
+
exports.sanitizeForFilename = sanitizeForFilename;
|
|
10
|
+
var _path = _interopRequireDefault(require("path"));
|
|
11
|
+
const MAX_FILENAME_LENGTH = 200;
|
|
12
|
+
const EXAMPLE_TITLE = /^Example\s*#\d+$/i;
|
|
13
|
+
const SPEC_FILE = /\.(js|ts)$/;
|
|
14
|
+
function sanitizeForFilename(name) {
|
|
15
|
+
if (!name) return 'test';
|
|
16
|
+
const cleaned = name.replace(/[^A-Za-z0-9._-]+/g, '-').replace(/-+/g, '-').replace(/^-|-$/g, '');
|
|
17
|
+
return cleaned || 'test';
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
// playwright-bdd nests Example rows under test.describe(scenario.name).
|
|
21
|
+
function extractOutlineScenarioName(titlePath = []) {
|
|
22
|
+
const segments = titlePath.filter(Boolean);
|
|
23
|
+
if (segments.length < 2 || !EXAMPLE_TITLE.test(segments[segments.length - 1])) {
|
|
24
|
+
return null;
|
|
25
|
+
}
|
|
26
|
+
for (let i = segments.length - 2; i >= 0; i--) {
|
|
27
|
+
const segment = segments[i];
|
|
28
|
+
if (segment && !SPEC_FILE.test(segment)) {
|
|
29
|
+
return segment;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
return null;
|
|
33
|
+
}
|
|
34
|
+
function resolveRuntimeLogBaseName({
|
|
35
|
+
workerIndex = 0,
|
|
36
|
+
retry = 0,
|
|
37
|
+
title = 'test',
|
|
38
|
+
titlePath = []
|
|
39
|
+
}) {
|
|
40
|
+
const scenarioName = extractOutlineScenarioName(titlePath);
|
|
41
|
+
const label = scenarioName ? `${scenarioName} ${title}` : title;
|
|
42
|
+
const base = `w${workerIndex}-${sanitizeForFilename(label)}-${retry}`;
|
|
43
|
+
if (base.length <= MAX_FILENAME_LENGTH) {
|
|
44
|
+
return base;
|
|
45
|
+
}
|
|
46
|
+
return base.slice(0, MAX_FILENAME_LENGTH).replace(/-$/, '');
|
|
47
|
+
}
|
|
48
|
+
function resolveRuntimeLogFilePath(logsDir, {
|
|
49
|
+
workerIndex,
|
|
50
|
+
retry,
|
|
51
|
+
title,
|
|
52
|
+
titlePath = []
|
|
53
|
+
}) {
|
|
54
|
+
const base = resolveRuntimeLogBaseName({
|
|
55
|
+
workerIndex,
|
|
56
|
+
retry,
|
|
57
|
+
title,
|
|
58
|
+
titlePath
|
|
59
|
+
});
|
|
60
|
+
return _path.default.join(logsDir, `${base}.jsonl`);
|
|
61
|
+
}
|
|
@@ -1,22 +1,17 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
|
+
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
|
|
3
4
|
Object.defineProperty(exports, "__esModule", {
|
|
4
5
|
value: true
|
|
5
6
|
});
|
|
6
|
-
exports.
|
|
7
|
-
|
|
8
|
-
|
|
7
|
+
exports.readRuntimeLogEvents = readRuntimeLogEvents;
|
|
8
|
+
exports.readRuntimeLogEventsForResult = readRuntimeLogEventsForResult;
|
|
9
|
+
var _fs = _interopRequireDefault(require("fs"));
|
|
10
|
+
var _fileUtils = require("../../../utils/fileUtils");
|
|
11
|
+
var _runtimeLogPath = require("./runtimeLogPath");
|
|
12
|
+
// Reads per-test runtime browser logs from the project runtime-logs folder.
|
|
9
13
|
// Used by the failure-tag classifier to correlate browser events with a test failure.
|
|
10
14
|
|
|
11
|
-
function attachmentBodyToString(body) {
|
|
12
|
-
if (!body) {
|
|
13
|
-
return '';
|
|
14
|
-
}
|
|
15
|
-
if (Buffer.isBuffer(body)) {
|
|
16
|
-
return body.toString('utf8');
|
|
17
|
-
}
|
|
18
|
-
return String(body);
|
|
19
|
-
}
|
|
20
15
|
function parseJsonlContent(raw) {
|
|
21
16
|
const trimmed = raw.trim();
|
|
22
17
|
if (!trimmed) {
|
|
@@ -36,10 +31,24 @@ function parseJsonlContent(raw) {
|
|
|
36
31
|
}
|
|
37
32
|
return events;
|
|
38
33
|
}
|
|
39
|
-
function
|
|
40
|
-
|
|
41
|
-
|
|
34
|
+
function readRuntimeLogEvents(filePath) {
|
|
35
|
+
if (!filePath || !_fs.default.existsSync(filePath)) {
|
|
36
|
+
return [];
|
|
37
|
+
}
|
|
38
|
+
const rawContent = (0, _fileUtils.readFileContents)(filePath);
|
|
39
|
+
if (typeof rawContent !== 'string') {
|
|
40
|
+
return [];
|
|
41
|
+
}
|
|
42
|
+
return parseJsonlContent(rawContent);
|
|
43
|
+
}
|
|
44
|
+
function readRuntimeLogEventsForResult(result, testTitle, logsDir) {
|
|
45
|
+
if (!logsDir) {
|
|
42
46
|
return [];
|
|
43
47
|
}
|
|
44
|
-
|
|
48
|
+
const filePath = (0, _runtimeLogPath.resolveRuntimeLogFilePath)(logsDir, {
|
|
49
|
+
workerIndex: (result === null || result === void 0 ? void 0 : result.workerIndex) ?? 0,
|
|
50
|
+
retry: (result === null || result === void 0 ? void 0 : result.retry) ?? 0,
|
|
51
|
+
title: testTitle
|
|
52
|
+
});
|
|
53
|
+
return readRuntimeLogEvents(filePath);
|
|
45
54
|
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
Object.defineProperty(exports, "__esModule", {
|
|
4
|
+
value: true
|
|
5
|
+
});
|
|
6
|
+
exports.CASE_PATTERNS = void 0;
|
|
7
|
+
// Case patterns: test-case / assertion failures from Playwright output.
|
|
8
|
+
// Matched against the failing result's stdout + stderr.
|
|
9
|
+
const CASE_PATTERNS = exports.CASE_PATTERNS = [
|
|
10
|
+
// Generator
|
|
11
|
+
/GeneratorError: Generator/i, /DataGeneratorConfigurationError: Data Generator configuration is missing for the profile/i, /HTTP error! status:/i, /status\s*:\s*DATA_GENERATION_ERROR/i];
|
|
@@ -3,6 +3,12 @@
|
|
|
3
3
|
Object.defineProperty(exports, "__esModule", {
|
|
4
4
|
value: true
|
|
5
5
|
});
|
|
6
|
+
Object.defineProperty(exports, "CASE_PATTERNS", {
|
|
7
|
+
enumerable: true,
|
|
8
|
+
get: function () {
|
|
9
|
+
return _casePatterns.CASE_PATTERNS;
|
|
10
|
+
}
|
|
11
|
+
});
|
|
6
12
|
Object.defineProperty(exports, "CODE_PATTERNS", {
|
|
7
13
|
enumerable: true,
|
|
8
14
|
get: function () {
|
|
@@ -23,4 +29,5 @@ Object.defineProperty(exports, "TOOL_PATTERNS", {
|
|
|
23
29
|
});
|
|
24
30
|
var _environmentPatterns = require("./environmentPatterns");
|
|
25
31
|
var _codePatterns = require("./codePatterns");
|
|
32
|
+
var _casePatterns = require("./casePatterns");
|
|
26
33
|
var _toolPatterns = require("./toolPatterns");
|
|
@@ -7,9 +7,7 @@ exports.TOOL_PATTERNS = void 0;
|
|
|
7
7
|
// Tool patterns: failures originating from the test tooling / data-generation
|
|
8
8
|
// layer (not the app under test). Matched against the failing result's
|
|
9
9
|
// stdout + stderr text. Add new patterns here to recognise more tool errors.
|
|
10
|
-
const TOOL_PATTERNS = exports.TOOL_PATTERNS = [/
|
|
11
|
-
// Generator
|
|
12
|
-
/GeneratorError: Generator/i, /DataGeneratorConfigurationError: Data Generator configuration is missing for the profile/i, /HTTP error! status:/i,
|
|
10
|
+
const TOOL_PATTERNS = exports.TOOL_PATTERNS = [/Error during login/i, /fetch failed/i,
|
|
13
11
|
// Configuration
|
|
14
12
|
/Config File Not Exists\. Please provide a/i, /Exception while getting the uat file from the application/i, /There is no beta feature configured with the name/i, /No config mapping found for stage:/i,
|
|
15
13
|
// Edition
|
|
@@ -13,10 +13,10 @@ var _ConfigurationHelper = require("../configuration/ConfigurationHelper");
|
|
|
13
13
|
var _failureTagClassifier = require("../helpers/failureTagClassifier");
|
|
14
14
|
const stage = (0, _ConfigurationHelper.getRunStage)();
|
|
15
15
|
|
|
16
|
-
// Walks the built suite tree and appends derived failure tags (CODE_failure, ENVIRONMENT_failure) into each spec's tags array.
|
|
16
|
+
// Walks the built suite tree and appends derived failure tags (CASE_failure, CODE_failure, ENVIRONMENT_failure) into each spec's tags array.
|
|
17
17
|
const applyFailureTags = node => {
|
|
18
18
|
for (const spec of node.specs ?? []) {
|
|
19
|
-
const failureTags = (0, _failureTagClassifier.classifyFailureTags)(spec.tests);
|
|
19
|
+
const failureTags = (0, _failureTagClassifier.classifyFailureTags)(spec.tests, spec.title);
|
|
20
20
|
if (failureTags.length > 0) {
|
|
21
21
|
spec.tags = (0, _failureTagClassifier.mergeFailureTags)(spec.tags, failureTags);
|
|
22
22
|
}
|
|
@@ -44,9 +44,21 @@ class CustomJsonReporter {
|
|
|
44
44
|
onTestEnd(test, result) {
|
|
45
45
|
var _result$errors, _result$steps;
|
|
46
46
|
const key = `${test.location.file}:${test.location.line}:${test.title}`;
|
|
47
|
+
const isFailure = !['passed', 'skipped'].includes(result.status);
|
|
48
|
+
let attachments = result.attachments ?? [];
|
|
49
|
+
if (isFailure) {
|
|
50
|
+
const metadata = (0, _failureTagClassifier.classifyFailureDetailsFromResult)({
|
|
51
|
+
status: result.status,
|
|
52
|
+
workerIndex: result.workerIndex,
|
|
53
|
+
retry: result.retry,
|
|
54
|
+
stdout: result.stdout.map(out => stdioEntry(out)),
|
|
55
|
+
stderr: result.stderr.map(err => stdioEntry(err))
|
|
56
|
+
}, test.title);
|
|
57
|
+
attachments = (0, _failureTagClassifier.upsertFailureTagMetadataAttachment)(attachments, metadata);
|
|
58
|
+
}
|
|
47
59
|
const testResult = {
|
|
48
60
|
status: result.status,
|
|
49
|
-
attachments
|
|
61
|
+
attachments,
|
|
50
62
|
startTime: result.startTime,
|
|
51
63
|
retry: result.retry,
|
|
52
64
|
stderr: result.stderr.map(err => stdioEntry(err)),
|