@lullabot/playwright-testing 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +61 -0
- package/bin/github-a11y-summary +5 -0
- package/bin/github-failure-summary +8 -0
- package/lib/accessibility-baseline-file.d.ts +47 -0
- package/lib/accessibility-baseline-file.js +205 -0
- package/lib/accessibility-baseline.d.ts +19 -0
- package/lib/accessibility-baseline.js +38 -0
- package/lib/accessible-screenshot.d.ts +181 -0
- package/lib/accessible-screenshot.js +519 -0
- package/lib/focus.d.ts +14 -0
- package/lib/focus.js +28 -0
- package/lib/fonts.d.ts +18 -0
- package/lib/fonts.js +24 -0
- package/lib/frames.d.ts +7 -0
- package/lib/frames.js +27 -0
- package/lib/github/a11y-summary.d.ts +55 -0
- package/lib/github/a11y-summary.js +383 -0
- package/lib/github/attachments.d.ts +98 -0
- package/lib/github/attachments.js +297 -0
- package/lib/github/failure-summary.d.ts +144 -0
- package/lib/github/failure-summary.js +567 -0
- package/lib/github/index.d.ts +6 -0
- package/lib/github/index.js +35 -0
- package/lib/github/report-paths.d.ts +38 -0
- package/lib/github/report-paths.js +200 -0
- package/lib/hover.d.ts +13 -0
- package/lib/hover.js +61 -0
- package/lib/images.d.ts +118 -0
- package/lib/images.js +260 -0
- package/lib/index.d.ts +13 -0
- package/lib/index.js +29 -0
- package/lib/interaction-states.d.ts +22 -0
- package/lib/interaction-states.js +75 -0
- package/lib/mock/index.d.ts +1 -0
- package/lib/mock/index.js +5 -0
- package/lib/mock/youtube.d.ts +5 -0
- package/lib/mock/youtube.js +38 -0
- package/lib/pseudo-state.d.ts +17 -0
- package/lib/pseudo-state.js +50 -0
- package/lib/videos.d.ts +134 -0
- package/lib/videos.js +349 -0
- package/lib/visualdiff.d.ts +154 -0
- package/lib/visualdiff.js +197 -0
- package/package.json +47 -0
- package/src/accessibility-baseline-file.test.ts +181 -0
- package/src/accessibility-baseline-file.ts +208 -0
- package/src/accessibility-baseline.test.ts +601 -0
- package/src/accessibility-baseline.ts +50 -0
- package/src/accessible-screenshot.test.ts +597 -0
- package/src/accessible-screenshot.ts +809 -0
- package/src/focus.test.ts +34 -0
- package/src/focus.ts +27 -0
- package/src/fonts.test.ts +17 -0
- package/src/fonts.ts +23 -0
- package/src/frames.test.ts +75 -0
- package/src/frames.ts +26 -0
- package/src/github/a11y-summary.test.ts +439 -0
- package/src/github/a11y-summary.ts +421 -0
- package/src/github/attachments.test.ts +248 -0
- package/src/github/attachments.ts +328 -0
- package/src/github/failure-summary.test.ts +636 -0
- package/src/github/failure-summary.ts +720 -0
- package/src/github/index.test.ts +24 -0
- package/src/github/index.ts +35 -0
- package/src/github/report-paths.test.ts +222 -0
- package/src/github/report-paths.ts +208 -0
- package/src/hover.test.ts +76 -0
- package/src/hover.ts +64 -0
- package/src/images.test.ts +355 -0
- package/src/images.ts +299 -0
- package/src/index.ts +13 -0
- package/src/interaction-states.test.ts +48 -0
- package/src/interaction-states.ts +94 -0
- package/src/mock/index.ts +1 -0
- package/src/mock/youtube.test.ts +38 -0
- package/src/mock/youtube.ts +39 -0
- package/src/pseudo-state.test.ts +83 -0
- package/src/pseudo-state.ts +69 -0
- package/src/videos.test.ts +637 -0
- package/src/videos.ts +389 -0
- package/src/visualdiff.test.ts +452 -0
- package/src/visualdiff.ts +381 -0
|
@@ -0,0 +1,567 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.FLAKE_MARKER_PREFIX = exports.FAILURE_MARKER_PREFIX = void 0;
|
|
37
|
+
exports.defuseMaskTriggers = defuseMaskTriggers;
|
|
38
|
+
exports.parseFailures = parseFailures;
|
|
39
|
+
exports.resolveImagePaths = resolveImagePaths;
|
|
40
|
+
exports.uploadImages = uploadImages;
|
|
41
|
+
exports.generateSummary = generateSummary;
|
|
42
|
+
exports.generateComment = generateComment;
|
|
43
|
+
exports.uploaderFromEnvironment = uploaderFromEnvironment;
|
|
44
|
+
exports.main = main;
|
|
45
|
+
const fs = __importStar(require("fs"));
|
|
46
|
+
const path = __importStar(require("path"));
|
|
47
|
+
const attachments_1 = require("./attachments");
|
|
48
|
+
const report_paths_1 = require("./report-paths");
|
|
49
|
+
/**
|
|
50
|
+
* Turn a Playwright JSON report into a job summary — and optionally a PR
|
|
51
|
+
* comment body — with the failure screenshots embedded inline.
|
|
52
|
+
*
|
|
53
|
+
* Playwright already captures everything worth looking at: `toHaveScreenshot`
|
|
54
|
+
* attaches `-expected`/`-actual`/`-diff` images when a comparison fails, and
|
|
55
|
+
* checkAccessibility() attaches a full-page screenshot with the violating
|
|
56
|
+
* elements outlined. Both normally end up zipped inside an artifact nobody
|
|
57
|
+
* downloads. Given an upload token they can be embedded where the failure is
|
|
58
|
+
* actually read.
|
|
59
|
+
*
|
|
60
|
+
* Without a token this still produces a useful text summary, so the feature
|
|
61
|
+
* degrades rather than disappearing.
|
|
62
|
+
*/
|
|
63
|
+
/** Attachment names Playwright gives the three snapshot comparison images. */
|
|
64
|
+
const SNAPSHOT_SUFFIXES = [
|
|
65
|
+
{ suffix: '-diff', kind: 'diff' },
|
|
66
|
+
{ suffix: '-actual', kind: 'actual' },
|
|
67
|
+
{ suffix: '-expected', kind: 'expected' },
|
|
68
|
+
{ suffix: '-previous', kind: 'previous' },
|
|
69
|
+
];
|
|
70
|
+
/** Attachment name used by checkAccessibility() for its highlighted screenshot. */
|
|
71
|
+
const A11Y_SCREENSHOT = 'a11y-violation-screenshot';
|
|
72
|
+
/**
|
|
73
|
+
* Prefix of the HTML comment carrying the failure count in a generated comment
|
|
74
|
+
* body. Invisible when rendered, and greppable by a workflow that has to decide
|
|
75
|
+
* whether the run is worth commenting on.
|
|
76
|
+
*/
|
|
77
|
+
exports.FAILURE_MARKER_PREFIX = '<!-- playwright-testing-failures: ';
|
|
78
|
+
/** Machine-readable flaky-test count used by the comment assembly action. */
|
|
79
|
+
exports.FLAKE_MARKER_PREFIX = '<!-- playwright-testing-flakes: ';
|
|
80
|
+
/**
|
|
81
|
+
* Neutralise text that the Actions runner would redact.
|
|
82
|
+
*
|
|
83
|
+
* Job summaries pass through the runner's secret masking (comments posted over
|
|
84
|
+
* the API do not). The masker replaces `Bearer <value>` with `***` and takes
|
|
85
|
+
* the following characters with it, so a test name or error containing the
|
|
86
|
+
* word can silently swallow the Markdown that follows it. A zero-width space
|
|
87
|
+
* inside the word reads identically and no longer matches.
|
|
88
|
+
*/
|
|
89
|
+
function defuseMaskTriggers(text) {
|
|
90
|
+
return text.replace(/Bearer(?=\s)/g, 'Bear\u200Ber');
|
|
91
|
+
}
|
|
92
|
+
/** Strip ANSI colour codes Playwright puts in error messages. */
|
|
93
|
+
function stripAnsi(text) {
|
|
94
|
+
return text.replace(/\u001b\[[0-9;]*m/g, '');
|
|
95
|
+
}
|
|
96
|
+
function classifyAttachment(name, contentType) {
|
|
97
|
+
if (!contentType.startsWith('image/'))
|
|
98
|
+
return null;
|
|
99
|
+
if (name === A11Y_SCREENSHOT)
|
|
100
|
+
return 'a11y';
|
|
101
|
+
const withoutExtension = name.replace(/\.[^.]+$/, '');
|
|
102
|
+
for (const { suffix, kind } of SNAPSHOT_SUFFIXES) {
|
|
103
|
+
if (withoutExtension.endsWith(suffix))
|
|
104
|
+
return kind;
|
|
105
|
+
}
|
|
106
|
+
return 'screenshot';
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Collect the images worth showing for one test result.
|
|
110
|
+
*
|
|
111
|
+
* Accessibility screenshots are kept even when the test passed, because a
|
|
112
|
+
* baselined violation still passes and its screenshot is still the thing you
|
|
113
|
+
* want to look at.
|
|
114
|
+
*/
|
|
115
|
+
function collectImages(attachments, include) {
|
|
116
|
+
const images = [];
|
|
117
|
+
for (const attachment of attachments) {
|
|
118
|
+
const name = String(attachment?.name ?? '');
|
|
119
|
+
const contentType = String(attachment?.contentType ?? '');
|
|
120
|
+
const kind = classifyAttachment(name, contentType);
|
|
121
|
+
if (!kind)
|
|
122
|
+
continue;
|
|
123
|
+
// Uploading three near-identical images per failure is rarely worth it;
|
|
124
|
+
// the diff is the one that shows what changed.
|
|
125
|
+
if (include === 'diff' && (kind === 'expected' || kind === 'actual' || kind === 'previous')) {
|
|
126
|
+
continue;
|
|
127
|
+
}
|
|
128
|
+
images.push({
|
|
129
|
+
name,
|
|
130
|
+
filePath: attachment?.path ? String(attachment.path) : undefined,
|
|
131
|
+
body: attachment?.body ? String(attachment.body) : undefined,
|
|
132
|
+
contentType,
|
|
133
|
+
kind,
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
return images;
|
|
137
|
+
}
|
|
138
|
+
function firstError(result) {
|
|
139
|
+
const message = result?.error?.message ?? result?.errors?.[0]?.message;
|
|
140
|
+
if (typeof message !== 'string' || !message.trim())
|
|
141
|
+
return undefined;
|
|
142
|
+
const cleaned = stripAnsi(message).trim().split('\n')[0];
|
|
143
|
+
return cleaned.length > 300 ? `${cleaned.slice(0, 300)}…` : cleaned;
|
|
144
|
+
}
|
|
145
|
+
function walkSuites(suites, parentFile, include, out) {
|
|
146
|
+
for (const suite of suites ?? []) {
|
|
147
|
+
const file = suite.file || parentFile;
|
|
148
|
+
for (const spec of suite.specs ?? []) {
|
|
149
|
+
for (const test of spec.tests ?? []) {
|
|
150
|
+
const results = test.results ?? [];
|
|
151
|
+
const result = results[results.length - 1];
|
|
152
|
+
if (!result)
|
|
153
|
+
continue;
|
|
154
|
+
// Playwright reports each retry as another result. A passing final
|
|
155
|
+
// attempt therefore hides the failed attempt unless we deliberately
|
|
156
|
+
// retain it and give the test its Playwright outcome: flaky.
|
|
157
|
+
const failedAttempts = results.slice(0, -1).filter((attempt) => (attempt?.status !== 'passed' && attempt?.status !== 'skipped'));
|
|
158
|
+
// JSON reports expose Playwright's computed outcome on the test. Keep
|
|
159
|
+
// the attempt-based fallback for older or hand-written reports that do
|
|
160
|
+
// not carry it.
|
|
161
|
+
const flaky = test.status === 'flaky' || (test.status === undefined && result.status === 'passed' && failedAttempts.length > 0);
|
|
162
|
+
const reportedResult = flaky ? failedAttempts[failedAttempts.length - 1] : result;
|
|
163
|
+
const images = collectImages(reportedResult.attachments ?? [], include);
|
|
164
|
+
const failed = result.status !== 'passed' && result.status !== 'skipped';
|
|
165
|
+
const hasA11yImage = images.some(image => image.kind === 'a11y');
|
|
166
|
+
if (!failed && !flaky && !hasA11yImage)
|
|
167
|
+
continue;
|
|
168
|
+
out.push({
|
|
169
|
+
title: spec.title,
|
|
170
|
+
file,
|
|
171
|
+
line: spec.line ?? 1,
|
|
172
|
+
status: flaky ? 'flaky' : String(result.status ?? 'unknown'),
|
|
173
|
+
error: firstError(reportedResult),
|
|
174
|
+
images,
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
walkSuites(suite.suites ?? [], file, include, out);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Parse a Playwright JSON report into the failures worth reporting.
|
|
183
|
+
*/
|
|
184
|
+
function parseFailures(reportPath, include = 'diff') {
|
|
185
|
+
const report = JSON.parse(fs.readFileSync(reportPath, 'utf8'));
|
|
186
|
+
const tests = [];
|
|
187
|
+
walkSuites(report.suites ?? [], '', include, tests);
|
|
188
|
+
const totalFailed = tests.filter(test => (test.status !== 'passed' && test.status !== 'skipped' && test.status !== 'flaky')).length;
|
|
189
|
+
const totalFlaky = tests.filter(test => test.status === 'flaky').length;
|
|
190
|
+
const totalImages = tests.reduce((sum, test) => sum + test.images.length, 0);
|
|
191
|
+
return { tests, totalFailed, totalFlaky, totalImages };
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Point every image at a file this process can open, and note the ones where
|
|
195
|
+
* that was not possible.
|
|
196
|
+
*
|
|
197
|
+
* Runs whether or not uploads are switched on: an unreachable attachment is
|
|
198
|
+
* worth reporting even on a fork build that was never going to upload it,
|
|
199
|
+
* because it is the same misconfiguration either way.
|
|
200
|
+
*/
|
|
201
|
+
function resolveImagePaths(report, resolve) {
|
|
202
|
+
const summary = { remapped: 0, used: [], unreadable: [] };
|
|
203
|
+
for (const test of report.tests) {
|
|
204
|
+
for (const image of test.images) {
|
|
205
|
+
if (!image.filePath)
|
|
206
|
+
continue;
|
|
207
|
+
const resolution = resolve(image.filePath);
|
|
208
|
+
if (!resolution.found) {
|
|
209
|
+
image.unreadable = true;
|
|
210
|
+
summary.unreadable.push(image.filePath);
|
|
211
|
+
continue;
|
|
212
|
+
}
|
|
213
|
+
if (resolution.path === image.filePath)
|
|
214
|
+
continue;
|
|
215
|
+
summary.remapped++;
|
|
216
|
+
image.filePath = resolution.path;
|
|
217
|
+
const prefix = resolution.prefix;
|
|
218
|
+
if (prefix && !summary.used.some(used => used.from === prefix.from && used.to === prefix.to)) {
|
|
219
|
+
summary.used.push(prefix);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
return summary;
|
|
224
|
+
}
|
|
225
|
+
/**
|
|
226
|
+
* Upload every collected image, annotating each one with its URL. Images the
|
|
227
|
+
* uploader declines are simply left without a URL, which the renderers treat
|
|
228
|
+
* as "link to the artifact instead".
|
|
229
|
+
*/
|
|
230
|
+
async function uploadImages(report, uploader) {
|
|
231
|
+
for (const test of report.tests) {
|
|
232
|
+
for (const image of test.images) {
|
|
233
|
+
let url = null;
|
|
234
|
+
if (image.filePath) {
|
|
235
|
+
url = await uploader.upload(image.filePath, path.basename(image.filePath));
|
|
236
|
+
}
|
|
237
|
+
else if (image.body) {
|
|
238
|
+
url = await uploader.uploadBuffer(Buffer.from(image.body, 'base64'), `${image.name}${extensionFor(image.contentType)}`, image.contentType);
|
|
239
|
+
}
|
|
240
|
+
if (url) {
|
|
241
|
+
image.url = url;
|
|
242
|
+
continue;
|
|
243
|
+
}
|
|
244
|
+
// Every one of these is a null, and they need different things said
|
|
245
|
+
// about them. resolveImagePaths() has already checked the file is there,
|
|
246
|
+
// so 'unreadable' here means the read itself failed; the size reasons
|
|
247
|
+
// are the uploader declining this one image rather than stopping.
|
|
248
|
+
switch (uploader.lastSkipReason) {
|
|
249
|
+
case 'unreadable':
|
|
250
|
+
image.unreadable = true;
|
|
251
|
+
break;
|
|
252
|
+
case 'too-large':
|
|
253
|
+
case 'budget':
|
|
254
|
+
image.oversize = true;
|
|
255
|
+
break;
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
/** Inlined attachments carry no file name, so derive one from the content type. */
|
|
261
|
+
function extensionFor(contentType) {
|
|
262
|
+
if (contentType === 'image/png')
|
|
263
|
+
return '.png';
|
|
264
|
+
if (contentType === 'image/jpeg')
|
|
265
|
+
return '.jpg';
|
|
266
|
+
if (contentType === 'image/gif')
|
|
267
|
+
return '.gif';
|
|
268
|
+
if (contentType === 'image/webp')
|
|
269
|
+
return '.webp';
|
|
270
|
+
return '';
|
|
271
|
+
}
|
|
272
|
+
function headline(report) {
|
|
273
|
+
if (report.tests.length === 0)
|
|
274
|
+
return ':white_check_mark: No failing tests or flaky tests with screenshots.';
|
|
275
|
+
// Nothing failed, but an accessibility check still captured something worth
|
|
276
|
+
// seeing — a baselined violation passes and is still screenshotted.
|
|
277
|
+
const parts = report.totalFailed === 0
|
|
278
|
+
? [':white_check_mark: No failing tests']
|
|
279
|
+
: [`**${report.totalFailed}** failing test(s)`];
|
|
280
|
+
if (report.totalFlaky > 0)
|
|
281
|
+
parts.push(`:warning: **${report.totalFlaky}** flaky test(s)`);
|
|
282
|
+
if (report.totalImages > 0)
|
|
283
|
+
parts.push(`**${report.totalImages}** screenshot(s)`);
|
|
284
|
+
return parts.join(' · ');
|
|
285
|
+
}
|
|
286
|
+
/** Every image across every test, which is what the diagnostics count. */
|
|
287
|
+
function allImages(report) {
|
|
288
|
+
return report.tests.flatMap(test => test.images);
|
|
289
|
+
}
|
|
290
|
+
/**
|
|
291
|
+
* Say why a set of images was not uploaded, or nothing when they were.
|
|
292
|
+
*
|
|
293
|
+
* "No token" and "the files were not where the report said" are different
|
|
294
|
+
* problems with different fixes, and until they are named apart a reader has
|
|
295
|
+
* no way to tell which one they have. An absent token is the dominant reason
|
|
296
|
+
* when it applies: nothing was going to be uploaded regardless.
|
|
297
|
+
*/
|
|
298
|
+
function missingImageReason(images, uploadReason) {
|
|
299
|
+
if (uploadReason)
|
|
300
|
+
return uploadReason;
|
|
301
|
+
const reasons = [];
|
|
302
|
+
const unreadable = images.filter(image => image.unreadable).length;
|
|
303
|
+
if (unreadable > 0) {
|
|
304
|
+
reasons.push(`${unreadable} could not be read from the path recorded in the report`);
|
|
305
|
+
}
|
|
306
|
+
const oversize = images.filter(image => image.oversize).length;
|
|
307
|
+
if (oversize > 0) {
|
|
308
|
+
reasons.push(`${oversize} too large to upload`);
|
|
309
|
+
}
|
|
310
|
+
return reasons.length > 0 ? reasons.join('; ') : undefined;
|
|
311
|
+
}
|
|
312
|
+
/**
|
|
313
|
+
* Render the job summary: a heading, a headline, and one collapsed block per
|
|
314
|
+
* test holding its images.
|
|
315
|
+
*
|
|
316
|
+
* Deliberately table-free. Test titles and error messages are arbitrary text,
|
|
317
|
+
* and the runner's masking can eat a cell delimiter and corrupt a whole row.
|
|
318
|
+
*/
|
|
319
|
+
function generateSummary(report, options = {}) {
|
|
320
|
+
// A baselined accessibility violation is screenshotted without failing
|
|
321
|
+
// anything, so a green run can still have something to show here.
|
|
322
|
+
const heading = report.totalFailed > 0
|
|
323
|
+
? '## Test Failures'
|
|
324
|
+
: report.totalFlaky > 0 ? '## Test Flakes' : '## Test Screenshots';
|
|
325
|
+
const lines = [`${heading}\n`, `${headline(report)}\n`];
|
|
326
|
+
if (report.tests.length === 0)
|
|
327
|
+
return lines.join('\n');
|
|
328
|
+
// Name the path boundary where it will be read, rather than leaving the
|
|
329
|
+
// reader to conclude they forgot the token.
|
|
330
|
+
const unreadable = allImages(report).filter(image => image.unreadable);
|
|
331
|
+
if (unreadable.length > 0) {
|
|
332
|
+
const example = defuseMaskTriggers(unreadable[0].filePath ?? '');
|
|
333
|
+
lines.push(`> :warning: **${unreadable.length} screenshot(s) could not be read.** The report records them ` +
|
|
334
|
+
`under \`${example}\`, which does not exist where this step ran. Playwright ran somewhere else — ` +
|
|
335
|
+
'a container, most likely — so run this command there too, or pass ' +
|
|
336
|
+
'`--path-prefix=CONTAINER_PATH:LOCAL_PATH` to map one onto the other.\n');
|
|
337
|
+
}
|
|
338
|
+
for (const test of report.tests) {
|
|
339
|
+
const title = defuseMaskTriggers(test.title);
|
|
340
|
+
lines.push(`### ${title}\n`);
|
|
341
|
+
lines.push(`\`${test.file}:${test.line}\` — ${test.status}\n`);
|
|
342
|
+
if (test.error) {
|
|
343
|
+
lines.push(`> ${defuseMaskTriggers(test.error)}\n`);
|
|
344
|
+
}
|
|
345
|
+
if (test.images.length === 0)
|
|
346
|
+
continue;
|
|
347
|
+
// The images go in the pull request comment, not here. A job summary is
|
|
348
|
+
// rendered once, when the job finishes, and an attachment uploaded seconds
|
|
349
|
+
// earlier is not resolvable yet — so it renders as a dead link and the
|
|
350
|
+
// cached result never improves, however often the page is reloaded.
|
|
351
|
+
const embedded = test.images.filter(image => image.url).length;
|
|
352
|
+
if (embedded > 0) {
|
|
353
|
+
lines.push(`${embedded} screenshot(s) uploaded — see the pull request comment.\n`);
|
|
354
|
+
}
|
|
355
|
+
else {
|
|
356
|
+
const hint = options.artifactHint ?? 'the Playwright report artifact';
|
|
357
|
+
const reason = missingImageReason(test.images, options.uploadReason);
|
|
358
|
+
const because = reason ? ` (${reason})` : '';
|
|
359
|
+
lines.push(`${test.images.length} screenshot(s) captured, not uploaded${because} — download ${hint}.\n`);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
return lines.join('\n');
|
|
363
|
+
}
|
|
364
|
+
/** A collapsed block of images for one test, used in the comment. */
|
|
365
|
+
function renderImageBlock(test) {
|
|
366
|
+
const embedded = test.images.filter(image => image.url);
|
|
367
|
+
if (embedded.length === 0)
|
|
368
|
+
return [];
|
|
369
|
+
const title = defuseMaskTriggers(test.title);
|
|
370
|
+
const lines = ['<details>', `<summary>Screenshots (${embedded.length})</summary>\n`];
|
|
371
|
+
for (const image of embedded) {
|
|
372
|
+
lines.push(`**${image.kind}**\n`);
|
|
373
|
+
const src = escapeAttribute(image.url ?? '');
|
|
374
|
+
lines.push(`<img src="${src}" alt="${image.kind} for ${escapeAttribute(title)}" width="640">\n`);
|
|
375
|
+
}
|
|
376
|
+
lines.push('</details>\n');
|
|
377
|
+
return lines;
|
|
378
|
+
}
|
|
379
|
+
function generateComment(report, options = {}) {
|
|
380
|
+
const heading = options.title ?? 'Playwright results';
|
|
381
|
+
const lines = [`### ${heading}\n`, `${headline(report)}\n`];
|
|
382
|
+
if (report.tests.length > 0) {
|
|
383
|
+
for (const test of report.tests.slice(0, 10)) {
|
|
384
|
+
const title = defuseMaskTriggers(test.title);
|
|
385
|
+
const images = test.images.length > 0 ? ` — ${test.images.length} screenshot(s)` : '';
|
|
386
|
+
lines.push(`- \`${test.status}\` ${title}${images}`);
|
|
387
|
+
}
|
|
388
|
+
if (report.tests.length > 10) {
|
|
389
|
+
lines.push(`- …and ${report.tests.length - 10} more`);
|
|
390
|
+
}
|
|
391
|
+
lines.push('');
|
|
392
|
+
for (const test of report.tests) {
|
|
393
|
+
const block = renderImageBlock(test);
|
|
394
|
+
if (block.length === 0)
|
|
395
|
+
continue;
|
|
396
|
+
lines.push(`**${defuseMaskTriggers(test.title)}**\n`);
|
|
397
|
+
lines.push(...block);
|
|
398
|
+
}
|
|
399
|
+
// An empty comment where images were expected reads as a broken feature.
|
|
400
|
+
// Whatever the reason, it belongs where the images would have been.
|
|
401
|
+
const missing = allImages(report).filter(image => !image.url);
|
|
402
|
+
const reason = missingImageReason(missing, options.uploadReason);
|
|
403
|
+
if (missing.length > 0 && reason) {
|
|
404
|
+
lines.push(`_${missing.length} screenshot(s) not shown — ${reason}._\n`);
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
if (options.summaryUrl) {
|
|
408
|
+
lines.push(`[Full run](${options.summaryUrl})\n`);
|
|
409
|
+
}
|
|
410
|
+
// A machine-readable count, so a workflow assembling several of these can
|
|
411
|
+
// decide whether to post at all without grepping prose — the empty-state
|
|
412
|
+
// sentence contains the words "failing test" too.
|
|
413
|
+
lines.push(`${options.failureMarkerPrefix ?? exports.FAILURE_MARKER_PREFIX}${report.totalFailed} -->\n`);
|
|
414
|
+
lines.push(`${options.flakeMarkerPrefix ?? exports.FLAKE_MARKER_PREFIX}${report.totalFlaky} -->\n`);
|
|
415
|
+
return lines.join('\n');
|
|
416
|
+
}
|
|
417
|
+
function escapeAttribute(value) {
|
|
418
|
+
return value.replace(/&/g, '&').replace(/"/g, '"').replace(/</g, '<');
|
|
419
|
+
}
|
|
420
|
+
function parseArgs(args) {
|
|
421
|
+
const options = {
|
|
422
|
+
reportPath: 'test-results/results.json',
|
|
423
|
+
include: 'diff',
|
|
424
|
+
pathPrefixes: [],
|
|
425
|
+
};
|
|
426
|
+
for (const arg of args) {
|
|
427
|
+
// Repeatable: a project can have more than one mount to translate.
|
|
428
|
+
if (arg.startsWith('--path-prefix=')) {
|
|
429
|
+
const prefix = (0, report_paths_1.parsePathPrefix)(arg.slice('--path-prefix='.length));
|
|
430
|
+
if (prefix) {
|
|
431
|
+
options.pathPrefixes.push(prefix);
|
|
432
|
+
}
|
|
433
|
+
else {
|
|
434
|
+
console.error(`Ignoring --path-prefix: expected FROM:TO, got ${arg.slice('--path-prefix='.length)}`);
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
if (arg.startsWith('--report-path='))
|
|
438
|
+
options.reportPath = arg.slice('--report-path='.length);
|
|
439
|
+
if (arg.startsWith('--comment-path='))
|
|
440
|
+
options.commentPath = arg.slice('--comment-path='.length);
|
|
441
|
+
if (arg.startsWith('--title='))
|
|
442
|
+
options.title = arg.slice('--title='.length);
|
|
443
|
+
if (arg.startsWith('--include=')) {
|
|
444
|
+
const value = arg.slice('--include='.length);
|
|
445
|
+
options.include = value === 'all' ? 'all' : 'diff';
|
|
446
|
+
}
|
|
447
|
+
if (arg.startsWith('--max-uploads=')) {
|
|
448
|
+
const value = Number.parseInt(arg.slice('--max-uploads='.length), 10);
|
|
449
|
+
if (Number.isFinite(value) && value >= 0)
|
|
450
|
+
options.maxUploads = value;
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
return options;
|
|
454
|
+
}
|
|
455
|
+
/**
|
|
456
|
+
* Raise a warning where someone will see it.
|
|
457
|
+
*
|
|
458
|
+
* A `::warning::` command has to go to stdout for the runner to pick it up,
|
|
459
|
+
* which is only safe once the summary itself is going to a file — otherwise it
|
|
460
|
+
* would land in the middle of the Markdown. Outside Actions, or when the
|
|
461
|
+
* summary is on stdout, stderr is the only sensible place.
|
|
462
|
+
*/
|
|
463
|
+
function warn(message) {
|
|
464
|
+
const inActions = process.env.GITHUB_ACTIONS === 'true' && !!process.env.GITHUB_STEP_SUMMARY;
|
|
465
|
+
if (inActions) {
|
|
466
|
+
process.stdout.write(`::warning title=Playwright screenshots::${escapeAnnotation(message)}\n`);
|
|
467
|
+
}
|
|
468
|
+
else {
|
|
469
|
+
console.error(`Warning: ${message}`);
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
/** Workflow commands are line-based, so anything that ends a line is encoded. */
|
|
473
|
+
function escapeAnnotation(message) {
|
|
474
|
+
return message.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A');
|
|
475
|
+
}
|
|
476
|
+
/** Build the run's job summary URL, which is the best anchor a comment can link to. */
|
|
477
|
+
function summaryUrl() {
|
|
478
|
+
const { GITHUB_SERVER_URL, GITHUB_REPOSITORY, GITHUB_RUN_ID } = process.env;
|
|
479
|
+
if (!GITHUB_SERVER_URL || !GITHUB_REPOSITORY || !GITHUB_RUN_ID)
|
|
480
|
+
return undefined;
|
|
481
|
+
return `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}`;
|
|
482
|
+
}
|
|
483
|
+
function uploaderFromEnvironment(overrides = {}) {
|
|
484
|
+
return new attachments_1.AttachmentUploader({
|
|
485
|
+
token: process.env.SCREENSHOT_GITHUB_TOKEN,
|
|
486
|
+
repositoryId: process.env.GITHUB_REPOSITORY_ID,
|
|
487
|
+
log: message => console.error(message),
|
|
488
|
+
...overrides,
|
|
489
|
+
});
|
|
490
|
+
}
|
|
491
|
+
async function main(args = process.argv.slice(2), adapter = {}) {
|
|
492
|
+
if (args.includes('--help') || args.includes('-h')) {
|
|
493
|
+
process.stdout.write(`Usage: ${adapter.commandName ?? 'playwright-testing-failure-summary'} [options]\n\n` +
|
|
494
|
+
'Options:\n' +
|
|
495
|
+
' --report-path=PATH Playwright JSON report (default: test-results/results.json)\n' +
|
|
496
|
+
' --comment-path=PATH Write a pull request comment body\n' +
|
|
497
|
+
' --include=diff|all Select comparison images (default: diff)\n' +
|
|
498
|
+
' --path-prefix=FROM:TO Remap attachment paths; repeatable\n' +
|
|
499
|
+
' --max-uploads=COUNT Limit uploaded screenshots\n' +
|
|
500
|
+
' --title=TEXT Set the comment heading\n' +
|
|
501
|
+
' -h, --help Show this help\n');
|
|
502
|
+
return;
|
|
503
|
+
}
|
|
504
|
+
const options = parseArgs(args);
|
|
505
|
+
const resolvedPath = path.resolve(options.reportPath);
|
|
506
|
+
// A missing report is not an error: the suite may not have run at all.
|
|
507
|
+
if (!fs.existsSync(resolvedPath)) {
|
|
508
|
+
console.error(`Playwright JSON report not found at ${resolvedPath} — skipping failure summary.`);
|
|
509
|
+
return;
|
|
510
|
+
}
|
|
511
|
+
const report = parseFailures(resolvedPath, options.include);
|
|
512
|
+
// The report records where the run wrote its attachments, which is not
|
|
513
|
+
// necessarily anywhere this process can reach. Settle that before uploading,
|
|
514
|
+
// so an unreachable file is reported as one rather than as a silent skip.
|
|
515
|
+
const paths = resolveImagePaths(report, (0, report_paths_1.createPathResolver)({ reportPath: resolvedPath, prefixes: options.pathPrefixes }));
|
|
516
|
+
if (paths.remapped > 0) {
|
|
517
|
+
const mappings = paths.used.map(prefix => `${prefix.from} → ${prefix.to}`).join(', ');
|
|
518
|
+
console.error(`Remapped ${paths.remapped} attachment path(s): ${mappings || 'no mapping recorded'}`);
|
|
519
|
+
}
|
|
520
|
+
if (paths.unreadable.length > 0) {
|
|
521
|
+
warn(`${paths.unreadable.length} screenshot(s) could not be read, starting with ${paths.unreadable[0]}. ` +
|
|
522
|
+
'Playwright ran somewhere this command cannot reach — a container, most likely. Run it there too, ' +
|
|
523
|
+
'or pass --path-prefix=CONTAINER_PATH:LOCAL_PATH.');
|
|
524
|
+
}
|
|
525
|
+
const uploader = uploaderFromEnvironment(options.maxUploads === undefined ? {} : { maxUploads: options.maxUploads });
|
|
526
|
+
if (uploader.enabled) {
|
|
527
|
+
await uploadImages(report, uploader);
|
|
528
|
+
const stats = uploader.getStats();
|
|
529
|
+
console.error(`Uploaded ${stats.uploaded} screenshot(s), skipped ${stats.skipped}.`);
|
|
530
|
+
if (uploader.disabledReason) {
|
|
531
|
+
// Reaching an undocumented endpoint that no longer answers is a real
|
|
532
|
+
// fault, unlike an absent token on a fork build.
|
|
533
|
+
warn(`Screenshot uploads stopped early — ${uploader.disabledReason}.`);
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
else {
|
|
537
|
+
console.error(`Screenshot uploads are off — ${uploader.disabledReason}.`);
|
|
538
|
+
}
|
|
539
|
+
const uploadReason = uploader.disabledReason ?? undefined;
|
|
540
|
+
const summary = generateSummary(report, { uploadReason });
|
|
541
|
+
const summaryFile = process.env.GITHUB_STEP_SUMMARY;
|
|
542
|
+
if (summaryFile) {
|
|
543
|
+
fs.appendFileSync(summaryFile, summary);
|
|
544
|
+
console.error('Failure summary written to $GITHUB_STEP_SUMMARY');
|
|
545
|
+
}
|
|
546
|
+
else {
|
|
547
|
+
process.stdout.write(summary);
|
|
548
|
+
}
|
|
549
|
+
if (options.commentPath) {
|
|
550
|
+
const comment = generateComment(report, {
|
|
551
|
+
summaryUrl: summaryUrl(),
|
|
552
|
+
title: options.title,
|
|
553
|
+
uploadReason,
|
|
554
|
+
failureMarkerPrefix: adapter.failureMarkerPrefix,
|
|
555
|
+
flakeMarkerPrefix: adapter.flakeMarkerPrefix,
|
|
556
|
+
});
|
|
557
|
+
fs.writeFileSync(path.resolve(options.commentPath), comment);
|
|
558
|
+
console.error(`Comment body written to ${options.commentPath}`);
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
// Auto-invoke when run directly (node lib/github/failure-summary.js).
|
|
562
|
+
if (require.main === module) {
|
|
563
|
+
main().catch((error) => {
|
|
564
|
+
console.error(error);
|
|
565
|
+
process.exit(1);
|
|
566
|
+
});
|
|
567
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export { generateAnnotations as generateA11yAnnotations, generateSummary as generateA11ySummary, main as runA11ySummary, parseA11yResults, } from './a11y-summary';
|
|
2
|
+
export type { A11yReport, A11yTestResult } from './a11y-summary';
|
|
3
|
+
export * from './attachments';
|
|
4
|
+
export { FAILURE_MARKER_PREFIX, FLAKE_MARKER_PREFIX, defuseMaskTriggers, generateComment as generateFailureComment, generateSummary as generateFailureSummary, main as runFailureSummary, parseFailures, resolveImagePaths, uploaderFromEnvironment, uploadImages, } from './failure-summary';
|
|
5
|
+
export type { FailedTest, FailureImage, FailureReport, FailureCommentOptions, FailureSummaryMainOptions, ImageKind, IncludeMode, PathResolutionSummary, SummaryOptions, } from './failure-summary';
|
|
6
|
+
export * from './report-paths';
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
exports.uploadImages = exports.uploaderFromEnvironment = exports.resolveImagePaths = exports.parseFailures = exports.runFailureSummary = exports.generateFailureSummary = exports.generateFailureComment = exports.defuseMaskTriggers = exports.FLAKE_MARKER_PREFIX = exports.FAILURE_MARKER_PREFIX = exports.parseA11yResults = exports.runA11ySummary = exports.generateA11ySummary = exports.generateA11yAnnotations = void 0;
|
|
18
|
+
var a11y_summary_1 = require("./a11y-summary");
|
|
19
|
+
Object.defineProperty(exports, "generateA11yAnnotations", { enumerable: true, get: function () { return a11y_summary_1.generateAnnotations; } });
|
|
20
|
+
Object.defineProperty(exports, "generateA11ySummary", { enumerable: true, get: function () { return a11y_summary_1.generateSummary; } });
|
|
21
|
+
Object.defineProperty(exports, "runA11ySummary", { enumerable: true, get: function () { return a11y_summary_1.main; } });
|
|
22
|
+
Object.defineProperty(exports, "parseA11yResults", { enumerable: true, get: function () { return a11y_summary_1.parseA11yResults; } });
|
|
23
|
+
__exportStar(require("./attachments"), exports);
|
|
24
|
+
var failure_summary_1 = require("./failure-summary");
|
|
25
|
+
Object.defineProperty(exports, "FAILURE_MARKER_PREFIX", { enumerable: true, get: function () { return failure_summary_1.FAILURE_MARKER_PREFIX; } });
|
|
26
|
+
Object.defineProperty(exports, "FLAKE_MARKER_PREFIX", { enumerable: true, get: function () { return failure_summary_1.FLAKE_MARKER_PREFIX; } });
|
|
27
|
+
Object.defineProperty(exports, "defuseMaskTriggers", { enumerable: true, get: function () { return failure_summary_1.defuseMaskTriggers; } });
|
|
28
|
+
Object.defineProperty(exports, "generateFailureComment", { enumerable: true, get: function () { return failure_summary_1.generateComment; } });
|
|
29
|
+
Object.defineProperty(exports, "generateFailureSummary", { enumerable: true, get: function () { return failure_summary_1.generateSummary; } });
|
|
30
|
+
Object.defineProperty(exports, "runFailureSummary", { enumerable: true, get: function () { return failure_summary_1.main; } });
|
|
31
|
+
Object.defineProperty(exports, "parseFailures", { enumerable: true, get: function () { return failure_summary_1.parseFailures; } });
|
|
32
|
+
Object.defineProperty(exports, "resolveImagePaths", { enumerable: true, get: function () { return failure_summary_1.resolveImagePaths; } });
|
|
33
|
+
Object.defineProperty(exports, "uploaderFromEnvironment", { enumerable: true, get: function () { return failure_summary_1.uploaderFromEnvironment; } });
|
|
34
|
+
Object.defineProperty(exports, "uploadImages", { enumerable: true, get: function () { return failure_summary_1.uploadImages; } });
|
|
35
|
+
__exportStar(require("./report-paths"), exports);
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
export interface PathPrefix {
|
|
2
|
+
/** Absolute path as recorded in the report — typically a container path. */
|
|
3
|
+
from: string;
|
|
4
|
+
/** The same directory as seen from here. */
|
|
5
|
+
to: string;
|
|
6
|
+
}
|
|
7
|
+
export interface PathResolution {
|
|
8
|
+
/** The path to read. Unchanged when the recorded one was already fine. */
|
|
9
|
+
path: string;
|
|
10
|
+
/** Whether a readable file was found. */
|
|
11
|
+
found: boolean;
|
|
12
|
+
/** The mapping that made it readable, when one was needed. */
|
|
13
|
+
prefix?: PathPrefix;
|
|
14
|
+
}
|
|
15
|
+
export interface PathResolverOptions {
|
|
16
|
+
/** Where the JSON report was read from, on this side of the boundary. */
|
|
17
|
+
reportPath: string;
|
|
18
|
+
/** Explicit mappings. Tried before anything is worked out. */
|
|
19
|
+
prefixes?: PathPrefix[];
|
|
20
|
+
/** Existence check, injectable for tests. */
|
|
21
|
+
exists?: (filePath: string) => boolean;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Parse a `FROM:TO` pair, as `--path-prefix` takes it.
|
|
25
|
+
*
|
|
26
|
+
* Split on the first colon: `TO` is a local absolute path and may itself
|
|
27
|
+
* contain one on Windows, whereas `FROM` comes from a container and will not.
|
|
28
|
+
*/
|
|
29
|
+
export declare function parsePathPrefix(value: string): PathPrefix | null;
|
|
30
|
+
/**
|
|
31
|
+
* Build the function that turns a recorded path into a readable one.
|
|
32
|
+
*
|
|
33
|
+
* A path that already resolves is left alone, so this costs one `stat` per
|
|
34
|
+
* attachment when the report and the files are on the same side. A mapping
|
|
35
|
+
* worked out for one attachment is kept and tried first for the rest, so the
|
|
36
|
+
* search runs once per run rather than once per image.
|
|
37
|
+
*/
|
|
38
|
+
export declare function createPathResolver(options: PathResolverOptions): (filePath: string) => PathResolution;
|