@lullabot/playwright-testing 0.1.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/github-a11y-summary +2 -2
- package/bin/github-failure-summary +4 -4
- package/lib/accessibility-baseline-file.d.ts +6 -6
- package/lib/accessibility-baseline-file.js +29 -23
- package/lib/accessibility-baseline.js +5 -3
- package/lib/accessible-screenshot.d.ts +4 -4
- package/lib/accessible-screenshot.js +88 -76
- package/lib/focus.js +4 -1
- package/lib/frames.js +4 -4
- package/lib/github/a11y-summary.js +74 -79
- package/lib/github/attachments.d.ts +1 -1
- package/lib/github/attachments.js +36 -34
- package/lib/github/failure-summary.d.ts +4 -4
- package/lib/github/failure-summary.js +139 -107
- package/lib/github/index.d.ts +6 -6
- package/lib/github/report-paths.js +17 -9
- package/lib/hover.js +17 -17
- package/lib/images.js +15 -12
- package/lib/index.d.ts +13 -13
- package/lib/interaction-states.d.ts +2 -2
- package/lib/interaction-states.js +7 -7
- package/lib/mock/index.d.ts +1 -1
- package/lib/mock/youtube.d.ts +1 -1
- package/lib/mock/youtube.js +1 -1
- package/lib/pseudo-state.d.ts +2 -2
- package/lib/pseudo-state.js +15 -7
- package/lib/videos.js +21 -10
- package/lib/visualdiff.d.ts +4 -4
- package/lib/visualdiff.js +35 -27
- package/package.json +3 -3
- package/src/accessibility-baseline-file.test.ts +190 -154
- package/src/accessibility-baseline-file.ts +107 -83
- package/src/accessibility-baseline.test.ts +633 -433
- package/src/accessibility-baseline.ts +41 -21
- package/src/accessible-screenshot.test.ts +551 -423
- package/src/accessible-screenshot.ts +334 -245
- package/src/focus.test.ts +23 -23
- package/src/focus.ts +7 -2
- package/src/fonts.test.ts +13 -13
- package/src/fonts.ts +1 -1
- package/src/frames.test.ts +10 -8
- package/src/frames.ts +5 -5
- package/src/github/a11y-summary.test.ts +436 -332
- package/src/github/a11y-summary.ts +189 -179
- package/src/github/attachments.test.ts +228 -167
- package/src/github/attachments.ts +144 -121
- package/src/github/failure-summary.test.ts +659 -445
- package/src/github/failure-summary.ts +376 -274
- package/src/github/index.test.ts +14 -14
- package/src/github/index.ts +6 -6
- package/src/github/report-paths.test.ts +211 -156
- package/src/github/report-paths.ts +75 -61
- package/src/hover.test.ts +50 -46
- package/src/hover.ts +18 -18
- package/src/images.test.ts +251 -206
- package/src/images.ts +52 -43
- package/src/index.ts +13 -13
- package/src/interaction-states.test.ts +72 -42
- package/src/interaction-states.ts +43 -29
- package/src/mock/index.ts +1 -1
- package/src/mock/youtube.test.ts +26 -26
- package/src/mock/youtube.ts +4 -6
- package/src/pseudo-state.test.ts +58 -51
- package/src/pseudo-state.ts +32 -27
- package/src/videos.test.ts +559 -399
- package/src/videos.ts +68 -44
- package/src/visualdiff.test.ts +416 -315
- package/src/visualdiff.ts +140 -99
|
@@ -62,21 +62,21 @@ const report_paths_1 = require("./report-paths");
|
|
|
62
62
|
*/
|
|
63
63
|
/** Attachment names Playwright gives the three snapshot comparison images. */
|
|
64
64
|
const SNAPSHOT_SUFFIXES = [
|
|
65
|
-
{ suffix:
|
|
66
|
-
{ suffix:
|
|
67
|
-
{ suffix:
|
|
68
|
-
{ suffix:
|
|
65
|
+
{ suffix: "-diff", kind: "diff" },
|
|
66
|
+
{ suffix: "-actual", kind: "actual" },
|
|
67
|
+
{ suffix: "-expected", kind: "expected" },
|
|
68
|
+
{ suffix: "-previous", kind: "previous" },
|
|
69
69
|
];
|
|
70
70
|
/** Attachment name used by checkAccessibility() for its highlighted screenshot. */
|
|
71
|
-
const A11Y_SCREENSHOT =
|
|
71
|
+
const A11Y_SCREENSHOT = "a11y-violation-screenshot";
|
|
72
72
|
/**
|
|
73
73
|
* Prefix of the HTML comment carrying the failure count in a generated comment
|
|
74
74
|
* body. Invisible when rendered, and greppable by a workflow that has to decide
|
|
75
75
|
* whether the run is worth commenting on.
|
|
76
76
|
*/
|
|
77
|
-
exports.FAILURE_MARKER_PREFIX =
|
|
77
|
+
exports.FAILURE_MARKER_PREFIX = "<!-- playwright-testing-failures: ";
|
|
78
78
|
/** Machine-readable flaky-test count used by the comment assembly action. */
|
|
79
|
-
exports.FLAKE_MARKER_PREFIX =
|
|
79
|
+
exports.FLAKE_MARKER_PREFIX = "<!-- playwright-testing-flakes: ";
|
|
80
80
|
/**
|
|
81
81
|
* Neutralise text that the Actions runner would redact.
|
|
82
82
|
*
|
|
@@ -87,23 +87,24 @@ exports.FLAKE_MARKER_PREFIX = '<!-- playwright-testing-flakes: ';
|
|
|
87
87
|
* inside the word reads identically and no longer matches.
|
|
88
88
|
*/
|
|
89
89
|
function defuseMaskTriggers(text) {
|
|
90
|
-
return text.replace(/Bearer(?=\s)/g,
|
|
90
|
+
return text.replace(/Bearer(?=\s)/g, "Bear\u200Ber");
|
|
91
91
|
}
|
|
92
92
|
/** Strip ANSI colour codes Playwright puts in error messages. */
|
|
93
93
|
function stripAnsi(text) {
|
|
94
|
-
|
|
94
|
+
// eslint-disable-next-line no-control-regex -- ANSI colour codes contain ESC.
|
|
95
|
+
return text.replace(/\u001b\[[0-9;]*m/g, "");
|
|
95
96
|
}
|
|
96
97
|
function classifyAttachment(name, contentType) {
|
|
97
|
-
if (!contentType.startsWith(
|
|
98
|
+
if (!contentType.startsWith("image/"))
|
|
98
99
|
return null;
|
|
99
100
|
if (name === A11Y_SCREENSHOT)
|
|
100
|
-
return
|
|
101
|
-
const withoutExtension = name.replace(/\.[^.]+$/,
|
|
101
|
+
return "a11y";
|
|
102
|
+
const withoutExtension = name.replace(/\.[^.]+$/, "");
|
|
102
103
|
for (const { suffix, kind } of SNAPSHOT_SUFFIXES) {
|
|
103
104
|
if (withoutExtension.endsWith(suffix))
|
|
104
105
|
return kind;
|
|
105
106
|
}
|
|
106
|
-
return
|
|
107
|
+
return "screenshot";
|
|
107
108
|
}
|
|
108
109
|
/**
|
|
109
110
|
* Collect the images worth showing for one test result.
|
|
@@ -115,14 +116,15 @@ function classifyAttachment(name, contentType) {
|
|
|
115
116
|
function collectImages(attachments, include) {
|
|
116
117
|
const images = [];
|
|
117
118
|
for (const attachment of attachments) {
|
|
118
|
-
const name = String(attachment?.name ??
|
|
119
|
-
const contentType = String(attachment?.contentType ??
|
|
119
|
+
const name = String(attachment?.name ?? "");
|
|
120
|
+
const contentType = String(attachment?.contentType ?? "");
|
|
120
121
|
const kind = classifyAttachment(name, contentType);
|
|
121
122
|
if (!kind)
|
|
122
123
|
continue;
|
|
123
124
|
// Uploading three near-identical images per failure is rarely worth it;
|
|
124
125
|
// the diff is the one that shows what changed.
|
|
125
|
-
if (include ===
|
|
126
|
+
if (include === "diff" &&
|
|
127
|
+
(kind === "expected" || kind === "actual" || kind === "previous")) {
|
|
126
128
|
continue;
|
|
127
129
|
}
|
|
128
130
|
images.push({
|
|
@@ -137,9 +139,9 @@ function collectImages(attachments, include) {
|
|
|
137
139
|
}
|
|
138
140
|
function firstError(result) {
|
|
139
141
|
const message = result?.error?.message ?? result?.errors?.[0]?.message;
|
|
140
|
-
if (typeof message !==
|
|
142
|
+
if (typeof message !== "string" || !message.trim())
|
|
141
143
|
return undefined;
|
|
142
|
-
const cleaned = stripAnsi(message).trim().split(
|
|
144
|
+
const cleaned = stripAnsi(message).trim().split("\n")[0];
|
|
143
145
|
return cleaned.length > 300 ? `${cleaned.slice(0, 300)}…` : cleaned;
|
|
144
146
|
}
|
|
145
147
|
function walkSuites(suites, parentFile, include, out) {
|
|
@@ -154,22 +156,29 @@ function walkSuites(suites, parentFile, include, out) {
|
|
|
154
156
|
// Playwright reports each retry as another result. A passing final
|
|
155
157
|
// attempt therefore hides the failed attempt unless we deliberately
|
|
156
158
|
// retain it and give the test its Playwright outcome: flaky.
|
|
157
|
-
const failedAttempts = results
|
|
159
|
+
const failedAttempts = results
|
|
160
|
+
.slice(0, -1)
|
|
161
|
+
.filter((attempt) => attempt?.status !== "passed" && attempt?.status !== "skipped");
|
|
158
162
|
// JSON reports expose Playwright's computed outcome on the test. Keep
|
|
159
163
|
// the attempt-based fallback for older or hand-written reports that do
|
|
160
164
|
// not carry it.
|
|
161
|
-
const flaky = test.status ===
|
|
162
|
-
|
|
165
|
+
const flaky = test.status === "flaky" ||
|
|
166
|
+
(test.status === undefined &&
|
|
167
|
+
result.status === "passed" &&
|
|
168
|
+
failedAttempts.length > 0);
|
|
169
|
+
const reportedResult = flaky
|
|
170
|
+
? failedAttempts[failedAttempts.length - 1]
|
|
171
|
+
: result;
|
|
163
172
|
const images = collectImages(reportedResult.attachments ?? [], include);
|
|
164
|
-
const failed = result.status !==
|
|
165
|
-
const hasA11yImage = images.some(image => image.kind ===
|
|
173
|
+
const failed = result.status !== "passed" && result.status !== "skipped";
|
|
174
|
+
const hasA11yImage = images.some((image) => image.kind === "a11y");
|
|
166
175
|
if (!failed && !flaky && !hasA11yImage)
|
|
167
176
|
continue;
|
|
168
177
|
out.push({
|
|
169
178
|
title: spec.title,
|
|
170
179
|
file,
|
|
171
180
|
line: spec.line ?? 1,
|
|
172
|
-
status: flaky ?
|
|
181
|
+
status: flaky ? "flaky" : String(result.status ?? "unknown"),
|
|
173
182
|
error: firstError(reportedResult),
|
|
174
183
|
images,
|
|
175
184
|
});
|
|
@@ -181,12 +190,14 @@ function walkSuites(suites, parentFile, include, out) {
|
|
|
181
190
|
/**
|
|
182
191
|
* Parse a Playwright JSON report into the failures worth reporting.
|
|
183
192
|
*/
|
|
184
|
-
function parseFailures(reportPath, include =
|
|
185
|
-
const report = JSON.parse(fs.readFileSync(reportPath,
|
|
193
|
+
function parseFailures(reportPath, include = "diff") {
|
|
194
|
+
const report = JSON.parse(fs.readFileSync(reportPath, "utf8"));
|
|
186
195
|
const tests = [];
|
|
187
|
-
walkSuites(report.suites ?? [],
|
|
188
|
-
const totalFailed = tests.filter(test =>
|
|
189
|
-
|
|
196
|
+
walkSuites(report.suites ?? [], "", include, tests);
|
|
197
|
+
const totalFailed = tests.filter((test) => test.status !== "passed" &&
|
|
198
|
+
test.status !== "skipped" &&
|
|
199
|
+
test.status !== "flaky").length;
|
|
200
|
+
const totalFlaky = tests.filter((test) => test.status === "flaky").length;
|
|
190
201
|
const totalImages = tests.reduce((sum, test) => sum + test.images.length, 0);
|
|
191
202
|
return { tests, totalFailed, totalFlaky, totalImages };
|
|
192
203
|
}
|
|
@@ -199,7 +210,11 @@ function parseFailures(reportPath, include = 'diff') {
|
|
|
199
210
|
* because it is the same misconfiguration either way.
|
|
200
211
|
*/
|
|
201
212
|
function resolveImagePaths(report, resolve) {
|
|
202
|
-
const summary = {
|
|
213
|
+
const summary = {
|
|
214
|
+
remapped: 0,
|
|
215
|
+
used: [],
|
|
216
|
+
unreadable: [],
|
|
217
|
+
};
|
|
203
218
|
for (const test of report.tests) {
|
|
204
219
|
for (const image of test.images) {
|
|
205
220
|
if (!image.filePath)
|
|
@@ -215,7 +230,8 @@ function resolveImagePaths(report, resolve) {
|
|
|
215
230
|
summary.remapped++;
|
|
216
231
|
image.filePath = resolution.path;
|
|
217
232
|
const prefix = resolution.prefix;
|
|
218
|
-
if (prefix &&
|
|
233
|
+
if (prefix &&
|
|
234
|
+
!summary.used.some((used) => used.from === prefix.from && used.to === prefix.to)) {
|
|
219
235
|
summary.used.push(prefix);
|
|
220
236
|
}
|
|
221
237
|
}
|
|
@@ -235,7 +251,7 @@ async function uploadImages(report, uploader) {
|
|
|
235
251
|
url = await uploader.upload(image.filePath, path.basename(image.filePath));
|
|
236
252
|
}
|
|
237
253
|
else if (image.body) {
|
|
238
|
-
url = await uploader.uploadBuffer(Buffer.from(image.body,
|
|
254
|
+
url = await uploader.uploadBuffer(Buffer.from(image.body, "base64"), `${image.name}${extensionFor(image.contentType)}`, image.contentType);
|
|
239
255
|
}
|
|
240
256
|
if (url) {
|
|
241
257
|
image.url = url;
|
|
@@ -246,11 +262,11 @@ async function uploadImages(report, uploader) {
|
|
|
246
262
|
// so 'unreadable' here means the read itself failed; the size reasons
|
|
247
263
|
// are the uploader declining this one image rather than stopping.
|
|
248
264
|
switch (uploader.lastSkipReason) {
|
|
249
|
-
case
|
|
265
|
+
case "unreadable":
|
|
250
266
|
image.unreadable = true;
|
|
251
267
|
break;
|
|
252
|
-
case
|
|
253
|
-
case
|
|
268
|
+
case "too-large":
|
|
269
|
+
case "budget":
|
|
254
270
|
image.oversize = true;
|
|
255
271
|
break;
|
|
256
272
|
}
|
|
@@ -259,33 +275,33 @@ async function uploadImages(report, uploader) {
|
|
|
259
275
|
}
|
|
260
276
|
/** Inlined attachments carry no file name, so derive one from the content type. */
|
|
261
277
|
function extensionFor(contentType) {
|
|
262
|
-
if (contentType ===
|
|
263
|
-
return
|
|
264
|
-
if (contentType ===
|
|
265
|
-
return
|
|
266
|
-
if (contentType ===
|
|
267
|
-
return
|
|
268
|
-
if (contentType ===
|
|
269
|
-
return
|
|
270
|
-
return
|
|
278
|
+
if (contentType === "image/png")
|
|
279
|
+
return ".png";
|
|
280
|
+
if (contentType === "image/jpeg")
|
|
281
|
+
return ".jpg";
|
|
282
|
+
if (contentType === "image/gif")
|
|
283
|
+
return ".gif";
|
|
284
|
+
if (contentType === "image/webp")
|
|
285
|
+
return ".webp";
|
|
286
|
+
return "";
|
|
271
287
|
}
|
|
272
288
|
function headline(report) {
|
|
273
289
|
if (report.tests.length === 0)
|
|
274
|
-
return
|
|
290
|
+
return ":white_check_mark: No failing tests or flaky tests with screenshots.";
|
|
275
291
|
// Nothing failed, but an accessibility check still captured something worth
|
|
276
292
|
// seeing — a baselined violation passes and is still screenshotted.
|
|
277
293
|
const parts = report.totalFailed === 0
|
|
278
|
-
? [
|
|
294
|
+
? [":white_check_mark: No failing tests"]
|
|
279
295
|
: [`**${report.totalFailed}** failing test(s)`];
|
|
280
296
|
if (report.totalFlaky > 0)
|
|
281
297
|
parts.push(`:warning: **${report.totalFlaky}** flaky test(s)`);
|
|
282
298
|
if (report.totalImages > 0)
|
|
283
299
|
parts.push(`**${report.totalImages}** screenshot(s)`);
|
|
284
|
-
return parts.join(
|
|
300
|
+
return parts.join(" · ");
|
|
285
301
|
}
|
|
286
302
|
/** Every image across every test, which is what the diagnostics count. */
|
|
287
303
|
function allImages(report) {
|
|
288
|
-
return report.tests.flatMap(test => test.images);
|
|
304
|
+
return report.tests.flatMap((test) => test.images);
|
|
289
305
|
}
|
|
290
306
|
/**
|
|
291
307
|
* Say why a set of images was not uploaded, or nothing when they were.
|
|
@@ -299,15 +315,15 @@ function missingImageReason(images, uploadReason) {
|
|
|
299
315
|
if (uploadReason)
|
|
300
316
|
return uploadReason;
|
|
301
317
|
const reasons = [];
|
|
302
|
-
const unreadable = images.filter(image => image.unreadable).length;
|
|
318
|
+
const unreadable = images.filter((image) => image.unreadable).length;
|
|
303
319
|
if (unreadable > 0) {
|
|
304
320
|
reasons.push(`${unreadable} could not be read from the path recorded in the report`);
|
|
305
321
|
}
|
|
306
|
-
const oversize = images.filter(image => image.oversize).length;
|
|
322
|
+
const oversize = images.filter((image) => image.oversize).length;
|
|
307
323
|
if (oversize > 0) {
|
|
308
324
|
reasons.push(`${oversize} too large to upload`);
|
|
309
325
|
}
|
|
310
|
-
return reasons.length > 0 ? reasons.join(
|
|
326
|
+
return reasons.length > 0 ? reasons.join("; ") : undefined;
|
|
311
327
|
}
|
|
312
328
|
/**
|
|
313
329
|
* Render the job summary: a heading, a headline, and one collapsed block per
|
|
@@ -320,20 +336,22 @@ function generateSummary(report, options = {}) {
|
|
|
320
336
|
// A baselined accessibility violation is screenshotted without failing
|
|
321
337
|
// anything, so a green run can still have something to show here.
|
|
322
338
|
const heading = report.totalFailed > 0
|
|
323
|
-
?
|
|
324
|
-
: report.totalFlaky > 0
|
|
339
|
+
? "## Test Failures"
|
|
340
|
+
: report.totalFlaky > 0
|
|
341
|
+
? "## Test Flakes"
|
|
342
|
+
: "## Test Screenshots";
|
|
325
343
|
const lines = [`${heading}\n`, `${headline(report)}\n`];
|
|
326
344
|
if (report.tests.length === 0)
|
|
327
|
-
return lines.join(
|
|
345
|
+
return lines.join("\n");
|
|
328
346
|
// Name the path boundary where it will be read, rather than leaving the
|
|
329
347
|
// reader to conclude they forgot the token.
|
|
330
|
-
const unreadable = allImages(report).filter(image => image.unreadable);
|
|
348
|
+
const unreadable = allImages(report).filter((image) => image.unreadable);
|
|
331
349
|
if (unreadable.length > 0) {
|
|
332
|
-
const example = defuseMaskTriggers(unreadable[0].filePath ??
|
|
350
|
+
const example = defuseMaskTriggers(unreadable[0].filePath ?? "");
|
|
333
351
|
lines.push(`> :warning: **${unreadable.length} screenshot(s) could not be read.** The report records them ` +
|
|
334
352
|
`under \`${example}\`, which does not exist where this step ran. Playwright ran somewhere else — ` +
|
|
335
|
-
|
|
336
|
-
|
|
353
|
+
"a container, most likely — so run this command there too, or pass " +
|
|
354
|
+
"`--path-prefix=CONTAINER_PATH:LOCAL_PATH` to map one onto the other.\n");
|
|
337
355
|
}
|
|
338
356
|
for (const test of report.tests) {
|
|
339
357
|
const title = defuseMaskTriggers(test.title);
|
|
@@ -348,47 +366,50 @@ function generateSummary(report, options = {}) {
|
|
|
348
366
|
// rendered once, when the job finishes, and an attachment uploaded seconds
|
|
349
367
|
// earlier is not resolvable yet — so it renders as a dead link and the
|
|
350
368
|
// cached result never improves, however often the page is reloaded.
|
|
351
|
-
const embedded = test.images.filter(image => image.url).length;
|
|
369
|
+
const embedded = test.images.filter((image) => image.url).length;
|
|
352
370
|
if (embedded > 0) {
|
|
353
371
|
lines.push(`${embedded} screenshot(s) uploaded — see the pull request comment.\n`);
|
|
354
372
|
}
|
|
355
373
|
else {
|
|
356
|
-
const hint = options.artifactHint ??
|
|
374
|
+
const hint = options.artifactHint ?? "the Playwright report artifact";
|
|
357
375
|
const reason = missingImageReason(test.images, options.uploadReason);
|
|
358
|
-
const because = reason ? ` (${reason})` :
|
|
376
|
+
const because = reason ? ` (${reason})` : "";
|
|
359
377
|
lines.push(`${test.images.length} screenshot(s) captured, not uploaded${because} — download ${hint}.\n`);
|
|
360
378
|
}
|
|
361
379
|
}
|
|
362
|
-
return lines.join(
|
|
380
|
+
return lines.join("\n");
|
|
363
381
|
}
|
|
364
382
|
/** A collapsed block of images for one test, used in the comment. */
|
|
365
383
|
function renderImageBlock(test) {
|
|
366
|
-
const embedded = test.images.filter(image => image.url);
|
|
384
|
+
const embedded = test.images.filter((image) => image.url);
|
|
367
385
|
if (embedded.length === 0)
|
|
368
386
|
return [];
|
|
369
387
|
const title = defuseMaskTriggers(test.title);
|
|
370
|
-
const lines = [
|
|
388
|
+
const lines = [
|
|
389
|
+
"<details>",
|
|
390
|
+
`<summary>Screenshots (${embedded.length})</summary>\n`,
|
|
391
|
+
];
|
|
371
392
|
for (const image of embedded) {
|
|
372
393
|
lines.push(`**${image.kind}**\n`);
|
|
373
|
-
const src = escapeAttribute(image.url ??
|
|
394
|
+
const src = escapeAttribute(image.url ?? "");
|
|
374
395
|
lines.push(`<img src="${src}" alt="${image.kind} for ${escapeAttribute(title)}" width="640">\n`);
|
|
375
396
|
}
|
|
376
|
-
lines.push(
|
|
397
|
+
lines.push("</details>\n");
|
|
377
398
|
return lines;
|
|
378
399
|
}
|
|
379
400
|
function generateComment(report, options = {}) {
|
|
380
|
-
const heading = options.title ??
|
|
401
|
+
const heading = options.title ?? "Playwright results";
|
|
381
402
|
const lines = [`### ${heading}\n`, `${headline(report)}\n`];
|
|
382
403
|
if (report.tests.length > 0) {
|
|
383
404
|
for (const test of report.tests.slice(0, 10)) {
|
|
384
405
|
const title = defuseMaskTriggers(test.title);
|
|
385
|
-
const images = test.images.length > 0 ? ` — ${test.images.length} screenshot(s)` :
|
|
406
|
+
const images = test.images.length > 0 ? ` — ${test.images.length} screenshot(s)` : "";
|
|
386
407
|
lines.push(`- \`${test.status}\` ${title}${images}`);
|
|
387
408
|
}
|
|
388
409
|
if (report.tests.length > 10) {
|
|
389
410
|
lines.push(`- …and ${report.tests.length - 10} more`);
|
|
390
411
|
}
|
|
391
|
-
lines.push(
|
|
412
|
+
lines.push("");
|
|
392
413
|
for (const test of report.tests) {
|
|
393
414
|
const block = renderImageBlock(test);
|
|
394
415
|
if (block.length === 0)
|
|
@@ -398,7 +419,7 @@ function generateComment(report, options = {}) {
|
|
|
398
419
|
}
|
|
399
420
|
// An empty comment where images were expected reads as a broken feature.
|
|
400
421
|
// Whatever the reason, it belongs where the images would have been.
|
|
401
|
-
const missing = allImages(report).filter(image => !image.url);
|
|
422
|
+
const missing = allImages(report).filter((image) => !image.url);
|
|
402
423
|
const reason = missingImageReason(missing, options.uploadReason);
|
|
403
424
|
if (missing.length > 0 && reason) {
|
|
404
425
|
lines.push(`_${missing.length} screenshot(s) not shown — ${reason}._\n`);
|
|
@@ -412,40 +433,43 @@ function generateComment(report, options = {}) {
|
|
|
412
433
|
// sentence contains the words "failing test" too.
|
|
413
434
|
lines.push(`${options.failureMarkerPrefix ?? exports.FAILURE_MARKER_PREFIX}${report.totalFailed} -->\n`);
|
|
414
435
|
lines.push(`${options.flakeMarkerPrefix ?? exports.FLAKE_MARKER_PREFIX}${report.totalFlaky} -->\n`);
|
|
415
|
-
return lines.join(
|
|
436
|
+
return lines.join("\n");
|
|
416
437
|
}
|
|
417
438
|
function escapeAttribute(value) {
|
|
418
|
-
return value
|
|
439
|
+
return value
|
|
440
|
+
.replace(/&/g, "&")
|
|
441
|
+
.replace(/"/g, """)
|
|
442
|
+
.replace(/</g, "<");
|
|
419
443
|
}
|
|
420
444
|
function parseArgs(args) {
|
|
421
445
|
const options = {
|
|
422
|
-
reportPath:
|
|
423
|
-
include:
|
|
446
|
+
reportPath: "test-results/results.json",
|
|
447
|
+
include: "diff",
|
|
424
448
|
pathPrefixes: [],
|
|
425
449
|
};
|
|
426
450
|
for (const arg of args) {
|
|
427
451
|
// Repeatable: a project can have more than one mount to translate.
|
|
428
|
-
if (arg.startsWith(
|
|
429
|
-
const prefix = (0, report_paths_1.parsePathPrefix)(arg.slice(
|
|
452
|
+
if (arg.startsWith("--path-prefix=")) {
|
|
453
|
+
const prefix = (0, report_paths_1.parsePathPrefix)(arg.slice("--path-prefix=".length));
|
|
430
454
|
if (prefix) {
|
|
431
455
|
options.pathPrefixes.push(prefix);
|
|
432
456
|
}
|
|
433
457
|
else {
|
|
434
|
-
console.error(`Ignoring --path-prefix: expected FROM:TO, got ${arg.slice(
|
|
458
|
+
console.error(`Ignoring --path-prefix: expected FROM:TO, got ${arg.slice("--path-prefix=".length)}`);
|
|
435
459
|
}
|
|
436
460
|
}
|
|
437
|
-
if (arg.startsWith(
|
|
438
|
-
options.reportPath = arg.slice(
|
|
439
|
-
if (arg.startsWith(
|
|
440
|
-
options.commentPath = arg.slice(
|
|
441
|
-
if (arg.startsWith(
|
|
442
|
-
options.title = arg.slice(
|
|
443
|
-
if (arg.startsWith(
|
|
444
|
-
const value = arg.slice(
|
|
445
|
-
options.include = value ===
|
|
461
|
+
if (arg.startsWith("--report-path="))
|
|
462
|
+
options.reportPath = arg.slice("--report-path=".length);
|
|
463
|
+
if (arg.startsWith("--comment-path="))
|
|
464
|
+
options.commentPath = arg.slice("--comment-path=".length);
|
|
465
|
+
if (arg.startsWith("--title="))
|
|
466
|
+
options.title = arg.slice("--title=".length);
|
|
467
|
+
if (arg.startsWith("--include=")) {
|
|
468
|
+
const value = arg.slice("--include=".length);
|
|
469
|
+
options.include = value === "all" ? "all" : "diff";
|
|
446
470
|
}
|
|
447
|
-
if (arg.startsWith(
|
|
448
|
-
const value = Number.parseInt(arg.slice(
|
|
471
|
+
if (arg.startsWith("--max-uploads=")) {
|
|
472
|
+
const value = Number.parseInt(arg.slice("--max-uploads=".length), 10);
|
|
449
473
|
if (Number.isFinite(value) && value >= 0)
|
|
450
474
|
options.maxUploads = value;
|
|
451
475
|
}
|
|
@@ -461,7 +485,7 @@ function parseArgs(args) {
|
|
|
461
485
|
* summary is on stdout, stderr is the only sensible place.
|
|
462
486
|
*/
|
|
463
487
|
function warn(message) {
|
|
464
|
-
const inActions = process.env.GITHUB_ACTIONS ===
|
|
488
|
+
const inActions = process.env.GITHUB_ACTIONS === "true" && !!process.env.GITHUB_STEP_SUMMARY;
|
|
465
489
|
if (inActions) {
|
|
466
490
|
process.stdout.write(`::warning title=Playwright screenshots::${escapeAnnotation(message)}\n`);
|
|
467
491
|
}
|
|
@@ -471,7 +495,10 @@ function warn(message) {
|
|
|
471
495
|
}
|
|
472
496
|
/** Workflow commands are line-based, so anything that ends a line is encoded. */
|
|
473
497
|
function escapeAnnotation(message) {
|
|
474
|
-
return message
|
|
498
|
+
return message
|
|
499
|
+
.replace(/%/g, "%25")
|
|
500
|
+
.replace(/\r/g, "%0D")
|
|
501
|
+
.replace(/\n/g, "%0A");
|
|
475
502
|
}
|
|
476
503
|
/** Build the run's job summary URL, which is the best anchor a comment can link to. */
|
|
477
504
|
function summaryUrl() {
|
|
@@ -484,21 +511,21 @@ function uploaderFromEnvironment(overrides = {}) {
|
|
|
484
511
|
return new attachments_1.AttachmentUploader({
|
|
485
512
|
token: process.env.SCREENSHOT_GITHUB_TOKEN,
|
|
486
513
|
repositoryId: process.env.GITHUB_REPOSITORY_ID,
|
|
487
|
-
log: message => console.error(message),
|
|
514
|
+
log: (message) => console.error(message),
|
|
488
515
|
...overrides,
|
|
489
516
|
});
|
|
490
517
|
}
|
|
491
518
|
async function main(args = process.argv.slice(2), adapter = {}) {
|
|
492
|
-
if (args.includes(
|
|
493
|
-
process.stdout.write(`Usage: ${adapter.commandName ??
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
519
|
+
if (args.includes("--help") || args.includes("-h")) {
|
|
520
|
+
process.stdout.write(`Usage: ${adapter.commandName ?? "playwright-testing-failure-summary"} [options]\n\n` +
|
|
521
|
+
"Options:\n" +
|
|
522
|
+
" --report-path=PATH Playwright JSON report (default: test-results/results.json)\n" +
|
|
523
|
+
" --comment-path=PATH Write a pull request comment body\n" +
|
|
524
|
+
" --include=diff|all Select comparison images (default: diff)\n" +
|
|
525
|
+
" --path-prefix=FROM:TO Remap attachment paths; repeatable\n" +
|
|
526
|
+
" --max-uploads=COUNT Limit uploaded screenshots\n" +
|
|
527
|
+
" --title=TEXT Set the comment heading\n" +
|
|
528
|
+
" -h, --help Show this help\n");
|
|
502
529
|
return;
|
|
503
530
|
}
|
|
504
531
|
const options = parseArgs(args);
|
|
@@ -512,15 +539,20 @@ async function main(args = process.argv.slice(2), adapter = {}) {
|
|
|
512
539
|
// The report records where the run wrote its attachments, which is not
|
|
513
540
|
// necessarily anywhere this process can reach. Settle that before uploading,
|
|
514
541
|
// so an unreachable file is reported as one rather than as a silent skip.
|
|
515
|
-
const paths = resolveImagePaths(report, (0, report_paths_1.createPathResolver)({
|
|
542
|
+
const paths = resolveImagePaths(report, (0, report_paths_1.createPathResolver)({
|
|
543
|
+
reportPath: resolvedPath,
|
|
544
|
+
prefixes: options.pathPrefixes,
|
|
545
|
+
}));
|
|
516
546
|
if (paths.remapped > 0) {
|
|
517
|
-
const mappings = paths.used
|
|
518
|
-
|
|
547
|
+
const mappings = paths.used
|
|
548
|
+
.map((prefix) => `${prefix.from} → ${prefix.to}`)
|
|
549
|
+
.join(", ");
|
|
550
|
+
console.error(`Remapped ${paths.remapped} attachment path(s): ${mappings || "no mapping recorded"}`);
|
|
519
551
|
}
|
|
520
552
|
if (paths.unreadable.length > 0) {
|
|
521
553
|
warn(`${paths.unreadable.length} screenshot(s) could not be read, starting with ${paths.unreadable[0]}. ` +
|
|
522
|
-
|
|
523
|
-
|
|
554
|
+
"Playwright ran somewhere this command cannot reach — a container, most likely. Run it there too, " +
|
|
555
|
+
"or pass --path-prefix=CONTAINER_PATH:LOCAL_PATH.");
|
|
524
556
|
}
|
|
525
557
|
const uploader = uploaderFromEnvironment(options.maxUploads === undefined ? {} : { maxUploads: options.maxUploads });
|
|
526
558
|
if (uploader.enabled) {
|
|
@@ -541,7 +573,7 @@ async function main(args = process.argv.slice(2), adapter = {}) {
|
|
|
541
573
|
const summaryFile = process.env.GITHUB_STEP_SUMMARY;
|
|
542
574
|
if (summaryFile) {
|
|
543
575
|
fs.appendFileSync(summaryFile, summary);
|
|
544
|
-
console.error(
|
|
576
|
+
console.error("Failure summary written to $GITHUB_STEP_SUMMARY");
|
|
545
577
|
}
|
|
546
578
|
else {
|
|
547
579
|
process.stdout.write(summary);
|
package/lib/github/index.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
export { generateAnnotations as generateA11yAnnotations, generateSummary as generateA11ySummary, main as runA11ySummary, parseA11yResults, } from
|
|
2
|
-
export type { A11yReport, A11yTestResult } from
|
|
3
|
-
export * from
|
|
4
|
-
export { FAILURE_MARKER_PREFIX, FLAKE_MARKER_PREFIX, defuseMaskTriggers, generateComment as generateFailureComment, generateSummary as generateFailureSummary, main as runFailureSummary, parseFailures, resolveImagePaths, uploaderFromEnvironment, uploadImages, } from
|
|
5
|
-
export type { FailedTest, FailureImage, FailureReport, FailureCommentOptions, FailureSummaryMainOptions, ImageKind, IncludeMode, PathResolutionSummary, SummaryOptions, } from
|
|
6
|
-
export * from
|
|
1
|
+
export { generateAnnotations as generateA11yAnnotations, generateSummary as generateA11ySummary, main as runA11ySummary, parseA11yResults, } from "./a11y-summary";
|
|
2
|
+
export type { A11yReport, A11yTestResult } from "./a11y-summary";
|
|
3
|
+
export * from "./attachments";
|
|
4
|
+
export { FAILURE_MARKER_PREFIX, FLAKE_MARKER_PREFIX, defuseMaskTriggers, generateComment as generateFailureComment, generateSummary as generateFailureSummary, main as runFailureSummary, parseFailures, resolveImagePaths, uploaderFromEnvironment, uploadImages, } from "./failure-summary";
|
|
5
|
+
export type { FailedTest, FailureImage, FailureReport, FailureCommentOptions, FailureSummaryMainOptions, ImageKind, IncludeMode, PathResolutionSummary, SummaryOptions, } from "./failure-summary";
|
|
6
|
+
export * from "./report-paths";
|
|
@@ -83,7 +83,7 @@ const MIN_TAIL_COMPONENTS = 2;
|
|
|
83
83
|
* contain one on Windows, whereas `FROM` comes from a container and will not.
|
|
84
84
|
*/
|
|
85
85
|
function parsePathPrefix(value) {
|
|
86
|
-
const separator = value.indexOf(
|
|
86
|
+
const separator = value.indexOf(":");
|
|
87
87
|
if (separator <= 0)
|
|
88
88
|
return null;
|
|
89
89
|
const from = value.slice(0, separator).trim();
|
|
@@ -111,7 +111,9 @@ function createPathResolver(options) {
|
|
|
111
111
|
// Longest tail first, so the most specific match wins. Stop one short of
|
|
112
112
|
// the whole path: something has to be left to rewrite.
|
|
113
113
|
for (let length = target.components.length - 1; length >= MIN_TAIL_COMPONENTS; length--) {
|
|
114
|
-
const tail = target.components
|
|
114
|
+
const tail = target.components
|
|
115
|
+
.slice(target.components.length - length)
|
|
116
|
+
.join("/");
|
|
115
117
|
for (const root of searchRoots) {
|
|
116
118
|
const candidate = `${trimTrailingSeparators(root)}/${tail}`;
|
|
117
119
|
if (!exists(candidate))
|
|
@@ -119,7 +121,10 @@ function createPathResolver(options) {
|
|
|
119
121
|
return {
|
|
120
122
|
path: candidate,
|
|
121
123
|
found: true,
|
|
122
|
-
prefix: {
|
|
124
|
+
prefix: {
|
|
125
|
+
from: joinPath(target, target.components.length - length),
|
|
126
|
+
to: root,
|
|
127
|
+
},
|
|
123
128
|
};
|
|
124
129
|
}
|
|
125
130
|
}
|
|
@@ -153,7 +158,7 @@ function applyPrefix(filePath, prefix) {
|
|
|
153
158
|
}
|
|
154
159
|
const rest = target.components.slice(from.components.length);
|
|
155
160
|
const to = trimTrailingSeparators(prefix.to);
|
|
156
|
-
return rest.length === 0 ? to : `${to}/${rest.join(
|
|
161
|
+
return rest.length === 0 ? to : `${to}/${rest.join("/")}`;
|
|
157
162
|
}
|
|
158
163
|
/**
|
|
159
164
|
* Trim trailing separators.
|
|
@@ -164,7 +169,7 @@ function applyPrefix(filePath, prefix) {
|
|
|
164
169
|
*/
|
|
165
170
|
function trimTrailingSeparators(value) {
|
|
166
171
|
let end = value.length;
|
|
167
|
-
while (end > 0 && (value[end - 1] ===
|
|
172
|
+
while (end > 0 && (value[end - 1] === "/" || value[end - 1] === "\\"))
|
|
168
173
|
end--;
|
|
169
174
|
return value.slice(0, end);
|
|
170
175
|
}
|
|
@@ -176,15 +181,18 @@ function trimTrailingSeparators(value) {
|
|
|
176
181
|
*/
|
|
177
182
|
function splitPath(value) {
|
|
178
183
|
const match = /^([A-Za-z]:[\\/]|[\\/])?/.exec(value);
|
|
179
|
-
const prefix = match?.[1] ??
|
|
184
|
+
const prefix = match?.[1] ?? "";
|
|
180
185
|
return {
|
|
181
|
-
root: prefix.replace(/\\/g,
|
|
182
|
-
components: value
|
|
186
|
+
root: prefix.replace(/\\/g, "/"),
|
|
187
|
+
components: value
|
|
188
|
+
.slice(prefix.length)
|
|
189
|
+
.split(/[\\/]+/)
|
|
190
|
+
.filter(Boolean),
|
|
183
191
|
};
|
|
184
192
|
}
|
|
185
193
|
/** Rebuild a path from its first `count` components. */
|
|
186
194
|
function joinPath(split, count) {
|
|
187
|
-
return split.root + split.components.slice(0, count).join(
|
|
195
|
+
return split.root + split.components.slice(0, count).join("/");
|
|
188
196
|
}
|
|
189
197
|
/** A directory and every directory above it. */
|
|
190
198
|
function ancestors(directory) {
|