@lullabot/playwright-testing 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/github-a11y-summary +2 -2
- package/bin/github-failure-summary +4 -4
- package/lib/accessibility-baseline-file.d.ts +6 -6
- package/lib/accessibility-baseline-file.js +29 -23
- package/lib/accessibility-baseline.js +5 -3
- package/lib/accessible-screenshot.d.ts +4 -4
- package/lib/accessible-screenshot.js +88 -76
- package/lib/focus.js +4 -1
- package/lib/frames.js +4 -4
- package/lib/github/a11y-summary.js +74 -79
- package/lib/github/attachments.d.ts +1 -1
- package/lib/github/attachments.js +36 -34
- package/lib/github/failure-summary.d.ts +4 -4
- package/lib/github/failure-summary.js +139 -107
- package/lib/github/index.d.ts +6 -6
- package/lib/github/report-paths.js +17 -9
- package/lib/hover.js +17 -17
- package/lib/images.js +15 -12
- package/lib/index.d.ts +13 -13
- package/lib/interaction-states.d.ts +2 -2
- package/lib/interaction-states.js +7 -7
- package/lib/mock/index.d.ts +1 -1
- package/lib/mock/youtube.d.ts +1 -1
- package/lib/mock/youtube.js +1 -1
- package/lib/pseudo-state.d.ts +2 -2
- package/lib/pseudo-state.js +15 -7
- package/lib/videos.js +21 -10
- package/lib/visualdiff.d.ts +4 -4
- package/lib/visualdiff.js +35 -27
- package/package.json +3 -3
- package/src/accessibility-baseline-file.test.ts +190 -154
- package/src/accessibility-baseline-file.ts +107 -83
- package/src/accessibility-baseline.test.ts +633 -433
- package/src/accessibility-baseline.ts +41 -21
- package/src/accessible-screenshot.test.ts +551 -423
- package/src/accessible-screenshot.ts +334 -245
- package/src/focus.test.ts +23 -23
- package/src/focus.ts +7 -2
- package/src/fonts.test.ts +13 -13
- package/src/fonts.ts +1 -1
- package/src/frames.test.ts +10 -8
- package/src/frames.ts +5 -5
- package/src/github/a11y-summary.test.ts +436 -332
- package/src/github/a11y-summary.ts +189 -179
- package/src/github/attachments.test.ts +228 -167
- package/src/github/attachments.ts +144 -121
- package/src/github/failure-summary.test.ts +659 -445
- package/src/github/failure-summary.ts +376 -274
- package/src/github/index.test.ts +14 -14
- package/src/github/index.ts +6 -6
- package/src/github/report-paths.test.ts +211 -156
- package/src/github/report-paths.ts +75 -61
- package/src/hover.test.ts +50 -46
- package/src/hover.ts +18 -18
- package/src/images.test.ts +251 -206
- package/src/images.ts +52 -43
- package/src/index.ts +13 -13
- package/src/interaction-states.test.ts +72 -42
- package/src/interaction-states.ts +43 -29
- package/src/mock/index.ts +1 -1
- package/src/mock/youtube.test.ts +26 -26
- package/src/mock/youtube.ts +4 -6
- package/src/pseudo-state.test.ts +58 -51
- package/src/pseudo-state.ts +32 -27
- package/src/videos.test.ts +559 -399
- package/src/videos.ts +68 -44
- package/src/visualdiff.test.ts +416 -315
- package/src/visualdiff.ts +140 -99
|
@@ -1,7 +1,12 @@
|
|
|
1
|
-
import * as fs from
|
|
2
|
-
import * as path from
|
|
3
|
-
import { AttachmentUploader, AttachmentUploaderOptions } from
|
|
4
|
-
import {
|
|
1
|
+
import * as fs from "fs";
|
|
2
|
+
import * as path from "path";
|
|
3
|
+
import { AttachmentUploader, AttachmentUploaderOptions } from "./attachments";
|
|
4
|
+
import {
|
|
5
|
+
PathPrefix,
|
|
6
|
+
PathResolution,
|
|
7
|
+
createPathResolver,
|
|
8
|
+
parsePathPrefix,
|
|
9
|
+
} from "./report-paths";
|
|
5
10
|
|
|
6
11
|
/**
|
|
7
12
|
* Turn a Playwright JSON report into a job summary — and optionally a PR
|
|
@@ -20,76 +25,77 @@ import { PathPrefix, PathResolution, createPathResolver, parsePathPrefix } from
|
|
|
20
25
|
|
|
21
26
|
/** Attachment names Playwright gives the three snapshot comparison images. */
|
|
22
27
|
const SNAPSHOT_SUFFIXES: Array<{ suffix: string; kind: ImageKind }> = [
|
|
23
|
-
{ suffix:
|
|
24
|
-
{ suffix:
|
|
25
|
-
{ suffix:
|
|
26
|
-
{ suffix:
|
|
27
|
-
]
|
|
28
|
+
{ suffix: "-diff", kind: "diff" },
|
|
29
|
+
{ suffix: "-actual", kind: "actual" },
|
|
30
|
+
{ suffix: "-expected", kind: "expected" },
|
|
31
|
+
{ suffix: "-previous", kind: "previous" },
|
|
32
|
+
];
|
|
28
33
|
|
|
29
34
|
/** Attachment name used by checkAccessibility() for its highlighted screenshot. */
|
|
30
|
-
const A11Y_SCREENSHOT =
|
|
35
|
+
const A11Y_SCREENSHOT = "a11y-violation-screenshot";
|
|
31
36
|
|
|
32
|
-
export type ImageKind =
|
|
37
|
+
export type ImageKind =
|
|
38
|
+
"diff" | "actual" | "expected" | "previous" | "a11y" | "screenshot";
|
|
33
39
|
|
|
34
40
|
export interface FailureImage {
|
|
35
41
|
/** Attachment name as it appears in the report. */
|
|
36
|
-
name: string
|
|
42
|
+
name: string;
|
|
37
43
|
/** Path on disk. Attachments the reporter inlined have none. */
|
|
38
|
-
filePath?: string
|
|
44
|
+
filePath?: string;
|
|
39
45
|
/**
|
|
40
46
|
* Base64 bytes, for attachments added with `testInfo.attach({ body })`. The
|
|
41
47
|
* JSON reporter inlines those rather than writing them out, which is how the
|
|
42
48
|
* accessibility screenshots arrive.
|
|
43
49
|
*/
|
|
44
|
-
body?: string
|
|
45
|
-
contentType: string
|
|
46
|
-
kind: ImageKind
|
|
50
|
+
body?: string;
|
|
51
|
+
contentType: string;
|
|
52
|
+
kind: ImageKind;
|
|
47
53
|
/** Populated once the image has been uploaded. */
|
|
48
|
-
url?: string
|
|
54
|
+
url?: string;
|
|
49
55
|
/**
|
|
50
56
|
* Set when the recorded path could not be found on this side of a container
|
|
51
57
|
* boundary. Distinguishes "there was nothing to upload with" from "there was
|
|
52
58
|
* nowhere to upload to", which otherwise render identically.
|
|
53
59
|
*/
|
|
54
|
-
unreadable?: boolean
|
|
60
|
+
unreadable?: boolean;
|
|
55
61
|
/**
|
|
56
62
|
* Set when the uploader would not carry the image: over GitHub's 10 MB
|
|
57
63
|
* ceiling, or over what is left of the run's byte budget. Uploading carried
|
|
58
64
|
* on around it, so this is one image's story rather than the run's.
|
|
59
65
|
*/
|
|
60
|
-
oversize?: boolean
|
|
66
|
+
oversize?: boolean;
|
|
61
67
|
}
|
|
62
68
|
|
|
63
69
|
export interface FailedTest {
|
|
64
|
-
title: string
|
|
65
|
-
file: string
|
|
66
|
-
line: number
|
|
70
|
+
title: string;
|
|
71
|
+
file: string;
|
|
72
|
+
line: number;
|
|
67
73
|
/** Display status, including the synthetic `flaky` status for a retry that passed. */
|
|
68
|
-
status: string
|
|
74
|
+
status: string;
|
|
69
75
|
/** First error message, trimmed to something a comment can carry. */
|
|
70
|
-
error?: string
|
|
71
|
-
images: FailureImage[]
|
|
76
|
+
error?: string;
|
|
77
|
+
images: FailureImage[];
|
|
72
78
|
}
|
|
73
79
|
|
|
74
80
|
export interface FailureReport {
|
|
75
|
-
tests: FailedTest[]
|
|
76
|
-
totalFailed: number
|
|
77
|
-
totalFlaky: number
|
|
78
|
-
totalImages: number
|
|
81
|
+
tests: FailedTest[];
|
|
82
|
+
totalFailed: number;
|
|
83
|
+
totalFlaky: number;
|
|
84
|
+
totalImages: number;
|
|
79
85
|
}
|
|
80
86
|
|
|
81
87
|
/** Which snapshot images to include. `diff` is the one that shows the problem. */
|
|
82
|
-
export type IncludeMode =
|
|
88
|
+
export type IncludeMode = "diff" | "all";
|
|
83
89
|
|
|
84
90
|
/**
|
|
85
91
|
* Prefix of the HTML comment carrying the failure count in a generated comment
|
|
86
92
|
* body. Invisible when rendered, and greppable by a workflow that has to decide
|
|
87
93
|
* whether the run is worth commenting on.
|
|
88
94
|
*/
|
|
89
|
-
export const FAILURE_MARKER_PREFIX =
|
|
95
|
+
export const FAILURE_MARKER_PREFIX = "<!-- playwright-testing-failures: ";
|
|
90
96
|
|
|
91
97
|
/** Machine-readable flaky-test count used by the comment assembly action. */
|
|
92
|
-
export const FLAKE_MARKER_PREFIX =
|
|
98
|
+
export const FLAKE_MARKER_PREFIX = "<!-- playwright-testing-flakes: ";
|
|
93
99
|
|
|
94
100
|
/**
|
|
95
101
|
* Neutralise text that the Actions runner would redact.
|
|
@@ -101,23 +107,27 @@ export const FLAKE_MARKER_PREFIX = '<!-- playwright-testing-flakes: '
|
|
|
101
107
|
* inside the word reads identically and no longer matches.
|
|
102
108
|
*/
|
|
103
109
|
export function defuseMaskTriggers(text: string): string {
|
|
104
|
-
return text.replace(/Bearer(?=\s)/g,
|
|
110
|
+
return text.replace(/Bearer(?=\s)/g, "Bear\u200Ber");
|
|
105
111
|
}
|
|
106
112
|
|
|
107
113
|
/** Strip ANSI colour codes Playwright puts in error messages. */
|
|
108
114
|
function stripAnsi(text: string): string {
|
|
109
|
-
|
|
115
|
+
// eslint-disable-next-line no-control-regex -- ANSI colour codes contain ESC.
|
|
116
|
+
return text.replace(/\u001b\[[0-9;]*m/g, "");
|
|
110
117
|
}
|
|
111
118
|
|
|
112
|
-
function classifyAttachment(
|
|
113
|
-
|
|
114
|
-
|
|
119
|
+
function classifyAttachment(
|
|
120
|
+
name: string,
|
|
121
|
+
contentType: string,
|
|
122
|
+
): ImageKind | null {
|
|
123
|
+
if (!contentType.startsWith("image/")) return null;
|
|
124
|
+
if (name === A11Y_SCREENSHOT) return "a11y";
|
|
115
125
|
|
|
116
|
-
const withoutExtension = name.replace(/\.[^.]+$/,
|
|
126
|
+
const withoutExtension = name.replace(/\.[^.]+$/, "");
|
|
117
127
|
for (const { suffix, kind } of SNAPSHOT_SUFFIXES) {
|
|
118
|
-
if (withoutExtension.endsWith(suffix)) return kind
|
|
128
|
+
if (withoutExtension.endsWith(suffix)) return kind;
|
|
119
129
|
}
|
|
120
|
-
return
|
|
130
|
+
return "screenshot";
|
|
121
131
|
}
|
|
122
132
|
|
|
123
133
|
/**
|
|
@@ -127,19 +137,25 @@ function classifyAttachment(name: string, contentType: string): ImageKind | null
|
|
|
127
137
|
* baselined violation still passes and its screenshot is still the thing you
|
|
128
138
|
* want to look at.
|
|
129
139
|
*/
|
|
130
|
-
function collectImages(
|
|
131
|
-
|
|
140
|
+
function collectImages(
|
|
141
|
+
attachments: any[],
|
|
142
|
+
include: IncludeMode,
|
|
143
|
+
): FailureImage[] {
|
|
144
|
+
const images: FailureImage[] = [];
|
|
132
145
|
|
|
133
146
|
for (const attachment of attachments) {
|
|
134
|
-
const name = String(attachment?.name ??
|
|
135
|
-
const contentType = String(attachment?.contentType ??
|
|
136
|
-
const kind = classifyAttachment(name, contentType)
|
|
137
|
-
if (!kind) continue
|
|
147
|
+
const name = String(attachment?.name ?? "");
|
|
148
|
+
const contentType = String(attachment?.contentType ?? "");
|
|
149
|
+
const kind = classifyAttachment(name, contentType);
|
|
150
|
+
if (!kind) continue;
|
|
138
151
|
|
|
139
152
|
// Uploading three near-identical images per failure is rarely worth it;
|
|
140
153
|
// the diff is the one that shows what changed.
|
|
141
|
-
if (
|
|
142
|
-
|
|
154
|
+
if (
|
|
155
|
+
include === "diff" &&
|
|
156
|
+
(kind === "expected" || kind === "actual" || kind === "previous")
|
|
157
|
+
) {
|
|
158
|
+
continue;
|
|
143
159
|
}
|
|
144
160
|
|
|
145
161
|
images.push({
|
|
@@ -148,88 +164,107 @@ function collectImages(attachments: any[], include: IncludeMode): FailureImage[]
|
|
|
148
164
|
body: attachment?.body ? String(attachment.body) : undefined,
|
|
149
165
|
contentType,
|
|
150
166
|
kind,
|
|
151
|
-
})
|
|
167
|
+
});
|
|
152
168
|
}
|
|
153
169
|
|
|
154
|
-
return images
|
|
170
|
+
return images;
|
|
155
171
|
}
|
|
156
172
|
|
|
157
173
|
function firstError(result: any): string | undefined {
|
|
158
|
-
const message = result?.error?.message ?? result?.errors?.[0]?.message
|
|
159
|
-
if (typeof message !==
|
|
160
|
-
const cleaned = stripAnsi(message).trim().split(
|
|
161
|
-
return cleaned.length > 300 ? `${cleaned.slice(0, 300)}…` : cleaned
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
function walkSuites(
|
|
174
|
+
const message = result?.error?.message ?? result?.errors?.[0]?.message;
|
|
175
|
+
if (typeof message !== "string" || !message.trim()) return undefined;
|
|
176
|
+
const cleaned = stripAnsi(message).trim().split("\n")[0];
|
|
177
|
+
return cleaned.length > 300 ? `${cleaned.slice(0, 300)}…` : cleaned;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function walkSuites(
|
|
181
|
+
suites: any[],
|
|
182
|
+
parentFile: string,
|
|
183
|
+
include: IncludeMode,
|
|
184
|
+
out: FailedTest[],
|
|
185
|
+
): void {
|
|
165
186
|
for (const suite of suites ?? []) {
|
|
166
|
-
const file = suite.file || parentFile
|
|
187
|
+
const file = suite.file || parentFile;
|
|
167
188
|
|
|
168
189
|
for (const spec of suite.specs ?? []) {
|
|
169
190
|
for (const test of spec.tests ?? []) {
|
|
170
|
-
const results = test.results ?? []
|
|
171
|
-
const result = results[results.length - 1]
|
|
172
|
-
if (!result) continue
|
|
191
|
+
const results = test.results ?? [];
|
|
192
|
+
const result = results[results.length - 1];
|
|
193
|
+
if (!result) continue;
|
|
173
194
|
|
|
174
195
|
// Playwright reports each retry as another result. A passing final
|
|
175
196
|
// attempt therefore hides the failed attempt unless we deliberately
|
|
176
197
|
// retain it and give the test its Playwright outcome: flaky.
|
|
177
|
-
const failedAttempts = results
|
|
178
|
-
|
|
179
|
-
|
|
198
|
+
const failedAttempts = results
|
|
199
|
+
.slice(0, -1)
|
|
200
|
+
.filter(
|
|
201
|
+
(attempt: any) =>
|
|
202
|
+
attempt?.status !== "passed" && attempt?.status !== "skipped",
|
|
203
|
+
);
|
|
180
204
|
// JSON reports expose Playwright's computed outcome on the test. Keep
|
|
181
205
|
// the attempt-based fallback for older or hand-written reports that do
|
|
182
206
|
// not carry it.
|
|
183
|
-
const flaky =
|
|
184
|
-
test.status ===
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
const
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
207
|
+
const flaky =
|
|
208
|
+
test.status === "flaky" ||
|
|
209
|
+
(test.status === undefined &&
|
|
210
|
+
result.status === "passed" &&
|
|
211
|
+
failedAttempts.length > 0);
|
|
212
|
+
const reportedResult = flaky
|
|
213
|
+
? failedAttempts[failedAttempts.length - 1]
|
|
214
|
+
: result;
|
|
215
|
+
|
|
216
|
+
const images = collectImages(reportedResult.attachments ?? [], include);
|
|
217
|
+
const failed =
|
|
218
|
+
result.status !== "passed" && result.status !== "skipped";
|
|
219
|
+
const hasA11yImage = images.some((image) => image.kind === "a11y");
|
|
220
|
+
if (!failed && !flaky && !hasA11yImage) continue;
|
|
192
221
|
|
|
193
222
|
out.push({
|
|
194
223
|
title: spec.title,
|
|
195
224
|
file,
|
|
196
225
|
line: spec.line ?? 1,
|
|
197
|
-
status: flaky ?
|
|
226
|
+
status: flaky ? "flaky" : String(result.status ?? "unknown"),
|
|
198
227
|
error: firstError(reportedResult),
|
|
199
228
|
images,
|
|
200
|
-
})
|
|
229
|
+
});
|
|
201
230
|
}
|
|
202
231
|
}
|
|
203
232
|
|
|
204
|
-
walkSuites(suite.suites ?? [], file, include, out)
|
|
233
|
+
walkSuites(suite.suites ?? [], file, include, out);
|
|
205
234
|
}
|
|
206
235
|
}
|
|
207
236
|
|
|
208
237
|
/**
|
|
209
238
|
* Parse a Playwright JSON report into the failures worth reporting.
|
|
210
239
|
*/
|
|
211
|
-
export function parseFailures(
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
const
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
const
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
240
|
+
export function parseFailures(
|
|
241
|
+
reportPath: string,
|
|
242
|
+
include: IncludeMode = "diff",
|
|
243
|
+
): FailureReport {
|
|
244
|
+
const report = JSON.parse(fs.readFileSync(reportPath, "utf8"));
|
|
245
|
+
const tests: FailedTest[] = [];
|
|
246
|
+
walkSuites(report.suites ?? [], "", include, tests);
|
|
247
|
+
|
|
248
|
+
const totalFailed = tests.filter(
|
|
249
|
+
(test) =>
|
|
250
|
+
test.status !== "passed" &&
|
|
251
|
+
test.status !== "skipped" &&
|
|
252
|
+
test.status !== "flaky",
|
|
253
|
+
).length;
|
|
254
|
+
const totalFlaky = tests.filter((test) => test.status === "flaky").length;
|
|
255
|
+
const totalImages = tests.reduce((sum, test) => sum + test.images.length, 0);
|
|
256
|
+
|
|
257
|
+
return { tests, totalFailed, totalFlaky, totalImages };
|
|
223
258
|
}
|
|
224
259
|
|
|
225
260
|
/** What re-rooting the report's attachment paths achieved. */
|
|
226
261
|
export interface PathResolutionSummary {
|
|
227
262
|
/** Attachments whose recorded path had to be rewritten to be readable. */
|
|
228
|
-
remapped: number
|
|
263
|
+
remapped: number;
|
|
229
264
|
/** The mappings that were used, for the log line. */
|
|
230
|
-
used: PathPrefix[]
|
|
265
|
+
used: PathPrefix[];
|
|
231
266
|
/** Recorded paths with no readable file behind them, on either side. */
|
|
232
|
-
unreadable: string[]
|
|
267
|
+
unreadable: string[];
|
|
233
268
|
}
|
|
234
269
|
|
|
235
270
|
/**
|
|
@@ -244,32 +279,41 @@ export function resolveImagePaths(
|
|
|
244
279
|
report: FailureReport,
|
|
245
280
|
resolve: (filePath: string) => PathResolution,
|
|
246
281
|
): PathResolutionSummary {
|
|
247
|
-
const summary: PathResolutionSummary = {
|
|
282
|
+
const summary: PathResolutionSummary = {
|
|
283
|
+
remapped: 0,
|
|
284
|
+
used: [],
|
|
285
|
+
unreadable: [],
|
|
286
|
+
};
|
|
248
287
|
|
|
249
288
|
for (const test of report.tests) {
|
|
250
289
|
for (const image of test.images) {
|
|
251
|
-
if (!image.filePath) continue
|
|
290
|
+
if (!image.filePath) continue;
|
|
252
291
|
|
|
253
|
-
const resolution = resolve(image.filePath)
|
|
292
|
+
const resolution = resolve(image.filePath);
|
|
254
293
|
if (!resolution.found) {
|
|
255
|
-
image.unreadable = true
|
|
256
|
-
summary.unreadable.push(image.filePath)
|
|
257
|
-
continue
|
|
294
|
+
image.unreadable = true;
|
|
295
|
+
summary.unreadable.push(image.filePath);
|
|
296
|
+
continue;
|
|
258
297
|
}
|
|
259
298
|
|
|
260
|
-
if (resolution.path === image.filePath) continue
|
|
299
|
+
if (resolution.path === image.filePath) continue;
|
|
261
300
|
|
|
262
|
-
summary.remapped
|
|
263
|
-
image.filePath = resolution.path
|
|
301
|
+
summary.remapped++;
|
|
302
|
+
image.filePath = resolution.path;
|
|
264
303
|
|
|
265
|
-
const prefix = resolution.prefix
|
|
266
|
-
if (
|
|
267
|
-
|
|
304
|
+
const prefix = resolution.prefix;
|
|
305
|
+
if (
|
|
306
|
+
prefix &&
|
|
307
|
+
!summary.used.some(
|
|
308
|
+
(used) => used.from === prefix.from && used.to === prefix.to,
|
|
309
|
+
)
|
|
310
|
+
) {
|
|
311
|
+
summary.used.push(prefix);
|
|
268
312
|
}
|
|
269
313
|
}
|
|
270
314
|
}
|
|
271
315
|
|
|
272
|
-
return summary
|
|
316
|
+
return summary;
|
|
273
317
|
}
|
|
274
318
|
|
|
275
319
|
/**
|
|
@@ -283,21 +327,24 @@ export async function uploadImages(
|
|
|
283
327
|
): Promise<void> {
|
|
284
328
|
for (const test of report.tests) {
|
|
285
329
|
for (const image of test.images) {
|
|
286
|
-
let url: string | null = null
|
|
330
|
+
let url: string | null = null;
|
|
287
331
|
|
|
288
332
|
if (image.filePath) {
|
|
289
|
-
url = await uploader.upload(
|
|
333
|
+
url = await uploader.upload(
|
|
334
|
+
image.filePath,
|
|
335
|
+
path.basename(image.filePath),
|
|
336
|
+
);
|
|
290
337
|
} else if (image.body) {
|
|
291
338
|
url = await uploader.uploadBuffer(
|
|
292
|
-
Buffer.from(image.body,
|
|
339
|
+
Buffer.from(image.body, "base64"),
|
|
293
340
|
`${image.name}${extensionFor(image.contentType)}`,
|
|
294
341
|
image.contentType,
|
|
295
|
-
)
|
|
342
|
+
);
|
|
296
343
|
}
|
|
297
344
|
|
|
298
345
|
if (url) {
|
|
299
|
-
image.url = url
|
|
300
|
-
continue
|
|
346
|
+
image.url = url;
|
|
347
|
+
continue;
|
|
301
348
|
}
|
|
302
349
|
|
|
303
350
|
// Every one of these is a null, and they need different things said
|
|
@@ -305,13 +352,13 @@ export async function uploadImages(
|
|
|
305
352
|
// so 'unreadable' here means the read itself failed; the size reasons
|
|
306
353
|
// are the uploader declining this one image rather than stopping.
|
|
307
354
|
switch (uploader.lastSkipReason) {
|
|
308
|
-
case
|
|
309
|
-
image.unreadable = true
|
|
310
|
-
break
|
|
311
|
-
case
|
|
312
|
-
case
|
|
313
|
-
image.oversize = true
|
|
314
|
-
break
|
|
355
|
+
case "unreadable":
|
|
356
|
+
image.unreadable = true;
|
|
357
|
+
break;
|
|
358
|
+
case "too-large":
|
|
359
|
+
case "budget":
|
|
360
|
+
image.oversize = true;
|
|
361
|
+
break;
|
|
315
362
|
}
|
|
316
363
|
}
|
|
317
364
|
}
|
|
@@ -319,31 +366,35 @@ export async function uploadImages(
|
|
|
319
366
|
|
|
320
367
|
/** Inlined attachments carry no file name, so derive one from the content type. */
|
|
321
368
|
function extensionFor(contentType: string): string {
|
|
322
|
-
if (contentType ===
|
|
323
|
-
if (contentType ===
|
|
324
|
-
if (contentType ===
|
|
325
|
-
if (contentType ===
|
|
326
|
-
return
|
|
369
|
+
if (contentType === "image/png") return ".png";
|
|
370
|
+
if (contentType === "image/jpeg") return ".jpg";
|
|
371
|
+
if (contentType === "image/gif") return ".gif";
|
|
372
|
+
if (contentType === "image/webp") return ".webp";
|
|
373
|
+
return "";
|
|
327
374
|
}
|
|
328
375
|
|
|
329
376
|
function headline(report: FailureReport): string {
|
|
330
|
-
if (report.tests.length === 0)
|
|
377
|
+
if (report.tests.length === 0)
|
|
378
|
+
return ":white_check_mark: No failing tests or flaky tests with screenshots.";
|
|
331
379
|
|
|
332
380
|
// Nothing failed, but an accessibility check still captured something worth
|
|
333
381
|
// seeing — a baselined violation passes and is still screenshotted.
|
|
334
|
-
const parts =
|
|
335
|
-
|
|
336
|
-
|
|
382
|
+
const parts =
|
|
383
|
+
report.totalFailed === 0
|
|
384
|
+
? [":white_check_mark: No failing tests"]
|
|
385
|
+
: [`**${report.totalFailed}** failing test(s)`];
|
|
337
386
|
|
|
338
|
-
if (report.totalFlaky > 0)
|
|
387
|
+
if (report.totalFlaky > 0)
|
|
388
|
+
parts.push(`:warning: **${report.totalFlaky}** flaky test(s)`);
|
|
339
389
|
|
|
340
|
-
if (report.totalImages > 0)
|
|
341
|
-
|
|
390
|
+
if (report.totalImages > 0)
|
|
391
|
+
parts.push(`**${report.totalImages}** screenshot(s)`);
|
|
392
|
+
return parts.join(" · ");
|
|
342
393
|
}
|
|
343
394
|
|
|
344
395
|
/** Every image across every test, which is what the diagnostics count. */
|
|
345
396
|
function allImages(report: FailureReport): FailureImage[] {
|
|
346
|
-
return report.tests.flatMap(test => test.images)
|
|
397
|
+
return report.tests.flatMap((test) => test.images);
|
|
347
398
|
}
|
|
348
399
|
|
|
349
400
|
/**
|
|
@@ -354,31 +405,36 @@ function allImages(report: FailureReport): FailureImage[] {
|
|
|
354
405
|
* no way to tell which one they have. An absent token is the dominant reason
|
|
355
406
|
* when it applies: nothing was going to be uploaded regardless.
|
|
356
407
|
*/
|
|
357
|
-
function missingImageReason(
|
|
358
|
-
|
|
408
|
+
function missingImageReason(
|
|
409
|
+
images: FailureImage[],
|
|
410
|
+
uploadReason?: string,
|
|
411
|
+
): string | undefined {
|
|
412
|
+
if (uploadReason) return uploadReason;
|
|
359
413
|
|
|
360
|
-
const reasons: string[] = []
|
|
414
|
+
const reasons: string[] = [];
|
|
361
415
|
|
|
362
|
-
const unreadable = images.filter(image => image.unreadable).length
|
|
416
|
+
const unreadable = images.filter((image) => image.unreadable).length;
|
|
363
417
|
if (unreadable > 0) {
|
|
364
|
-
reasons.push(
|
|
418
|
+
reasons.push(
|
|
419
|
+
`${unreadable} could not be read from the path recorded in the report`,
|
|
420
|
+
);
|
|
365
421
|
}
|
|
366
422
|
|
|
367
|
-
const oversize = images.filter(image => image.oversize).length
|
|
423
|
+
const oversize = images.filter((image) => image.oversize).length;
|
|
368
424
|
if (oversize > 0) {
|
|
369
|
-
reasons.push(`${oversize} too large to upload`)
|
|
425
|
+
reasons.push(`${oversize} too large to upload`);
|
|
370
426
|
}
|
|
371
427
|
|
|
372
|
-
return reasons.length > 0 ? reasons.join(
|
|
428
|
+
return reasons.length > 0 ? reasons.join("; ") : undefined;
|
|
373
429
|
}
|
|
374
430
|
|
|
375
431
|
export interface SummaryOptions {
|
|
376
|
-
artifactHint?: string
|
|
432
|
+
artifactHint?: string;
|
|
377
433
|
/**
|
|
378
434
|
* Why uploading did not happen at all, phrased to sit inside a sentence —
|
|
379
435
|
* `no upload token configured`, say. Leave unset when uploads ran.
|
|
380
436
|
*/
|
|
381
|
-
uploadReason?: string
|
|
437
|
+
uploadReason?: string;
|
|
382
438
|
}
|
|
383
439
|
|
|
384
440
|
/**
|
|
@@ -388,74 +444,89 @@ export interface SummaryOptions {
|
|
|
388
444
|
* Deliberately table-free. Test titles and error messages are arbitrary text,
|
|
389
445
|
* and the runner's masking can eat a cell delimiter and corrupt a whole row.
|
|
390
446
|
*/
|
|
391
|
-
export function generateSummary(
|
|
447
|
+
export function generateSummary(
|
|
448
|
+
report: FailureReport,
|
|
449
|
+
options: SummaryOptions = {},
|
|
450
|
+
): string {
|
|
392
451
|
// A baselined accessibility violation is screenshotted without failing
|
|
393
452
|
// anything, so a green run can still have something to show here.
|
|
394
|
-
const heading =
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
453
|
+
const heading =
|
|
454
|
+
report.totalFailed > 0
|
|
455
|
+
? "## Test Failures"
|
|
456
|
+
: report.totalFlaky > 0
|
|
457
|
+
? "## Test Flakes"
|
|
458
|
+
: "## Test Screenshots";
|
|
459
|
+
const lines: string[] = [`${heading}\n`, `${headline(report)}\n`];
|
|
398
460
|
|
|
399
|
-
if (report.tests.length === 0) return lines.join(
|
|
461
|
+
if (report.tests.length === 0) return lines.join("\n");
|
|
400
462
|
|
|
401
463
|
// Name the path boundary where it will be read, rather than leaving the
|
|
402
464
|
// reader to conclude they forgot the token.
|
|
403
|
-
const unreadable = allImages(report).filter(image => image.unreadable)
|
|
465
|
+
const unreadable = allImages(report).filter((image) => image.unreadable);
|
|
404
466
|
if (unreadable.length > 0) {
|
|
405
|
-
const example = defuseMaskTriggers(unreadable[0].filePath ??
|
|
467
|
+
const example = defuseMaskTriggers(unreadable[0].filePath ?? "");
|
|
406
468
|
lines.push(
|
|
407
469
|
`> :warning: **${unreadable.length} screenshot(s) could not be read.** The report records them ` +
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
)
|
|
470
|
+
`under \`${example}\`, which does not exist where this step ran. Playwright ran somewhere else — ` +
|
|
471
|
+
"a container, most likely — so run this command there too, or pass " +
|
|
472
|
+
"`--path-prefix=CONTAINER_PATH:LOCAL_PATH` to map one onto the other.\n",
|
|
473
|
+
);
|
|
412
474
|
}
|
|
413
475
|
|
|
414
476
|
for (const test of report.tests) {
|
|
415
|
-
const title = defuseMaskTriggers(test.title)
|
|
416
|
-
lines.push(`### ${title}\n`)
|
|
417
|
-
lines.push(`\`${test.file}:${test.line}\` — ${test.status}\n`)
|
|
477
|
+
const title = defuseMaskTriggers(test.title);
|
|
478
|
+
lines.push(`### ${title}\n`);
|
|
479
|
+
lines.push(`\`${test.file}:${test.line}\` — ${test.status}\n`);
|
|
418
480
|
|
|
419
481
|
if (test.error) {
|
|
420
|
-
lines.push(`> ${defuseMaskTriggers(test.error)}\n`)
|
|
482
|
+
lines.push(`> ${defuseMaskTriggers(test.error)}\n`);
|
|
421
483
|
}
|
|
422
484
|
|
|
423
|
-
if (test.images.length === 0) continue
|
|
485
|
+
if (test.images.length === 0) continue;
|
|
424
486
|
|
|
425
487
|
// The images go in the pull request comment, not here. A job summary is
|
|
426
488
|
// rendered once, when the job finishes, and an attachment uploaded seconds
|
|
427
489
|
// earlier is not resolvable yet — so it renders as a dead link and the
|
|
428
490
|
// cached result never improves, however often the page is reloaded.
|
|
429
|
-
const embedded = test.images.filter(image => image.url).length
|
|
491
|
+
const embedded = test.images.filter((image) => image.url).length;
|
|
430
492
|
if (embedded > 0) {
|
|
431
|
-
lines.push(
|
|
493
|
+
lines.push(
|
|
494
|
+
`${embedded} screenshot(s) uploaded — see the pull request comment.\n`,
|
|
495
|
+
);
|
|
432
496
|
} else {
|
|
433
|
-
const hint = options.artifactHint ??
|
|
434
|
-
const reason = missingImageReason(test.images, options.uploadReason)
|
|
435
|
-
const because = reason ? ` (${reason})` :
|
|
436
|
-
lines.push(
|
|
497
|
+
const hint = options.artifactHint ?? "the Playwright report artifact";
|
|
498
|
+
const reason = missingImageReason(test.images, options.uploadReason);
|
|
499
|
+
const because = reason ? ` (${reason})` : "";
|
|
500
|
+
lines.push(
|
|
501
|
+
`${test.images.length} screenshot(s) captured, not uploaded${because} — download ${hint}.\n`,
|
|
502
|
+
);
|
|
437
503
|
}
|
|
438
504
|
}
|
|
439
505
|
|
|
440
|
-
return lines.join(
|
|
506
|
+
return lines.join("\n");
|
|
441
507
|
}
|
|
442
508
|
|
|
443
509
|
/** A collapsed block of images for one test, used in the comment. */
|
|
444
510
|
function renderImageBlock(test: FailedTest): string[] {
|
|
445
|
-
const embedded = test.images.filter(image => image.url)
|
|
446
|
-
if (embedded.length === 0) return []
|
|
511
|
+
const embedded = test.images.filter((image) => image.url);
|
|
512
|
+
if (embedded.length === 0) return [];
|
|
447
513
|
|
|
448
|
-
const title = defuseMaskTriggers(test.title)
|
|
449
|
-
const lines = [
|
|
514
|
+
const title = defuseMaskTriggers(test.title);
|
|
515
|
+
const lines = [
|
|
516
|
+
"<details>",
|
|
517
|
+
`<summary>Screenshots (${embedded.length})</summary>\n`,
|
|
518
|
+
];
|
|
450
519
|
|
|
451
520
|
for (const image of embedded) {
|
|
452
|
-
lines.push(`**${image.kind}**\n`)
|
|
453
|
-
const src = escapeAttribute(image.url ??
|
|
454
|
-
lines.push(
|
|
521
|
+
lines.push(`**${image.kind}**\n`);
|
|
522
|
+
const src = escapeAttribute(image.url ?? "");
|
|
523
|
+
lines.push(
|
|
524
|
+
`<img src="${src}" alt="${image.kind} for ${escapeAttribute(title)}" width="640">\n`,
|
|
525
|
+
);
|
|
455
526
|
}
|
|
456
527
|
|
|
457
|
-
lines.push(
|
|
458
|
-
return lines
|
|
528
|
+
lines.push("</details>\n");
|
|
529
|
+
return lines;
|
|
459
530
|
}
|
|
460
531
|
|
|
461
532
|
/**
|
|
@@ -471,106 +542,119 @@ function renderImageBlock(test: FailedTest): string[] {
|
|
|
471
542
|
* broken in the email even though they are fine on the web.
|
|
472
543
|
*/
|
|
473
544
|
export interface FailureCommentOptions {
|
|
474
|
-
summaryUrl?: string
|
|
475
|
-
title?: string
|
|
476
|
-
uploadReason?: string
|
|
545
|
+
summaryUrl?: string;
|
|
546
|
+
title?: string;
|
|
547
|
+
uploadReason?: string;
|
|
477
548
|
/** Adapter namespace for the machine-readable failure marker. */
|
|
478
|
-
failureMarkerPrefix?: string
|
|
549
|
+
failureMarkerPrefix?: string;
|
|
479
550
|
/** Adapter namespace for the machine-readable flaky-test marker. */
|
|
480
|
-
flakeMarkerPrefix?: string
|
|
551
|
+
flakeMarkerPrefix?: string;
|
|
481
552
|
}
|
|
482
553
|
|
|
483
554
|
export function generateComment(
|
|
484
555
|
report: FailureReport,
|
|
485
556
|
options: FailureCommentOptions = {},
|
|
486
557
|
): string {
|
|
487
|
-
const heading = options.title ??
|
|
488
|
-
const lines: string[] = [`### ${heading}\n`, `${headline(report)}\n`]
|
|
558
|
+
const heading = options.title ?? "Playwright results";
|
|
559
|
+
const lines: string[] = [`### ${heading}\n`, `${headline(report)}\n`];
|
|
489
560
|
|
|
490
561
|
if (report.tests.length > 0) {
|
|
491
562
|
for (const test of report.tests.slice(0, 10)) {
|
|
492
|
-
const title = defuseMaskTriggers(test.title)
|
|
493
|
-
const images =
|
|
494
|
-
|
|
563
|
+
const title = defuseMaskTriggers(test.title);
|
|
564
|
+
const images =
|
|
565
|
+
test.images.length > 0 ? ` — ${test.images.length} screenshot(s)` : "";
|
|
566
|
+
lines.push(`- \`${test.status}\` ${title}${images}`);
|
|
495
567
|
}
|
|
496
568
|
if (report.tests.length > 10) {
|
|
497
|
-
lines.push(`- …and ${report.tests.length - 10} more`)
|
|
569
|
+
lines.push(`- …and ${report.tests.length - 10} more`);
|
|
498
570
|
}
|
|
499
|
-
lines.push(
|
|
571
|
+
lines.push("");
|
|
500
572
|
|
|
501
573
|
for (const test of report.tests) {
|
|
502
|
-
const block = renderImageBlock(test)
|
|
503
|
-
if (block.length === 0) continue
|
|
504
|
-
lines.push(`**${defuseMaskTriggers(test.title)}**\n`)
|
|
505
|
-
lines.push(...block)
|
|
574
|
+
const block = renderImageBlock(test);
|
|
575
|
+
if (block.length === 0) continue;
|
|
576
|
+
lines.push(`**${defuseMaskTriggers(test.title)}**\n`);
|
|
577
|
+
lines.push(...block);
|
|
506
578
|
}
|
|
507
579
|
|
|
508
580
|
// An empty comment where images were expected reads as a broken feature.
|
|
509
581
|
// Whatever the reason, it belongs where the images would have been.
|
|
510
|
-
const missing = allImages(report).filter(image => !image.url)
|
|
511
|
-
const reason = missingImageReason(missing, options.uploadReason)
|
|
582
|
+
const missing = allImages(report).filter((image) => !image.url);
|
|
583
|
+
const reason = missingImageReason(missing, options.uploadReason);
|
|
512
584
|
if (missing.length > 0 && reason) {
|
|
513
|
-
lines.push(`_${missing.length} screenshot(s) not shown — ${reason}._\n`)
|
|
585
|
+
lines.push(`_${missing.length} screenshot(s) not shown — ${reason}._\n`);
|
|
514
586
|
}
|
|
515
587
|
}
|
|
516
588
|
|
|
517
589
|
if (options.summaryUrl) {
|
|
518
|
-
lines.push(`[Full run](${options.summaryUrl})\n`)
|
|
590
|
+
lines.push(`[Full run](${options.summaryUrl})\n`);
|
|
519
591
|
}
|
|
520
592
|
|
|
521
593
|
// A machine-readable count, so a workflow assembling several of these can
|
|
522
594
|
// decide whether to post at all without grepping prose — the empty-state
|
|
523
595
|
// sentence contains the words "failing test" too.
|
|
524
|
-
lines.push(
|
|
525
|
-
|
|
596
|
+
lines.push(
|
|
597
|
+
`${options.failureMarkerPrefix ?? FAILURE_MARKER_PREFIX}${report.totalFailed} -->\n`,
|
|
598
|
+
);
|
|
599
|
+
lines.push(
|
|
600
|
+
`${options.flakeMarkerPrefix ?? FLAKE_MARKER_PREFIX}${report.totalFlaky} -->\n`,
|
|
601
|
+
);
|
|
526
602
|
|
|
527
|
-
return lines.join(
|
|
603
|
+
return lines.join("\n");
|
|
528
604
|
}
|
|
529
605
|
|
|
530
606
|
function escapeAttribute(value: string): string {
|
|
531
|
-
return value
|
|
607
|
+
return value
|
|
608
|
+
.replace(/&/g, "&")
|
|
609
|
+
.replace(/"/g, """)
|
|
610
|
+
.replace(/</g, "<");
|
|
532
611
|
}
|
|
533
612
|
|
|
534
613
|
interface CliOptions {
|
|
535
|
-
reportPath: string
|
|
536
|
-
commentPath?: string
|
|
537
|
-
include: IncludeMode
|
|
538
|
-
maxUploads?: number
|
|
539
|
-
title?: string
|
|
540
|
-
pathPrefixes: PathPrefix[]
|
|
614
|
+
reportPath: string;
|
|
615
|
+
commentPath?: string;
|
|
616
|
+
include: IncludeMode;
|
|
617
|
+
maxUploads?: number;
|
|
618
|
+
title?: string;
|
|
619
|
+
pathPrefixes: PathPrefix[];
|
|
541
620
|
}
|
|
542
621
|
|
|
543
622
|
function parseArgs(args: string[]): CliOptions {
|
|
544
623
|
const options: CliOptions = {
|
|
545
|
-
reportPath:
|
|
546
|
-
include:
|
|
624
|
+
reportPath: "test-results/results.json",
|
|
625
|
+
include: "diff",
|
|
547
626
|
pathPrefixes: [],
|
|
548
|
-
}
|
|
627
|
+
};
|
|
549
628
|
|
|
550
629
|
for (const arg of args) {
|
|
551
630
|
// Repeatable: a project can have more than one mount to translate.
|
|
552
|
-
if (arg.startsWith(
|
|
553
|
-
const prefix = parsePathPrefix(arg.slice(
|
|
631
|
+
if (arg.startsWith("--path-prefix=")) {
|
|
632
|
+
const prefix = parsePathPrefix(arg.slice("--path-prefix=".length));
|
|
554
633
|
if (prefix) {
|
|
555
|
-
options.pathPrefixes.push(prefix)
|
|
634
|
+
options.pathPrefixes.push(prefix);
|
|
556
635
|
} else {
|
|
557
|
-
console.error(
|
|
636
|
+
console.error(
|
|
637
|
+
`Ignoring --path-prefix: expected FROM:TO, got ${arg.slice("--path-prefix=".length)}`,
|
|
638
|
+
);
|
|
558
639
|
}
|
|
559
640
|
}
|
|
560
|
-
if (arg.startsWith(
|
|
561
|
-
|
|
562
|
-
if (arg.startsWith(
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
options.
|
|
641
|
+
if (arg.startsWith("--report-path="))
|
|
642
|
+
options.reportPath = arg.slice("--report-path=".length);
|
|
643
|
+
if (arg.startsWith("--comment-path="))
|
|
644
|
+
options.commentPath = arg.slice("--comment-path=".length);
|
|
645
|
+
if (arg.startsWith("--title="))
|
|
646
|
+
options.title = arg.slice("--title=".length);
|
|
647
|
+
if (arg.startsWith("--include=")) {
|
|
648
|
+
const value = arg.slice("--include=".length);
|
|
649
|
+
options.include = value === "all" ? "all" : "diff";
|
|
566
650
|
}
|
|
567
|
-
if (arg.startsWith(
|
|
568
|
-
const value = Number.parseInt(arg.slice(
|
|
569
|
-
if (Number.isFinite(value) && value >= 0) options.maxUploads = value
|
|
651
|
+
if (arg.startsWith("--max-uploads=")) {
|
|
652
|
+
const value = Number.parseInt(arg.slice("--max-uploads=".length), 10);
|
|
653
|
+
if (Number.isFinite(value) && value >= 0) options.maxUploads = value;
|
|
570
654
|
}
|
|
571
655
|
}
|
|
572
656
|
|
|
573
|
-
return options
|
|
657
|
+
return options;
|
|
574
658
|
}
|
|
575
659
|
|
|
576
660
|
/**
|
|
@@ -582,24 +666,31 @@ function parseArgs(args: string[]): CliOptions {
|
|
|
582
666
|
* summary is on stdout, stderr is the only sensible place.
|
|
583
667
|
*/
|
|
584
668
|
function warn(message: string): void {
|
|
585
|
-
const inActions =
|
|
669
|
+
const inActions =
|
|
670
|
+
process.env.GITHUB_ACTIONS === "true" && !!process.env.GITHUB_STEP_SUMMARY;
|
|
586
671
|
if (inActions) {
|
|
587
|
-
process.stdout.write(
|
|
672
|
+
process.stdout.write(
|
|
673
|
+
`::warning title=Playwright screenshots::${escapeAnnotation(message)}\n`,
|
|
674
|
+
);
|
|
588
675
|
} else {
|
|
589
|
-
console.error(`Warning: ${message}`)
|
|
676
|
+
console.error(`Warning: ${message}`);
|
|
590
677
|
}
|
|
591
678
|
}
|
|
592
679
|
|
|
593
680
|
/** Workflow commands are line-based, so anything that ends a line is encoded. */
|
|
594
681
|
function escapeAnnotation(message: string): string {
|
|
595
|
-
return message
|
|
682
|
+
return message
|
|
683
|
+
.replace(/%/g, "%25")
|
|
684
|
+
.replace(/\r/g, "%0D")
|
|
685
|
+
.replace(/\n/g, "%0A");
|
|
596
686
|
}
|
|
597
687
|
|
|
598
688
|
/** Build the run's job summary URL, which is the best anchor a comment can link to. */
|
|
599
689
|
function summaryUrl(): string | undefined {
|
|
600
|
-
const { GITHUB_SERVER_URL, GITHUB_REPOSITORY, GITHUB_RUN_ID } = process.env
|
|
601
|
-
if (!GITHUB_SERVER_URL || !GITHUB_REPOSITORY || !GITHUB_RUN_ID)
|
|
602
|
-
|
|
690
|
+
const { GITHUB_SERVER_URL, GITHUB_REPOSITORY, GITHUB_RUN_ID } = process.env;
|
|
691
|
+
if (!GITHUB_SERVER_URL || !GITHUB_REPOSITORY || !GITHUB_RUN_ID)
|
|
692
|
+
return undefined;
|
|
693
|
+
return `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}`;
|
|
603
694
|
}
|
|
604
695
|
|
|
605
696
|
export function uploaderFromEnvironment(
|
|
@@ -608,9 +699,9 @@ export function uploaderFromEnvironment(
|
|
|
608
699
|
return new AttachmentUploader({
|
|
609
700
|
token: process.env.SCREENSHOT_GITHUB_TOKEN,
|
|
610
701
|
repositoryId: process.env.GITHUB_REPOSITORY_ID,
|
|
611
|
-
log: message => console.error(message),
|
|
702
|
+
log: (message) => console.error(message),
|
|
612
703
|
...overrides,
|
|
613
|
-
})
|
|
704
|
+
});
|
|
614
705
|
}
|
|
615
706
|
|
|
616
707
|
/**
|
|
@@ -618,84 +709,95 @@ export function uploaderFromEnvironment(
|
|
|
618
709
|
* uploaded twice for the same run.
|
|
619
710
|
*/
|
|
620
711
|
export interface FailureSummaryMainOptions {
|
|
621
|
-
commandName?: string
|
|
622
|
-
failureMarkerPrefix?: string
|
|
623
|
-
flakeMarkerPrefix?: string
|
|
712
|
+
commandName?: string;
|
|
713
|
+
failureMarkerPrefix?: string;
|
|
714
|
+
flakeMarkerPrefix?: string;
|
|
624
715
|
}
|
|
625
716
|
|
|
626
717
|
export async function main(
|
|
627
718
|
args: string[] = process.argv.slice(2),
|
|
628
719
|
adapter: FailureSummaryMainOptions = {},
|
|
629
720
|
): Promise<void> {
|
|
630
|
-
if (args.includes(
|
|
721
|
+
if (args.includes("--help") || args.includes("-h")) {
|
|
631
722
|
process.stdout.write(
|
|
632
|
-
`Usage: ${adapter.commandName ??
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
)
|
|
642
|
-
return
|
|
723
|
+
`Usage: ${adapter.commandName ?? "playwright-testing-failure-summary"} [options]\n\n` +
|
|
724
|
+
"Options:\n" +
|
|
725
|
+
" --report-path=PATH Playwright JSON report (default: test-results/results.json)\n" +
|
|
726
|
+
" --comment-path=PATH Write a pull request comment body\n" +
|
|
727
|
+
" --include=diff|all Select comparison images (default: diff)\n" +
|
|
728
|
+
" --path-prefix=FROM:TO Remap attachment paths; repeatable\n" +
|
|
729
|
+
" --max-uploads=COUNT Limit uploaded screenshots\n" +
|
|
730
|
+
" --title=TEXT Set the comment heading\n" +
|
|
731
|
+
" -h, --help Show this help\n",
|
|
732
|
+
);
|
|
733
|
+
return;
|
|
643
734
|
}
|
|
644
735
|
|
|
645
|
-
const options = parseArgs(args)
|
|
646
|
-
const resolvedPath = path.resolve(options.reportPath)
|
|
736
|
+
const options = parseArgs(args);
|
|
737
|
+
const resolvedPath = path.resolve(options.reportPath);
|
|
647
738
|
|
|
648
739
|
// A missing report is not an error: the suite may not have run at all.
|
|
649
740
|
if (!fs.existsSync(resolvedPath)) {
|
|
650
|
-
console.error(
|
|
651
|
-
|
|
741
|
+
console.error(
|
|
742
|
+
`Playwright JSON report not found at ${resolvedPath} — skipping failure summary.`,
|
|
743
|
+
);
|
|
744
|
+
return;
|
|
652
745
|
}
|
|
653
746
|
|
|
654
|
-
const report = parseFailures(resolvedPath, options.include)
|
|
747
|
+
const report = parseFailures(resolvedPath, options.include);
|
|
655
748
|
|
|
656
749
|
// The report records where the run wrote its attachments, which is not
|
|
657
750
|
// necessarily anywhere this process can reach. Settle that before uploading,
|
|
658
751
|
// so an unreachable file is reported as one rather than as a silent skip.
|
|
659
752
|
const paths = resolveImagePaths(
|
|
660
753
|
report,
|
|
661
|
-
createPathResolver({
|
|
662
|
-
|
|
754
|
+
createPathResolver({
|
|
755
|
+
reportPath: resolvedPath,
|
|
756
|
+
prefixes: options.pathPrefixes,
|
|
757
|
+
}),
|
|
758
|
+
);
|
|
663
759
|
if (paths.remapped > 0) {
|
|
664
|
-
const mappings = paths.used
|
|
665
|
-
|
|
760
|
+
const mappings = paths.used
|
|
761
|
+
.map((prefix) => `${prefix.from} → ${prefix.to}`)
|
|
762
|
+
.join(", ");
|
|
763
|
+
console.error(
|
|
764
|
+
`Remapped ${paths.remapped} attachment path(s): ${mappings || "no mapping recorded"}`,
|
|
765
|
+
);
|
|
666
766
|
}
|
|
667
767
|
if (paths.unreadable.length > 0) {
|
|
668
768
|
warn(
|
|
669
769
|
`${paths.unreadable.length} screenshot(s) could not be read, starting with ${paths.unreadable[0]}. ` +
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
)
|
|
770
|
+
"Playwright ran somewhere this command cannot reach — a container, most likely. Run it there too, " +
|
|
771
|
+
"or pass --path-prefix=CONTAINER_PATH:LOCAL_PATH.",
|
|
772
|
+
);
|
|
673
773
|
}
|
|
674
774
|
|
|
675
775
|
const uploader = uploaderFromEnvironment(
|
|
676
776
|
options.maxUploads === undefined ? {} : { maxUploads: options.maxUploads },
|
|
677
|
-
)
|
|
777
|
+
);
|
|
678
778
|
if (uploader.enabled) {
|
|
679
|
-
await uploadImages(report, uploader)
|
|
680
|
-
const stats = uploader.getStats()
|
|
681
|
-
console.error(
|
|
779
|
+
await uploadImages(report, uploader);
|
|
780
|
+
const stats = uploader.getStats();
|
|
781
|
+
console.error(
|
|
782
|
+
`Uploaded ${stats.uploaded} screenshot(s), skipped ${stats.skipped}.`,
|
|
783
|
+
);
|
|
682
784
|
if (uploader.disabledReason) {
|
|
683
785
|
// Reaching an undocumented endpoint that no longer answers is a real
|
|
684
786
|
// fault, unlike an absent token on a fork build.
|
|
685
|
-
warn(`Screenshot uploads stopped early — ${uploader.disabledReason}.`)
|
|
787
|
+
warn(`Screenshot uploads stopped early — ${uploader.disabledReason}.`);
|
|
686
788
|
}
|
|
687
789
|
} else {
|
|
688
|
-
console.error(`Screenshot uploads are off — ${uploader.disabledReason}.`)
|
|
790
|
+
console.error(`Screenshot uploads are off — ${uploader.disabledReason}.`);
|
|
689
791
|
}
|
|
690
792
|
|
|
691
|
-
const uploadReason = uploader.disabledReason ?? undefined
|
|
692
|
-
const summary = generateSummary(report, { uploadReason })
|
|
693
|
-
const summaryFile = process.env.GITHUB_STEP_SUMMARY
|
|
793
|
+
const uploadReason = uploader.disabledReason ?? undefined;
|
|
794
|
+
const summary = generateSummary(report, { uploadReason });
|
|
795
|
+
const summaryFile = process.env.GITHUB_STEP_SUMMARY;
|
|
694
796
|
if (summaryFile) {
|
|
695
|
-
fs.appendFileSync(summaryFile, summary)
|
|
696
|
-
console.error(
|
|
797
|
+
fs.appendFileSync(summaryFile, summary);
|
|
798
|
+
console.error("Failure summary written to $GITHUB_STEP_SUMMARY");
|
|
697
799
|
} else {
|
|
698
|
-
process.stdout.write(summary)
|
|
800
|
+
process.stdout.write(summary);
|
|
699
801
|
}
|
|
700
802
|
|
|
701
803
|
if (options.commentPath) {
|
|
@@ -705,16 +807,16 @@ export async function main(
|
|
|
705
807
|
uploadReason,
|
|
706
808
|
failureMarkerPrefix: adapter.failureMarkerPrefix,
|
|
707
809
|
flakeMarkerPrefix: adapter.flakeMarkerPrefix,
|
|
708
|
-
})
|
|
709
|
-
fs.writeFileSync(path.resolve(options.commentPath), comment)
|
|
710
|
-
console.error(`Comment body written to ${options.commentPath}`)
|
|
810
|
+
});
|
|
811
|
+
fs.writeFileSync(path.resolve(options.commentPath), comment);
|
|
812
|
+
console.error(`Comment body written to ${options.commentPath}`);
|
|
711
813
|
}
|
|
712
814
|
}
|
|
713
815
|
|
|
714
816
|
// Auto-invoke when run directly (node lib/github/failure-summary.js).
|
|
715
817
|
if (require.main === module) {
|
|
716
818
|
main().catch((error: unknown) => {
|
|
717
|
-
console.error(error)
|
|
718
|
-
process.exit(1)
|
|
719
|
-
})
|
|
819
|
+
console.error(error);
|
|
820
|
+
process.exit(1);
|
|
821
|
+
});
|
|
720
822
|
}
|