@lullabot/playwright-testing 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/github-a11y-summary +2 -2
- package/bin/github-failure-summary +4 -4
- package/lib/accessibility-baseline-file.d.ts +6 -6
- package/lib/accessibility-baseline-file.js +29 -23
- package/lib/accessibility-baseline.js +5 -3
- package/lib/accessible-screenshot.d.ts +4 -4
- package/lib/accessible-screenshot.js +88 -76
- package/lib/focus.js +4 -1
- package/lib/frames.js +4 -4
- package/lib/github/a11y-summary.js +74 -79
- package/lib/github/attachments.d.ts +1 -1
- package/lib/github/attachments.js +36 -34
- package/lib/github/failure-summary.d.ts +4 -4
- package/lib/github/failure-summary.js +139 -107
- package/lib/github/index.d.ts +6 -6
- package/lib/github/report-paths.js +17 -9
- package/lib/hover.js +17 -17
- package/lib/images.js +15 -12
- package/lib/index.d.ts +13 -13
- package/lib/interaction-states.d.ts +2 -2
- package/lib/interaction-states.js +7 -7
- package/lib/mock/index.d.ts +1 -1
- package/lib/mock/youtube.d.ts +1 -1
- package/lib/mock/youtube.js +1 -1
- package/lib/pseudo-state.d.ts +2 -2
- package/lib/pseudo-state.js +15 -7
- package/lib/videos.js +21 -10
- package/lib/visualdiff.d.ts +4 -4
- package/lib/visualdiff.js +35 -27
- package/package.json +3 -3
- package/src/accessibility-baseline-file.test.ts +190 -154
- package/src/accessibility-baseline-file.ts +107 -83
- package/src/accessibility-baseline.test.ts +633 -433
- package/src/accessibility-baseline.ts +41 -21
- package/src/accessible-screenshot.test.ts +551 -423
- package/src/accessible-screenshot.ts +334 -245
- package/src/focus.test.ts +23 -23
- package/src/focus.ts +7 -2
- package/src/fonts.test.ts +13 -13
- package/src/fonts.ts +1 -1
- package/src/frames.test.ts +10 -8
- package/src/frames.ts +5 -5
- package/src/github/a11y-summary.test.ts +436 -332
- package/src/github/a11y-summary.ts +189 -179
- package/src/github/attachments.test.ts +228 -167
- package/src/github/attachments.ts +144 -121
- package/src/github/failure-summary.test.ts +659 -445
- package/src/github/failure-summary.ts +376 -274
- package/src/github/index.test.ts +14 -14
- package/src/github/index.ts +6 -6
- package/src/github/report-paths.test.ts +211 -156
- package/src/github/report-paths.ts +75 -61
- package/src/hover.test.ts +50 -46
- package/src/hover.ts +18 -18
- package/src/images.test.ts +251 -206
- package/src/images.ts +52 -43
- package/src/index.ts +13 -13
- package/src/interaction-states.test.ts +72 -42
- package/src/interaction-states.ts +43 -29
- package/src/mock/index.ts +1 -1
- package/src/mock/youtube.test.ts +26 -26
- package/src/mock/youtube.ts +4 -6
- package/src/pseudo-state.test.ts +58 -51
- package/src/pseudo-state.ts +32 -27
- package/src/videos.test.ts +559 -399
- package/src/videos.ts +68 -44
- package/src/visualdiff.test.ts +416 -315
- package/src/visualdiff.ts +140 -99
|
@@ -1,601 +1,801 @@
|
|
|
1
|
-
import { describe, it, expect, vi, beforeEach, afterEach } from
|
|
1
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
|
|
2
2
|
|
|
3
3
|
// Mock AxeBuilder — must be hoisted since vi.mock is hoisted
|
|
4
|
-
const {
|
|
5
|
-
const mockWithTags = vi.fn().mockReturnThis()
|
|
6
|
-
const mockExclude = vi.fn().mockReturnThis()
|
|
7
|
-
const mockOptions = vi.fn().mockReturnThis()
|
|
8
|
-
const mockAnalyze = vi.fn()
|
|
4
|
+
const { mockAnalyze, MockAxeBuilder } = vi.hoisted(() => {
|
|
5
|
+
const mockWithTags = vi.fn().mockReturnThis();
|
|
6
|
+
const mockExclude = vi.fn().mockReturnThis();
|
|
7
|
+
const mockOptions = vi.fn().mockReturnThis();
|
|
8
|
+
const mockAnalyze = vi.fn();
|
|
9
9
|
|
|
10
10
|
class MockAxeBuilder {
|
|
11
|
-
withTags = mockWithTags
|
|
12
|
-
exclude = mockExclude
|
|
13
|
-
options = mockOptions
|
|
14
|
-
analyze = mockAnalyze
|
|
15
|
-
constructor(_args: any) {}
|
|
11
|
+
withTags = mockWithTags;
|
|
12
|
+
exclude = mockExclude;
|
|
13
|
+
options = mockOptions;
|
|
14
|
+
analyze = mockAnalyze;
|
|
16
15
|
}
|
|
17
16
|
|
|
18
|
-
return {
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
17
|
+
return {
|
|
18
|
+
mockWithTags,
|
|
19
|
+
mockExclude,
|
|
20
|
+
mockOptions,
|
|
21
|
+
mockAnalyze,
|
|
22
|
+
MockAxeBuilder,
|
|
23
|
+
};
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
vi.mock("@axe-core/playwright", () => ({
|
|
22
27
|
default: MockAxeBuilder,
|
|
23
|
-
}))
|
|
28
|
+
}));
|
|
24
29
|
|
|
25
30
|
// Mock @playwright/test
|
|
26
|
-
const { mockToMatchSnapshot, mockToBe, mockExpectSoft, mockExpectHard } =
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
|
|
31
|
+
const { mockToMatchSnapshot, mockToBe, mockExpectSoft, mockExpectHard } =
|
|
32
|
+
vi.hoisted(() => {
|
|
33
|
+
const mockToMatchSnapshot = vi.fn();
|
|
34
|
+
const mockToBe = vi.fn();
|
|
35
|
+
const mockExpectSoft = vi.fn(() => ({
|
|
36
|
+
toMatchSnapshot: mockToMatchSnapshot,
|
|
37
|
+
toHaveScreenshot: vi.fn(),
|
|
38
|
+
}));
|
|
39
|
+
const mockExpectHard = vi.fn(() => ({
|
|
40
|
+
toMatchSnapshot: mockToMatchSnapshot,
|
|
41
|
+
toBe: mockToBe,
|
|
42
|
+
}));
|
|
43
|
+
return { mockToMatchSnapshot, mockToBe, mockExpectSoft, mockExpectHard };
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
vi.mock("@playwright/test", () => {
|
|
41
47
|
const expectFn = Object.assign(mockExpectHard, {
|
|
42
48
|
soft: mockExpectSoft,
|
|
43
|
-
})
|
|
49
|
+
});
|
|
44
50
|
return {
|
|
45
51
|
expect: expectFn,
|
|
46
52
|
Locator: {},
|
|
47
53
|
Page: {},
|
|
48
54
|
TestInfo: {},
|
|
49
|
-
}
|
|
50
|
-
})
|
|
51
|
-
|
|
52
|
-
import { checkAccessibility } from
|
|
53
|
-
import {
|
|
54
|
-
|
|
55
|
-
|
|
55
|
+
};
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
import { checkAccessibility } from "./accessible-screenshot";
|
|
59
|
+
import {
|
|
60
|
+
defineAccessibilityBaseline,
|
|
61
|
+
AccessibilityBaseline,
|
|
62
|
+
} from "./accessibility-baseline";
|
|
63
|
+
|
|
64
|
+
function makeAxeResults(
|
|
65
|
+
overrides?: Partial<{ violations: any[]; passes: any[] }>,
|
|
66
|
+
) {
|
|
56
67
|
return {
|
|
57
68
|
violations: [],
|
|
58
69
|
passes: [],
|
|
59
70
|
incomplete: [],
|
|
60
71
|
inapplicable: [],
|
|
61
72
|
...overrides,
|
|
62
|
-
}
|
|
73
|
+
};
|
|
63
74
|
}
|
|
64
75
|
|
|
65
|
-
function makeTestInfo(opts?: {
|
|
76
|
+
function makeTestInfo(opts?: {
|
|
77
|
+
updateSnapshots?: "all" | "changed" | "missing" | "none";
|
|
78
|
+
snapshotsDir?: string;
|
|
79
|
+
title?: string;
|
|
80
|
+
}) {
|
|
66
81
|
// Default to `updateSnapshots: 'all'` so that, in the absence of an
|
|
67
82
|
// explicit `baseline` option, the dispatch routes to snapshot mode (the
|
|
68
83
|
// legacy behavior these tests were originally written against). Tests
|
|
69
84
|
// that want to exercise on-disk baseline mode override this.
|
|
70
|
-
const dir = opts?.snapshotsDir ??
|
|
71
|
-
const title = opts?.title ??
|
|
85
|
+
const dir = opts?.snapshotsDir ?? "/tmp/__a11y_test_snapshots__";
|
|
86
|
+
const title = opts?.title ?? "mocked test";
|
|
72
87
|
return {
|
|
73
88
|
testId: `mocked-${Math.random()}`,
|
|
74
89
|
title,
|
|
75
|
-
titlePath: [
|
|
76
|
-
annotations: [] as Array<{ type: string
|
|
90
|
+
titlePath: ["file.spec.ts", title],
|
|
91
|
+
annotations: [] as Array<{ type: string; description?: string }>,
|
|
77
92
|
attach: vi.fn().mockResolvedValue(undefined),
|
|
78
|
-
snapshotPath: (...segs: string[]) => `${dir}/${segs.join(
|
|
79
|
-
config: { updateSnapshots: opts?.updateSnapshots ??
|
|
80
|
-
}
|
|
93
|
+
snapshotPath: (...segs: string[]) => `${dir}/${segs.join("/")}`,
|
|
94
|
+
config: { updateSnapshots: opts?.updateSnapshots ?? "all" },
|
|
95
|
+
};
|
|
81
96
|
}
|
|
82
97
|
|
|
83
98
|
function makePage() {
|
|
84
99
|
return {
|
|
85
100
|
evaluate: vi.fn().mockResolvedValue(undefined),
|
|
86
|
-
screenshot: vi.fn().mockResolvedValue(Buffer.from(
|
|
87
|
-
}
|
|
101
|
+
screenshot: vi.fn().mockResolvedValue(Buffer.from("fake-png")),
|
|
102
|
+
};
|
|
88
103
|
}
|
|
89
104
|
|
|
90
|
-
function makeViolation(
|
|
105
|
+
function makeViolation(
|
|
106
|
+
rule: string,
|
|
107
|
+
targets: string[][],
|
|
108
|
+
extras?: { description?: string; impact?: string; helpUrl?: string },
|
|
109
|
+
) {
|
|
91
110
|
return {
|
|
92
111
|
id: rule,
|
|
93
112
|
description: extras?.description ?? `Description for ${rule}`,
|
|
94
|
-
impact: extras?.impact ??
|
|
95
|
-
helpUrl:
|
|
96
|
-
|
|
113
|
+
impact: extras?.impact ?? "serious",
|
|
114
|
+
helpUrl:
|
|
115
|
+
extras?.helpUrl ?? `https://dequeuniversity.com/rules/axe/4.8/${rule}`,
|
|
116
|
+
nodes: targets.map((targetArr) => ({
|
|
97
117
|
target: targetArr,
|
|
98
118
|
})),
|
|
99
|
-
}
|
|
119
|
+
};
|
|
100
120
|
}
|
|
101
121
|
|
|
102
|
-
describe(
|
|
122
|
+
describe("accessibility baseline", () => {
|
|
103
123
|
beforeEach(() => {
|
|
104
|
-
vi.clearAllMocks()
|
|
105
|
-
mockAnalyze.mockResolvedValue(makeAxeResults())
|
|
106
|
-
})
|
|
124
|
+
vi.clearAllMocks();
|
|
125
|
+
mockAnalyze.mockResolvedValue(makeAxeResults());
|
|
126
|
+
});
|
|
107
127
|
|
|
108
|
-
describe(
|
|
109
|
-
it(
|
|
128
|
+
describe("defineAccessibilityBaseline", () => {
|
|
129
|
+
it("returns the entries unchanged", () => {
|
|
110
130
|
const entries: AccessibilityBaseline = [
|
|
111
|
-
{
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
131
|
+
{
|
|
132
|
+
rule: "color-contrast",
|
|
133
|
+
targets: ["#sidebar .tag"],
|
|
134
|
+
reason: "Known issue",
|
|
135
|
+
willBeFixedIn: "PROJ-123",
|
|
136
|
+
},
|
|
137
|
+
];
|
|
138
|
+
expect(defineAccessibilityBaseline(entries)).toBe(entries);
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
it("requires a substantive reason and tracking reference for every waiver", () => {
|
|
142
|
+
expect(() =>
|
|
143
|
+
defineAccessibilityBaseline([
|
|
144
|
+
{
|
|
145
|
+
rule: "image-alt",
|
|
146
|
+
targets: ["img"],
|
|
147
|
+
reason: "",
|
|
148
|
+
willBeFixedIn: "PROJ-1",
|
|
149
|
+
},
|
|
150
|
+
]),
|
|
151
|
+
).toThrow("reason");
|
|
152
|
+
expect(() =>
|
|
153
|
+
defineAccessibilityBaseline([
|
|
154
|
+
{
|
|
155
|
+
rule: "image-alt",
|
|
156
|
+
targets: ["img"],
|
|
157
|
+
reason: "Known issue",
|
|
158
|
+
willBeFixedIn: "TODO",
|
|
159
|
+
},
|
|
160
|
+
]),
|
|
161
|
+
).toThrow("willBeFixedIn");
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
it("rejects duplicate waivers for the same rule and target", () => {
|
|
165
|
+
expect(() =>
|
|
166
|
+
defineAccessibilityBaseline([
|
|
167
|
+
{
|
|
168
|
+
rule: "image-alt",
|
|
169
|
+
targets: ["img.hero"],
|
|
170
|
+
reason: "First",
|
|
171
|
+
willBeFixedIn: "PROJ-1",
|
|
172
|
+
},
|
|
173
|
+
{
|
|
174
|
+
rule: "image-alt",
|
|
175
|
+
targets: ["img.hero"],
|
|
176
|
+
reason: "Second",
|
|
177
|
+
willBeFixedIn: "PROJ-2",
|
|
178
|
+
},
|
|
179
|
+
]),
|
|
180
|
+
).toThrow("duplicate");
|
|
181
|
+
});
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
describe("baseline matching", () => {
|
|
185
|
+
it("suppresses violations that match a baseline entry (exact rule + target)", async () => {
|
|
135
186
|
const wcagResults = makeAxeResults({
|
|
136
|
-
violations: [makeViolation(
|
|
137
|
-
})
|
|
138
|
-
mockAnalyze.mockResolvedValue(wcagResults)
|
|
187
|
+
violations: [makeViolation("color-contrast", [["#sidebar .tag"]])],
|
|
188
|
+
});
|
|
189
|
+
mockAnalyze.mockResolvedValue(wcagResults);
|
|
139
190
|
|
|
140
|
-
const testInfo = makeTestInfo()
|
|
191
|
+
const testInfo = makeTestInfo();
|
|
141
192
|
const baseline: AccessibilityBaseline = [
|
|
142
|
-
{
|
|
143
|
-
|
|
193
|
+
{
|
|
194
|
+
rule: "color-contrast",
|
|
195
|
+
targets: ["#sidebar .tag"],
|
|
196
|
+
reason: "Known issue",
|
|
197
|
+
willBeFixedIn: "PROJ-123",
|
|
198
|
+
},
|
|
199
|
+
];
|
|
144
200
|
|
|
145
201
|
await checkAccessibility(makePage() as any, testInfo as any, {
|
|
146
|
-
bestPracticeMode:
|
|
202
|
+
bestPracticeMode: "off",
|
|
147
203
|
baseline,
|
|
148
|
-
})
|
|
204
|
+
});
|
|
149
205
|
|
|
150
206
|
// Should not call toBe (no failure)
|
|
151
|
-
expect(mockToBe).not.toHaveBeenCalled()
|
|
207
|
+
expect(mockToBe).not.toHaveBeenCalled();
|
|
152
208
|
// Should not call toMatchSnapshot (baseline mode skips it)
|
|
153
|
-
expect(mockToMatchSnapshot).not.toHaveBeenCalled()
|
|
154
|
-
})
|
|
209
|
+
expect(mockToMatchSnapshot).not.toHaveBeenCalled();
|
|
210
|
+
});
|
|
155
211
|
|
|
156
|
-
it(
|
|
212
|
+
it("suppresses violations with normalized ID match (--UNIQUE-ID replacement)", async () => {
|
|
157
213
|
const wcagResults = makeAxeResults({
|
|
158
|
-
violations: [makeViolation(
|
|
159
|
-
})
|
|
160
|
-
mockAnalyze.mockResolvedValue(wcagResults)
|
|
214
|
+
violations: [makeViolation("color-contrast", [["#edit-field--42"]])],
|
|
215
|
+
});
|
|
216
|
+
mockAnalyze.mockResolvedValue(wcagResults);
|
|
161
217
|
|
|
162
|
-
const testInfo = makeTestInfo()
|
|
218
|
+
const testInfo = makeTestInfo();
|
|
163
219
|
const baseline: AccessibilityBaseline = [
|
|
164
|
-
{
|
|
165
|
-
|
|
220
|
+
{
|
|
221
|
+
rule: "color-contrast",
|
|
222
|
+
targets: ["#edit-field--UNIQUE-ID"],
|
|
223
|
+
reason: "Dynamic ID",
|
|
224
|
+
willBeFixedIn: "PROJ-456",
|
|
225
|
+
},
|
|
226
|
+
];
|
|
166
227
|
|
|
167
228
|
await checkAccessibility(makePage() as any, testInfo as any, {
|
|
168
|
-
bestPracticeMode:
|
|
229
|
+
bestPracticeMode: "off",
|
|
169
230
|
baseline,
|
|
170
|
-
})
|
|
231
|
+
});
|
|
171
232
|
|
|
172
|
-
expect(mockToBe).not.toHaveBeenCalled()
|
|
173
|
-
expect(mockToMatchSnapshot).not.toHaveBeenCalled()
|
|
174
|
-
})
|
|
233
|
+
expect(mockToBe).not.toHaveBeenCalled();
|
|
234
|
+
expect(mockToMatchSnapshot).not.toHaveBeenCalled();
|
|
235
|
+
});
|
|
175
236
|
|
|
176
|
-
it(
|
|
237
|
+
it("fails on unmatched violations with full details", async () => {
|
|
177
238
|
const wcagResults = makeAxeResults({
|
|
178
|
-
violations: [
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
239
|
+
violations: [
|
|
240
|
+
makeViolation(
|
|
241
|
+
"color-contrast",
|
|
242
|
+
[["#sidebar .tag", ".footer__copyright"]],
|
|
243
|
+
{
|
|
244
|
+
description:
|
|
245
|
+
"Elements must meet minimum color contrast ratio thresholds",
|
|
246
|
+
impact: "serious",
|
|
247
|
+
helpUrl:
|
|
248
|
+
"https://dequeuniversity.com/rules/axe/4.8/color-contrast",
|
|
249
|
+
},
|
|
250
|
+
),
|
|
251
|
+
],
|
|
252
|
+
});
|
|
253
|
+
mockAnalyze.mockResolvedValue(wcagResults);
|
|
254
|
+
|
|
255
|
+
const testInfo = makeTestInfo();
|
|
187
256
|
await checkAccessibility(makePage() as any, testInfo as any, {
|
|
188
|
-
bestPracticeMode:
|
|
257
|
+
bestPracticeMode: "off",
|
|
189
258
|
baseline: [],
|
|
190
|
-
})
|
|
259
|
+
});
|
|
191
260
|
|
|
192
|
-
expect(mockToBe).toHaveBeenCalled()
|
|
261
|
+
expect(mockToBe).toHaveBeenCalled();
|
|
193
262
|
// First arg is null, second is the message string
|
|
194
263
|
const failureMessage = mockExpectHard.mock.calls.find(
|
|
195
|
-
(call: any[]) => call.length >= 2 && typeof call[1] ===
|
|
196
|
-
)?.[1] as string
|
|
197
|
-
|
|
198
|
-
expect(failureMessage).toContain(
|
|
199
|
-
expect(failureMessage).toContain(
|
|
200
|
-
expect(failureMessage).toContain(
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
expect(failureMessage).toContain(
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
264
|
+
(call: any[]) => call.length >= 2 && typeof call[1] === "string",
|
|
265
|
+
)?.[1] as string;
|
|
266
|
+
|
|
267
|
+
expect(failureMessage).toContain("color-contrast");
|
|
268
|
+
expect(failureMessage).toContain("serious");
|
|
269
|
+
expect(failureMessage).toContain(
|
|
270
|
+
"Elements must meet minimum color contrast ratio thresholds",
|
|
271
|
+
);
|
|
272
|
+
expect(failureMessage).toContain(
|
|
273
|
+
"https://dequeuniversity.com/rules/axe/4.8/color-contrast",
|
|
274
|
+
);
|
|
275
|
+
expect(failureMessage).toContain("#sidebar .tag");
|
|
276
|
+
expect(failureMessage).toContain(".footer__copyright");
|
|
277
|
+
});
|
|
278
|
+
});
|
|
279
|
+
|
|
280
|
+
describe("stale detection", () => {
|
|
281
|
+
it("reports stale baseline entries with annotations", async () => {
|
|
282
|
+
const wcagResults = makeAxeResults({ violations: [] });
|
|
283
|
+
mockAnalyze.mockResolvedValue(wcagResults);
|
|
284
|
+
|
|
285
|
+
const testInfo = makeTestInfo();
|
|
213
286
|
const baseline: AccessibilityBaseline = [
|
|
214
|
-
{
|
|
215
|
-
|
|
287
|
+
{
|
|
288
|
+
rule: "color-contrast",
|
|
289
|
+
targets: ["#old-element"],
|
|
290
|
+
reason: "Was broken",
|
|
291
|
+
willBeFixedIn: "PROJ-789",
|
|
292
|
+
},
|
|
293
|
+
];
|
|
216
294
|
|
|
217
295
|
await checkAccessibility(makePage() as any, testInfo as any, {
|
|
218
|
-
bestPracticeMode:
|
|
296
|
+
bestPracticeMode: "off",
|
|
219
297
|
baseline,
|
|
220
|
-
})
|
|
221
|
-
|
|
222
|
-
const staleAnnotations = testInfo.annotations.filter(
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
expect(staleAnnotations
|
|
226
|
-
expect(staleAnnotations[0].description).toContain(
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
298
|
+
});
|
|
299
|
+
|
|
300
|
+
const staleAnnotations = testInfo.annotations.filter(
|
|
301
|
+
(a) => a.type === "Stale a11y baseline entry",
|
|
302
|
+
);
|
|
303
|
+
expect(staleAnnotations).toHaveLength(1);
|
|
304
|
+
expect(staleAnnotations[0].description).toContain("color-contrast");
|
|
305
|
+
expect(staleAnnotations[0].description).toContain("#old-element");
|
|
306
|
+
expect(staleAnnotations[0].description).toContain("no longer detected");
|
|
307
|
+
});
|
|
308
|
+
});
|
|
309
|
+
|
|
310
|
+
describe("snapshot bypass", () => {
|
|
311
|
+
it("does NOT call toMatchSnapshot when baseline is active", async () => {
|
|
312
|
+
const wcagResults = makeAxeResults({ violations: [] });
|
|
313
|
+
mockAnalyze.mockResolvedValue(wcagResults);
|
|
314
|
+
|
|
315
|
+
const testInfo = makeTestInfo();
|
|
236
316
|
await checkAccessibility(makePage() as any, testInfo as any, {
|
|
237
|
-
bestPracticeMode:
|
|
317
|
+
bestPracticeMode: "off",
|
|
238
318
|
baseline: [],
|
|
239
|
-
})
|
|
319
|
+
});
|
|
240
320
|
|
|
241
|
-
expect(mockToMatchSnapshot).not.toHaveBeenCalled()
|
|
242
|
-
})
|
|
321
|
+
expect(mockToMatchSnapshot).not.toHaveBeenCalled();
|
|
322
|
+
});
|
|
243
323
|
|
|
244
|
-
it(
|
|
245
|
-
const testInfo = makeTestInfo()
|
|
324
|
+
it("calls toMatchSnapshot when no baseline is provided", async () => {
|
|
325
|
+
const testInfo = makeTestInfo();
|
|
246
326
|
await checkAccessibility(makePage() as any, testInfo as any, {
|
|
247
|
-
bestPracticeMode:
|
|
248
|
-
})
|
|
327
|
+
bestPracticeMode: "off",
|
|
328
|
+
});
|
|
249
329
|
|
|
250
|
-
expect(mockToMatchSnapshot).toHaveBeenCalled()
|
|
251
|
-
})
|
|
252
|
-
})
|
|
330
|
+
expect(mockToMatchSnapshot).toHaveBeenCalled();
|
|
331
|
+
});
|
|
332
|
+
});
|
|
253
333
|
|
|
254
|
-
describe(
|
|
255
|
-
it(
|
|
334
|
+
describe("copy-pasteable baseline entry in failure message", () => {
|
|
335
|
+
it("includes a copy-pasteable entry for unmatched violations", async () => {
|
|
256
336
|
const wcagResults = makeAxeResults({
|
|
257
|
-
violations: [makeViolation(
|
|
258
|
-
})
|
|
259
|
-
mockAnalyze.mockResolvedValue(wcagResults)
|
|
337
|
+
violations: [makeViolation("image-alt", [["img.hero"]])],
|
|
338
|
+
});
|
|
339
|
+
mockAnalyze.mockResolvedValue(wcagResults);
|
|
260
340
|
|
|
261
|
-
const testInfo = makeTestInfo()
|
|
341
|
+
const testInfo = makeTestInfo();
|
|
262
342
|
await checkAccessibility(makePage() as any, testInfo as any, {
|
|
263
|
-
bestPracticeMode:
|
|
343
|
+
bestPracticeMode: "off",
|
|
264
344
|
baseline: [],
|
|
265
|
-
})
|
|
345
|
+
});
|
|
266
346
|
|
|
267
347
|
const failureMessage = mockExpectHard.mock.calls.find(
|
|
268
|
-
(call: any[]) => call.length >= 2 && typeof call[1] ===
|
|
269
|
-
)?.[1] as string
|
|
270
|
-
|
|
271
|
-
expect(failureMessage).toContain("rule: 'image-alt'")
|
|
272
|
-
expect(failureMessage).toContain("targets: ['img.hero']")
|
|
273
|
-
expect(failureMessage).toContain(
|
|
274
|
-
expect(failureMessage).toContain(
|
|
275
|
-
expect(failureMessage).toContain(
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
348
|
+
(call: any[]) => call.length >= 2 && typeof call[1] === "string",
|
|
349
|
+
)?.[1] as string;
|
|
350
|
+
|
|
351
|
+
expect(failureMessage).toContain("rule: 'image-alt'");
|
|
352
|
+
expect(failureMessage).toContain("targets: ['img.hero']");
|
|
353
|
+
expect(failureMessage).toContain("reason:");
|
|
354
|
+
expect(failureMessage).toContain("willBeFixedIn:");
|
|
355
|
+
expect(failureMessage).toContain(
|
|
356
|
+
"Add to your baseline to accept this violation:",
|
|
357
|
+
);
|
|
358
|
+
});
|
|
359
|
+
});
|
|
360
|
+
|
|
361
|
+
describe("baseline annotations", () => {
|
|
362
|
+
it("pushes Baselined a11y violation annotations for matched violations", async () => {
|
|
281
363
|
const wcagResults = makeAxeResults({
|
|
282
|
-
violations: [makeViolation(
|
|
283
|
-
})
|
|
284
|
-
mockAnalyze.mockResolvedValue(wcagResults)
|
|
364
|
+
violations: [makeViolation("color-contrast", [["#sidebar .tag"]])],
|
|
365
|
+
});
|
|
366
|
+
mockAnalyze.mockResolvedValue(wcagResults);
|
|
285
367
|
|
|
286
|
-
const testInfo = makeTestInfo()
|
|
368
|
+
const testInfo = makeTestInfo();
|
|
287
369
|
const baseline: AccessibilityBaseline = [
|
|
288
|
-
{
|
|
289
|
-
|
|
370
|
+
{
|
|
371
|
+
rule: "color-contrast",
|
|
372
|
+
targets: ["#sidebar .tag"],
|
|
373
|
+
reason: "Known issue",
|
|
374
|
+
willBeFixedIn: "PROJ-123",
|
|
375
|
+
},
|
|
376
|
+
];
|
|
290
377
|
|
|
291
378
|
await checkAccessibility(makePage() as any, testInfo as any, {
|
|
292
|
-
bestPracticeMode:
|
|
379
|
+
bestPracticeMode: "off",
|
|
293
380
|
baseline,
|
|
294
|
-
})
|
|
295
|
-
|
|
296
|
-
const baselinedAnnotations = testInfo.annotations.filter(
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
381
|
+
});
|
|
382
|
+
|
|
383
|
+
const baselinedAnnotations = testInfo.annotations.filter(
|
|
384
|
+
(a) => a.type === "Baselined a11y violation",
|
|
385
|
+
);
|
|
386
|
+
expect(baselinedAnnotations).toHaveLength(1);
|
|
387
|
+
expect(baselinedAnnotations[0].description).toBe(
|
|
388
|
+
"color-contrast: Known issue — PROJ-123",
|
|
389
|
+
);
|
|
390
|
+
});
|
|
391
|
+
|
|
392
|
+
it("pushes Stale a11y baseline entry annotations for unmatched baseline entries", async () => {
|
|
393
|
+
const wcagResults = makeAxeResults({ violations: [] });
|
|
394
|
+
mockAnalyze.mockResolvedValue(wcagResults);
|
|
395
|
+
|
|
396
|
+
const testInfo = makeTestInfo();
|
|
306
397
|
const baseline: AccessibilityBaseline = [
|
|
307
|
-
{
|
|
308
|
-
|
|
398
|
+
{
|
|
399
|
+
rule: "old-rule",
|
|
400
|
+
targets: [".gone"],
|
|
401
|
+
reason: "Was broken",
|
|
402
|
+
willBeFixedIn: "PROJ-999",
|
|
403
|
+
},
|
|
404
|
+
];
|
|
309
405
|
|
|
310
406
|
await checkAccessibility(makePage() as any, testInfo as any, {
|
|
311
|
-
bestPracticeMode:
|
|
407
|
+
bestPracticeMode: "off",
|
|
312
408
|
baseline,
|
|
313
|
-
})
|
|
314
|
-
|
|
315
|
-
const staleAnnotations = testInfo.annotations.filter(
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
409
|
+
});
|
|
410
|
+
|
|
411
|
+
const staleAnnotations = testInfo.annotations.filter(
|
|
412
|
+
(a) => a.type === "Stale a11y baseline entry",
|
|
413
|
+
);
|
|
414
|
+
expect(staleAnnotations).toHaveLength(1);
|
|
415
|
+
expect(staleAnnotations[0].description).toBe(
|
|
416
|
+
"old-rule on .gone — no longer detected",
|
|
417
|
+
);
|
|
418
|
+
});
|
|
419
|
+
});
|
|
420
|
+
|
|
421
|
+
describe("baseline summary annotation", () => {
|
|
422
|
+
it("includes correct counts in summary annotation", async () => {
|
|
323
423
|
const wcagResults = makeAxeResults({
|
|
324
424
|
violations: [
|
|
325
|
-
makeViolation(
|
|
326
|
-
makeViolation(
|
|
425
|
+
makeViolation("color-contrast", [["#sidebar .tag"]]),
|
|
426
|
+
makeViolation("image-alt", [["img.hero"]]),
|
|
327
427
|
],
|
|
328
|
-
})
|
|
329
|
-
mockAnalyze.mockResolvedValue(wcagResults)
|
|
428
|
+
});
|
|
429
|
+
mockAnalyze.mockResolvedValue(wcagResults);
|
|
330
430
|
|
|
331
|
-
const testInfo = makeTestInfo()
|
|
431
|
+
const testInfo = makeTestInfo();
|
|
332
432
|
const baseline: AccessibilityBaseline = [
|
|
333
|
-
{
|
|
334
|
-
|
|
433
|
+
{
|
|
434
|
+
rule: "color-contrast",
|
|
435
|
+
targets: ["#sidebar .tag"],
|
|
436
|
+
reason: "Known",
|
|
437
|
+
willBeFixedIn: "PROJ-1",
|
|
438
|
+
},
|
|
439
|
+
];
|
|
335
440
|
|
|
336
441
|
await checkAccessibility(makePage() as any, testInfo as any, {
|
|
337
|
-
bestPracticeMode:
|
|
442
|
+
bestPracticeMode: "off",
|
|
338
443
|
baseline,
|
|
339
|
-
})
|
|
444
|
+
});
|
|
340
445
|
|
|
341
446
|
const summaryAnnotations = testInfo.annotations.filter(
|
|
342
|
-
a =>
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
expect(summaryAnnotations
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
447
|
+
(a) =>
|
|
448
|
+
a.type === "Accessibility" && a.description?.includes("baselined"),
|
|
449
|
+
);
|
|
450
|
+
expect(summaryAnnotations).toHaveLength(1);
|
|
451
|
+
expect(summaryAnnotations[0].description).toBe(
|
|
452
|
+
"WCAG scan: 1 new violations (1 baselined)",
|
|
453
|
+
);
|
|
454
|
+
});
|
|
455
|
+
});
|
|
456
|
+
|
|
457
|
+
describe("snapshot-mode baseline suggestions", () => {
|
|
458
|
+
it("attaches baseline suggestions when violations exist in snapshot mode", async () => {
|
|
351
459
|
const wcagResults = makeAxeResults({
|
|
352
|
-
violations: [makeViolation(
|
|
353
|
-
})
|
|
354
|
-
mockAnalyze.mockResolvedValue(wcagResults)
|
|
460
|
+
violations: [makeViolation("color-contrast", [["#sidebar .tag"]])],
|
|
461
|
+
});
|
|
462
|
+
mockAnalyze.mockResolvedValue(wcagResults);
|
|
355
463
|
|
|
356
|
-
const testInfo = makeTestInfo()
|
|
464
|
+
const testInfo = makeTestInfo();
|
|
357
465
|
await checkAccessibility(makePage() as any, testInfo as any, {
|
|
358
|
-
bestPracticeMode:
|
|
359
|
-
})
|
|
466
|
+
bestPracticeMode: "off",
|
|
467
|
+
});
|
|
360
468
|
|
|
361
469
|
// Should attach a11y-baseline-suggestions
|
|
362
470
|
expect(testInfo.attach).toHaveBeenCalledWith(
|
|
363
|
-
|
|
471
|
+
"a11y-baseline-suggestions",
|
|
364
472
|
expect.objectContaining({
|
|
365
|
-
contentType:
|
|
366
|
-
})
|
|
367
|
-
)
|
|
473
|
+
contentType: "text/plain",
|
|
474
|
+
}),
|
|
475
|
+
);
|
|
368
476
|
|
|
369
477
|
// Check the attachment body contains the suggestion
|
|
370
478
|
const attachCall = testInfo.attach.mock.calls.find(
|
|
371
|
-
(call: any[]) => call[0] ===
|
|
372
|
-
)
|
|
373
|
-
expect(attachCall).toBeTruthy()
|
|
374
|
-
const body = attachCall![1].body as string
|
|
375
|
-
expect(body).toContain("rule: 'color-contrast'")
|
|
376
|
-
expect(body).toContain("targets: ['#sidebar .tag']")
|
|
479
|
+
(call: any[]) => call[0] === "a11y-baseline-suggestions",
|
|
480
|
+
);
|
|
481
|
+
expect(attachCall).toBeTruthy();
|
|
482
|
+
const body = attachCall![1].body as string;
|
|
483
|
+
expect(body).toContain("rule: 'color-contrast'");
|
|
484
|
+
expect(body).toContain("targets: ['#sidebar .tag']");
|
|
377
485
|
|
|
378
486
|
// Should push annotation about switching to baseline mode
|
|
379
|
-
const baselineHintAnnotation = testInfo.annotations.find(
|
|
380
|
-
a
|
|
381
|
-
)
|
|
382
|
-
expect(baselineHintAnnotation).toBeTruthy()
|
|
383
|
-
expect(baselineHintAnnotation!.type).toBe(
|
|
384
|
-
})
|
|
385
|
-
|
|
386
|
-
it(
|
|
387
|
-
const wcagResults = makeAxeResults({ violations: [] })
|
|
388
|
-
mockAnalyze.mockResolvedValue(wcagResults)
|
|
389
|
-
|
|
390
|
-
const testInfo = makeTestInfo()
|
|
487
|
+
const baselineHintAnnotation = testInfo.annotations.find((a) =>
|
|
488
|
+
a.description?.includes("switch to baseline mode"),
|
|
489
|
+
);
|
|
490
|
+
expect(baselineHintAnnotation).toBeTruthy();
|
|
491
|
+
expect(baselineHintAnnotation!.type).toBe("Accessibility");
|
|
492
|
+
});
|
|
493
|
+
|
|
494
|
+
it("does not attach baseline suggestions when no violations in snapshot mode", async () => {
|
|
495
|
+
const wcagResults = makeAxeResults({ violations: [] });
|
|
496
|
+
mockAnalyze.mockResolvedValue(wcagResults);
|
|
497
|
+
|
|
498
|
+
const testInfo = makeTestInfo();
|
|
391
499
|
await checkAccessibility(makePage() as any, testInfo as any, {
|
|
392
|
-
bestPracticeMode:
|
|
393
|
-
})
|
|
500
|
+
bestPracticeMode: "off",
|
|
501
|
+
});
|
|
394
502
|
|
|
395
503
|
expect(testInfo.attach).not.toHaveBeenCalledWith(
|
|
396
|
-
|
|
397
|
-
expect.anything()
|
|
398
|
-
)
|
|
399
|
-
})
|
|
400
|
-
})
|
|
504
|
+
"a11y-baseline-suggestions",
|
|
505
|
+
expect.anything(),
|
|
506
|
+
);
|
|
507
|
+
});
|
|
508
|
+
});
|
|
401
509
|
|
|
402
|
-
describe(
|
|
403
|
-
let tmpDir: string
|
|
404
|
-
let originalCI: string | undefined
|
|
510
|
+
describe("on-disk baseline (new default for snapshotless tests)", () => {
|
|
511
|
+
let tmpDir: string;
|
|
512
|
+
let originalCI: string | undefined;
|
|
405
513
|
|
|
406
514
|
beforeEach(async () => {
|
|
407
|
-
const fs = await import(
|
|
408
|
-
const os = await import(
|
|
409
|
-
const path = await import(
|
|
410
|
-
tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(),
|
|
411
|
-
originalCI = process.env.CI
|
|
412
|
-
delete process.env.CI
|
|
413
|
-
})
|
|
515
|
+
const fs = await import("fs");
|
|
516
|
+
const os = await import("os");
|
|
517
|
+
const path = await import("path");
|
|
518
|
+
tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), "a11y-disp-"));
|
|
519
|
+
originalCI = process.env.CI;
|
|
520
|
+
delete process.env.CI;
|
|
521
|
+
});
|
|
414
522
|
|
|
415
523
|
afterEach(async () => {
|
|
416
|
-
const fs = await import(
|
|
417
|
-
await fs.promises.rm(tmpDir, { recursive: true, force: true })
|
|
418
|
-
if (originalCI === undefined) delete process.env.CI
|
|
419
|
-
else process.env.CI = originalCI
|
|
420
|
-
})
|
|
421
|
-
|
|
422
|
-
it(
|
|
423
|
-
mockAnalyze.mockResolvedValue(makeAxeResults({ violations: [] }))
|
|
424
|
-
|
|
425
|
-
const testInfo = makeTestInfo({
|
|
524
|
+
const fs = await import("fs");
|
|
525
|
+
await fs.promises.rm(tmpDir, { recursive: true, force: true });
|
|
526
|
+
if (originalCI === undefined) delete process.env.CI;
|
|
527
|
+
else process.env.CI = originalCI;
|
|
528
|
+
});
|
|
529
|
+
|
|
530
|
+
it("seeds an empty baseline file when no violations and no snapshot exist (local)", async () => {
|
|
531
|
+
mockAnalyze.mockResolvedValue(makeAxeResults({ violations: [] }));
|
|
532
|
+
|
|
533
|
+
const testInfo = makeTestInfo({
|
|
534
|
+
updateSnapshots: "none",
|
|
535
|
+
snapshotsDir: tmpDir,
|
|
536
|
+
title: "clean test",
|
|
537
|
+
});
|
|
426
538
|
await checkAccessibility(makePage() as any, testInfo as any, {
|
|
427
|
-
bestPracticeMode:
|
|
428
|
-
})
|
|
539
|
+
bestPracticeMode: "off",
|
|
540
|
+
});
|
|
429
541
|
|
|
430
|
-
const fs = await import(
|
|
431
|
-
const path = await import(
|
|
432
|
-
const file = path.join(tmpDir,
|
|
433
|
-
const written = JSON.parse(await fs.promises.readFile(file,
|
|
434
|
-
expect(written.note).toBe(
|
|
435
|
-
expect(written.violations).toEqual([])
|
|
542
|
+
const fs = await import("fs");
|
|
543
|
+
const path = await import("path");
|
|
544
|
+
const file = path.join(tmpDir, "clean-test-1.a11y-baseline.json");
|
|
545
|
+
const written = JSON.parse(await fs.promises.readFile(file, "utf8"));
|
|
546
|
+
expect(written.note).toBe("No accessibility violations found");
|
|
547
|
+
expect(written.violations).toEqual([]);
|
|
436
548
|
|
|
437
549
|
// Snapshot mode is NOT used; on-disk baseline mode used assertBaseline.
|
|
438
|
-
expect(mockToMatchSnapshot).not.toHaveBeenCalled()
|
|
550
|
+
expect(mockToMatchSnapshot).not.toHaveBeenCalled();
|
|
439
551
|
// Test does not fail (no toBe call against a "baseline file present" sentinel).
|
|
440
|
-
expect(mockToBe).not.toHaveBeenCalled()
|
|
441
|
-
})
|
|
442
|
-
|
|
443
|
-
it(
|
|
444
|
-
mockAnalyze.mockResolvedValue(
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
552
|
+
expect(mockToBe).not.toHaveBeenCalled();
|
|
553
|
+
});
|
|
554
|
+
|
|
555
|
+
it("seeds a TODO-prompted baseline file when violations exist and no snapshot (local)", async () => {
|
|
556
|
+
mockAnalyze.mockResolvedValue(
|
|
557
|
+
makeAxeResults({
|
|
558
|
+
violations: [makeViolation("color-contrast", [["#footer .legal"]])],
|
|
559
|
+
}),
|
|
560
|
+
);
|
|
561
|
+
|
|
562
|
+
const testInfo = makeTestInfo({
|
|
563
|
+
updateSnapshots: "none",
|
|
564
|
+
snapshotsDir: tmpDir,
|
|
565
|
+
title: "with violations",
|
|
566
|
+
});
|
|
449
567
|
await checkAccessibility(makePage() as any, testInfo as any, {
|
|
450
|
-
bestPracticeMode:
|
|
451
|
-
})
|
|
452
|
-
|
|
453
|
-
const fs = await import(
|
|
454
|
-
const path = await import(
|
|
455
|
-
const file = path.join(tmpDir,
|
|
456
|
-
const written = JSON.parse(await fs.promises.readFile(file,
|
|
457
|
-
expect(written.note).toMatch(/TODO/)
|
|
568
|
+
bestPracticeMode: "off",
|
|
569
|
+
});
|
|
570
|
+
|
|
571
|
+
const fs = await import("fs");
|
|
572
|
+
const path = await import("path");
|
|
573
|
+
const file = path.join(tmpDir, "with-violations-1.a11y-baseline.json");
|
|
574
|
+
const written = JSON.parse(await fs.promises.readFile(file, "utf8"));
|
|
575
|
+
expect(written.note).toMatch(/TODO/);
|
|
458
576
|
expect(written.violations).toEqual([
|
|
459
|
-
{
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
577
|
+
{
|
|
578
|
+
rule: "color-contrast",
|
|
579
|
+
targets: ["#footer .legal"],
|
|
580
|
+
reason: "TODO",
|
|
581
|
+
willBeFixedIn: "TODO",
|
|
582
|
+
},
|
|
583
|
+
]);
|
|
584
|
+
expect(mockToBe).not.toHaveBeenCalled();
|
|
585
|
+
});
|
|
586
|
+
|
|
587
|
+
it("fails the test on CI when seeding (mirrors Playwright missing-snapshot behaviour)", async () => {
|
|
588
|
+
process.env.CI = "1";
|
|
589
|
+
mockAnalyze.mockResolvedValue(
|
|
590
|
+
makeAxeResults({
|
|
591
|
+
violations: [makeViolation("color-contrast", [["#x"]])],
|
|
592
|
+
}),
|
|
593
|
+
);
|
|
594
|
+
|
|
595
|
+
const testInfo = makeTestInfo({
|
|
596
|
+
updateSnapshots: "none",
|
|
597
|
+
snapshotsDir: tmpDir,
|
|
598
|
+
title: "on ci",
|
|
599
|
+
});
|
|
471
600
|
await checkAccessibility(makePage() as any, testInfo as any, {
|
|
472
|
-
bestPracticeMode:
|
|
473
|
-
})
|
|
601
|
+
bestPracticeMode: "off",
|
|
602
|
+
});
|
|
474
603
|
|
|
475
604
|
// File is still seeded.
|
|
476
|
-
const fs = await import(
|
|
477
|
-
const path = await import(
|
|
478
|
-
const file = path.join(tmpDir,
|
|
479
|
-
expect(fs.existsSync(file)).toBe(true)
|
|
605
|
+
const fs = await import("fs");
|
|
606
|
+
const path = await import("path");
|
|
607
|
+
const file = path.join(tmpDir, "on-ci-1.a11y-baseline.json");
|
|
608
|
+
expect(fs.existsSync(file)).toBe(true);
|
|
480
609
|
|
|
481
610
|
// Test fails with the seed-and-commit directive.
|
|
482
|
-
expect(mockToBe).toHaveBeenCalled()
|
|
611
|
+
expect(mockToBe).toHaveBeenCalled();
|
|
483
612
|
const failureMessage = mockExpectHard.mock.calls.find(
|
|
484
|
-
(call: any[]) =>
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
613
|
+
(call: any[]) =>
|
|
614
|
+
call.length >= 2 &&
|
|
615
|
+
typeof call[1] === "string" &&
|
|
616
|
+
call[1].includes("a11y baseline file was missing"),
|
|
617
|
+
)?.[1] as string;
|
|
618
|
+
expect(failureMessage).toContain(file);
|
|
619
|
+
expect(failureMessage).toMatch(/commit it/i);
|
|
620
|
+
});
|
|
621
|
+
|
|
622
|
+
it("loads an existing on-disk baseline JSON and matches violations against it", async () => {
|
|
623
|
+
mockAnalyze.mockResolvedValue(
|
|
624
|
+
makeAxeResults({
|
|
625
|
+
violations: [makeViolation("color-contrast", [["#known"]])],
|
|
626
|
+
}),
|
|
627
|
+
);
|
|
494
628
|
|
|
495
629
|
// Pre-write a baseline.
|
|
496
|
-
const fs = await import(
|
|
497
|
-
const path = await import(
|
|
498
|
-
const file = path.join(tmpDir,
|
|
499
|
-
await fs.promises.writeFile(
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
630
|
+
const fs = await import("fs");
|
|
631
|
+
const path = await import("path");
|
|
632
|
+
const file = path.join(tmpDir, "known-test-1.a11y-baseline.json");
|
|
633
|
+
await fs.promises.writeFile(
|
|
634
|
+
file,
|
|
635
|
+
JSON.stringify({
|
|
636
|
+
note: "Known issue tracked in PROJ-1.",
|
|
637
|
+
violations: [
|
|
638
|
+
{
|
|
639
|
+
rule: "color-contrast",
|
|
640
|
+
targets: ["#known"],
|
|
641
|
+
reason: "Known",
|
|
642
|
+
willBeFixedIn: "PROJ-1",
|
|
643
|
+
},
|
|
644
|
+
],
|
|
645
|
+
}),
|
|
646
|
+
);
|
|
647
|
+
|
|
648
|
+
const testInfo = makeTestInfo({
|
|
649
|
+
updateSnapshots: "none",
|
|
650
|
+
snapshotsDir: tmpDir,
|
|
651
|
+
title: "known test",
|
|
652
|
+
});
|
|
505
653
|
await checkAccessibility(makePage() as any, testInfo as any, {
|
|
506
|
-
bestPracticeMode:
|
|
507
|
-
})
|
|
654
|
+
bestPracticeMode: "off",
|
|
655
|
+
});
|
|
508
656
|
|
|
509
657
|
// Violation matched -> no failure.
|
|
510
|
-
expect(mockToBe).not.toHaveBeenCalled()
|
|
511
|
-
const baselined = testInfo.annotations.filter(
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
const
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
658
|
+
expect(mockToBe).not.toHaveBeenCalled();
|
|
659
|
+
const baselined = testInfo.annotations.filter(
|
|
660
|
+
(a) => a.type === "Baselined a11y violation",
|
|
661
|
+
);
|
|
662
|
+
expect(baselined).toHaveLength(1);
|
|
663
|
+
});
|
|
664
|
+
|
|
665
|
+
it("rejects a persisted waiver whose metadata is still TODO", async () => {
|
|
666
|
+
mockAnalyze.mockResolvedValue(
|
|
667
|
+
makeAxeResults({
|
|
668
|
+
violations: [makeViolation("color-contrast", [["#known"]])],
|
|
669
|
+
}),
|
|
670
|
+
);
|
|
671
|
+
|
|
672
|
+
const fs = await import("fs");
|
|
673
|
+
const path = await import("path");
|
|
674
|
+
const file = path.join(tmpDir, "unfinished-test-1.a11y-baseline.json");
|
|
675
|
+
await fs.promises.writeFile(
|
|
676
|
+
file,
|
|
677
|
+
JSON.stringify({
|
|
678
|
+
note: "Seeded",
|
|
679
|
+
violations: [
|
|
680
|
+
{
|
|
681
|
+
rule: "color-contrast",
|
|
682
|
+
targets: ["#known"],
|
|
683
|
+
reason: "TODO",
|
|
684
|
+
willBeFixedIn: "TODO",
|
|
685
|
+
},
|
|
686
|
+
],
|
|
687
|
+
}),
|
|
688
|
+
);
|
|
689
|
+
|
|
690
|
+
const testInfo = makeTestInfo({
|
|
691
|
+
updateSnapshots: "none",
|
|
692
|
+
snapshotsDir: tmpDir,
|
|
693
|
+
title: "unfinished test",
|
|
694
|
+
});
|
|
695
|
+
await expect(
|
|
696
|
+
checkAccessibility(makePage() as any, testInfo as any, {
|
|
697
|
+
bestPracticeMode: "off",
|
|
698
|
+
}),
|
|
699
|
+
).rejects.toThrow("requires a reason");
|
|
700
|
+
});
|
|
701
|
+
|
|
702
|
+
it("produces per-call counter files for multi-call tests", async () => {
|
|
703
|
+
mockAnalyze.mockResolvedValue(makeAxeResults({ violations: [] }));
|
|
704
|
+
|
|
705
|
+
const testInfo = makeTestInfo({
|
|
706
|
+
updateSnapshots: "none",
|
|
707
|
+
snapshotsDir: tmpDir,
|
|
708
|
+
title: "multi call",
|
|
709
|
+
});
|
|
710
|
+
await checkAccessibility(makePage() as any, testInfo as any, {
|
|
711
|
+
bestPracticeMode: "off",
|
|
712
|
+
});
|
|
713
|
+
await checkAccessibility(makePage() as any, testInfo as any, {
|
|
714
|
+
bestPracticeMode: "off",
|
|
715
|
+
});
|
|
716
|
+
|
|
717
|
+
const fs = await import("fs");
|
|
718
|
+
const path = await import("path");
|
|
719
|
+
const f1 = path.join(tmpDir, "multi-call-1.a11y-baseline.json");
|
|
720
|
+
const f2 = path.join(tmpDir, "multi-call-2.a11y-baseline.json");
|
|
721
|
+
expect(fs.existsSync(f1)).toBe(true);
|
|
722
|
+
expect(fs.existsSync(f2)).toBe(true);
|
|
723
|
+
});
|
|
724
|
+
|
|
725
|
+
it("still uses snapshot mode when a legacy snapshot file exists for the test", async () => {
|
|
726
|
+
mockAnalyze.mockResolvedValue(makeAxeResults({ violations: [] }));
|
|
551
727
|
|
|
552
728
|
// Pre-create a Playwright-style snapshot file in the snapshots dir.
|
|
553
|
-
const fs = await import(
|
|
554
|
-
const path = await import(
|
|
729
|
+
const fs = await import("fs");
|
|
730
|
+
const path = await import("path");
|
|
555
731
|
await fs.promises.writeFile(
|
|
556
|
-
path.join(tmpDir,
|
|
557
|
-
|
|
558
|
-
)
|
|
559
|
-
|
|
560
|
-
const testInfo = makeTestInfo({
|
|
561
|
-
|
|
732
|
+
path.join(tmpDir, "legacy-test-1-chromium-linux.txt"),
|
|
733
|
+
"[]\n",
|
|
734
|
+
);
|
|
735
|
+
|
|
736
|
+
const testInfo = makeTestInfo({
|
|
737
|
+
updateSnapshots: "none",
|
|
738
|
+
snapshotsDir: tmpDir,
|
|
739
|
+
title: "legacy test",
|
|
740
|
+
});
|
|
741
|
+
await checkAccessibility(makePage() as any, testInfo as any, {
|
|
742
|
+
bestPracticeMode: "off",
|
|
743
|
+
});
|
|
562
744
|
|
|
563
745
|
// Snapshot mode -> toMatchSnapshot was invoked.
|
|
564
|
-
expect(mockToMatchSnapshot).toHaveBeenCalled()
|
|
565
|
-
})
|
|
746
|
+
expect(mockToMatchSnapshot).toHaveBeenCalled();
|
|
747
|
+
});
|
|
566
748
|
|
|
567
749
|
it("uses on-disk baseline mode when updateSnapshots is 'missing' (Playwright's default)", async () => {
|
|
568
|
-
mockAnalyze.mockResolvedValue(makeAxeResults({ violations: [] }))
|
|
750
|
+
mockAnalyze.mockResolvedValue(makeAxeResults({ violations: [] }));
|
|
569
751
|
|
|
570
|
-
const testInfo = makeTestInfo({
|
|
571
|
-
|
|
752
|
+
const testInfo = makeTestInfo({
|
|
753
|
+
updateSnapshots: "missing",
|
|
754
|
+
snapshotsDir: tmpDir,
|
|
755
|
+
title: "default config",
|
|
756
|
+
});
|
|
757
|
+
await checkAccessibility(makePage() as any, testInfo as any, {
|
|
758
|
+
bestPracticeMode: "off",
|
|
759
|
+
});
|
|
572
760
|
|
|
573
761
|
// Playwright's default config value is 'missing' — not an explicit user
|
|
574
762
|
// opt-in via --update-snapshots. It must route to baseline mode so new
|
|
575
763
|
// a11y tests default to baseline mode as documented.
|
|
576
|
-
const fs = await import(
|
|
577
|
-
const path = await import(
|
|
578
|
-
const file = path.join(tmpDir,
|
|
579
|
-
expect(fs.existsSync(file)).toBe(true)
|
|
580
|
-
expect(mockToMatchSnapshot).not.toHaveBeenCalled()
|
|
581
|
-
})
|
|
764
|
+
const fs = await import("fs");
|
|
765
|
+
const path = await import("path");
|
|
766
|
+
const file = path.join(tmpDir, "default-config-1.a11y-baseline.json");
|
|
767
|
+
expect(fs.existsSync(file)).toBe(true);
|
|
768
|
+
expect(mockToMatchSnapshot).not.toHaveBeenCalled();
|
|
769
|
+
});
|
|
582
770
|
|
|
583
771
|
it("uses snapshot mode when updateSnapshots is 'changed' (explicit --update-snapshots)", async () => {
|
|
584
|
-
mockAnalyze.mockResolvedValue(makeAxeResults({ violations: [] }))
|
|
772
|
+
mockAnalyze.mockResolvedValue(makeAxeResults({ violations: [] }));
|
|
585
773
|
|
|
586
|
-
const testInfo = makeTestInfo({
|
|
587
|
-
|
|
774
|
+
const testInfo = makeTestInfo({
|
|
775
|
+
updateSnapshots: "changed",
|
|
776
|
+
snapshotsDir: tmpDir,
|
|
777
|
+
title: "changed mode",
|
|
778
|
+
});
|
|
779
|
+
await checkAccessibility(makePage() as any, testInfo as any, {
|
|
780
|
+
bestPracticeMode: "off",
|
|
781
|
+
});
|
|
588
782
|
|
|
589
|
-
expect(mockToMatchSnapshot).toHaveBeenCalled()
|
|
590
|
-
})
|
|
783
|
+
expect(mockToMatchSnapshot).toHaveBeenCalled();
|
|
784
|
+
});
|
|
591
785
|
|
|
592
786
|
it("uses snapshot mode when updateSnapshots is 'all' (explicit --update-snapshots=all)", async () => {
|
|
593
|
-
mockAnalyze.mockResolvedValue(makeAxeResults({ violations: [] }))
|
|
787
|
+
mockAnalyze.mockResolvedValue(makeAxeResults({ violations: [] }));
|
|
594
788
|
|
|
595
|
-
const testInfo = makeTestInfo({
|
|
596
|
-
|
|
789
|
+
const testInfo = makeTestInfo({
|
|
790
|
+
updateSnapshots: "all",
|
|
791
|
+
snapshotsDir: tmpDir,
|
|
792
|
+
title: "all mode",
|
|
793
|
+
});
|
|
794
|
+
await checkAccessibility(makePage() as any, testInfo as any, {
|
|
795
|
+
bestPracticeMode: "off",
|
|
796
|
+
});
|
|
597
797
|
|
|
598
|
-
expect(mockToMatchSnapshot).toHaveBeenCalled()
|
|
599
|
-
})
|
|
600
|
-
})
|
|
601
|
-
})
|
|
798
|
+
expect(mockToMatchSnapshot).toHaveBeenCalled();
|
|
799
|
+
});
|
|
800
|
+
});
|
|
801
|
+
});
|