@lullabot/playwright-testing 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/bin/github-a11y-summary +2 -2
  2. package/bin/github-failure-summary +4 -4
  3. package/lib/accessibility-baseline-file.d.ts +6 -6
  4. package/lib/accessibility-baseline-file.js +29 -23
  5. package/lib/accessibility-baseline.js +5 -3
  6. package/lib/accessible-screenshot.d.ts +4 -4
  7. package/lib/accessible-screenshot.js +110 -88
  8. package/lib/focus.js +4 -1
  9. package/lib/frames.js +4 -4
  10. package/lib/github/a11y-summary.js +74 -79
  11. package/lib/github/attachments.d.ts +1 -1
  12. package/lib/github/attachments.js +36 -34
  13. package/lib/github/failure-summary.d.ts +4 -4
  14. package/lib/github/failure-summary.js +140 -108
  15. package/lib/github/index.d.ts +6 -6
  16. package/lib/github/report-paths.js +17 -9
  17. package/lib/hover.js +17 -17
  18. package/lib/images.js +15 -12
  19. package/lib/index.d.ts +13 -13
  20. package/lib/interaction-states.d.ts +2 -2
  21. package/lib/interaction-states.js +7 -7
  22. package/lib/mock/index.d.ts +1 -1
  23. package/lib/mock/youtube.d.ts +1 -1
  24. package/lib/mock/youtube.js +1 -1
  25. package/lib/pseudo-state.d.ts +2 -2
  26. package/lib/pseudo-state.js +15 -7
  27. package/lib/videos.js +21 -10
  28. package/lib/visualdiff.d.ts +4 -4
  29. package/lib/visualdiff.js +35 -27
  30. package/package.json +3 -3
  31. package/src/accessibility-baseline-file.test.ts +190 -154
  32. package/src/accessibility-baseline-file.ts +107 -83
  33. package/src/accessibility-baseline.test.ts +869 -433
  34. package/src/accessibility-baseline.ts +41 -21
  35. package/src/accessible-screenshot.test.ts +551 -423
  36. package/src/accessible-screenshot.ts +358 -251
  37. package/src/focus.test.ts +23 -23
  38. package/src/focus.ts +7 -2
  39. package/src/fonts.test.ts +13 -13
  40. package/src/fonts.ts +1 -1
  41. package/src/frames.test.ts +10 -8
  42. package/src/frames.ts +5 -5
  43. package/src/github/a11y-summary.test.ts +436 -332
  44. package/src/github/a11y-summary.ts +189 -179
  45. package/src/github/attachments.test.ts +228 -167
  46. package/src/github/attachments.ts +144 -121
  47. package/src/github/failure-summary.test.ts +660 -445
  48. package/src/github/failure-summary.ts +376 -274
  49. package/src/github/index.test.ts +14 -14
  50. package/src/github/index.ts +6 -6
  51. package/src/github/report-paths.test.ts +211 -156
  52. package/src/github/report-paths.ts +75 -61
  53. package/src/hover.test.ts +50 -46
  54. package/src/hover.ts +18 -18
  55. package/src/images.test.ts +251 -206
  56. package/src/images.ts +52 -43
  57. package/src/index.ts +13 -13
  58. package/src/interaction-states.test.ts +72 -42
  59. package/src/interaction-states.ts +43 -29
  60. package/src/mock/index.ts +1 -1
  61. package/src/mock/youtube.test.ts +26 -26
  62. package/src/mock/youtube.ts +4 -6
  63. package/src/pseudo-state.test.ts +58 -51
  64. package/src/pseudo-state.ts +32 -27
  65. package/src/videos.test.ts +559 -399
  66. package/src/videos.ts +68 -44
  67. package/src/visualdiff.test.ts +416 -315
  68. package/src/visualdiff.ts +140 -99
@@ -1,601 +1,1037 @@
1
- import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
1
+ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
2
2
 
3
3
  // Mock AxeBuilder — must be hoisted since vi.mock is hoisted
4
- const { mockWithTags, mockExclude, mockOptions, mockAnalyze, MockAxeBuilder } = vi.hoisted(() => {
5
- const mockWithTags = vi.fn().mockReturnThis()
6
- const mockExclude = vi.fn().mockReturnThis()
7
- const mockOptions = vi.fn().mockReturnThis()
8
- const mockAnalyze = vi.fn()
4
+ const { mockAnalyze, MockAxeBuilder } = vi.hoisted(() => {
5
+ const mockWithTags = vi.fn().mockReturnThis();
6
+ const mockExclude = vi.fn().mockReturnThis();
7
+ const mockOptions = vi.fn().mockReturnThis();
8
+ const mockAnalyze = vi.fn();
9
9
 
10
10
  class MockAxeBuilder {
11
- withTags = mockWithTags
12
- exclude = mockExclude
13
- options = mockOptions
14
- analyze = mockAnalyze
15
- constructor(_args: any) {}
11
+ withTags = mockWithTags;
12
+ exclude = mockExclude;
13
+ options = mockOptions;
14
+ analyze = mockAnalyze;
16
15
  }
17
16
 
18
- return { mockWithTags, mockExclude, mockOptions, mockAnalyze, MockAxeBuilder }
19
- })
20
-
21
- vi.mock('@axe-core/playwright', () => ({
17
+ return {
18
+ mockWithTags,
19
+ mockExclude,
20
+ mockOptions,
21
+ mockAnalyze,
22
+ MockAxeBuilder,
23
+ };
24
+ });
25
+
26
+ vi.mock("@axe-core/playwright", () => ({
22
27
  default: MockAxeBuilder,
23
- }))
28
+ }));
24
29
 
25
30
  // Mock @playwright/test
26
- const { mockToMatchSnapshot, mockToBe, mockExpectSoft, mockExpectHard } = vi.hoisted(() => {
27
- const mockToMatchSnapshot = vi.fn()
28
- const mockToBe = vi.fn()
29
- const mockExpectSoft = vi.fn(() => ({
30
- toMatchSnapshot: mockToMatchSnapshot,
31
- toHaveScreenshot: vi.fn(),
32
- }))
33
- const mockExpectHard = vi.fn((_val?: any, _msg?: string) => ({
34
- toMatchSnapshot: mockToMatchSnapshot,
35
- toBe: mockToBe,
36
- }))
37
- return { mockToMatchSnapshot, mockToBe, mockExpectSoft, mockExpectHard }
38
- })
39
-
40
- vi.mock('@playwright/test', () => {
31
+ const { mockToMatchSnapshot, mockToBe, mockExpectSoft, mockExpectHard } =
32
+ vi.hoisted(() => {
33
+ const mockToMatchSnapshot = vi.fn();
34
+ const mockToBe = vi.fn();
35
+ const mockExpectSoft = vi.fn(() => ({
36
+ toMatchSnapshot: mockToMatchSnapshot,
37
+ toHaveScreenshot: vi.fn(),
38
+ }));
39
+ const mockExpectHard = vi.fn(() => ({
40
+ toMatchSnapshot: mockToMatchSnapshot,
41
+ toBe: mockToBe,
42
+ }));
43
+ return { mockToMatchSnapshot, mockToBe, mockExpectSoft, mockExpectHard };
44
+ });
45
+
46
+ vi.mock("@playwright/test", () => {
41
47
  const expectFn = Object.assign(mockExpectHard, {
42
48
  soft: mockExpectSoft,
43
- })
49
+ });
44
50
  return {
45
51
  expect: expectFn,
46
52
  Locator: {},
47
53
  Page: {},
48
54
  TestInfo: {},
49
- }
50
- })
51
-
52
- import { checkAccessibility } from './accessible-screenshot'
53
- import { defineAccessibilityBaseline, AccessibilityBaseline } from './accessibility-baseline'
54
-
55
- function makeAxeResults(overrides?: Partial<{ violations: any[], passes: any[] }>) {
55
+ };
56
+ });
57
+
58
+ import { checkAccessibility } from "./accessible-screenshot";
59
+ import {
60
+ defineAccessibilityBaseline,
61
+ AccessibilityBaseline,
62
+ } from "./accessibility-baseline";
63
+
64
+ function makeAxeResults(
65
+ overrides?: Partial<{ violations: any[]; passes: any[] }>,
66
+ ) {
56
67
  return {
57
68
  violations: [],
58
69
  passes: [],
59
70
  incomplete: [],
60
71
  inapplicable: [],
61
72
  ...overrides,
62
- }
73
+ };
63
74
  }
64
75
 
65
- function makeTestInfo(opts?: { updateSnapshots?: 'all' | 'changed' | 'missing' | 'none', snapshotsDir?: string, title?: string }) {
76
+ function makeTestInfo(opts?: {
77
+ updateSnapshots?: "all" | "changed" | "missing" | "none";
78
+ snapshotsDir?: string;
79
+ title?: string;
80
+ }) {
66
81
  // Default to `updateSnapshots: 'all'` so that, in the absence of an
67
82
  // explicit `baseline` option, the dispatch routes to snapshot mode (the
68
83
  // legacy behavior these tests were originally written against). Tests
69
84
  // that want to exercise on-disk baseline mode override this.
70
- const dir = opts?.snapshotsDir ?? '/tmp/__a11y_test_snapshots__'
71
- const title = opts?.title ?? 'mocked test'
85
+ const dir = opts?.snapshotsDir ?? "/tmp/__a11y_test_snapshots__";
86
+ const title = opts?.title ?? "mocked test";
72
87
  return {
73
88
  testId: `mocked-${Math.random()}`,
74
89
  title,
75
- titlePath: ['file.spec.ts', title],
76
- annotations: [] as Array<{ type: string, description?: string }>,
90
+ titlePath: ["file.spec.ts", title],
91
+ annotations: [] as Array<{ type: string; description?: string }>,
77
92
  attach: vi.fn().mockResolvedValue(undefined),
78
- snapshotPath: (...segs: string[]) => `${dir}/${segs.join('/')}`,
79
- config: { updateSnapshots: opts?.updateSnapshots ?? 'all' },
80
- }
93
+ snapshotPath: (...segs: string[]) => `${dir}/${segs.join("/")}`,
94
+ config: { updateSnapshots: opts?.updateSnapshots ?? "all" },
95
+ };
81
96
  }
82
97
 
83
98
  function makePage() {
84
99
  return {
85
100
  evaluate: vi.fn().mockResolvedValue(undefined),
86
- screenshot: vi.fn().mockResolvedValue(Buffer.from('fake-png')),
87
- }
101
+ screenshot: vi.fn().mockResolvedValue(Buffer.from("fake-png")),
102
+ };
88
103
  }
89
104
 
90
- function makeViolation(rule: string, targets: string[][], extras?: { description?: string, impact?: string, helpUrl?: string }) {
105
+ function makeViolation(
106
+ rule: string,
107
+ targets: string[][],
108
+ extras?: { description?: string; impact?: string; helpUrl?: string },
109
+ ) {
91
110
  return {
92
111
  id: rule,
93
112
  description: extras?.description ?? `Description for ${rule}`,
94
- impact: extras?.impact ?? 'serious',
95
- helpUrl: extras?.helpUrl ?? `https://dequeuniversity.com/rules/axe/4.8/${rule}`,
96
- nodes: targets.map(targetArr => ({
113
+ impact: extras?.impact ?? "serious",
114
+ helpUrl:
115
+ extras?.helpUrl ?? `https://dequeuniversity.com/rules/axe/4.8/${rule}`,
116
+ nodes: targets.map((targetArr) => ({
97
117
  target: targetArr,
98
118
  })),
99
- }
119
+ };
100
120
  }
101
121
 
102
- describe('accessibility baseline', () => {
122
+ describe("accessibility baseline", () => {
103
123
  beforeEach(() => {
104
- vi.clearAllMocks()
105
- mockAnalyze.mockResolvedValue(makeAxeResults())
106
- })
124
+ vi.clearAllMocks();
125
+ mockAnalyze.mockResolvedValue(makeAxeResults());
126
+ });
107
127
 
108
- describe('defineAccessibilityBaseline', () => {
109
- it('returns the entries unchanged', () => {
128
+ describe("defineAccessibilityBaseline", () => {
129
+ it("returns the entries unchanged", () => {
110
130
  const entries: AccessibilityBaseline = [
111
- { rule: 'color-contrast', targets: ['#sidebar .tag'], reason: 'Known issue', willBeFixedIn: 'PROJ-123' },
112
- ]
113
- expect(defineAccessibilityBaseline(entries)).toBe(entries)
114
- })
115
-
116
- it('requires a substantive reason and tracking reference for every waiver', () => {
117
- expect(() => defineAccessibilityBaseline([
118
- {rule: 'image-alt', targets: ['img'], reason: '', willBeFixedIn: 'PROJ-1'},
119
- ])).toThrow('reason')
120
- expect(() => defineAccessibilityBaseline([
121
- {rule: 'image-alt', targets: ['img'], reason: 'Known issue', willBeFixedIn: 'TODO'},
122
- ])).toThrow('willBeFixedIn')
123
- })
124
-
125
- it('rejects duplicate waivers for the same rule and target', () => {
126
- expect(() => defineAccessibilityBaseline([
127
- {rule: 'image-alt', targets: ['img.hero'], reason: 'First', willBeFixedIn: 'PROJ-1'},
128
- {rule: 'image-alt', targets: ['img.hero'], reason: 'Second', willBeFixedIn: 'PROJ-2'},
129
- ])).toThrow('duplicate')
130
- })
131
- })
132
-
133
- describe('baseline matching', () => {
134
- it('suppresses violations that match a baseline entry (exact rule + target)', async () => {
131
+ {
132
+ rule: "color-contrast",
133
+ targets: ["#sidebar .tag"],
134
+ reason: "Known issue",
135
+ willBeFixedIn: "PROJ-123",
136
+ },
137
+ ];
138
+ expect(defineAccessibilityBaseline(entries)).toBe(entries);
139
+ });
140
+
141
+ it("requires a substantive reason and tracking reference for every waiver", () => {
142
+ expect(() =>
143
+ defineAccessibilityBaseline([
144
+ {
145
+ rule: "image-alt",
146
+ targets: ["img"],
147
+ reason: "",
148
+ willBeFixedIn: "PROJ-1",
149
+ },
150
+ ]),
151
+ ).toThrow("reason");
152
+ expect(() =>
153
+ defineAccessibilityBaseline([
154
+ {
155
+ rule: "image-alt",
156
+ targets: ["img"],
157
+ reason: "Known issue",
158
+ willBeFixedIn: "TODO",
159
+ },
160
+ ]),
161
+ ).toThrow("willBeFixedIn");
162
+ });
163
+
164
+ it("rejects duplicate waivers for the same rule and target", () => {
165
+ expect(() =>
166
+ defineAccessibilityBaseline([
167
+ {
168
+ rule: "image-alt",
169
+ targets: ["img.hero"],
170
+ reason: "First",
171
+ willBeFixedIn: "PROJ-1",
172
+ },
173
+ {
174
+ rule: "image-alt",
175
+ targets: ["img.hero"],
176
+ reason: "Second",
177
+ willBeFixedIn: "PROJ-2",
178
+ },
179
+ ]),
180
+ ).toThrow("duplicate");
181
+ });
182
+ });
183
+
184
+ describe("baseline matching", () => {
185
+ it("suppresses violations that match a baseline entry (exact rule + target)", async () => {
135
186
  const wcagResults = makeAxeResults({
136
- violations: [makeViolation('color-contrast', [['#sidebar .tag']])],
137
- })
138
- mockAnalyze.mockResolvedValue(wcagResults)
187
+ violations: [makeViolation("color-contrast", [["#sidebar .tag"]])],
188
+ });
189
+ mockAnalyze.mockResolvedValue(wcagResults);
139
190
 
140
- const testInfo = makeTestInfo()
191
+ const testInfo = makeTestInfo();
141
192
  const baseline: AccessibilityBaseline = [
142
- { rule: 'color-contrast', targets: ['#sidebar .tag'], reason: 'Known issue', willBeFixedIn: 'PROJ-123' },
143
- ]
193
+ {
194
+ rule: "color-contrast",
195
+ targets: ["#sidebar .tag"],
196
+ reason: "Known issue",
197
+ willBeFixedIn: "PROJ-123",
198
+ },
199
+ ];
144
200
 
145
201
  await checkAccessibility(makePage() as any, testInfo as any, {
146
- bestPracticeMode: 'off',
202
+ bestPracticeMode: "off",
147
203
  baseline,
148
- })
204
+ });
149
205
 
150
206
  // Should not call toBe (no failure)
151
- expect(mockToBe).not.toHaveBeenCalled()
207
+ expect(mockToBe).not.toHaveBeenCalled();
152
208
  // Should not call toMatchSnapshot (baseline mode skips it)
153
- expect(mockToMatchSnapshot).not.toHaveBeenCalled()
154
- })
209
+ expect(mockToMatchSnapshot).not.toHaveBeenCalled();
210
+ });
155
211
 
156
- it('suppresses violations with normalized ID match (--UNIQUE-ID replacement)', async () => {
212
+ it("suppresses violations with normalized ID match (--UNIQUE-ID replacement)", async () => {
157
213
  const wcagResults = makeAxeResults({
158
- violations: [makeViolation('color-contrast', [['#edit-field--42']])],
159
- })
160
- mockAnalyze.mockResolvedValue(wcagResults)
214
+ violations: [makeViolation("color-contrast", [["#edit-field--42"]])],
215
+ });
216
+ mockAnalyze.mockResolvedValue(wcagResults);
161
217
 
162
- const testInfo = makeTestInfo()
218
+ const testInfo = makeTestInfo();
163
219
  const baseline: AccessibilityBaseline = [
164
- { rule: 'color-contrast', targets: ['#edit-field--UNIQUE-ID'], reason: 'Dynamic ID', willBeFixedIn: 'PROJ-456' },
165
- ]
220
+ {
221
+ rule: "color-contrast",
222
+ targets: ["#edit-field--UNIQUE-ID"],
223
+ reason: "Dynamic ID",
224
+ willBeFixedIn: "PROJ-456",
225
+ },
226
+ ];
166
227
 
167
228
  await checkAccessibility(makePage() as any, testInfo as any, {
168
- bestPracticeMode: 'off',
229
+ bestPracticeMode: "off",
169
230
  baseline,
170
- })
231
+ });
171
232
 
172
- expect(mockToBe).not.toHaveBeenCalled()
173
- expect(mockToMatchSnapshot).not.toHaveBeenCalled()
174
- })
233
+ expect(mockToBe).not.toHaveBeenCalled();
234
+ expect(mockToMatchSnapshot).not.toHaveBeenCalled();
235
+ });
175
236
 
176
- it('fails on unmatched violations with full details', async () => {
237
+ it("fails on unmatched violations with full details", async () => {
177
238
  const wcagResults = makeAxeResults({
178
- violations: [makeViolation('color-contrast', [['#sidebar .tag', '.footer__copyright']], {
179
- description: 'Elements must meet minimum color contrast ratio thresholds',
180
- impact: 'serious',
181
- helpUrl: 'https://dequeuniversity.com/rules/axe/4.8/color-contrast',
182
- })],
183
- })
184
- mockAnalyze.mockResolvedValue(wcagResults)
185
-
186
- const testInfo = makeTestInfo()
239
+ violations: [
240
+ makeViolation(
241
+ "color-contrast",
242
+ [["#sidebar .tag", ".footer__copyright"]],
243
+ {
244
+ description:
245
+ "Elements must meet minimum color contrast ratio thresholds",
246
+ impact: "serious",
247
+ helpUrl:
248
+ "https://dequeuniversity.com/rules/axe/4.8/color-contrast",
249
+ },
250
+ ),
251
+ ],
252
+ });
253
+ mockAnalyze.mockResolvedValue(wcagResults);
254
+
255
+ const testInfo = makeTestInfo();
187
256
  await checkAccessibility(makePage() as any, testInfo as any, {
188
- bestPracticeMode: 'off',
257
+ bestPracticeMode: "off",
189
258
  baseline: [],
190
- })
259
+ });
191
260
 
192
- expect(mockToBe).toHaveBeenCalled()
261
+ expect(mockToBe).toHaveBeenCalled();
193
262
  // First arg is null, second is the message string
194
263
  const failureMessage = mockExpectHard.mock.calls.find(
195
- (call: any[]) => call.length >= 2 && typeof call[1] === 'string'
196
- )?.[1] as string
197
-
198
- expect(failureMessage).toContain('color-contrast')
199
- expect(failureMessage).toContain('serious')
200
- expect(failureMessage).toContain('Elements must meet minimum color contrast ratio thresholds')
201
- expect(failureMessage).toContain('https://dequeuniversity.com/rules/axe/4.8/color-contrast')
202
- expect(failureMessage).toContain('#sidebar .tag')
203
- expect(failureMessage).toContain('.footer__copyright')
204
- })
205
- })
206
-
207
- describe('stale detection', () => {
208
- it('reports stale baseline entries with annotations', async () => {
209
- const wcagResults = makeAxeResults({ violations: [] })
210
- mockAnalyze.mockResolvedValue(wcagResults)
211
-
212
- const testInfo = makeTestInfo()
264
+ (call: any[]) => call.length >= 2 && typeof call[1] === "string",
265
+ )?.[1] as string;
266
+
267
+ expect(failureMessage).toContain("color-contrast");
268
+ expect(failureMessage).toContain("serious");
269
+ expect(failureMessage).toContain(
270
+ "Elements must meet minimum color contrast ratio thresholds",
271
+ );
272
+ expect(failureMessage).toContain(
273
+ "https://dequeuniversity.com/rules/axe/4.8/color-contrast",
274
+ );
275
+ expect(failureMessage).toContain("#sidebar .tag");
276
+ expect(failureMessage).toContain(".footer__copyright");
277
+ });
278
+ });
279
+
280
+ describe("stale detection", () => {
281
+ it("reports stale baseline entries with annotations", async () => {
282
+ const wcagResults = makeAxeResults({ violations: [] });
283
+ mockAnalyze.mockResolvedValue(wcagResults);
284
+
285
+ const testInfo = makeTestInfo();
213
286
  const baseline: AccessibilityBaseline = [
214
- { rule: 'color-contrast', targets: ['#old-element'], reason: 'Was broken', willBeFixedIn: 'PROJ-789' },
215
- ]
287
+ {
288
+ rule: "color-contrast",
289
+ targets: ["#old-element"],
290
+ reason: "Was broken",
291
+ willBeFixedIn: "PROJ-789",
292
+ },
293
+ ];
216
294
 
217
295
  await checkAccessibility(makePage() as any, testInfo as any, {
218
- bestPracticeMode: 'off',
296
+ bestPracticeMode: "off",
219
297
  baseline,
220
- })
221
-
222
- const staleAnnotations = testInfo.annotations.filter(a => a.type === 'Stale a11y baseline entry')
223
- expect(staleAnnotations).toHaveLength(1)
224
- expect(staleAnnotations[0].description).toContain('color-contrast')
225
- expect(staleAnnotations[0].description).toContain('#old-element')
226
- expect(staleAnnotations[0].description).toContain('no longer detected')
227
- })
228
- })
229
-
230
- describe('snapshot bypass', () => {
231
- it('does NOT call toMatchSnapshot when baseline is active', async () => {
232
- const wcagResults = makeAxeResults({ violations: [] })
233
- mockAnalyze.mockResolvedValue(wcagResults)
234
-
235
- const testInfo = makeTestInfo()
298
+ });
299
+
300
+ const staleAnnotations = testInfo.annotations.filter(
301
+ (a) => a.type === "Stale a11y baseline entry",
302
+ );
303
+ expect(staleAnnotations).toHaveLength(1);
304
+ expect(staleAnnotations[0].description).toContain("color-contrast");
305
+ expect(staleAnnotations[0].description).toContain("#old-element");
306
+ expect(staleAnnotations[0].description).toContain("no longer detected");
307
+ });
308
+
309
+ it.each([
310
+ { name: "best-practice", bp: ["page-has-heading-one"], wcag: [] },
311
+ { name: "WCAG", bp: [], wcag: ["image-alt"] },
312
+ { name: "both scans", bp: ["page-has-heading-one"], wcag: ["image-alt"] },
313
+ ])(
314
+ "does not mark entries matched by $name as stale",
315
+ async ({ bp, wcag }) => {
316
+ const rules = [...bp, ...wcag];
317
+ const baseline: AccessibilityBaseline = rules.map((rule) => ({
318
+ rule,
319
+ targets: ["#known"],
320
+ reason: "Known issue",
321
+ willBeFixedIn: "PROJ-353",
322
+ }));
323
+ mockAnalyze
324
+ .mockResolvedValueOnce(
325
+ makeAxeResults({
326
+ violations: bp.map((rule) => makeViolation(rule, [["#known"]])),
327
+ }),
328
+ )
329
+ .mockResolvedValueOnce(
330
+ makeAxeResults({
331
+ violations: wcag.map((rule) => makeViolation(rule, [["#known"]])),
332
+ }),
333
+ );
334
+
335
+ const testInfo = makeTestInfo();
336
+ await checkAccessibility(makePage() as any, testInfo as any, {
337
+ baseline,
338
+ });
339
+
340
+ expect(
341
+ testInfo.annotations.filter(
342
+ (a) => a.type === "Stale a11y baseline entry",
343
+ ),
344
+ ).toEqual([]);
345
+ expect(
346
+ testInfo.annotations.filter(
347
+ (a) => a.type === "Baselined a11y violation",
348
+ ),
349
+ ).toHaveLength(rules.length);
350
+ expect(
351
+ testInfo.annotations.filter((a) =>
352
+ a.description?.includes("new violations"),
353
+ ),
354
+ ).toEqual([
355
+ {
356
+ type: "Accessibility",
357
+ description: `Best-practice scan: 0 new violations (${bp.length} baselined)`,
358
+ },
359
+ {
360
+ type: "Accessibility",
361
+ description: `WCAG scan: 0 new violations (${wcag.length} baselined)`,
362
+ },
363
+ ]);
364
+ expect(mockToBe).not.toHaveBeenCalled();
365
+ },
366
+ );
367
+
368
+ it("reports entries unmatched by either scan only once", async () => {
369
+ const testInfo = makeTestInfo();
370
+ await checkAccessibility(makePage() as any, testInfo as any, {
371
+ baseline: [
372
+ {
373
+ rule: "page-has-heading-one",
374
+ targets: ["html"],
375
+ reason: "Was broken",
376
+ willBeFixedIn: "PROJ-353",
377
+ },
378
+ ],
379
+ });
380
+
381
+ expect(
382
+ testInfo.annotations.filter(
383
+ (a) => a.type === "Stale a11y baseline entry",
384
+ ),
385
+ ).toEqual([
386
+ {
387
+ type: "Stale a11y baseline entry",
388
+ description: "page-has-heading-one on html — no longer detected",
389
+ },
390
+ ]);
391
+ });
392
+
393
+ it("does not carry baseline matches into later checks in the same test", async () => {
394
+ const baseline: AccessibilityBaseline = [
395
+ {
396
+ rule: "page-has-heading-one",
397
+ targets: ["html"],
398
+ reason: "Known issue",
399
+ willBeFixedIn: "PROJ-353",
400
+ },
401
+ ];
402
+ mockAnalyze.mockResolvedValueOnce(
403
+ makeAxeResults({
404
+ violations: [makeViolation("page-has-heading-one", [["html"]])],
405
+ }),
406
+ );
407
+ const testInfo = makeTestInfo();
236
408
  await checkAccessibility(makePage() as any, testInfo as any, {
237
- bestPracticeMode: 'off',
409
+ baseline,
410
+ });
411
+ expect(
412
+ testInfo.annotations.filter(
413
+ (a) => a.type === "Stale a11y baseline entry",
414
+ ),
415
+ ).toEqual([]);
416
+
417
+ await checkAccessibility(makePage() as any, testInfo as any, {
418
+ baseline,
419
+ });
420
+ expect(
421
+ testInfo.annotations.filter(
422
+ (a) => a.type === "Stale a11y baseline entry",
423
+ ),
424
+ ).toHaveLength(1);
425
+ });
426
+
427
+ it("reports stale entries before an unmatched WCAG violation throws", async () => {
428
+ const baseline: AccessibilityBaseline = [
429
+ {
430
+ rule: "page-has-heading-one",
431
+ targets: ["html"],
432
+ reason: "Known issue",
433
+ willBeFixedIn: "PROJ-353",
434
+ },
435
+ {
436
+ rule: "heading-order",
437
+ targets: ["h3"],
438
+ reason: "Was broken",
439
+ willBeFixedIn: "PROJ-354",
440
+ },
441
+ ];
442
+ mockAnalyze
443
+ .mockResolvedValueOnce(
444
+ makeAxeResults({
445
+ violations: [makeViolation("page-has-heading-one", [["html"]])],
446
+ }),
447
+ )
448
+ .mockResolvedValueOnce(
449
+ makeAxeResults({
450
+ violations: [makeViolation("image-alt", [["img"]])],
451
+ }),
452
+ );
453
+ mockToBe.mockImplementationOnce(() => {
454
+ throw new Error("new WCAG violation");
455
+ });
456
+ const testInfo = makeTestInfo();
457
+
458
+ await expect(
459
+ checkAccessibility(makePage() as any, testInfo as any, { baseline }),
460
+ ).rejects.toThrow("new WCAG violation");
461
+
462
+ expect(
463
+ testInfo.annotations.filter(
464
+ (a) => a.type === "Stale a11y baseline entry",
465
+ ),
466
+ ).toEqual([
467
+ {
468
+ type: "Stale a11y baseline entry",
469
+ description: "heading-order on h3 — no longer detected",
470
+ },
471
+ ]);
472
+ expect(mockExpectHard).toHaveBeenCalledWith(
473
+ null,
474
+ expect.stringContaining("image-alt"),
475
+ );
476
+ });
477
+ });
478
+
479
+ describe("snapshot bypass", () => {
480
+ it("does NOT call toMatchSnapshot when baseline is active", async () => {
481
+ const wcagResults = makeAxeResults({ violations: [] });
482
+ mockAnalyze.mockResolvedValue(wcagResults);
483
+
484
+ const testInfo = makeTestInfo();
485
+ await checkAccessibility(makePage() as any, testInfo as any, {
486
+ bestPracticeMode: "off",
238
487
  baseline: [],
239
- })
488
+ });
240
489
 
241
- expect(mockToMatchSnapshot).not.toHaveBeenCalled()
242
- })
490
+ expect(mockToMatchSnapshot).not.toHaveBeenCalled();
491
+ });
243
492
 
244
- it('calls toMatchSnapshot when no baseline is provided', async () => {
245
- const testInfo = makeTestInfo()
493
+ it("calls toMatchSnapshot when no baseline is provided", async () => {
494
+ const testInfo = makeTestInfo();
246
495
  await checkAccessibility(makePage() as any, testInfo as any, {
247
- bestPracticeMode: 'off',
248
- })
496
+ bestPracticeMode: "off",
497
+ });
249
498
 
250
- expect(mockToMatchSnapshot).toHaveBeenCalled()
251
- })
252
- })
499
+ expect(mockToMatchSnapshot).toHaveBeenCalled();
500
+ });
501
+ });
253
502
 
254
- describe('copy-pasteable baseline entry in failure message', () => {
255
- it('includes a copy-pasteable entry for unmatched violations', async () => {
503
+ describe("copy-pasteable baseline entry in failure message", () => {
504
+ it("includes a copy-pasteable entry for unmatched violations", async () => {
256
505
  const wcagResults = makeAxeResults({
257
- violations: [makeViolation('image-alt', [['img.hero']])],
258
- })
259
- mockAnalyze.mockResolvedValue(wcagResults)
506
+ violations: [makeViolation("image-alt", [["img.hero"]])],
507
+ });
508
+ mockAnalyze.mockResolvedValue(wcagResults);
260
509
 
261
- const testInfo = makeTestInfo()
510
+ const testInfo = makeTestInfo();
262
511
  await checkAccessibility(makePage() as any, testInfo as any, {
263
- bestPracticeMode: 'off',
512
+ bestPracticeMode: "off",
264
513
  baseline: [],
265
- })
514
+ });
266
515
 
267
516
  const failureMessage = mockExpectHard.mock.calls.find(
268
- (call: any[]) => call.length >= 2 && typeof call[1] === 'string'
269
- )?.[1] as string
270
-
271
- expect(failureMessage).toContain("rule: 'image-alt'")
272
- expect(failureMessage).toContain("targets: ['img.hero']")
273
- expect(failureMessage).toContain('reason:')
274
- expect(failureMessage).toContain('willBeFixedIn:')
275
- expect(failureMessage).toContain('Add to your baseline to accept this violation:')
276
- })
277
- })
278
-
279
- describe('baseline annotations', () => {
280
- it('pushes Baselined a11y violation annotations for matched violations', async () => {
517
+ (call: any[]) => call.length >= 2 && typeof call[1] === "string",
518
+ )?.[1] as string;
519
+
520
+ expect(failureMessage).toContain("rule: 'image-alt'");
521
+ expect(failureMessage).toContain("targets: ['img.hero']");
522
+ expect(failureMessage).toContain("reason:");
523
+ expect(failureMessage).toContain("willBeFixedIn:");
524
+ expect(failureMessage).toContain(
525
+ "Add to your baseline to accept this violation:",
526
+ );
527
+ });
528
+ });
529
+
530
+ describe("baseline annotations", () => {
531
+ it("pushes Baselined a11y violation annotations for matched violations", async () => {
281
532
  const wcagResults = makeAxeResults({
282
- violations: [makeViolation('color-contrast', [['#sidebar .tag']])],
283
- })
284
- mockAnalyze.mockResolvedValue(wcagResults)
533
+ violations: [makeViolation("color-contrast", [["#sidebar .tag"]])],
534
+ });
535
+ mockAnalyze.mockResolvedValue(wcagResults);
285
536
 
286
- const testInfo = makeTestInfo()
537
+ const testInfo = makeTestInfo();
287
538
  const baseline: AccessibilityBaseline = [
288
- { rule: 'color-contrast', targets: ['#sidebar .tag'], reason: 'Known issue', willBeFixedIn: 'PROJ-123' },
289
- ]
539
+ {
540
+ rule: "color-contrast",
541
+ targets: ["#sidebar .tag"],
542
+ reason: "Known issue",
543
+ willBeFixedIn: "PROJ-123",
544
+ },
545
+ ];
290
546
 
291
547
  await checkAccessibility(makePage() as any, testInfo as any, {
292
- bestPracticeMode: 'off',
548
+ bestPracticeMode: "off",
293
549
  baseline,
294
- })
295
-
296
- const baselinedAnnotations = testInfo.annotations.filter(a => a.type === 'Baselined a11y violation')
297
- expect(baselinedAnnotations).toHaveLength(1)
298
- expect(baselinedAnnotations[0].description).toBe('color-contrast: Known issue — PROJ-123')
299
- })
300
-
301
- it('pushes Stale a11y baseline entry annotations for unmatched baseline entries', async () => {
302
- const wcagResults = makeAxeResults({ violations: [] })
303
- mockAnalyze.mockResolvedValue(wcagResults)
304
-
305
- const testInfo = makeTestInfo()
550
+ });
551
+
552
+ const baselinedAnnotations = testInfo.annotations.filter(
553
+ (a) => a.type === "Baselined a11y violation",
554
+ );
555
+ expect(baselinedAnnotations).toHaveLength(1);
556
+ expect(baselinedAnnotations[0].description).toBe(
557
+ "color-contrast: Known issue — PROJ-123",
558
+ );
559
+ });
560
+
561
+ it("pushes Stale a11y baseline entry annotations for unmatched baseline entries", async () => {
562
+ const wcagResults = makeAxeResults({ violations: [] });
563
+ mockAnalyze.mockResolvedValue(wcagResults);
564
+
565
+ const testInfo = makeTestInfo();
306
566
  const baseline: AccessibilityBaseline = [
307
- { rule: 'old-rule', targets: ['.gone'], reason: 'Was broken', willBeFixedIn: 'PROJ-999' },
308
- ]
567
+ {
568
+ rule: "old-rule",
569
+ targets: [".gone"],
570
+ reason: "Was broken",
571
+ willBeFixedIn: "PROJ-999",
572
+ },
573
+ ];
309
574
 
310
575
  await checkAccessibility(makePage() as any, testInfo as any, {
311
- bestPracticeMode: 'off',
576
+ bestPracticeMode: "off",
312
577
  baseline,
313
- })
314
-
315
- const staleAnnotations = testInfo.annotations.filter(a => a.type === 'Stale a11y baseline entry')
316
- expect(staleAnnotations).toHaveLength(1)
317
- expect(staleAnnotations[0].description).toBe('old-rule on .gone — no longer detected')
318
- })
319
- })
320
-
321
- describe('baseline summary annotation', () => {
322
- it('includes correct counts in summary annotation', async () => {
578
+ });
579
+
580
+ const staleAnnotations = testInfo.annotations.filter(
581
+ (a) => a.type === "Stale a11y baseline entry",
582
+ );
583
+ expect(staleAnnotations).toHaveLength(1);
584
+ expect(staleAnnotations[0].description).toBe(
585
+ "old-rule on .gone — no longer detected",
586
+ );
587
+ });
588
+ });
589
+
590
+ describe("baseline summary annotation", () => {
591
+ it("includes correct counts in summary annotation", async () => {
323
592
  const wcagResults = makeAxeResults({
324
593
  violations: [
325
- makeViolation('color-contrast', [['#sidebar .tag']]),
326
- makeViolation('image-alt', [['img.hero']]),
594
+ makeViolation("color-contrast", [["#sidebar .tag"]]),
595
+ makeViolation("image-alt", [["img.hero"]]),
327
596
  ],
328
- })
329
- mockAnalyze.mockResolvedValue(wcagResults)
597
+ });
598
+ mockAnalyze.mockResolvedValue(wcagResults);
330
599
 
331
- const testInfo = makeTestInfo()
600
+ const testInfo = makeTestInfo();
332
601
  const baseline: AccessibilityBaseline = [
333
- { rule: 'color-contrast', targets: ['#sidebar .tag'], reason: 'Known', willBeFixedIn: 'PROJ-1' },
334
- ]
602
+ {
603
+ rule: "color-contrast",
604
+ targets: ["#sidebar .tag"],
605
+ reason: "Known",
606
+ willBeFixedIn: "PROJ-1",
607
+ },
608
+ ];
335
609
 
336
610
  await checkAccessibility(makePage() as any, testInfo as any, {
337
- bestPracticeMode: 'off',
611
+ bestPracticeMode: "off",
338
612
  baseline,
339
- })
613
+ });
340
614
 
341
615
  const summaryAnnotations = testInfo.annotations.filter(
342
- a => a.type === 'Accessibility' && a.description?.includes('baselined')
343
- )
344
- expect(summaryAnnotations).toHaveLength(1)
345
- expect(summaryAnnotations[0].description).toBe('WCAG scan: 1 new violations (1 baselined)')
346
- })
347
- })
348
-
349
- describe('snapshot-mode baseline suggestions', () => {
350
- it('attaches baseline suggestions when violations exist in snapshot mode', async () => {
616
+ (a) =>
617
+ a.type === "Accessibility" && a.description?.includes("baselined"),
618
+ );
619
+ expect(summaryAnnotations).toHaveLength(1);
620
+ expect(summaryAnnotations[0].description).toBe(
621
+ "WCAG scan: 1 new violations (1 baselined)",
622
+ );
623
+ });
624
+ });
625
+
626
+ describe("snapshot-mode baseline suggestions", () => {
627
+ it("attaches baseline suggestions when violations exist in snapshot mode", async () => {
351
628
  const wcagResults = makeAxeResults({
352
- violations: [makeViolation('color-contrast', [['#sidebar .tag']])],
353
- })
354
- mockAnalyze.mockResolvedValue(wcagResults)
629
+ violations: [makeViolation("color-contrast", [["#sidebar .tag"]])],
630
+ });
631
+ mockAnalyze.mockResolvedValue(wcagResults);
355
632
 
356
- const testInfo = makeTestInfo()
633
+ const testInfo = makeTestInfo();
357
634
  await checkAccessibility(makePage() as any, testInfo as any, {
358
- bestPracticeMode: 'off',
359
- })
635
+ bestPracticeMode: "off",
636
+ });
360
637
 
361
638
  // Should attach a11y-baseline-suggestions
362
639
  expect(testInfo.attach).toHaveBeenCalledWith(
363
- 'a11y-baseline-suggestions',
640
+ "a11y-baseline-suggestions",
364
641
  expect.objectContaining({
365
- contentType: 'text/plain',
366
- })
367
- )
642
+ contentType: "text/plain",
643
+ }),
644
+ );
368
645
 
369
646
  // Check the attachment body contains the suggestion
370
647
  const attachCall = testInfo.attach.mock.calls.find(
371
- (call: any[]) => call[0] === 'a11y-baseline-suggestions'
372
- )
373
- expect(attachCall).toBeTruthy()
374
- const body = attachCall![1].body as string
375
- expect(body).toContain("rule: 'color-contrast'")
376
- expect(body).toContain("targets: ['#sidebar .tag']")
648
+ (call: any[]) => call[0] === "a11y-baseline-suggestions",
649
+ );
650
+ expect(attachCall).toBeTruthy();
651
+ const body = attachCall![1].body as string;
652
+ expect(body).toContain("rule: 'color-contrast'");
653
+ expect(body).toContain("targets: ['#sidebar .tag']");
377
654
 
378
655
  // Should push annotation about switching to baseline mode
379
- const baselineHintAnnotation = testInfo.annotations.find(
380
- a => a.description?.includes('switch to baseline mode')
381
- )
382
- expect(baselineHintAnnotation).toBeTruthy()
383
- expect(baselineHintAnnotation!.type).toBe('Accessibility')
384
- })
385
-
386
- it('does not attach baseline suggestions when no violations in snapshot mode', async () => {
387
- const wcagResults = makeAxeResults({ violations: [] })
388
- mockAnalyze.mockResolvedValue(wcagResults)
389
-
390
- const testInfo = makeTestInfo()
656
+ const baselineHintAnnotation = testInfo.annotations.find((a) =>
657
+ a.description?.includes("switch to baseline mode"),
658
+ );
659
+ expect(baselineHintAnnotation).toBeTruthy();
660
+ expect(baselineHintAnnotation!.type).toBe("Accessibility");
661
+ });
662
+
663
+ it("does not attach baseline suggestions when no violations in snapshot mode", async () => {
664
+ const wcagResults = makeAxeResults({ violations: [] });
665
+ mockAnalyze.mockResolvedValue(wcagResults);
666
+
667
+ const testInfo = makeTestInfo();
391
668
  await checkAccessibility(makePage() as any, testInfo as any, {
392
- bestPracticeMode: 'off',
393
- })
669
+ bestPracticeMode: "off",
670
+ });
394
671
 
395
672
  expect(testInfo.attach).not.toHaveBeenCalledWith(
396
- 'a11y-baseline-suggestions',
397
- expect.anything()
398
- )
399
- })
400
- })
673
+ "a11y-baseline-suggestions",
674
+ expect.anything(),
675
+ );
676
+ });
677
+ });
401
678
 
402
- describe('on-disk baseline (new default for snapshotless tests)', () => {
403
- let tmpDir: string
404
- let originalCI: string | undefined
679
+ describe("on-disk baseline (new default for snapshotless tests)", () => {
680
+ let tmpDir: string;
681
+ let originalCI: string | undefined;
405
682
 
406
683
  beforeEach(async () => {
407
- const fs = await import('fs')
408
- const os = await import('os')
409
- const path = await import('path')
410
- tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'a11y-disp-'))
411
- originalCI = process.env.CI
412
- delete process.env.CI
413
- })
684
+ const fs = await import("fs");
685
+ const os = await import("os");
686
+ const path = await import("path");
687
+ tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), "a11y-disp-"));
688
+ originalCI = process.env.CI;
689
+ delete process.env.CI;
690
+ });
414
691
 
415
692
  afterEach(async () => {
416
- const fs = await import('fs')
417
- await fs.promises.rm(tmpDir, { recursive: true, force: true })
418
- if (originalCI === undefined) delete process.env.CI
419
- else process.env.CI = originalCI
420
- })
421
-
422
- it('seeds an empty baseline file when no violations and no snapshot exist (local)', async () => {
423
- mockAnalyze.mockResolvedValue(makeAxeResults({ violations: [] }))
424
-
425
- const testInfo = makeTestInfo({ updateSnapshots: 'none', snapshotsDir: tmpDir, title: 'clean test' })
693
+ const fs = await import("fs");
694
+ await fs.promises.rm(tmpDir, { recursive: true, force: true });
695
+ if (originalCI === undefined) delete process.env.CI;
696
+ else process.env.CI = originalCI;
697
+ });
698
+
699
+ it("seeds an empty baseline file when no violations and no snapshot exist (local)", async () => {
700
+ mockAnalyze.mockResolvedValue(makeAxeResults({ violations: [] }));
701
+
702
+ const testInfo = makeTestInfo({
703
+ updateSnapshots: "none",
704
+ snapshotsDir: tmpDir,
705
+ title: "clean test",
706
+ });
426
707
  await checkAccessibility(makePage() as any, testInfo as any, {
427
- bestPracticeMode: 'off',
428
- })
708
+ bestPracticeMode: "off",
709
+ });
429
710
 
430
- const fs = await import('fs')
431
- const path = await import('path')
432
- const file = path.join(tmpDir, 'clean-test-1.a11y-baseline.json')
433
- const written = JSON.parse(await fs.promises.readFile(file, 'utf8'))
434
- expect(written.note).toBe('No accessibility violations found')
435
- expect(written.violations).toEqual([])
711
+ const fs = await import("fs");
712
+ const path = await import("path");
713
+ const file = path.join(tmpDir, "clean-test-1.a11y-baseline.json");
714
+ const written = JSON.parse(await fs.promises.readFile(file, "utf8"));
715
+ expect(written.note).toBe("No accessibility violations found");
716
+ expect(written.violations).toEqual([]);
436
717
 
437
718
  // Snapshot mode is NOT used; on-disk baseline mode used assertBaseline.
438
- expect(mockToMatchSnapshot).not.toHaveBeenCalled()
719
+ expect(mockToMatchSnapshot).not.toHaveBeenCalled();
439
720
  // Test does not fail (no toBe call against a "baseline file present" sentinel).
440
- expect(mockToBe).not.toHaveBeenCalled()
441
- })
442
-
443
- it('seeds a TODO-prompted baseline file when violations exist and no snapshot (local)', async () => {
444
- mockAnalyze.mockResolvedValue(makeAxeResults({
445
- violations: [makeViolation('color-contrast', [['#footer .legal']])],
446
- }))
447
-
448
- const testInfo = makeTestInfo({ updateSnapshots: 'none', snapshotsDir: tmpDir, title: 'with violations' })
721
+ expect(mockToBe).not.toHaveBeenCalled();
722
+ });
723
+
724
+ it("seeds a TODO-prompted baseline file when violations exist and no snapshot (local)", async () => {
725
+ mockAnalyze.mockResolvedValue(
726
+ makeAxeResults({
727
+ violations: [makeViolation("color-contrast", [["#footer .legal"]])],
728
+ }),
729
+ );
730
+
731
+ const testInfo = makeTestInfo({
732
+ updateSnapshots: "none",
733
+ snapshotsDir: tmpDir,
734
+ title: "with violations",
735
+ });
449
736
  await checkAccessibility(makePage() as any, testInfo as any, {
450
- bestPracticeMode: 'off',
451
- })
452
-
453
- const fs = await import('fs')
454
- const path = await import('path')
455
- const file = path.join(tmpDir, 'with-violations-1.a11y-baseline.json')
456
- const written = JSON.parse(await fs.promises.readFile(file, 'utf8'))
457
- expect(written.note).toMatch(/TODO/)
737
+ bestPracticeMode: "off",
738
+ });
739
+
740
+ const fs = await import("fs");
741
+ const path = await import("path");
742
+ const file = path.join(tmpDir, "with-violations-1.a11y-baseline.json");
743
+ const written = JSON.parse(await fs.promises.readFile(file, "utf8"));
744
+ expect(written.note).toMatch(/TODO/);
458
745
  expect(written.violations).toEqual([
459
- { rule: 'color-contrast', targets: ['#footer .legal'], reason: 'TODO', willBeFixedIn: 'TODO' },
460
- ])
461
- expect(mockToBe).not.toHaveBeenCalled()
462
- })
463
-
464
- it('fails the test on CI when seeding (mirrors Playwright missing-snapshot behaviour)', async () => {
465
- process.env.CI = '1'
466
- mockAnalyze.mockResolvedValue(makeAxeResults({
467
- violations: [makeViolation('color-contrast', [['#x']])],
468
- }))
469
-
470
- const testInfo = makeTestInfo({ updateSnapshots: 'none', snapshotsDir: tmpDir, title: 'on ci' })
746
+ {
747
+ rule: "color-contrast",
748
+ targets: ["#footer .legal"],
749
+ reason: "TODO",
750
+ willBeFixedIn: "TODO",
751
+ },
752
+ ]);
753
+ expect(mockToBe).not.toHaveBeenCalled();
754
+ });
755
+
756
+ it("fails the test on CI when seeding (mirrors Playwright missing-snapshot behaviour)", async () => {
757
+ process.env.CI = "1";
758
+ mockAnalyze.mockResolvedValue(
759
+ makeAxeResults({
760
+ violations: [makeViolation("color-contrast", [["#x"]])],
761
+ }),
762
+ );
763
+
764
+ const testInfo = makeTestInfo({
765
+ updateSnapshots: "none",
766
+ snapshotsDir: tmpDir,
767
+ title: "on ci",
768
+ });
471
769
  await checkAccessibility(makePage() as any, testInfo as any, {
472
- bestPracticeMode: 'off',
473
- })
770
+ bestPracticeMode: "off",
771
+ });
474
772
 
475
773
  // File is still seeded.
476
- const fs = await import('fs')
477
- const path = await import('path')
478
- const file = path.join(tmpDir, 'on-ci-1.a11y-baseline.json')
479
- expect(fs.existsSync(file)).toBe(true)
774
+ const fs = await import("fs");
775
+ const path = await import("path");
776
+ const file = path.join(tmpDir, "on-ci-1.a11y-baseline.json");
777
+ expect(fs.existsSync(file)).toBe(true);
480
778
 
481
779
  // Test fails with the seed-and-commit directive.
482
- expect(mockToBe).toHaveBeenCalled()
780
+ expect(mockToBe).toHaveBeenCalled();
483
781
  const failureMessage = mockExpectHard.mock.calls.find(
484
- (call: any[]) => call.length >= 2 && typeof call[1] === 'string' && call[1].includes('a11y baseline file was missing')
485
- )?.[1] as string
486
- expect(failureMessage).toContain(file)
487
- expect(failureMessage).toMatch(/commit it/i)
488
- })
489
-
490
- it('loads an existing on-disk baseline JSON and matches violations against it', async () => {
491
- mockAnalyze.mockResolvedValue(makeAxeResults({
492
- violations: [makeViolation('color-contrast', [['#known']])],
493
- }))
782
+ (call: any[]) =>
783
+ call.length >= 2 &&
784
+ typeof call[1] === "string" &&
785
+ call[1].includes("a11y baseline file was missing"),
786
+ )?.[1] as string;
787
+ expect(failureMessage).toContain(file);
788
+ expect(failureMessage).toMatch(/commit it/i);
789
+ });
790
+
791
+ it("loads an existing on-disk baseline JSON and matches violations against it", async () => {
792
+ mockAnalyze.mockResolvedValue(
793
+ makeAxeResults({
794
+ violations: [makeViolation("color-contrast", [["#known"]])],
795
+ }),
796
+ );
494
797
 
495
798
  // Pre-write a baseline.
496
- const fs = await import('fs')
497
- const path = await import('path')
498
- const file = path.join(tmpDir, 'known-test-1.a11y-baseline.json')
499
- await fs.promises.writeFile(file, JSON.stringify({
500
- note: 'Known issue tracked in PROJ-1.',
501
- violations: [{ rule: 'color-contrast', targets: ['#known'], reason: 'Known', willBeFixedIn: 'PROJ-1' }],
502
- }))
503
-
504
- const testInfo = makeTestInfo({ updateSnapshots: 'none', snapshotsDir: tmpDir, title: 'known test' })
799
+ const fs = await import("fs");
800
+ const path = await import("path");
801
+ const file = path.join(tmpDir, "known-test-1.a11y-baseline.json");
802
+ await fs.promises.writeFile(
803
+ file,
804
+ JSON.stringify({
805
+ note: "Known issue tracked in PROJ-1.",
806
+ violations: [
807
+ {
808
+ rule: "color-contrast",
809
+ targets: ["#known"],
810
+ reason: "Known",
811
+ willBeFixedIn: "PROJ-1",
812
+ },
813
+ ],
814
+ }),
815
+ );
816
+
817
+ const testInfo = makeTestInfo({
818
+ updateSnapshots: "none",
819
+ snapshotsDir: tmpDir,
820
+ title: "known test",
821
+ });
505
822
  await checkAccessibility(makePage() as any, testInfo as any, {
506
- bestPracticeMode: 'off',
507
- })
823
+ bestPracticeMode: "off",
824
+ });
508
825
 
509
826
  // Violation matched -> no failure.
510
- expect(mockToBe).not.toHaveBeenCalled()
511
- const baselined = testInfo.annotations.filter(a => a.type === 'Baselined a11y violation')
512
- expect(baselined).toHaveLength(1)
513
- })
514
-
515
- it('rejects a persisted waiver whose metadata is still TODO', async () => {
516
- mockAnalyze.mockResolvedValue(makeAxeResults({
517
- violations: [makeViolation('color-contrast', [['#known']])],
518
- }))
519
-
520
- const fs = await import('fs')
521
- const path = await import('path')
522
- const file = path.join(tmpDir, 'unfinished-test-1.a11y-baseline.json')
523
- await fs.promises.writeFile(file, JSON.stringify({
524
- note: 'Seeded',
525
- violations: [{rule: 'color-contrast', targets: ['#known'], reason: 'TODO', willBeFixedIn: 'TODO'}],
526
- }))
527
-
528
- const testInfo = makeTestInfo({updateSnapshots: 'none', snapshotsDir: tmpDir, title: 'unfinished test'})
529
- await expect(checkAccessibility(makePage() as any, testInfo as any, {
530
- bestPracticeMode: 'off',
531
- })).rejects.toThrow('requires a reason')
532
- })
533
-
534
- it('produces per-call counter files for multi-call tests', async () => {
535
- mockAnalyze.mockResolvedValue(makeAxeResults({ violations: [] }))
536
-
537
- const testInfo = makeTestInfo({ updateSnapshots: 'none', snapshotsDir: tmpDir, title: 'multi call' })
538
- await checkAccessibility(makePage() as any, testInfo as any, { bestPracticeMode: 'off' })
539
- await checkAccessibility(makePage() as any, testInfo as any, { bestPracticeMode: 'off' })
540
-
541
- const fs = await import('fs')
542
- const path = await import('path')
543
- const f1 = path.join(tmpDir, 'multi-call-1.a11y-baseline.json')
544
- const f2 = path.join(tmpDir, 'multi-call-2.a11y-baseline.json')
545
- expect(fs.existsSync(f1)).toBe(true)
546
- expect(fs.existsSync(f2)).toBe(true)
547
- })
548
-
549
- it('still uses snapshot mode when a legacy snapshot file exists for the test', async () => {
550
- mockAnalyze.mockResolvedValue(makeAxeResults({ violations: [] }))
827
+ expect(mockToBe).not.toHaveBeenCalled();
828
+ const baselined = testInfo.annotations.filter(
829
+ (a) => a.type === "Baselined a11y violation",
830
+ );
831
+ expect(baselined).toHaveLength(1);
832
+ });
833
+
834
+ it("rejects a persisted waiver whose metadata is still TODO", async () => {
835
+ mockAnalyze.mockResolvedValue(
836
+ makeAxeResults({
837
+ violations: [makeViolation("color-contrast", [["#known"]])],
838
+ }),
839
+ );
840
+
841
+ const fs = await import("fs");
842
+ const path = await import("path");
843
+ const file = path.join(tmpDir, "unfinished-test-1.a11y-baseline.json");
844
+ await fs.promises.writeFile(
845
+ file,
846
+ JSON.stringify({
847
+ note: "Seeded",
848
+ violations: [
849
+ {
850
+ rule: "color-contrast",
851
+ targets: ["#known"],
852
+ reason: "TODO",
853
+ willBeFixedIn: "TODO",
854
+ },
855
+ ],
856
+ }),
857
+ );
858
+
859
+ const testInfo = makeTestInfo({
860
+ updateSnapshots: "none",
861
+ snapshotsDir: tmpDir,
862
+ title: "unfinished test",
863
+ });
864
+ await expect(
865
+ checkAccessibility(makePage() as any, testInfo as any, {
866
+ bestPracticeMode: "off",
867
+ }),
868
+ ).rejects.toThrow("requires a reason");
869
+ });
870
+
871
+ it("keeps stale detection independent for each scan-specific on-disk baseline", async () => {
872
+ const fs = await import("fs");
873
+ const path = await import("path");
874
+ await fs.promises.writeFile(
875
+ path.join(tmpDir, "separate-scans-1.a11y-baseline-best-practice.json"),
876
+ JSON.stringify({
877
+ note: "Known best-practice violation.",
878
+ violations: [
879
+ {
880
+ rule: "page-has-heading-one",
881
+ targets: ["html"],
882
+ reason: "Known",
883
+ willBeFixedIn: "PROJ-353",
884
+ },
885
+ ],
886
+ }),
887
+ );
888
+ await fs.promises.writeFile(
889
+ path.join(tmpDir, "separate-scans-1.a11y-baseline.json"),
890
+ JSON.stringify({
891
+ note: "Previously broken image.",
892
+ violations: [
893
+ {
894
+ rule: "image-alt",
895
+ targets: ["img"],
896
+ reason: "Was broken",
897
+ willBeFixedIn: "PROJ-354",
898
+ },
899
+ ],
900
+ }),
901
+ );
902
+ mockAnalyze.mockResolvedValueOnce(
903
+ makeAxeResults({
904
+ violations: [makeViolation("page-has-heading-one", [["html"]])],
905
+ }),
906
+ );
907
+
908
+ const testInfo = makeTestInfo({
909
+ updateSnapshots: "none",
910
+ snapshotsDir: tmpDir,
911
+ title: "separate scans",
912
+ });
913
+ await checkAccessibility(makePage() as any, testInfo as any);
914
+
915
+ expect(
916
+ testInfo.annotations.filter(
917
+ (a) => a.type === "Baselined a11y violation",
918
+ ),
919
+ ).toEqual([
920
+ {
921
+ type: "Baselined a11y violation",
922
+ description: "page-has-heading-one: Known — PROJ-353",
923
+ },
924
+ ]);
925
+ expect(
926
+ testInfo.annotations.filter(
927
+ (a) => a.type === "Stale a11y baseline entry",
928
+ ),
929
+ ).toEqual([
930
+ {
931
+ type: "Stale a11y baseline entry",
932
+ description: "image-alt on img — no longer detected",
933
+ },
934
+ ]);
935
+ expect(mockToBe).not.toHaveBeenCalled();
936
+ });
937
+
938
+ it("produces per-call counter files for multi-call tests", async () => {
939
+ mockAnalyze.mockResolvedValue(makeAxeResults({ violations: [] }));
940
+
941
+ const testInfo = makeTestInfo({
942
+ updateSnapshots: "none",
943
+ snapshotsDir: tmpDir,
944
+ title: "multi call",
945
+ });
946
+ await checkAccessibility(makePage() as any, testInfo as any, {
947
+ bestPracticeMode: "off",
948
+ });
949
+ await checkAccessibility(makePage() as any, testInfo as any, {
950
+ bestPracticeMode: "off",
951
+ });
952
+
953
+ const fs = await import("fs");
954
+ const path = await import("path");
955
+ const f1 = path.join(tmpDir, "multi-call-1.a11y-baseline.json");
956
+ const f2 = path.join(tmpDir, "multi-call-2.a11y-baseline.json");
957
+ expect(fs.existsSync(f1)).toBe(true);
958
+ expect(fs.existsSync(f2)).toBe(true);
959
+ });
960
+
961
+ it("still uses snapshot mode when a legacy snapshot file exists for the test", async () => {
962
+ mockAnalyze.mockResolvedValue(makeAxeResults({ violations: [] }));
551
963
 
552
964
  // Pre-create a Playwright-style snapshot file in the snapshots dir.
553
- const fs = await import('fs')
554
- const path = await import('path')
965
+ const fs = await import("fs");
966
+ const path = await import("path");
555
967
  await fs.promises.writeFile(
556
- path.join(tmpDir, 'legacy-test-1-chromium-linux.txt'),
557
- '[]\n',
558
- )
559
-
560
- const testInfo = makeTestInfo({ updateSnapshots: 'none', snapshotsDir: tmpDir, title: 'legacy test' })
561
- await checkAccessibility(makePage() as any, testInfo as any, { bestPracticeMode: 'off' })
968
+ path.join(tmpDir, "legacy-test-1-chromium-linux.txt"),
969
+ "[]\n",
970
+ );
971
+
972
+ const testInfo = makeTestInfo({
973
+ updateSnapshots: "none",
974
+ snapshotsDir: tmpDir,
975
+ title: "legacy test",
976
+ });
977
+ await checkAccessibility(makePage() as any, testInfo as any, {
978
+ bestPracticeMode: "off",
979
+ });
562
980
 
563
981
  // Snapshot mode -> toMatchSnapshot was invoked.
564
- expect(mockToMatchSnapshot).toHaveBeenCalled()
565
- })
982
+ expect(mockToMatchSnapshot).toHaveBeenCalled();
983
+ });
566
984
 
567
985
  it("uses on-disk baseline mode when updateSnapshots is 'missing' (Playwright's default)", async () => {
568
- mockAnalyze.mockResolvedValue(makeAxeResults({ violations: [] }))
986
+ mockAnalyze.mockResolvedValue(makeAxeResults({ violations: [] }));
569
987
 
570
- const testInfo = makeTestInfo({ updateSnapshots: 'missing', snapshotsDir: tmpDir, title: 'default config' })
571
- await checkAccessibility(makePage() as any, testInfo as any, { bestPracticeMode: 'off' })
988
+ const testInfo = makeTestInfo({
989
+ updateSnapshots: "missing",
990
+ snapshotsDir: tmpDir,
991
+ title: "default config",
992
+ });
993
+ await checkAccessibility(makePage() as any, testInfo as any, {
994
+ bestPracticeMode: "off",
995
+ });
572
996
 
573
997
  // Playwright's default config value is 'missing' — not an explicit user
574
998
  // opt-in via --update-snapshots. It must route to baseline mode so new
575
999
  // a11y tests default to baseline mode as documented.
576
- const fs = await import('fs')
577
- const path = await import('path')
578
- const file = path.join(tmpDir, 'default-config-1.a11y-baseline.json')
579
- expect(fs.existsSync(file)).toBe(true)
580
- expect(mockToMatchSnapshot).not.toHaveBeenCalled()
581
- })
1000
+ const fs = await import("fs");
1001
+ const path = await import("path");
1002
+ const file = path.join(tmpDir, "default-config-1.a11y-baseline.json");
1003
+ expect(fs.existsSync(file)).toBe(true);
1004
+ expect(mockToMatchSnapshot).not.toHaveBeenCalled();
1005
+ });
582
1006
 
583
1007
  it("uses snapshot mode when updateSnapshots is 'changed' (explicit --update-snapshots)", async () => {
584
- mockAnalyze.mockResolvedValue(makeAxeResults({ violations: [] }))
1008
+ mockAnalyze.mockResolvedValue(makeAxeResults({ violations: [] }));
585
1009
 
586
- const testInfo = makeTestInfo({ updateSnapshots: 'changed', snapshotsDir: tmpDir, title: 'changed mode' })
587
- await checkAccessibility(makePage() as any, testInfo as any, { bestPracticeMode: 'off' })
1010
+ const testInfo = makeTestInfo({
1011
+ updateSnapshots: "changed",
1012
+ snapshotsDir: tmpDir,
1013
+ title: "changed mode",
1014
+ });
1015
+ await checkAccessibility(makePage() as any, testInfo as any, {
1016
+ bestPracticeMode: "off",
1017
+ });
588
1018
 
589
- expect(mockToMatchSnapshot).toHaveBeenCalled()
590
- })
1019
+ expect(mockToMatchSnapshot).toHaveBeenCalled();
1020
+ });
591
1021
 
592
1022
  it("uses snapshot mode when updateSnapshots is 'all' (explicit --update-snapshots=all)", async () => {
593
- mockAnalyze.mockResolvedValue(makeAxeResults({ violations: [] }))
1023
+ mockAnalyze.mockResolvedValue(makeAxeResults({ violations: [] }));
594
1024
 
595
- const testInfo = makeTestInfo({ updateSnapshots: 'all', snapshotsDir: tmpDir, title: 'all mode' })
596
- await checkAccessibility(makePage() as any, testInfo as any, { bestPracticeMode: 'off' })
1025
+ const testInfo = makeTestInfo({
1026
+ updateSnapshots: "all",
1027
+ snapshotsDir: tmpDir,
1028
+ title: "all mode",
1029
+ });
1030
+ await checkAccessibility(makePage() as any, testInfo as any, {
1031
+ bestPracticeMode: "off",
1032
+ });
597
1033
 
598
- expect(mockToMatchSnapshot).toHaveBeenCalled()
599
- })
600
- })
601
- })
1034
+ expect(mockToMatchSnapshot).toHaveBeenCalled();
1035
+ });
1036
+ });
1037
+ });