@lullabot/playwright-testing 1.0.1 → 1.1.0

package/lib/accessible-screenshot.js

@@ -38,6 +38,9 @@ async function checkAccessibility(page, testInfo, options) {
     if (!testInfo.annotations.some((a) => a.type === "@a11y")) {
         testInfo.annotations.push({ type: "@a11y" });
     }
+    // An in-code baseline is shared by both scans. Track matches across this
+    // check so that a rule detected by either scan is not reported as stale.
+    const sharedBaselineMatches = new Set();
     if (bestPracticeMode !== "off") {
         const bpResults = await runBestPracticeScan(page, testInfo, {
             exclude: [...exclude, ...bestPracticeExclude],
@@ -52,7 +55,7 @@ async function checkAccessibility(page, testInfo, options) {
             scan: "best-practice",
             expectFn: test_1.expect.soft,
             scanLabel: "Best-practice scan",
-        }, baseline);
+        }, baseline, sharedBaselineMatches);
     }
     const wcagScanResults = await runWcagScan(page, testInfo, {
         wcagTags,
@@ -68,7 +71,7 @@ async function checkAccessibility(page, testInfo, options) {
         scan: "wcag",
         expectFn: test_1.expect,
         scanLabel: "WCAG scan",
-    }, baseline);
+    }, baseline, sharedBaselineMatches);
 }
 /**
  * Dispatch the assertion for one scan to the correct mode:
@@ -85,9 +88,9 @@ async function checkAccessibility(page, testInfo, options) {
  *    test (matching Playwright's missing-snapshot behaviour); locally,
  *    seeding passes so the first run is green.
  */
-async function dispatchAssertion(ctx, inCodeBaseline) {
+async function dispatchAssertion(ctx, inCodeBaseline, sharedBaselineMatches) {
     if (inCodeBaseline) {
-        return assertBaseline(ctx, inCodeBaseline);
+        return assertBaseline(ctx, inCodeBaseline, sharedBaselineMatches);
     }
     if (await (0, accessibility_baseline_file_js_1.snapshotExists)(ctx.testInfo)) {
         return assertSnapshot(ctx);
@@ -214,7 +217,7 @@ async function screenshotViolatingElements(page, testInfo, results) {
 /**
  * Assert violations against a baseline allowlist (in-code or on-disk).
  */
-function assertBaseline(ctx, baseline) {
+function assertBaseline(ctx, baseline, sharedBaselineMatches) {
     (0, accessibility_baseline_js_1.validateAccessibilityBaseline)(baseline);
     const { testInfo, results, scanLabel, expectFn } = ctx;
     const allViolations = extractNormalizedViolations(results);
@@ -229,6 +232,7 @@ function assertBaseline(ctx, baseline) {
         });
         if (baselineIndex >= 0) {
             matchedBaselineIndices.add(baselineIndex);
+            sharedBaselineMatches?.add(baselineIndex);
             const entry = baseline[baselineIndex];
             testInfo.annotations.push({
                 type: "Baselined a11y violation",
@@ -239,15 +243,21 @@ function assertBaseline(ctx, baseline) {
             unmatchedViolations.push(violation);
         }
     }
-    // Report stale baseline entries.
-    baseline.forEach((entry, idx) => {
-        if (!matchedBaselineIndices.has(idx)) {
-            testInfo.annotations.push({
-                type: "Stale a11y baseline entry",
-                description: `${entry.rule} on ${entry.targets.join(", ")} — no longer detected`,
-            });
-        }
-    });
+    // The WCAG scan always runs last, even when best-practice is disabled.
+    // Report shared entries once after both scans have contributed matches,
+    // before the hard assertion can throw. On-disk baselines belong to one
+    // scan and continue to report stale entries independently.
+    if (!sharedBaselineMatches || ctx.scan === "wcag") {
+        const matches = sharedBaselineMatches ?? matchedBaselineIndices;
+        baseline.forEach((entry, idx) => {
+            if (!matches.has(idx)) {
+                testInfo.annotations.push({
+                    type: "Stale a11y baseline entry",
+                    description: `${entry.rule} on ${entry.targets.join(", ")} — no longer detected`,
+                });
+            }
+        });
+    }
     // Summary annotation for baseline mode.
     const baselinedCount = matchedBaselineIndices.size;
     testInfo.annotations.push({

package/lib/github/failure-summary.js

@@ -292,7 +292,7 @@ function headline(report) {
     // seeing — a baselined violation passes and is still screenshotted.
     const parts = report.totalFailed === 0
         ? [":white_check_mark: No failing tests"]
-        : [`**${report.totalFailed}** failing test(s)`];
+        : [`:x: **${report.totalFailed}** failing test(s)`];
     if (report.totalFlaky > 0)
         parts.push(`:warning: **${report.totalFlaky}** flaky test(s)`);
     if (report.totalImages > 0)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lullabot/playwright-testing",
-  "version": "1.0.1",
+  "version": "1.1.0",
   "description": "Framework-neutral Playwright testing utilities",
   "license": "Apache-2.0",
   "repository": {

package/src/accessibility-baseline.test.ts

@@ -305,6 +305,175 @@ describe("accessibility baseline", () => {
       expect(staleAnnotations[0].description).toContain("#old-element");
       expect(staleAnnotations[0].description).toContain("no longer detected");
     });
+
+    it.each([
+      { name: "best-practice", bp: ["page-has-heading-one"], wcag: [] },
+      { name: "WCAG", bp: [], wcag: ["image-alt"] },
+      { name: "both scans", bp: ["page-has-heading-one"], wcag: ["image-alt"] },
+    ])(
+      "does not mark entries matched by $name as stale",
+      async ({ bp, wcag }) => {
+        const rules = [...bp, ...wcag];
+        const baseline: AccessibilityBaseline = rules.map((rule) => ({
+          rule,
+          targets: ["#known"],
+          reason: "Known issue",
+          willBeFixedIn: "PROJ-353",
+        }));
+        mockAnalyze
+          .mockResolvedValueOnce(
+            makeAxeResults({
+              violations: bp.map((rule) => makeViolation(rule, [["#known"]])),
+            }),
+          )
+          .mockResolvedValueOnce(
+            makeAxeResults({
+              violations: wcag.map((rule) => makeViolation(rule, [["#known"]])),
+            }),
+          );
+
+        const testInfo = makeTestInfo();
+        await checkAccessibility(makePage() as any, testInfo as any, {
+          baseline,
+        });
+
+        expect(
+          testInfo.annotations.filter(
+            (a) => a.type === "Stale a11y baseline entry",
+          ),
+        ).toEqual([]);
+        expect(
+          testInfo.annotations.filter(
+            (a) => a.type === "Baselined a11y violation",
+          ),
+        ).toHaveLength(rules.length);
+        expect(
+          testInfo.annotations.filter((a) =>
+            a.description?.includes("new violations"),
+          ),
+        ).toEqual([
+          {
+            type: "Accessibility",
+            description: `Best-practice scan: 0 new violations (${bp.length} baselined)`,
+          },
+          {
+            type: "Accessibility",
+            description: `WCAG scan: 0 new violations (${wcag.length} baselined)`,
+          },
+        ]);
+        expect(mockToBe).not.toHaveBeenCalled();
+      },
+    );
+
+    it("reports entries unmatched by either scan only once", async () => {
+      const testInfo = makeTestInfo();
+      await checkAccessibility(makePage() as any, testInfo as any, {
+        baseline: [
+          {
+            rule: "page-has-heading-one",
+            targets: ["html"],
+            reason: "Was broken",
+            willBeFixedIn: "PROJ-353",
+          },
+        ],
+      });
+
+      expect(
+        testInfo.annotations.filter(
+          (a) => a.type === "Stale a11y baseline entry",
+        ),
+      ).toEqual([
+        {
+          type: "Stale a11y baseline entry",
+          description: "page-has-heading-one on html — no longer detected",
+        },
+      ]);
+    });
+
+    it("does not carry baseline matches into later checks in the same test", async () => {
+      const baseline: AccessibilityBaseline = [
+        {
+          rule: "page-has-heading-one",
+          targets: ["html"],
+          reason: "Known issue",
+          willBeFixedIn: "PROJ-353",
+        },
+      ];
+      mockAnalyze.mockResolvedValueOnce(
+        makeAxeResults({
+          violations: [makeViolation("page-has-heading-one", [["html"]])],
+        }),
+      );
+      const testInfo = makeTestInfo();
+      await checkAccessibility(makePage() as any, testInfo as any, {
+        baseline,
+      });
+      expect(
+        testInfo.annotations.filter(
+          (a) => a.type === "Stale a11y baseline entry",
+        ),
+      ).toEqual([]);
+
+      await checkAccessibility(makePage() as any, testInfo as any, {
+        baseline,
+      });
+      expect(
+        testInfo.annotations.filter(
+          (a) => a.type === "Stale a11y baseline entry",
+        ),
+      ).toHaveLength(1);
+    });
+
+    it("reports stale entries before an unmatched WCAG violation throws", async () => {
+      const baseline: AccessibilityBaseline = [
+        {
+          rule: "page-has-heading-one",
+          targets: ["html"],
+          reason: "Known issue",
+          willBeFixedIn: "PROJ-353",
+        },
+        {
+          rule: "heading-order",
+          targets: ["h3"],
+          reason: "Was broken",
+          willBeFixedIn: "PROJ-354",
+        },
+      ];
+      mockAnalyze
+        .mockResolvedValueOnce(
+          makeAxeResults({
+            violations: [makeViolation("page-has-heading-one", [["html"]])],
+          }),
+        )
+        .mockResolvedValueOnce(
+          makeAxeResults({
+            violations: [makeViolation("image-alt", [["img"]])],
+          }),
+        );
+      mockToBe.mockImplementationOnce(() => {
+        throw new Error("new WCAG violation");
+      });
+      const testInfo = makeTestInfo();
+
+      await expect(
+        checkAccessibility(makePage() as any, testInfo as any, { baseline }),
+      ).rejects.toThrow("new WCAG violation");
+
+      expect(
+        testInfo.annotations.filter(
+          (a) => a.type === "Stale a11y baseline entry",
+        ),
+      ).toEqual([
+        {
+          type: "Stale a11y baseline entry",
+          description: "heading-order on h3 — no longer detected",
+        },
+      ]);
+      expect(mockExpectHard).toHaveBeenCalledWith(
+        null,
+        expect.stringContaining("image-alt"),
+      );
+    });
   });
 
   describe("snapshot bypass", () => {
@@ -699,6 +868,73 @@ describe("accessibility baseline", () => {
       ).rejects.toThrow("requires a reason");
     });
 
+    it("keeps stale detection independent for each scan-specific on-disk baseline", async () => {
+      const fs = await import("fs");
+      const path = await import("path");
+      await fs.promises.writeFile(
+        path.join(tmpDir, "separate-scans-1.a11y-baseline-best-practice.json"),
+        JSON.stringify({
+          note: "Known best-practice violation.",
+          violations: [
+            {
+              rule: "page-has-heading-one",
+              targets: ["html"],
+              reason: "Known",
+              willBeFixedIn: "PROJ-353",
+            },
+          ],
+        }),
+      );
+      await fs.promises.writeFile(
+        path.join(tmpDir, "separate-scans-1.a11y-baseline.json"),
+        JSON.stringify({
+          note: "Previously broken image.",
+          violations: [
+            {
+              rule: "image-alt",
+              targets: ["img"],
+              reason: "Was broken",
+              willBeFixedIn: "PROJ-354",
+            },
+          ],
+        }),
+      );
+      mockAnalyze.mockResolvedValueOnce(
+        makeAxeResults({
+          violations: [makeViolation("page-has-heading-one", [["html"]])],
+        }),
+      );
+
+      const testInfo = makeTestInfo({
+        updateSnapshots: "none",
+        snapshotsDir: tmpDir,
+        title: "separate scans",
+      });
+      await checkAccessibility(makePage() as any, testInfo as any);
+
+      expect(
+        testInfo.annotations.filter(
+          (a) => a.type === "Baselined a11y violation",
+        ),
+      ).toEqual([
+        {
+          type: "Baselined a11y violation",
+          description: "page-has-heading-one: Known — PROJ-353",
+        },
+      ]);
+      expect(
+        testInfo.annotations.filter(
+          (a) => a.type === "Stale a11y baseline entry",
+        ),
+      ).toEqual([
+        {
+          type: "Stale a11y baseline entry",
+          description: "image-alt on img — no longer detected",
+        },
+      ]);
+      expect(mockToBe).not.toHaveBeenCalled();
+    });
+
     it("produces per-call counter files for multi-call tests", async () => {
       mockAnalyze.mockResolvedValue(makeAxeResults({ violations: [] }));
 

package/src/accessible-screenshot.ts

@@ -253,6 +253,10 @@ export async function checkAccessibility(
     testInfo.annotations.push({ type: "@a11y" });
   }
 
+  // An in-code baseline is shared by both scans. Track matches across this
+  // check so that a rule detected by either scan is not reported as stale.
+  const sharedBaselineMatches = new Set<number>();
+
   if (bestPracticeMode !== "off") {
     const bpResults = await runBestPracticeScan(page, testInfo, {
       exclude: [...exclude, ...bestPracticeExclude],
@@ -270,6 +274,7 @@ export async function checkAccessibility(
         scanLabel: "Best-practice scan",
       },
       baseline,
+      sharedBaselineMatches,
     );
   }
 
@@ -292,6 +297,7 @@ export async function checkAccessibility(
       scanLabel: "WCAG scan",
     },
     baseline,
+    sharedBaselineMatches,
   );
 }
 
@@ -321,9 +327,10 @@ interface ScanContext {
 async function dispatchAssertion(
   ctx: ScanContext,
   inCodeBaseline?: AccessibilityBaseline,
+  sharedBaselineMatches?: Set<number>,
 ): Promise<void> {
   if (inCodeBaseline) {
-    return assertBaseline(ctx, inCodeBaseline);
+    return assertBaseline(ctx, inCodeBaseline, sharedBaselineMatches);
   }
 
   if (await snapshotExists(ctx.testInfo)) {
@@ -497,7 +504,11 @@ async function screenshotViolatingElements(
 /**
  * Assert violations against a baseline allowlist (in-code or on-disk).
  */
-function assertBaseline(ctx: ScanContext, baseline: AccessibilityBaseline) {
+function assertBaseline(
+  ctx: ScanContext,
+  baseline: AccessibilityBaseline,
+  sharedBaselineMatches?: Set<number>,
+) {
   validateAccessibilityBaseline(baseline);
   const { testInfo, results, scanLabel, expectFn } = ctx;
   const allViolations = extractNormalizedViolations(results);
@@ -517,6 +528,7 @@ function assertBaseline(ctx: ScanContext, baseline: AccessibilityBaseline) {
 
     if (baselineIndex >= 0) {
       matchedBaselineIndices.add(baselineIndex);
+      sharedBaselineMatches?.add(baselineIndex);
       const entry = baseline[baselineIndex];
       testInfo.annotations.push({
         type: "Baselined a11y violation",
@@ -527,15 +539,21 @@ function assertBaseline(ctx: ScanContext, baseline: AccessibilityBaseline) {
     }
   }
 
-  // Report stale baseline entries.
-  baseline.forEach((entry, idx) => {
-    if (!matchedBaselineIndices.has(idx)) {
-      testInfo.annotations.push({
-        type: "Stale a11y baseline entry",
-        description: `${entry.rule} on ${entry.targets.join(", ")} — no longer detected`,
-      });
-    }
-  });
+  // The WCAG scan always runs last, even when best-practice is disabled.
+  // Report shared entries once after both scans have contributed matches,
+  // before the hard assertion can throw. On-disk baselines belong to one
+  // scan and continue to report stale entries independently.
+  if (!sharedBaselineMatches || ctx.scan === "wcag") {
+    const matches = sharedBaselineMatches ?? matchedBaselineIndices;
+    baseline.forEach((entry, idx) => {
+      if (!matches.has(idx)) {
+        testInfo.annotations.push({
+          type: "Stale a11y baseline entry",
+          description: `${entry.rule} on ${entry.targets.join(", ")} — no longer detected`,
+        });
+      }
+    });
+  }
 
   // Summary annotation for baseline mode.
   const baselinedCount = matchedBaselineIndices.size;

package/src/github/failure-summary.test.ts

@@ -245,6 +245,7 @@ describe("generateSummary", () => {
     // attachment resolves, and that dead render is cached for good.
     const summary = generateSummary(reportWith("https://example.test/a"));
 
+    expect(summary).toContain(":x: **1** failing test(s)");
     expect(summary).not.toContain("<img");
     expect(summary).toContain(
       "1 screenshot(s) uploaded — see the pull request comment.",
@@ -396,7 +397,7 @@ describe("generateComment", () => {
       summaryUrl: "https://example.test/run/1",
     });
 
-    expect(comment).toContain("**1** failing test(s)");
+    expect(comment).toContain(":x: **1** failing test(s)");
     expect(comment).toContain("homepage matches");
     expect(comment).toContain("(https://example.test/run/1)");
   });

package/src/github/failure-summary.ts

@@ -382,7 +382,7 @@ function headline(report: FailureReport): string {
   const parts =
     report.totalFailed === 0
       ? [":white_check_mark: No failing tests"]
-      : [`**${report.totalFailed}** failing test(s)`];
+      : [`:x: **${report.totalFailed}** failing test(s)`];
 
   if (report.totalFlaky > 0)
     parts.push(`:warning: **${report.totalFlaky}** flaky test(s)`);