@lullabot/playwright-testing 0.1.0

package/src/focus.test.ts

@@ -0,0 +1,34 @@
+import {afterEach, describe, expect, it, vi} from 'vitest'
+import {blurActiveElement} from './focus'
+
+describe('blurActiveElement', () => {
+  afterEach(() => vi.unstubAllGlobals())
+
+  it('blurs a genuinely focused element', async () => {
+    const activeElement = {blur: vi.fn()}
+    const document = {activeElement, body: {}, documentElement: {}}
+    const page = {
+      evaluate: vi.fn((callback: () => boolean) => {
+        vi.stubGlobal('document', document)
+        return callback()
+      }),
+    }
+
+    await expect(blurActiveElement(page as never)).resolves.toBe(true)
+    expect(activeElement.blur).toHaveBeenCalledOnce()
+  })
+
+  it('does not blur the document body fallback', async () => {
+    const body = {blur: vi.fn()}
+    const document = {activeElement: body, body, documentElement: {}}
+    const page = {
+      evaluate: vi.fn((callback: () => boolean) => {
+        vi.stubGlobal('document', document)
+        return callback()
+      }),
+    }
+
+    await expect(blurActiveElement(page as never)).resolves.toBe(false)
+    expect(body.blur).not.toHaveBeenCalled()
+  })
+})

package/src/focus.ts

@@ -0,0 +1,27 @@
+import {Page} from "@playwright/test";
+
+/**
+ * Remove keyboard focus from the active element.
+ *
+ * A focused form control paints a focus outline/ring. Whether an element still
+ * has focus when the screenshot is taken depends on test timing, so that ring
+ * appears in some runs and not others, producing a visual diff that is invisible
+ * to a human but trips the strict pixel budget. Blurring the active element
+ * first makes the focus state deterministic.
+ *
+ * @param page
+ * @returns `true` if an element was blurred, `false` if nothing was focused.
+ */
+export async function blurActiveElement(page: Page): Promise<boolean> {
+  return page.evaluate(() => {
+    const el = document.activeElement as HTMLElement | null;
+    // `document.activeElement` falls back to `<body>`/`<html>` when nothing is
+    // focused. Blurring that paints no focus ring but can still fire `focusout`
+    // handlers, so skip it and only blur a genuinely focused control.
+    if (!el || el === document.body || el === document.documentElement || typeof el.blur !== "function") {
+      return false;
+    }
+    el.blur();
+    return true;
+  });
+}

package/src/fonts.test.ts

@@ -0,0 +1,17 @@
+import {describe, expect, it, vi} from 'vitest'
+import {waitForFonts} from './fonts'
+
+describe('waitForFonts', () => {
+  it('waits for the browser font set to become ready', async () => {
+    const evaluate = vi.fn(async (callback: () => Promise<void>) => {
+      const ready = Promise.resolve()
+      vi.stubGlobal('document', {fonts: {ready}})
+      await callback()
+    })
+
+    await waitForFonts({evaluate} as never)
+
+    expect(evaluate).toHaveBeenCalledOnce()
+    vi.unstubAllGlobals()
+  })
+})

package/src/fonts.ts

@@ -0,0 +1,23 @@
+import {Page} from "@playwright/test";
+
+/**
+ * Wait for all web fonts to finish loading.
+ *
+ * Visual snapshots can flake when text is first painted with fallback font
+ * metrics before the web fonts finish loading. The fallback glyphs have
+ * different widths, so text wraps onto a different number of lines and the page
+ * renders at a slightly different height than it does once the real fonts are
+ * applied. By the time Playwright retries the layout has settled, which is why
+ * these failures usually only show up on the first attempt.
+ *
+ * Awaiting `document.fonts.ready` blocks until every font face the page has
+ * requested has finished loading (or failed), so the screenshot always captures
+ * the settled, web-font layout.
+ *
+ * @param page
+ */
+export async function waitForFonts(page: Page): Promise<void> {
+  await page.evaluate(async () => {
+    await document.fonts.ready;
+  });
+}

package/src/frames.test.ts

@@ -0,0 +1,75 @@
+import { describe, expect, it, vi } from 'vitest';
+
+import { waitForFrames } from './frames';
+
+function makeLocator({
+  connected = true,
+  visible = true,
+  frame = makeFrame(),
+}: {
+  connected?: boolean;
+  visible?: boolean;
+  frame?: ReturnType<typeof makeFrame> | null;
+} = {}) {
+  const element = {
+    contentFrame: vi.fn().mockResolvedValue(frame),
+  };
+
+  return {
+    evaluate: vi.fn().mockResolvedValue(connected),
+    isVisible: vi.fn().mockResolvedValue(visible),
+    scrollIntoViewIfNeeded: vi.fn().mockResolvedValue(undefined),
+    elementHandle: vi.fn().mockResolvedValue(element),
+    element,
+  };
+}
+
+function makeFrame() {
+  return {
+    waitForURL: vi.fn().mockResolvedValue(undefined),
+    waitForLoadState: vi.fn().mockResolvedValue(undefined),
+  };
+}
+
+function makePage(locators: ReturnType<typeof makeLocator>[]) {
+  return {
+    locator: vi.fn().mockReturnValue({
+      all: vi.fn().mockResolvedValue(locators),
+    }),
+  } as never;
+}
+
+describe('waitForFrames', () => {
+  it('waits for each visible frame URL and load state', async () => {
+    const first = makeLocator();
+    const second = makeLocator();
+
+    await waitForFrames(makePage([first, second]));
+
+    for (const locator of [first, second]) {
+      expect(locator.scrollIntoViewIfNeeded).toHaveBeenCalledTimes(2);
+      expect(locator.element.contentFrame).toHaveBeenCalledOnce();
+      expect(locator.element.contentFrame.mock.results[0].value).toBeInstanceOf(Promise);
+      const frame = await locator.element.contentFrame.mock.results[0].value;
+      expect(frame?.waitForURL).toHaveBeenCalledOnce();
+      expect(frame?.waitForLoadState).toHaveBeenCalledWith('load');
+    }
+  });
+
+  it('ignores frames that are detached or hidden', async () => {
+    const detached = makeLocator({ connected: false });
+    const hidden = makeLocator({ visible: false });
+
+    await waitForFrames(makePage([detached, hidden]));
+
+    expect(detached.isVisible).not.toHaveBeenCalled();
+    expect(detached.elementHandle).not.toHaveBeenCalled();
+    expect(hidden.elementHandle).not.toHaveBeenCalled();
+  });
+
+  it('tolerates an iframe that detaches before its content frame is resolved', async () => {
+    const locator = makeLocator({ frame: null });
+
+    await expect(waitForFrames(makePage([locator]))).resolves.toBeUndefined();
+  });
+});

package/src/frames.ts

@@ -0,0 +1,26 @@
+import {Page} from "@playwright/test";
+
+/**
+ * Wait for all frames to load.
+ *
+ * @param page
+ */
+export async function waitForFrames(page: Page): Promise<void> {
+  const locators = await page.locator('iframe');
+  // Trigger lazy-loading iframes. Since this should be fast, and we don't want to
+  // have to deal with concurrency bugs, we do this in serial.
+  for (const locator of await locators.all()) {
+    // Ensure iframes are connected to the DOM before trying to scroll to them.
+    // https://github.com/microsoft/playwright/issues/23758
+    if (await locator.evaluate(iframe => iframe.isConnected)) {
+      if (await locator.isVisible()) {
+        await locator.scrollIntoViewIfNeeded();
+        await locator.scrollIntoViewIfNeeded();
+        const element = await locator.elementHandle();
+        const frame = await element?.contentFrame();
+        await frame?.waitForURL(new RegExp('.*/.*', 'i'));
+        await frame?.waitForLoadState('load');
+      }
+    }
+  }
+}

package/src/github/a11y-summary.test.ts

@@ -0,0 +1,439 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import * as fs from 'fs'
+import * as os from 'os'
+import * as path from 'path'
+import { parseA11yResults, generateSummary, generateAnnotations } from './a11y-summary'
+
+// A minimal Playwright JSON report with a11y data.
+function makeReport(overrides: {
+  violations?: any[]
+  annotations?: any[]
+  attachments?: any[]
+  specTitle?: string
+  file?: string
+  line?: number
+} = {}) {
+  const {
+    violations = [],
+    annotations = [{ type: '@a11y' }],
+    attachments = [],
+    specTitle = 'homepage accessibility check',
+    file = 'tests/a11y.spec.ts',
+    line = 10,
+  } = overrides
+
+  return {
+    suites: [{
+      title: '',
+      file,
+      specs: [{
+        title: specTitle,
+        line,
+        tests: [{
+          annotations: [],
+          results: [{
+            // Playwright stores runtime annotations (from testInfo.annotations.push())
+            // at the result level, not the test level.
+            annotations,
+            attachments,
+          }],
+        }],
+      }],
+    }],
+  }
+}
+
+function makeWcagAttachment(violations: any[]) {
+  const body = Buffer.from(JSON.stringify({
+    violations: violations.map(v => ({
+      id: v.rule ?? 'color-contrast',
+      impact: v.impact ?? 'serious',
+      description: v.description ?? 'Elements must have sufficient color contrast',
+      helpUrl: v.helpUrl ?? 'https://dequeuniversity.com/rules/axe/4.7/color-contrast',
+      nodes: (v.targets ?? ['#main']).map((t: string) => ({ target: [t] })),
+    })),
+  })).toString('base64')
+
+  return {
+    name: 'a11y-wcag-scan-results',
+    body,
+    contentType: 'application/json',
+  }
+}
+
+describe('parseA11yResults', () => {
+  let tmpDir: string
+  let reportPath: string
+
+  beforeEach(() => {
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'a11y-summary-'))
+    reportPath = path.join(tmpDir, 'test-report.json')
+  })
+
+  afterEach(() => {
+    fs.rmSync(tmpDir, { recursive: true, force: true })
+  })
+
+  it('throws when report file does not exist', () => {
+    expect(() => parseA11yResults('/nonexistent/report.json')).toThrow(
+      'Playwright JSON report not found',
+    )
+  })
+
+  it('returns empty report when no a11y tests exist', () => {
+    const report = { suites: [{ specs: [{ tests: [{ annotations: [] }] }] }] }
+    fs.writeFileSync(reportPath, JSON.stringify(report))
+    const result = parseA11yResults(reportPath)
+    expect(result.tests).toHaveLength(0)
+    expect(result.totalViolations).toBe(0)
+  })
+
+  it('extracts tests with a11y annotations', () => {
+    const report = makeReport({
+      annotations: [
+        { type: '@a11y' },
+        { type: 'Accessibility', description: 'WCAG scan: 1 violations (50 rules passed)' },
+      ],
+      attachments: [makeWcagAttachment([{ rule: 'color-contrast', targets: ['#header'] }])],
+    })
+    fs.writeFileSync(reportPath, JSON.stringify(report))
+    const result = parseA11yResults(reportPath)
+    expect(result.tests).toHaveLength(1)
+    expect(result.tests[0].violations).toHaveLength(1)
+    expect(result.tests[0].violations[0].rule).toBe('color-contrast')
+    expect(result.totalViolations).toBe(1)
+  })
+
+  it('counts baselined and stale annotations', () => {
+    const report = makeReport({
+      annotations: [
+        { type: '@a11y' },
+        { type: 'Baselined a11y violation', description: 'color-contrast: known issue' },
+        { type: 'Baselined a11y violation', description: 'link-name: known issue' },
+        { type: 'Stale a11y baseline entry', description: 'image-alt on .hero — no longer detected' },
+      ],
+    })
+    fs.writeFileSync(reportPath, JSON.stringify(report))
+    const result = parseA11yResults(reportPath)
+    expect(result.totalBaselined).toBe(2)
+    expect(result.totalStale).toBe(1)
+  })
+
+  it('handles nested suites', () => {
+    const report = {
+      suites: [{
+        title: 'outer',
+        file: 'tests/a11y.spec.ts',
+        specs: [],
+        suites: [{
+          title: 'inner',
+          specs: [{
+            title: 'nested test',
+            line: 5,
+            tests: [{
+              annotations: [{ type: '@a11y' }, { type: 'Accessibility', description: 'passed' }],
+              results: [{ attachments: [] }],
+            }],
+          }],
+        }],
+      }],
+    }
+    fs.writeFileSync(reportPath, JSON.stringify(report))
+    const result = parseA11yResults(reportPath)
+    expect(result.tests).toHaveLength(1)
+    expect(result.tests[0].file).toBe('tests/a11y.spec.ts')
+  })
+})
+
+describe('generateSummary', () => {
+  it('shows green check when no violations', () => {
+    const md = generateSummary({
+      tests: [{
+        title: 'passes a11y',
+        file: 'tests/a11y.spec.ts',
+        line: 10,
+        annotations: [{ type: 'Accessibility', description: 'WCAG scan: 0 violations (50 rules passed)' }],
+        violations: [],
+      }],
+      totalViolations: 0,
+      totalBaselined: 0,
+      totalStale: 0,
+    })
+    expect(md).toContain(':white_check_mark:')
+    expect(md).toContain('All accessibility checks passed')
+  })
+
+  it('shows violation table when violations exist', () => {
+    const md = generateSummary({
+      tests: [{
+        title: 'homepage check',
+        file: 'tests/a11y.spec.ts',
+        line: 10,
+        annotations: [{ type: 'Accessibility', description: 'WCAG scan: 1 violations' }],
+        violations: [{
+          rule: 'color-contrast',
+          impact: 'serious',
+          description: 'Elements must have sufficient color contrast',
+          helpUrl: 'https://dequeuniversity.com/rules/axe/4.7/color-contrast',
+          targets: ['#header'],
+        }],
+      }],
+      totalViolations: 1,
+      totalBaselined: 0,
+      totalStale: 0,
+    })
+    expect(md).toContain('**1** violation(s)')
+    expect(md).toContain('color-contrast')
+    expect(md).toContain('| Rule |')
+    expect(md).toContain('`#header`')
+  })
+
+  it('returns message when no tests found', () => {
+    const md = generateSummary({ tests: [], totalViolations: 0, totalBaselined: 0, totalStale: 0 })
+    expect(md).toContain('No accessibility tests were found')
+  })
+
+  it('shows baselined and stale counts', () => {
+    const md = generateSummary({
+      tests: [{
+        title: 'baseline test',
+        file: 'test.ts',
+        line: 1,
+        annotations: [
+          { type: 'Baselined a11y violation', description: 'known issue' },
+          { type: 'Stale a11y baseline entry', description: 'no longer detected' },
+        ],
+        violations: [],
+      }],
+      totalViolations: 0,
+      totalBaselined: 1,
+      totalStale: 1,
+    })
+    expect(md).toContain('**1** baselined')
+    expect(md).toContain('**1** stale baseline entries')
+  })
+
+})
+
+describe('generateAnnotations', () => {
+  it('emits ::error for violations', () => {
+    const output = generateAnnotations({
+      tests: [{
+        title: 'test',
+        file: 'tests/a11y.spec.ts',
+        line: 15,
+        annotations: [],
+        violations: [{
+          rule: 'color-contrast',
+          impact: 'serious',
+          description: 'Elements must have sufficient color contrast',
+          helpUrl: 'https://example.com',
+          targets: ['#main'],
+        }],
+      }],
+      totalViolations: 1,
+      totalBaselined: 0,
+      totalStale: 0,
+    })
+    expect(output).toContain('::error file=tests/a11y.spec.ts,line=15,title=color-contrast (serious)')
+    expect(output).toContain('#main')
+  })
+
+  it('emits ::warning for stale baselines', () => {
+    const output = generateAnnotations({
+      tests: [{
+        title: 'test',
+        file: 'tests/a11y.spec.ts',
+        line: 10,
+        annotations: [{ type: 'Stale a11y baseline entry', description: 'no longer detected' }],
+        violations: [],
+      }],
+      totalViolations: 0,
+      totalBaselined: 0,
+      totalStale: 1,
+    })
+    expect(output).toContain('::warning file=tests/a11y.spec.ts,line=10,title=Stale a11y baseline')
+  })
+
+  it('returns empty string when no violations or stale entries', () => {
+    const output = generateAnnotations({
+      tests: [{
+        title: 'test',
+        file: 'test.ts',
+        line: 1,
+        annotations: [{ type: 'Accessibility', description: 'passed' }],
+        violations: [],
+      }],
+      totalViolations: 0,
+      totalBaselined: 0,
+      totalStale: 0,
+    })
+    expect(output).toBe('')
+  })
+})
+
+describe('workflow-command injection hardening', () => {
+  // Defence in depth: CSS target selectors are extracted by axe-core from
+  // page DOM, so a crafted page could inject workflow-command syntax.
+  it('escapes newlines and % in ::error output', () => {
+    const output = generateAnnotations({
+      tests: [{
+        title: 'test',
+        file: 'tests/a11y.spec.ts',
+        line: 15,
+        annotations: [],
+        violations: [{
+          rule: 'color-contrast',
+          impact: 'serious',
+          description: 'bad\ndescription with 100%',
+          helpUrl: 'https://example.com',
+          targets: ['#evil\n::error title=pwned::gotcha'],
+        }],
+      }],
+      totalViolations: 1,
+      totalBaselined: 0,
+      totalStale: 0,
+    })
+    // No raw newlines or literal '%' characters escape into the message body.
+    expect(output.split('\n')).toHaveLength(1)
+    expect(output).toContain('%0A')
+    expect(output).toContain('%25')
+    // The forged `::error` must not appear as a standalone command.
+    expect(output).not.toMatch(/\n::error title=pwned/)
+  })
+
+  it('escapes : and , in property values', () => {
+    const output = generateAnnotations({
+      tests: [{
+        title: 'test',
+        file: 'tests/with,comma:colon.spec.ts',
+        line: 1,
+        annotations: [],
+        violations: [{
+          rule: 'color-contrast',
+          impact: 'serious',
+          description: 'ok',
+          helpUrl: 'https://example.com',
+          targets: ['#main'],
+        }],
+      }],
+      totalViolations: 1,
+      totalBaselined: 0,
+      totalStale: 0,
+    })
+    expect(output).toContain('file=tests/with%2Ccomma%3Acolon.spec.ts')
+  })
+
+  it('escapes stale baseline warning content', () => {
+    const output = generateAnnotations({
+      tests: [{
+        title: 'test',
+        file: 'tests/a11y.spec.ts',
+        line: 10,
+        annotations: [{ type: 'Stale a11y baseline entry', description: 'stale\n::error::forged' }],
+        violations: [],
+      }],
+      totalViolations: 0,
+      totalBaselined: 0,
+      totalStale: 1,
+    })
+    expect(output.split('\n')).toHaveLength(1)
+    expect(output).toContain('%0A')
+  })
+})
+
+describe('markdown escaping in the summary table', () => {
+  it('escapes pipes and newlines in description cells', () => {
+    const md = generateSummary({
+      tests: [{
+        title: 'homepage',
+        file: 'tests/a11y.spec.ts',
+        line: 10,
+        annotations: [],
+        violations: [{
+          rule: 'color-contrast',
+          impact: 'serious',
+          description: 'pipe | in | description\nand a newline',
+          helpUrl: 'https://example.com',
+          targets: ['#main'],
+        }],
+      }],
+      totalViolations: 1,
+      totalBaselined: 0,
+      totalStale: 0,
+    })
+    // Find the row line — the one that starts with `| [color-contrast]`.
+    const row = md.split('\n').find(l => l.startsWith('| [color-contrast]'))
+    expect(row).toBeDefined()
+    // Row should still have exactly 5 pipe-separated segments (4 columns + trailing).
+    expect(row!.split(' | ').length).toBe(4)
+    expect(row).toContain('\\|')
+    // No literal newline inside the cell.
+    expect(row).not.toMatch(/\n/)
+  })
+
+  it('escapes backticks in target code spans', () => {
+    const md = generateSummary({
+      tests: [{
+        title: 'homepage',
+        file: 'tests/a11y.spec.ts',
+        line: 10,
+        annotations: [],
+        violations: [{
+          rule: 'color-contrast',
+          impact: 'serious',
+          description: 'bad',
+          helpUrl: 'https://example.com',
+          // A target containing a backtick — would escape a naive `${t}` code span.
+          targets: ['#foo`span'],
+        }],
+      }],
+      totalViolations: 1,
+      totalBaselined: 0,
+      totalStale: 0,
+    })
+    // The raw selector must not appear wrapped in single backticks
+    // (which would be the unescaped form).
+    expect(md).not.toMatch(/`#foo`span`/)
+    // It should be wrapped in double backticks with padding instead.
+    expect(md).toContain('`` #foo`span ``')
+  })
+})
+
+describe('main() error handling', () => {
+  let tmpDir: string
+  let origCwd: string
+  let origSummary: string | undefined
+
+  beforeEach(() => {
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'a11y-main-'))
+    origCwd = process.cwd()
+    origSummary = process.env.GITHUB_STEP_SUMMARY
+    delete process.env.GITHUB_STEP_SUMMARY
+    process.chdir(tmpDir)
+  })
+
+  afterEach(() => {
+    process.chdir(origCwd)
+    if (origSummary !== undefined) {
+      process.env.GITHUB_STEP_SUMMARY = origSummary
+    }
+    fs.rmSync(tmpDir, { recursive: true, force: true })
+  })
+
+  it('returns without throwing when the report is missing', async () => {
+    const { main } = await import('./a11y-summary')
+    expect(() =>
+      main(['--mode=summary', '--report-path=nonexistent.json']),
+    ).not.toThrow()
+  })
+
+  it('propagates parse errors so CI surfaces real problems', async () => {
+    const bad = path.join(tmpDir, 'bad.json')
+    fs.writeFileSync(bad, '{ this is not json')
+    const { main } = await import('./a11y-summary')
+    expect(() =>
+      main(['--mode=summary', `--report-path=${bad}`]),
+    ).toThrow()
+  })
+})