@lullabot/playwright-testing 0.1.0

package/src/github/index.test.ts

@@ -0,0 +1,24 @@
+import { describe, expect, it } from 'vitest'
+import {
+  AttachmentUploader,
+  createPathResolver,
+  generateA11yAnnotations,
+  generateA11ySummary,
+  generateFailureComment,
+  generateFailureSummary,
+  parseA11yResults,
+  parseFailures,
+} from './index'
+
+describe('GitHub package entry point', () => {
+  it('exports the reporting, attachment, and path-remapping APIs', () => {
+    expect(parseA11yResults).toBeTypeOf('function')
+    expect(generateA11ySummary).toBeTypeOf('function')
+    expect(generateA11yAnnotations).toBeTypeOf('function')
+    expect(parseFailures).toBeTypeOf('function')
+    expect(generateFailureSummary).toBeTypeOf('function')
+    expect(generateFailureComment).toBeTypeOf('function')
+    expect(AttachmentUploader).toBeTypeOf('function')
+    expect(createPathResolver).toBeTypeOf('function')
+  })
+})

package/src/github/index.ts

@@ -0,0 +1,35 @@
+export {
+  generateAnnotations as generateA11yAnnotations,
+  generateSummary as generateA11ySummary,
+  main as runA11ySummary,
+  parseA11yResults,
+} from './a11y-summary'
+export type { A11yReport, A11yTestResult } from './a11y-summary'
+
+export * from './attachments'
+
+export {
+  FAILURE_MARKER_PREFIX,
+  FLAKE_MARKER_PREFIX,
+  defuseMaskTriggers,
+  generateComment as generateFailureComment,
+  generateSummary as generateFailureSummary,
+  main as runFailureSummary,
+  parseFailures,
+  resolveImagePaths,
+  uploaderFromEnvironment,
+  uploadImages,
+} from './failure-summary'
+export type {
+  FailedTest,
+  FailureImage,
+  FailureReport,
+  FailureCommentOptions,
+  FailureSummaryMainOptions,
+  ImageKind,
+  IncludeMode,
+  PathResolutionSummary,
+  SummaryOptions,
+} from './failure-summary'
+
+export * from './report-paths'

package/src/github/report-paths.test.ts

@@ -0,0 +1,222 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import * as fs from 'fs'
+import * as os from 'os'
+import * as path from 'path'
+import { createPathResolver, parsePathPrefix } from './report-paths'
+
+let tmpDir: string
+
+beforeEach(() => {
+  tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'report-paths-test-'))
+})
+
+afterEach(() => {
+  fs.rmSync(tmpDir, { recursive: true, force: true })
+})
+
+/** Create a file, and every directory above it, under the temporary directory. */
+function touch(relativePath: string): string {
+  const filePath = path.join(tmpDir, relativePath)
+  fs.mkdirSync(path.dirname(filePath), { recursive: true })
+  fs.writeFileSync(filePath, 'png')
+  return filePath
+}
+
+describe('parsePathPrefix', () => {
+  it('splits on the first colon', () => {
+    expect(parsePathPrefix('/var/www/html:/home/runner/work/site/site')).toEqual({
+      from: '/var/www/html',
+      to: '/home/runner/work/site/site',
+    })
+  })
+
+  it('keeps a colon in the local path, as Windows drives have', () => {
+    expect(parsePathPrefix('/var/www/html:C:/checkout')).toEqual({
+      from: '/var/www/html',
+      to: 'C:/checkout',
+    })
+  })
+
+  it('rejects a value that is not a pair', () => {
+    expect(parsePathPrefix('/var/www/html')).toBeNull()
+    expect(parsePathPrefix(':/home/runner')).toBeNull()
+    expect(parsePathPrefix('/var/www/html:')).toBeNull()
+  })
+})
+
+describe('createPathResolver', () => {
+  it('leaves a path that already resolves alone', () => {
+    const filePath = touch('test/playwright/test-results/diff.png')
+    const resolve = createPathResolver({ reportPath: path.join(tmpDir, 'results.json') })
+
+    expect(resolve(filePath)).toEqual({ path: filePath, found: true })
+  })
+
+  it('re-roots a container path onto the checkout', () => {
+    touch('test/playwright/test-results/home-chromium/home-1-diff.png')
+    const resolve = createPathResolver({
+      reportPath: path.join(tmpDir, 'test/playwright/test-results/results.json'),
+    })
+
+    const resolution = resolve(
+      '/var/www/html/test/playwright/test-results/home-chromium/home-1-diff.png',
+    )
+
+    expect(resolution.found).toBe(true)
+    expect(resolution.path)
+      .toBe(path.join(tmpDir, 'test/playwright/test-results/home-chromium/home-1-diff.png'))
+    expect(resolution.prefix).toEqual({ from: '/var/www/html', to: tmpDir })
+  })
+
+  it('finds the files when the report sits outside the output directory', () => {
+    // The arrangement the integration test produces: the report is written
+    // beside the Playwright config while the images go to a separate output
+    // directory, so the two share nothing but the mount point.
+    touch('test/playwright/test-results-visual/fixture-chromium/fixture-diff.png')
+    const resolve = createPathResolver({
+      reportPath: path.join(tmpDir, 'test/playwright/visual-diff-results.json'),
+    })
+
+    const resolution = resolve(
+      '/var/www/html/test/playwright/test-results-visual/fixture-chromium/fixture-diff.png',
+    )
+
+    expect(resolution.found).toBe(true)
+    expect(resolution.path)
+      .toBe(path.join(tmpDir, 'test/playwright/test-results-visual/fixture-chromium/fixture-diff.png'))
+  })
+
+  it('does not assume a fixed offset between the report and the mount point', () => {
+    // This repository's own CI creates the project under a generated directory,
+    // so its depth below the workspace differs from the container's.
+    touch('tmp/pwtest-abc123/test/playwright/test-results/home-chromium/home-1-diff.png')
+    const resolve = createPathResolver({
+      reportPath: path.join(tmpDir, 'tmp/pwtest-abc123/test/playwright/test-results/results.json'),
+    })
+
+    const resolution = resolve(
+      '/var/www/html/test/playwright/test-results/home-chromium/home-1-diff.png',
+    )
+
+    expect(resolution.found).toBe(true)
+    expect(resolution.prefix).toEqual({
+      from: '/var/www/html',
+      to: path.join(tmpDir, 'tmp/pwtest-abc123'),
+    })
+  })
+
+  it('prefers the longest tail, so the most specific match wins', () => {
+    // A shallow decoy sharing the last two components must not beat the real
+    // file, which shares many more.
+    touch('decoy/fixture-chromium/fixture-diff.png')
+    touch('test/playwright/test-results/fixture-chromium/fixture-diff.png')
+
+    const resolve = createPathResolver({
+      reportPath: path.join(tmpDir, 'test/playwright/test-results/results.json'),
+    })
+
+    expect(resolve('/var/www/html/test/playwright/test-results/fixture-chromium/fixture-diff.png').path)
+      .toBe(path.join(tmpDir, 'test/playwright/test-results/fixture-chromium/fixture-diff.png'))
+  })
+
+  it('will not identify a file by its name alone', () => {
+    // One component of tail is a coincidence waiting to happen: some unrelated
+    // fixture-diff.png must not be served up as the failure's diff image.
+    touch('fixture-diff.png')
+    const resolve = createPathResolver({ reportPath: path.join(tmpDir, 'results.json') })
+
+    expect(resolve('/var/www/html/somewhere/else/fixture-diff.png').found).toBe(false)
+  })
+
+  it('reports a path it cannot find rather than inventing one', () => {
+    const resolve = createPathResolver({
+      reportPath: path.join(tmpDir, 'test/playwright/test-results/results.json'),
+    })
+
+    const resolution = resolve('/var/www/html/test/playwright/test-results/missing-diff.png')
+
+    expect(resolution).toEqual({
+      path: '/var/www/html/test/playwright/test-results/missing-diff.png',
+      found: false,
+    })
+  })
+
+  it('reuses a mapping it has already worked out', () => {
+    touch('test-results/one/a-diff.png')
+    touch('test-results/two/b-diff.png')
+
+    const looked: string[] = []
+    const resolve = createPathResolver({
+      reportPath: path.join(tmpDir, 'results.json'),
+      exists: filePath => {
+        looked.push(filePath)
+        return fs.existsSync(filePath)
+      },
+    })
+
+    resolve('/var/www/html/test-results/one/a-diff.png')
+    const afterFirst = looked.length
+    looked.length = 0
+
+    expect(resolve('/var/www/html/test-results/two/b-diff.png').found).toBe(true)
+    // The learned mapping is tried straight after the original path, rather
+    // than searching the tree again.
+    expect(looked.length).toBeLessThan(afterFirst)
+    expect(looked).toEqual([
+      '/var/www/html/test-results/two/b-diff.png',
+      path.join(tmpDir, 'test-results/two/b-diff.png'),
+    ])
+  })
+
+  it('prefers an explicit prefix over anything it would work out', () => {
+    touch('elsewhere/test-results/home-1-diff.png')
+    touch('test/playwright/test-results/home-1-diff.png')
+
+    const resolve = createPathResolver({
+      reportPath: path.join(tmpDir, 'test/playwright/test-results/results.json'),
+      prefixes: [{ from: '/var/www/html/test/playwright', to: path.join(tmpDir, 'elsewhere') }],
+    })
+
+    expect(resolve('/var/www/html/test/playwright/test-results/home-1-diff.png').path)
+      .toBe(path.join(tmpDir, 'elsewhere/test-results/home-1-diff.png'))
+  })
+
+  it('ignores a prefix the path is not under', () => {
+    touch('test-results/diff.png')
+    const resolve = createPathResolver({
+      reportPath: path.join(tmpDir, 'results.json'),
+      prefixes: [{ from: '/somewhere/else', to: tmpDir }],
+    })
+
+    // The explicit prefix does not apply, and one component of tail is too
+    // little to search on.
+    expect(resolve('/var/www/html/diff.png').found).toBe(false)
+  })
+
+  it('does not match a prefix part way through a directory name', () => {
+    touch('results/diff.png')
+    const resolve = createPathResolver({
+      reportPath: path.join(tmpDir, 'results.json'),
+      prefixes: [{ from: '/var/www/ht', to: tmpDir }],
+    })
+
+    // /var/www/html starts with the string /var/www/ht but is not under it,
+    // so the explicit mapping must not fire. The search still finds the file.
+    const resolution = resolve('/var/www/html/results/diff.png')
+    expect(resolution.path).toBe(path.join(tmpDir, 'results/diff.png'))
+    expect(resolution.prefix).toEqual({ from: '/var/www/html', to: tmpDir })
+  })
+
+  it('tolerates trailing slashes on the local side', () => {
+    touch('test-results/diff.png')
+    const resolve = createPathResolver({
+      reportPath: path.join(tmpDir, 'results.json'),
+      // A run of them, not just one: the trim is an index scan rather than a
+      // regex, because `/[\\/]+$/` backtracks quadratically over exactly this.
+      prefixes: [{ from: '/var/www/html', to: `${tmpDir}///` }],
+    })
+
+    expect(resolve('/var/www/html/test-results/diff.png').path)
+      .toBe(`${tmpDir}/test-results/diff.png`)
+  })
+})

package/src/github/report-paths.ts

@@ -0,0 +1,208 @@
+import * as fs from 'fs'
+import * as path from 'path'
+
+/**
+ * Translate the file paths a Playwright JSON report records into paths the
+ * process reading that report can actually open.
+ *
+ * The two are not always the same file system. Running Playwright inside a
+ * container — which is the normal arrangement for this package, where the suite
+ * runs in DDEV's web container — records every attachment under the path it had
+ * *there*, `/var/www/html/…`. A workflow step reading the report on the runner
+ * is on the other side of that boundary, and `/var/www/html` means nothing to
+ * it. The report is readable, the attachments are not, and nothing about the
+ * failure says why.
+ *
+ * The bind mount that creates the problem also solves it: the same files are
+ * visible from both sides under different prefixes, so the paths need
+ * re-rooting rather than fetching. What the two views share is the tail of the
+ * path — the part below the mount point — so the mount point is found by
+ * hanging progressively longer tails of a recorded path off the directories
+ * around the report until one of them names a file that exists.
+ *
+ * Every mapping is confirmed against the disk before it is used, so a wrong
+ * guess resolves to nothing rather than to the wrong image. The report's own
+ * `config` is deliberately not consulted: `rootDir` is the *test* directory
+ * rather than the project root, and `outputDir` need not contain the report,
+ * so neither reliably shares a tail with the path the report was read from.
+ *
+ * Symlinking the container path on the runner instead looks like it should
+ * work and does not: the Ubuntu runner image ships Apache, so /var/www/html
+ * already exists as a directory and `ln -sfn` quietly creates the link inside
+ * it. The command exits 0 and changes nothing.
+ */
+
+/**
+ * Shortest tail worth trying, in path components.
+ *
+ * A bare file name is too little to identify a file by — some unrelated
+ * `fixture-diff.png` elsewhere in the tree would match. One directory of
+ * context makes a coincidence unlikely, and longer tails are tried first
+ * regardless, so the most specific match always wins.
+ */
+const MIN_TAIL_COMPONENTS = 2
+
+export interface PathPrefix {
+  /** Absolute path as recorded in the report — typically a container path. */
+  from: string
+  /** The same directory as seen from here. */
+  to: string
+}
+
+export interface PathResolution {
+  /** The path to read. Unchanged when the recorded one was already fine. */
+  path: string
+  /** Whether a readable file was found. */
+  found: boolean
+  /** The mapping that made it readable, when one was needed. */
+  prefix?: PathPrefix
+}
+
+export interface PathResolverOptions {
+  /** Where the JSON report was read from, on this side of the boundary. */
+  reportPath: string
+  /** Explicit mappings. Tried before anything is worked out. */
+  prefixes?: PathPrefix[]
+  /** Existence check, injectable for tests. */
+  exists?: (filePath: string) => boolean
+}
+
+/**
+ * Parse a `FROM:TO` pair, as `--path-prefix` takes it.
+ *
+ * Split on the first colon: `TO` is a local absolute path and may itself
+ * contain one on Windows, whereas `FROM` comes from a container and will not.
+ */
+export function parsePathPrefix(value: string): PathPrefix | null {
+  const separator = value.indexOf(':')
+  if (separator <= 0) return null
+
+  const from = value.slice(0, separator).trim()
+  const to = value.slice(separator + 1).trim()
+  if (!from || !to) return null
+
+  return { from, to }
+}
+
+/**
+ * Build the function that turns a recorded path into a readable one.
+ *
+ * A path that already resolves is left alone, so this costs one `stat` per
+ * attachment when the report and the files are on the same side. A mapping
+ * worked out for one attachment is kept and tried first for the rest, so the
+ * search runs once per run rather than once per image.
+ */
+export function createPathResolver(
+  options: PathResolverOptions,
+): (filePath: string) => PathResolution {
+  const exists = options.exists ?? ((filePath: string) => fs.existsSync(filePath))
+  const explicit = options.prefixes ?? []
+  const searchRoots = ancestors(path.dirname(path.resolve(options.reportPath)))
+  const learned: PathPrefix[] = []
+
+  /** Hang tails of the recorded path off each directory around the report. */
+  function search(filePath: string): PathResolution | null {
+    const target = splitPath(filePath)
+
+    // Longest tail first, so the most specific match wins. Stop one short of
+    // the whole path: something has to be left to rewrite.
+    for (let length = target.components.length - 1; length >= MIN_TAIL_COMPONENTS; length--) {
+      const tail = target.components.slice(target.components.length - length).join('/')
+
+      for (const root of searchRoots) {
+        const candidate = `${trimTrailingSeparators(root)}/${tail}`
+        if (!exists(candidate)) continue
+
+        return {
+          path: candidate,
+          found: true,
+          prefix: { from: joinPath(target, target.components.length - length), to: root },
+        }
+      }
+    }
+
+    return null
+  }
+
+  return (filePath: string): PathResolution => {
+    if (exists(filePath)) return { path: filePath, found: true }
+
+    for (const prefix of [...explicit, ...learned]) {
+      const rewritten = applyPrefix(filePath, prefix)
+      if (rewritten && exists(rewritten)) return { path: rewritten, found: true, prefix }
+    }
+
+    const discovered = search(filePath)
+    if (!discovered) return { path: filePath, found: false }
+
+    if (discovered.prefix) learned.push(discovered.prefix)
+    return discovered
+  }
+}
+
+/** Swap one prefix for another, or null when the path is not under it. */
+function applyPrefix(filePath: string, prefix: PathPrefix): string | null {
+  const from = splitPath(prefix.from)
+  const target = splitPath(filePath)
+
+  if (target.components.length < from.components.length) return null
+  for (let index = 0; index < from.components.length; index++) {
+    if (target.components[index] !== from.components[index]) return null
+  }
+
+  const rest = target.components.slice(from.components.length)
+  const to = trimTrailingSeparators(prefix.to)
+  return rest.length === 0 ? to : `${to}/${rest.join('/')}`
+}
+
+/**
+ * Trim trailing separators.
+ *
+ * An index scan rather than `/[\\/]+$/`, because that pattern backtracks
+ * quadratically over a long run of separators — and these strings are paths
+ * out of a report this package did not write.
+ */
+function trimTrailingSeparators(value: string): string {
+  let end = value.length
+  while (end > 0 && (value[end - 1] === '/' || value[end - 1] === '\\')) end--
+  return value.slice(0, end)
+}
+
+interface SplitPath {
+  /** `/`, `C:/`, or empty for a relative path. */
+  root: string
+  components: string[]
+}
+
+/**
+ * Split a path into a root and its components, accepting either separator.
+ *
+ * The two sides of the boundary need not agree on the separator, and the
+ * recorded paths come from a report rather than from this platform.
+ */
+function splitPath(value: string): SplitPath {
+  const match = /^([A-Za-z]:[\\/]|[\\/])?/.exec(value)
+  const prefix = match?.[1] ?? ''
+  return {
+    root: prefix.replace(/\\/g, '/'),
+    components: value.slice(prefix.length).split(/[\\/]+/).filter(Boolean),
+  }
+}
+
+/** Rebuild a path from its first `count` components. */
+function joinPath(split: SplitPath, count: number): string {
+  return split.root + split.components.slice(0, count).join('/')
+}
+
+/** A directory and every directory above it. */
+function ancestors(directory: string): string[] {
+  const out: string[] = []
+  let current = directory
+
+  for (;;) {
+    out.push(current)
+    const parent = path.dirname(current)
+    if (parent === current) return out
+    current = parent
+  }
+}

package/src/hover.test.ts

@@ -0,0 +1,76 @@
+import {afterEach, describe, expect, it, vi} from 'vitest'
+import {clearHover} from './hover'
+
+function fakePage() {
+  let shield: any
+  const document = {
+    querySelector: vi.fn(() => shield),
+    createElement: vi.fn(() => {
+      shield = {
+        setAttribute: vi.fn(),
+        style: {cssText: ''},
+        remove: vi.fn(() => {
+          shield = undefined
+        }),
+      }
+      return shield
+    }),
+    documentElement: {
+      appendChild: vi.fn(),
+    },
+  }
+  const page = {
+    evaluate: vi.fn(async (callback: (attribute: string) => void, attribute: string) => {
+      vi.stubGlobal('document', document)
+      callback(attribute)
+    }),
+    mouse: {
+      move: vi.fn().mockResolvedValue(undefined),
+    },
+  }
+
+  return {document, page, getShield: () => shield}
+}
+
+describe('clearHover', () => {
+  afterEach(() => {
+    vi.unstubAllGlobals()
+  })
+
+  it('covers page content and moves the pointer onto the shield', async () => {
+    const {document, page, getShield} = fakePage()
+
+    const cleanup = await clearHover(page as any)
+    const shield = getShield()
+
+    expect(document.createElement).toHaveBeenCalledWith('playwright-testing-hover-shield')
+    expect(shield.setAttribute).toHaveBeenCalledWith('aria-hidden', 'true')
+    expect(shield.style.cssText).toContain('position: fixed !important')
+    expect(shield.style.cssText).toContain('pointer-events: auto !important')
+    expect(shield.style.cssText).toContain('opacity: 0 !important')
+    expect(document.documentElement.appendChild).toHaveBeenCalledWith(shield)
+    expect(page.mouse.move).toHaveBeenCalledWith(0, 0)
+
+    await cleanup()
+    expect(shield.remove).toHaveBeenCalledOnce()
+  })
+
+  it('makes cleanup idempotent', async () => {
+    const {page, getShield} = fakePage()
+    const cleanup = await clearHover(page as any)
+    const shield = getShield()
+
+    await cleanup()
+    await cleanup()
+
+    expect(shield.remove).toHaveBeenCalledOnce()
+  })
+
+  it('removes the shield when moving the pointer fails', async () => {
+    const {page, getShield} = fakePage()
+    page.mouse.move.mockRejectedValueOnce(new Error('mouse failed'))
+
+    await expect(clearHover(page as any)).rejects.toThrow('mouse failed')
+    expect(getShield()).toBeUndefined()
+  })
+})

package/src/hover.ts

@@ -0,0 +1,64 @@
+import {Page} from "@playwright/test";
+
+const HOVER_SHIELD_ATTRIBUTE = 'data-playwright-testing-hover-shield';
+
+/**
+ * Move the pointer away from page content for a deterministic screenshot.
+ *
+ * Pointer position survives clicks, navigation, and layout changes. That can
+ * leave an unrelated element under the pointer and activate its `:hover`
+ * styles. A transparent shield gives the pointer a neutral target regardless
+ * of what the page renders at the chosen coordinate.
+ *
+ * The returned cleanup function removes the shield. Call it after the
+ * screenshot so normal pointer hit testing is restored.
+ */
+export async function clearHover(page: Page): Promise<() => Promise<void>> {
+  await page.evaluate((attribute) => {
+    document.querySelector(`[${attribute}]`)?.remove();
+
+    const shield = document.createElement('playwright-testing-hover-shield');
+    shield.setAttribute(attribute, '');
+    shield.setAttribute('aria-hidden', 'true');
+    shield.style.cssText = [
+      'all: initial !important',
+      'position: fixed !important',
+      'inset: 0 !important',
+      'display: block !important',
+      'width: 100vw !important',
+      'height: 100vh !important',
+      'margin: 0 !important',
+      'padding: 0 !important',
+      'border: 0 !important',
+      'opacity: 0 !important',
+      'pointer-events: auto !important',
+      'z-index: 2147483647 !important',
+    ].join(';');
+    document.documentElement.appendChild(shield);
+  }, HOVER_SHIELD_ATTRIBUTE);
+
+  try {
+    // Playwright emits a mousemove even when the pointer was already at this
+    // coordinate, ensuring mouseleave/pointerleave handlers get a chance to
+    // respond to the new hit target.
+    await page.mouse.move(0, 0);
+  } catch (error) {
+    await removeHoverShield(page);
+    throw error;
+  }
+
+  let removed = false;
+  return async () => {
+    if (removed) {
+      return;
+    }
+    removed = true;
+    await removeHoverShield(page);
+  };
+}
+
+async function removeHoverShield(page: Page): Promise<void> {
+  await page.evaluate((attribute) => {
+    document.querySelector(`[${attribute}]`)?.remove();
+  }, HOVER_SHIELD_ATTRIBUTE);
+}