@lullabot/playwright-testing 0.1.0

package/lib/videos.js

@@ -0,0 +1,349 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.settleVideos = settleVideos;
+exports.restorePlayback = restorePlayback;
+exports.waitForVideos = waitForVideos;
+exports.restoreVideoPlayback = restoreVideoPlayback;
+/**
+ * Settle every <video> on the page so a screenshot of it is reproducible.
+ *
+ * A `<video>` is the one replaced element nothing else in the screenshot path
+ * covers. waitForImages() only looks at `img`, and Playwright's
+ * `animations: 'disabled'` only fast-forwards CSS transitions and Web
+ * Animations -- it does not touch media playback. That leaves two separate ways
+ * for a video to break a visual comparison:
+ *
+ * 1. **The frame is captured before it exists.** A full-page screenshot rasters
+ *    the whole document, including regions that were never in the viewport, so
+ *    it will happily capture a video that is still loading. The region paints as
+ *    an empty box on one capture and as the video on the next, which Playwright
+ *    reports as "Failed to take two consecutive stable screenshots" -- the run
+ *    burns its whole stability window and fails even though the image it ended
+ *    up with was correct. When the video is slower still, the captures agree on
+ *    the empty box, which is worse: it passes stability and bakes a blank
+ *    rectangle into the baseline if snapshots are being regenerated.
+ * 2. **The frame is captured while the video is playing.** An `autoplay muted
+ *    loop` video renders whatever moment the shutter caught. Chromium only
+ *    autoplays a muted video while it is on screen, so whether such a baseline
+ *    is reproducible at all currently rests on nothing more principled than
+ *    whether an earlier scroll happened to bring the video into the viewport.
+ *
+ * Both are fixed together: pin playback first, bring the video on screen so it
+ * both loads and composites, wait for a decodable frame, and rewind.
+ *
+ * Note this is deliberately *not* built on `requestVideoFrameCallback()`, which
+ * looks like the right signal and is not: it only fires when a frame is
+ * presented, and a paused off-screen video never presents one. Measured against
+ * a real page it never fired at all -- every call sat there until its timeout,
+ * adding the full timeout to each capture and reporting nothing.
+ */
+/**
+ * Settle every visible video, in the browser.
+ *
+ * This runs in the browser via `page.evaluate()`, which serializes the function
+ * source, so it must stay self-contained and reference nothing else in this
+ * module. It is exported separately from waitForVideos() so it can be tested
+ * directly.
+ *
+ * The order of operations matters and is not the obvious one:
+ *
+ * - Playback is pinned -- `autoplay` cleared and the element paused -- *before*
+ *   the video is scrolled into view, because bringing a muted `autoplay` video
+ *   on screen is exactly what starts it: measured on a real page, scrolling to
+ *   it first and pausing after left `currentTime` at 0.6s and climbing, while
+ *   pinning first held it at 0 through the same scroll. Pinning first turns the
+ *   scroll into a pure repaint.
+ * - The video is scrolled into view *before* its readiness is polled, because
+ *   for a lazily loaded video the scroll is what starts the load. Polling first
+ *   would sit there watching a `preload="none"` element that has been told not
+ *   to fetch anything, burn the whole timeout, and only then scroll it on
+ *   screen -- leaving the capture racing a load that had just begun.
+ *
+ * Videos are handled in serial: each one is scrolled into view in turn, and
+ * doing that concurrently would just mean the last one wins. Every scroll is
+ * undone -- the window offset and any scrollable ancestor `scrollIntoView()`
+ * moved -- so this composes with callers that care where the page is left;
+ * waitForImages(), in particular, does its own careful scroll back to the top.
+ *
+ * Playback state is not restored here, because the whole point is to leave the
+ * videos pinned at their first frame for the capture. It is recorded on each
+ * element so restorePlayback() can put it back afterwards.
+ *
+ * A video that never produces a frame is not an error. A page that legitimately
+ * references a missing or undecodable video should still be screenshotted and
+ * still have its accessibility checked, and the comparison is what fails. The
+ * wait is not silent either: the sources that never became ready are returned
+ * so the caller can report them.
+ *
+ * @param options.timeoutMs How long to wait for each video to become ready.
+ * @param options.pollMs How long to sleep between readiness checks.
+ * @param options.paintTimeoutMs How long to wait on a frame or a seek that may
+ *   never arrive.
+ * @returns The sources of the videos that never became ready, and whether the
+ *   page was scrolled at all.
+ */
+async function settleVideos(options) {
+    const timeoutMs = options.timeoutMs;
+    const pollMs = options.pollMs ?? 50;
+    const paintTimeoutMs = options.paintTimeoutMs ?? 1000;
+    // HTMLMediaElement readyStates: nothing loaded at all, and a frame exists for
+    // the current position.
+    const HAVE_NOTHING = 0;
+    const HAVE_CURRENT_DATA = 2;
+    const isCandidate = (video) => {
+        const rect = video.getBoundingClientRect();
+        if (rect.width === 0 || rect.height === 0) {
+            return false;
+        }
+        const style = getComputedStyle(video);
+        return style.visibility !== "hidden" && style.display !== "none";
+    };
+    // Taking a screenshot should not be what stops a video: a test may well go on
+    // to assert that it is playing. Record enough to put it back, on the element
+    // itself so a later evaluate() in the same page can find it, and only the
+    // first time so repeated captures do not record the pinned state as the
+    // original one. The key is spelled out here and in restorePlayback() rather
+    // than shared, because a reference to anything in this module would be a
+    // ReferenceError once this function is serialized into the browser.
+    const remember = (video) => {
+        if (!("__playwrightTestingVideoState" in video)) {
+            video["__playwrightTestingVideoState"] = {
+                autoplay: video.autoplay,
+                paused: video.paused,
+                currentTime: video.currentTime,
+            };
+        }
+    };
+    // A muted autoplay video plays whenever it is on screen, so clear the
+    // attribute as well as pausing -- otherwise Chromium restarts it the moment
+    // the scroll below makes it visible.
+    const pin = (video) => {
+        video.autoplay = false;
+        if (!video.paused) {
+            video.pause();
+        }
+    };
+    // Two frames, because the first only gets as far as scheduling the paint that
+    // the second then observes. The timeout is not decoration: Chromium stops
+    // running animation frames for a page it is not rendering -- one behind
+    // another tab of the same context, say -- and page.evaluate() has no timeout
+    // of its own, so an unguarded wait here hangs the whole capture.
+    const nextPaint = () => new Promise(resolve => {
+        const timer = setTimeout(resolve, paintTimeoutMs);
+        requestAnimationFrame(() => requestAnimationFrame(() => {
+            clearTimeout(timer);
+            resolve();
+        }));
+    });
+    // Assigning `currentTime` only *starts* a seek: the browser still has to
+    // decode the frame at the new position and present it, which lands several
+    // frames later when the position is not a keyframe. Waiting for `seeked` is
+    // what makes the rewind visible in the capture rather than a race.
+    const seeked = (video) => new Promise(resolve => {
+        const done = () => {
+            clearTimeout(timer);
+            video.removeEventListener("seeked", done);
+            resolve();
+        };
+        const timer = setTimeout(done, paintTimeoutMs);
+        video.addEventListener("seeked", done);
+    });
+    // scrollIntoView() scrolls every scrollable ancestor, not just the window, so
+    // record where each of them was. Without this a video inside a modal body, an
+    // off-canvas tray or a carousel track leaves that region showing something
+    // the baseline never had, and the pixel diff lands nowhere near the video.
+    const ancestorScrolls = (video) => {
+        const saved = [];
+        for (let node = video.parentElement; node; node = node.parentElement) {
+            if (node.scrollHeight > node.clientHeight || node.scrollWidth > node.clientWidth) {
+                saved.push({ element: node, top: node.scrollTop, left: node.scrollLeft });
+            }
+        }
+        return saved;
+    };
+    const sourceOf = (video) => {
+        const src = video.currentSrc || video.src
+            || video.querySelector("source")?.getAttribute("src") || "";
+        try {
+            return new URL(src, location.href).href;
+        }
+        catch {
+            return src;
+        }
+    };
+    const videos = Array.from(document.querySelectorAll("video")).filter(isCandidate);
+    if (videos.length === 0) {
+        return { notReady: [], scrolled: false };
+    }
+    const scrollX = window.scrollX;
+    const scrollY = window.scrollY;
+    const notReady = [];
+    const restoreWindowScroll = async () => {
+        window.scroll({ top: scrollY, left: scrollX, behavior: "instant" });
+        await nextPaint();
+        // window.scroll is async and can be dropped outright -- when application UI
+        // has locked the body, for example -- so confirm it landed and ask once more.
+        if (window.scrollX !== scrollX || window.scrollY !== scrollY) {
+            window.scroll({ top: scrollY, left: scrollX, behavior: "instant" });
+            await nextPaint();
+        }
+    };
+    try {
+        for (const video of videos) {
+            remember(video);
+            pin(video);
+            // Bringing the video on screen is both what starts a lazy load and what
+            // gives Chromium a reason to composite the first frame. Without it a
+            // full-page capture can raster the region before the frame is ready to
+            // paint.
+            const ancestors = ancestorScrolls(video);
+            try {
+                video.scrollIntoView({ block: "center", inline: "nearest", behavior: "instant" });
+                await nextPaint();
+                // `preload="none"` means the browser fetches nothing until playback is
+                // asked for, and nothing here ever asks. Scrolling alone does not override
+                // it, so say what is wanted and let the poll below wait for it.
+                if (video.readyState === HAVE_NOTHING) {
+                    video.preload = "auto";
+                    try {
+                        video.load();
+                    }
+                    catch {
+                        // Nothing loadable; reported as not ready below.
+                    }
+                }
+                // Each video gets its own deadline. A single deadline for the call would
+                // make `timeoutMs` a budget the first slow video can spend in full, so the
+                // second one is reported as having no frame without ever being waited for.
+                const deadline = Date.now() + timeoutMs;
+                const readyBeforeWait = video.readyState >= HAVE_CURRENT_DATA;
+                while (video.readyState < HAVE_CURRENT_DATA && Date.now() < deadline) {
+                    await new Promise(resolve => setTimeout(resolve, pollMs));
+                }
+                if (video.readyState < HAVE_CURRENT_DATA) {
+                    // Reported below, but still settled: a video that becomes ready between
+                    // here and the capture must not start playing off the back of `autoplay`.
+                    notReady.push(sourceOf(video));
+                }
+                else if (!readyBeforeWait) {
+                    // The frame only arrived during the wait, so spend a paint compositing it
+                    // while the video is still on screen. A video that was already ready was
+                    // composited by the paint after the scroll.
+                    await nextPaint();
+                }
+                // Belt and braces: `pin()` above should have kept it still through the
+                // scroll, and rewinding costs nothing if it did. `currentTime` throws on a
+                // source that is not seekable, such as a live stream.
+                pin(video);
+                try {
+                    if (video.currentTime !== 0) {
+                        video.currentTime = 0;
+                        await seeked(video);
+                        await nextPaint();
+                    }
+                }
+                catch {
+                    // Not seekable; whatever frame it is showing is the best available.
+                }
+            }
+            finally {
+                for (const { element, top, left } of ancestors) {
+                    element.scrollTop = top;
+                    element.scrollLeft = left;
+                }
+            }
+        }
+    }
+    finally {
+        await restoreWindowScroll();
+    }
+    return { notReady, scrolled: true };
+}
+/**
+ * Put back the playback state settleVideos() recorded, in the browser.
+ *
+ * Like settleVideos() this is serialized into the page by `page.evaluate()`, so
+ * it must stay self-contained, and it is exported separately from
+ * restoreVideoPlayback() so it can be tested directly.
+ *
+ * Videos that were not settled are left alone, and the recorded state is
+ * cleared as it is applied so a second call is a no-op rather than a rewind of
+ * whatever the test did in between.
+ */
+function restorePlayback() {
+    for (const video of Array.from(document.querySelectorAll("video"))) {
+        const saved = video["__playwrightTestingVideoState"];
+        if (!saved) {
+            continue;
+        }
+        delete video["__playwrightTestingVideoState"];
+        video.autoplay = saved.autoplay;
+        try {
+            if (video.currentTime !== saved.currentTime) {
+                video.currentTime = saved.currentTime;
+            }
+        }
+        catch {
+            // Not seekable; it was not seekable on the way in either.
+        }
+        if (!saved.paused && video.paused) {
+            const played = video.play();
+            if (played && typeof played.catch === "function") {
+                // Autoplay policy can refuse this. Restoring playback is best effort:
+                // the capture is already taken, and throwing here would fail a test for
+                // something it never asked for.
+                played.catch(() => { });
+            }
+        }
+    }
+}
+async function waitForVideos(page, options = {}) {
+    const timeoutMs = typeof options === "number" ? options : options.timeoutMs ?? 5000;
+    const afterScroll = typeof options === "number" ? undefined : options.afterScroll;
+    const notReady = [];
+    let scrolled = false;
+    for (const frame of page.frames()) {
+        try {
+            const result = await frame.evaluate(settleVideos, { timeoutMs });
+            notReady.push(...result.notReady);
+            scrolled = scrolled || result.scrolled;
+        }
+        catch {
+            // A frame that navigated or detached while we were working through the
+            // list has nothing left to settle.
+        }
+    }
+    if (scrolled && afterScroll) {
+        try {
+            await afterScroll(page);
+        }
+        catch (error) {
+            await restoreVideoPlayback(page);
+            throw error;
+        }
+    }
+    if (notReady.length > 0) {
+        console.warn(`waitForVideos: ${notReady.length} video(s) had no frame available within ${timeoutMs}ms and may be captured as an empty box: ${notReady.join(', ')}`);
+    }
+    return notReady;
+}
+/**
+ * Restore the playback state waitForVideos() pinned.
+ *
+ * Settling a video means pausing it, clearing `autoplay` and rewinding it, and
+ * a test that takes a screenshot has not asked for any of that to outlive the
+ * capture -- it may well go on to assert that the video is playing. Call this
+ * once the screenshot is taken to hand the page back as it was found.
+ *
+ * @param page
+ */
+async function restoreVideoPlayback(page) {
+    for (const frame of page.frames()) {
+        try {
+            await frame.evaluate(restorePlayback);
+        }
+        catch {
+            // Same as waitForVideos(): a frame that has gone away needs nothing.
+        }
+    }
+}

package/lib/visualdiff.d.ts

@@ -0,0 +1,154 @@
+import { type BrowserContext, type Page, type TestInfo } from '@playwright/test';
+import { type ScreenshotOptions } from "./accessible-screenshot.js";
+import type { AccessibilityBaseline } from './accessibility-baseline.js';
+import { type ForcedPseudoState } from './pseudo-state.js';
+import type { InteractionState } from './interaction-states.js';
+export interface VisualDiffExecutionContext {
+    page: Page;
+    testInfo: TestInfo;
+    testCase: VisualDiff;
+    group: VisualDiffGroup;
+    config?: VisualDiffUrlConfig;
+}
+export interface VisualDiffPreset {
+    /**
+     * Neutral adapter seam for screenshot defaults such as exclusions, browser
+     * thresholds, or post-scroll settling hooks.
+     */
+    screenshotOptions?: ScreenshotOptions | ((context: VisualDiffExecutionContext) => ScreenshotOptions | Promise<ScreenshotOptions>);
+}
+export declare function defineVisualDiffConfig(cases: VisualDiffUrlConfig, preset?: VisualDiffPreset): VisualDiffTestCases;
+export declare function defaultTestFunction(testCase: VisualDiff, group: VisualDiffGroup, config?: VisualDiffUrlConfig, preset?: VisualDiffPreset): ({ page, context }: {
+    page: Page;
+    context: BrowserContext;
+}, testInfo: TestInfo) => Promise<void>;
+/**
+ * Execute a set of visual diffs against groups of test cases.
+ */
+export declare class VisualDiffTestCases {
+    /**
+     * The configuration object containing all visual diff test cases.
+     * @private
+     */
+    private config;
+    private preset?;
+    /**
+     * Construct a new set of VisualDiffTestCases
+     *
+     * @param config The config that has been imported via "import ..."
+     */
+    constructor(config: VisualDiffUrlConfig, preset?: VisualDiffPreset);
+    /**
+     * Describe, execute, and skip test cases
+     *
+     * @param overriddenTestFunction An optional custom test function. Note: when
+     *   using a custom test function, automatic mask, interaction-state, and
+     *   pseudo-state handling is bypassed. You must apply them yourself.
+     */
+    describe(overriddenTestFunction?: (testCase: VisualDiff, group: VisualDiffGroup) => Function | void): void;
+}
+/**
+ * The top level configuration object.
+ */
+export type VisualDiffUrlConfig = {
+    name: string;
+    description?: string;
+    groups: VisualDiffGroup[];
+    /** Prefix prepended before group and case paths. */
+    pathPrefix?: string;
+    /** Fallback representative URL for every case. */
+    representativeUrl?: string;
+    /** Fallback network/page mock for every case. */
+    mockClass?: MockableConstructor;
+    /**
+     * CSS selectors for elements to mask globally across all test cases.
+     * Useful for dynamic content like copyright years that change over time.
+     * These are merged with any group-level and test-case-level masks.
+     */
+    mask?: string[];
+    /**
+     * The color of the overlay box for masked elements, in CSS color format.
+     * Can be overridden at the group or test-case level.
+     */
+    maskColor?: string;
+    /**
+     * Accessibility baseline for managing known violations.
+     * When provided, violations matching the baseline are suppressed and
+     * toMatchSnapshot() is skipped in favour of baseline-driven assertions.
+     */
+    a11yBaseline?: AccessibilityBaseline;
+    /**
+     * Cross-browser hover and focus states to apply for every test case. These
+     * are merged with group-level and test-case-level interaction states.
+     */
+    interactionStates?: VisualDiffInteractionState[];
+    /**
+     * Chromium-only pseudo-states to force for every test case. These are merged
+     * with group-level and test-case-level pseudo-states.
+     */
+    pseudoStates?: ForcedPseudoState[];
+};
+/**
+ * A group of Visual Diff test cases.
+ */
+export type VisualDiffGroup = BaseVisualDiff & {
+    pathPrefix?: string;
+    testCases: VisualDiff[];
+};
+export interface MockableConstructor {
+    new (): Mockable;
+}
+export interface Mockable {
+    mock(page: Page): Promise<void>;
+}
+/** A selector and real cross-browser interaction states to apply to it. */
+export interface VisualDiffInteractionState {
+    selector: string;
+    states: InteractionState[];
+}
+/**
+ * An individual test case.
+ */
+export type VisualDiff = BaseVisualDiff & {
+    path: string;
+};
+export type BaseVisualDiff = {
+    name: string;
+    description?: string;
+    representativeUrl?: string;
+    skip?: SkipTest;
+    mockClass?: MockableConstructor;
+    /** Accessibility baseline overriding less-specific configuration. */
+    a11yBaseline?: AccessibilityBaseline;
+    /**
+     * CSS selectors for elements to mask when taking screenshots.
+     * These are merged with any config-level and (for test cases) group-level masks.
+     */
+    mask?: string[];
+    /**
+     * The color of the overlay box for masked elements, in CSS color format.
+     * Overrides the mask color set at less-specific levels (config or group).
+     */
+    maskColor?: string;
+    /**
+     * Cross-browser hover and focus states to apply while capturing the
+     * screenshot and running its accessibility scan.
+     */
+    interactionStates?: VisualDiffInteractionState[];
+    /**
+     * Chromium-only pseudo-states to force while capturing the screenshot and
+     * running its accessibility scan.
+     */
+    pseudoStates?: ForcedPseudoState[];
+};
+/**
+ * A declaration that a test should be skipped.
+ *
+ * Nothing prevents calling test.skip() in a custom test function, but this
+ * type ensures that every skip has both a reason and a link to a ticket.
+ */
+export type SkipTest = {
+    reason: string;
+    willBeFixedIn: string;
+    callback?: (testCase: BaseVisualDiff) => boolean;
+};