donobu 2.15.0 → 2.16.0

package/dist/utils/PlaywrightUtils.js

@@ -1,5 +1,4 @@
 "use strict";
-// PlaywrightUtils.ts
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
@@ -10,23 +9,29 @@ const MiscUtils_1 = require("./MiscUtils");
 const path_1 = __importDefault(require("path"));
 const PageClosedException_1 = require("../exceptions/PageClosedException");
 const child_process_1 = require("child_process");
+const ControlPanel_1 = require("../managers/ControlPanel");
+const PageInspector_1 = require("../managers/PageInspector");
+const RunAccessibilityTestTool_1 = require("../tools/RunAccessibilityTestTool");
 /**
  * Miscellaneous utility functions for working with the Playwright SDK. If you are looking to
  * instantiate a Playwright instance, see PlaywrightSetup instead.
  */
 class PlaywrightUtils {
     /**
-     * Attempts to take a screenshot of the given page in PNG format, returning
-     * the raw byte array. If the operation fails, an empty array is returned.
+     * Attempts to take a screenshot of the given page, returning the raw byte
+     * array.
+     *
+     * If the operation fails, an empty array is returned.
+     *
      * Note that if the Donobu control panel is present in the page, it will be
      * excluded from the screenshot.
      */
-    static async takePngScreenshot(page) {
+    static async takeScreenshot(page, type = 'jpeg') {
         try {
-            const style = `#${PlaywrightUtils.DONOBU_CONTROL_PANEL_ELEMENT_ID} { display: none !important; }`;
+            const style = `#${ControlPanel_1.ControlPanel.DONOBU_CONTROL_PANEL_ELEMENT_ID} { display: none !important; }`;
             return await page.screenshot({
                 style,
-                type: 'png',
+                type: type,
             });
         }
         catch (error) {
@@ -47,56 +52,9 @@ class PlaywrightUtils {
      */
     static async generateSelectors(element) {
         return element.evaluate((elem) => {
-            return window.donobuGenerateSmartSelectors(elem);
+            return window.__donobu.generateSmartSelectors(elem);
         });
     }
-    /**
-     * Returns a JavaScript code snippet intended to run as an initialization
-     * script for `DonobuFlow` flows. This script helps other scripts to know
-     * which elements have mouse-click related event listeners attached to them
-     * via inspecting the {@code window.donobuGetClickableElements} element array.
-     */
-    static clickableElementsTrackerInitScript() {
-        return PlaywrightUtils.CLICKABLE_ELEMENTS_TRACKER_INIT_SCRIPT;
-    }
-    /**
-     * Returns a JavaScript code snippet intended to run as an initialization
-     * script for `DonobuFlow` flows. This script helps flows handle/track
-     * prompts/confirmations. This is done specially since these browser actions
-     * pause the Javascript main thread and also cause issues with running various
-     * Playwright actions. See `DonobuFlow.onDialog(Dialog)` for details.
-     */
-    static dialogPromptTrackerInitScript() {
-        return PlaywrightUtils.DIALOG_PROMPT_TRACKER_INIT_SCRIPT;
-    }
-    /**
-     * Returns a JavaScript code snippet intended to run as an initialization
-     * script for `DonobuFlow` flows. This is an in-page version of the
-     * `SelectorGenerator` class so that smart selectors can be generated in
-     * the web client. This is done so the
-     * {@link PlaywrightUtils.pageInteractionsTrackerInitScript()} can generate
-     * smart selectors on the fly and pass them to the `PageInteractionTracker`
-     * so the page interactions can be converted to synthetic `ToolCallResult`s.
-     */
-    static smartSelectorGeneratorInitScript() {
-        return PlaywrightUtils.SMART_SELECTOR_GENERATOR_INIT_SCRIPT;
-    }
-    /**
-     * This script is used by the `RunAccessibilityTestTool` to test the
-     * accessibility of a webpage. This script is injected on page load, rather
-     * than when running the `RunAccessibilityTestTool` because we need to bypass
-     * webpage Content Security Policy (CSP); this is done by injecting the
-     * script via `BrowserContext#addInitScript`.
-     */
-    static accessibilityTestInitScript() {
-        return PlaywrightUtils.ACCESSIBILITY_TEST_INIT_SCRIPT;
-    }
-    static pageInteractionsTrackerInitScript() {
-        return PlaywrightUtils.PAGE_INTERACTIONS_TRACKER_INIT_SCRIPT;
-    }
-    static donobuControlPanelInitScript() {
-        return PlaywrightUtils.DONOBU_CONTROL_PANEL_INIT_SCRIPT;
-    }
     /**
      * Sets up the given browser context so that it can be minimally used by the
      * rest of Donobu. This involves registering virtual routes for serving up
@@ -104,614 +62,10 @@ class PlaywrightUtils {
      * scripts.
      */
     static async setupBasicBrowserContext(browserContext) {
-        // Add Donobu CSS and SVG routes to the browser so that are not violating
-        // the CSP of any of the pages we attempt to render our custom stuff on.
-        await browserContext.route('**/donobu-virtual-mouse.svg', async (route) => {
-            await route.fulfill({
-                status: 200,
-                contentType: 'image/svg+xml',
-                body: MiscUtils_1.MiscUtils.getResourceFileAsString('donobu-virtual-mouse.svg'),
-            });
-        });
-        await browserContext.route('**/donobu.css', async (route) => {
-            await route.fulfill({
-                status: 200,
-                contentType: 'text/css',
-                body: MiscUtils_1.MiscUtils.getResourceFileAsString('donobu.css'),
-            });
-        });
-        await browserContext.addInitScript(PlaywrightUtils.accessibilityTestInitScript());
-        await browserContext.addInitScript(PlaywrightUtils.clickableElementsTrackerInitScript());
-        await browserContext.addInitScript(PlaywrightUtils.dialogPromptTrackerInitScript());
-        await browserContext.addInitScript(PlaywrightUtils.smartSelectorGeneratorInitScript());
-    }
-    /**
-     * Returns the given locator or its label, if it has one and is associated with a labelable element.
-     * Labelable elements include: button, input (except hidden), meter, output, progress, select, and textarea.
-     */
-    static async getLocatorOrItsLabel(element) {
-        const timeoutMilliseconds = 5000;
-        try {
-            // First check if the element itself is visible
-            try {
-                await element.waitFor({
-                    state: 'visible',
-                    timeout: timeoutMilliseconds,
-                });
-            }
-            catch (error) {
-                // If the element itself isn't visible, just return it without looking for labels
-                if (PlaywrightUtils.isPageClosedError(error)) {
-                    throw new PageClosedException_1.PageClosedException();
-                }
-                return element;
-            }
-            // Check if the element is a labelable element
-            // See https://developer.mozilla.org/en-US/docs/Web/HTML/Reference/Elements/label
-            const isLabelable = await element.evaluate((elem) => {
-                const tagName = elem.tagName.toLowerCase();
-                if (tagName === 'button' ||
-                    tagName === 'meter' ||
-                    tagName === 'output' ||
-                    tagName === 'progress' ||
-                    tagName === 'select' ||
-                    tagName === 'textarea') {
-                    return true;
-                }
-                if (tagName === 'input') {
-                    // Input is labelable except when type="hidden"
-                    const type = elem.getAttribute('type')?.toLowerCase() || 'text';
-                    return type !== 'hidden';
-                }
-                return false;
-            });
-            // If the element is not labelable, return the original locator
-            if (!isLabelable) {
-                return element;
-            }
-            // Try to get the ID attribute
-            const id = await element
-                .getAttribute('id', {
-                timeout: timeoutMilliseconds,
-            })
-                .catch(() => null);
-            if (id) {
-                try {
-                    // Get the underlying ElementHandle first
-                    const handle = await element.elementHandle({
-                        timeout: timeoutMilliseconds,
-                    });
-                    if (handle) {
-                        // From the handle, retrieve the frame that owns this element
-                        const frame = await handle.ownerFrame();
-                        if (frame) {
-                            // Look for label with matching 'for' attribute
-                            const labelByFor = frame.locator(`label[for="${id}"]`);
-                            // Check if the label exists and is visible
-                            if ((await labelByFor.count()) > 0) {
-                                try {
-                                    await labelByFor.waitFor({
-                                        state: 'visible',
-                                        timeout: timeoutMilliseconds,
-                                    });
-                                    return labelByFor;
-                                }
-                                catch (visibilityError) {
-                                    // If label exists but isn't visible, continue to other checks
-                                    if (PlaywrightUtils.isPageClosedError(visibilityError)) {
-                                        throw new PageClosedException_1.PageClosedException();
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-                catch (error) {
-                    // Handle errors in the ElementHandle/frame retrieval, but continue to other checks
-                    if (PlaywrightUtils.isPageClosedError(error)) {
-                        throw new PageClosedException_1.PageClosedException();
-                    }
-                }
-            }
-            // Fallback: look for a directly wrapping label (immediate ancestor)
-            // Using 'xpath=parent::label' to only get direct parent if it's a label
-            const directParentLabel = element.locator('xpath=parent::label');
-            if ((await directParentLabel.count()) > 0) {
-                try {
-                    await directParentLabel.waitFor({
-                        state: 'visible',
-                        timeout: timeoutMilliseconds,
-                    });
-                    return directParentLabel;
-                }
-                catch (visibilityError) {
-                    if (PlaywrightUtils.isPageClosedError(visibilityError)) {
-                        throw new PageClosedException_1.PageClosedException();
-                    }
-                }
-            }
-            // If direct parent isn't a label, check any ancestor label
-            const wrappingLabel = element.locator('xpath=ancestor::label[1]'); // Get closest ancestor label
-            if ((await wrappingLabel.count()) > 0) {
-                try {
-                    await wrappingLabel.waitFor({
-                        state: 'visible',
-                        timeout: timeoutMilliseconds,
-                    });
-                    return wrappingLabel;
-                }
-                catch (visibilityError) {
-                    if (PlaywrightUtils.isPageClosedError(visibilityError)) {
-                        throw new PageClosedException_1.PageClosedException();
-                    }
-                }
-            }
-            // Neither "for" nor wrapping label found or visible
-            return element;
-        }
-        catch (error) {
-            if (PlaywrightUtils.isPageClosedError(error)) {
-                throw new PageClosedException_1.PageClosedException();
-            }
-            throw error;
-        }
-    }
-    /**
-     * Reads and clears the desired next state as directed by the in-flow control
-     * panel. Note that the control panel does not have carte blanche control, as
-     * we only support the control panel signaling an intent to pause and resume a
-     * flow.
-     */
-    static async popControlPanelNextDesiredState(page) {
-        try {
-            const desiredNextState = await page.evaluate(() => {
-                const tmpState = window.donobuNextState;
-                window.donobuNextState = null;
-                return tmpState;
-            });
-            if (desiredNextState) {
-                return desiredNextState;
-            }
-        }
-        catch (error) {
-            if (PlaywrightUtils.isPageClosedError(error)) {
-                throw new PageClosedException_1.PageClosedException();
-            }
-            else {
-                throw error;
-            }
-        }
-        return null;
-    }
-    /**
-     * Hides the control panel by setting its display to 'none'. This ensures the
-     * panel is not visible and does not intercept mouse events.
-     */
-    static async hideControlPanel(focusedPage, flowMetadata) {
-        if (!flowMetadata.isControlPanelEnabled) {
-            return;
-        }
-        for (const page of focusedPage.context().pages()) {
-            try {
-                await page.evaluate((panelId) => {
-                    const element = document.querySelector(`#${panelId}`);
-                    if (element) {
-                        element.style.display = 'none';
-                    }
-                }, PlaywrightUtils.DONOBU_CONTROL_PANEL_ELEMENT_ID);
-            }
-            catch (error) {
-                if (PlaywrightUtils.isPageClosedError(error)) {
-                    // Pass; this can happen normally.
-                }
-                else {
-                    Logger_1.appLogger.error(`Unexpected exception while attempting to show control panel for flow '${flowMetadata.id}'`, error);
-                }
-            }
-        }
-    }
-    /**
-     * Shows the control panel by resetting its display property. Assumes the
-     * original display was 'block'.
-     */
-    static async showControlPanel(focusedPage, flowMetadata) {
-        if (!flowMetadata.isControlPanelEnabled) {
-            return;
-        }
-        for (const page of focusedPage.context().pages()) {
-            try {
-                await page.evaluate((panelId) => {
-                    const element = document.querySelector(`#${panelId}`);
-                    if (element) {
-                        element.style.display = 'block';
-                    }
-                }, PlaywrightUtils.DONOBU_CONTROL_PANEL_ELEMENT_ID);
-            }
-            catch (error) {
-                if (PlaywrightUtils.isPageClosedError(error)) {
-                    // Pass; this can happen normally.
-                }
-                else {
-                    Logger_1.appLogger.error(`Unexpected exception while attempting to show control panel for flow '${flowMetadata.id}'`, error);
-                }
-            }
-        }
-    }
-    /**
-     * Updates all control panels in the given browser context. If the provided
-     * headline is non-null, the headline in the control panel will be updated.
-     */
-    static async updateControlPanel(focusedPage, flowMetadata, headline) {
-        const controlPanelQuerySelector = `#${PlaywrightUtils.DONOBU_CONTROL_PANEL_ELEMENT_ID}`;
-        const controlPanelHeadlineQuerySelector = `#${PlaywrightUtils.DONOBU_CONTROL_PANEL_HEADLINE_ELEMENT_ID}`;
-        for (const page of focusedPage.context().pages()) {
-            try {
-                const controlPanel = page.locator(controlPanelQuerySelector).first();
-                if ((await controlPanel.count()) === 1) {
-                    await controlPanel.evaluate((element, state) => {
-                        element.donobuFlowState = state;
-                    }, flowMetadata.state.toString());
-                    if (headline) {
-                        const controlPanelHeadline = page
-                            .locator(controlPanelHeadlineQuerySelector)
-                            .first();
-                        if ((await controlPanelHeadline.count()) === 1) {
-                            const escapedMessage = headline.replace(/[\\"']/g, '\\$&');
-                            await controlPanelHeadline.evaluate((element, message) => {
-                                element.textContent = message;
-                            }, escapedMessage);
-                        }
-                    }
-                }
-            }
-            catch (error) {
-                if (PlaywrightUtils.isPageClosedError(error)) {
-                    // Pass; this can happen normally.
-                }
-                else {
-                    Logger_1.appLogger.error(`Unexpected exception while attempting to show control panel for flow '${flowMetadata.id}'`, error);
-                }
-            }
-        }
-    }
-    /**
-     * Returns all elements that have the
-     * {@link PlaywrightUtils.DONOBU_INTERACTABLE_ATTRIBUTE} HTML attribute.
-     **/
-    static async getAttributedInteractableElements(page) {
-        try {
-            const interactableElements = [];
-            const frames = page
-                .frames()
-                .filter((frame) => PlaywrightUtils.frameFilter(frame));
-            for (const frame of frames) {
-                const attributeMap = await frame.evaluate((interactableAttribute) => {
-                    const getOuterHTMLWithoutChildrenExceptForSelectElements = (element) => {
-                        if (element.tagName.toLowerCase() === 'select') {
-                            return element.outerHTML;
-                        }
-                        else {
-                            const tempElement = document.createElement('div');
-                            tempElement.appendChild(element.cloneNode(false));
-                            return tempElement.innerHTML;
-                        }
-                    };
-                    const attributeToHtmlMap = {};
-                    const elements = document.querySelectorAll(`[${interactableAttribute}]`);
-                    elements.forEach((element) => {
-                        const attributeValue = element.getAttribute(interactableAttribute);
-                        const outerHtml = getOuterHTMLWithoutChildrenExceptForSelectElements(element);
-                        if (attributeValue) {
-                            attributeToHtmlMap[attributeValue] = outerHtml;
-                        }
-                    });
-                    return attributeToHtmlMap;
-                }, PlaywrightUtils.DONOBU_INTERACTABLE_ATTRIBUTE);
-                Object.entries(attributeMap).forEach(([donobuAttributeValue, htmlSnippet]) => {
-                    interactableElements.push({
-                        donobuAttributeValue,
-                        htmlSnippet,
-                    });
-                });
-            }
-            return interactableElements;
-        }
-        catch (error) {
-            if (PlaywrightUtils.isPageClosedError(error)) {
-                throw new PageClosedException_1.PageClosedException();
-            }
-            else {
-                throw error;
-            }
-        }
-    }
-    /**
-     * Assigns a globally unique {@link PlaywrightUtils.DONOBU_INTERACTABLE_ATTRIBUTE}
-     * attribute value to each visible, interactable, element in the given page.
-     * Any pre-existing {@link PlaywrightUtils.DONOBU_INTERACTABLE_ATTRIBUTE}
-     * attributes will be removed.
-     */
-    static async attributeInteractableElements(page) {
-        try {
-            // Remove any preexisting attributes
-            await this.deattributeVisibleInteractableElements(page);
-            // Get viewport dimensions and scroll position properly
-            const viewportInfo = await page.evaluate(() => {
-                return {
-                    viewportWidth: window.innerWidth,
-                    viewportHeight: window.innerHeight,
-                    scrollX: window.scrollX || window.pageXOffset,
-                    scrollY: window.scrollY || window.pageYOffset,
-                };
-            });
-            // 1) Attribute elements in the main page
-            let annotationOffset = await page.evaluate(this.attributeElementsInContext, [0, PlaywrightUtils.DONOBU_INTERACTABLE_ATTRIBUTE]);
-            // 2) Check child frames, attributing elements if the frame is (partially) in view
-            const frames = page
-                .frames()
-                .filter((frame) => PlaywrightUtils.frameFilter(frame) && frame !== page.mainFrame());
-            for (const frame of frames) {
-                const elementHandle = await frame.frameElement();
-                if (!elementHandle) {
-                    continue;
-                }
-                const boundingBox = await elementHandle.boundingBox();
-                if (!boundingBox) {
-                    continue;
-                }
-                // boundingBox coordinates are already in viewport space, so we need to account for scroll
-                const isInViewport = boundingBox.x + boundingBox.width > 0 &&
-                    boundingBox.x < viewportInfo.viewportWidth &&
-                    boundingBox.y + boundingBox.height > 0 &&
-                    boundingBox.y < viewportInfo.viewportHeight;
-                if (isInViewport) {
-                    annotationOffset = await frame.evaluate(this.attributeElementsInContext, [
-                        annotationOffset,
-                        PlaywrightUtils.DONOBU_INTERACTABLE_ATTRIBUTE,
-                    ]);
-                }
-            }
-        }
-        catch (error) {
-            if (PlaywrightUtils.isPageClosedError(error)) {
-                throw new PageClosedException_1.PageClosedException();
-            }
-            else {
-                throw error;
-            }
-        }
-    }
-    /**
-     * Annotate all elements in the given page that have a
-     * {@link PlaywrightUtils.DONOBU_INTERACTABLE_ATTRIBUTE} HTML attribute.
-     *
-     * The annotations are placed in a shadow root to avoid site-specific CSS,
-     * each having a {@link PlaywrightUtils.DONOBU_ANNOTATION_ATTRIBUTE} attribute.
-     */
-    static async annotateInteractableElements(page) {
-        try {
-            // Filter frames as needed
-            const frames = page
-                .frames()
-                .filter((frame) => PlaywrightUtils.frameFilter(frame));
-            for (const frame of frames) {
-                await frame.evaluate(([interactableAttr, annotationAttr]) => {
-                    // 1) Ensure we have a shadow container in the main document
-                    let container = document.getElementById('annotation-shadow-container');
-                    if (!container) {
-                        container = document.createElement('div');
-                        container.id = 'annotation-shadow-container';
-                        // Position container so child elements can be absolutely placed
-                        Object.assign(container.style, {
-                            position: 'fixed', // anchor to viewport
-                            inset: '0', // stretch across it
-                            pointerEvents: 'none', // Let clicks pass through
-                            zIndex: '2147483647', // win every z-index fight
-                        });
-                        // Check if document.body exists before trying to append.
-                        if (document.body) {
-                            document.body.appendChild(container);
-                        }
-                        else if (document.documentElement) {
-                            // Fall back to document.documentElement if body does not exist.
-                            document.documentElement.appendChild(container);
-                        }
-                        else {
-                            // If neither exists, we can't proceed with annotations in this frame.
-                            console.warn(`Cannot create annotation container for ${window.location.href} since the document structure is incomplete`);
-                            return;
-                        }
-                        // Attach a shadow root
-                        const shadowRoot = container.attachShadow({ mode: 'open' });
-                        // Add a <style> element inside the shadow root to reset and define annotation styles
-                        const style = document.createElement('style');
-                        style.textContent = `
-                :host {
-                  all: initial; /* Reset styles in shadow root */
-                }
-                .annotation {
-                  position: absolute;
-                  z-index: 2147483647;
-                  background-color: black;
-                  color: white;
-                  width: 40px;
-                  height: 40px;
-                  border-radius: 50%;
-                  display: flex;
-                  align-items: center;
-                  justify-content: center;
-                  font-size: 14px;
-                  font-weight: bold;
-                  line-height: 20px;
-                  text-align: center;
-                  box-shadow: 0px 2px 4px rgba(0,0,0,0.2);
-                  border: 4px solid #FF4136;
-                  pointer-events: none;
-                }
-              `;
-                        shadowRoot.appendChild(style);
-                    }
-                    // Retrieve the shadow root to place annotation elements
-                    const containerEl = document.getElementById('annotation-shadow-container');
-                    if (!containerEl?.shadowRoot) {
-                        return;
-                    }
-                    const shadowRoot = containerEl.shadowRoot;
-                    // 2) Factory to create a new annotation inside the shadow root
-                    const createAnnotation = (value) => {
-                        const annotation = document.createElement('div');
-                        annotation.classList.add('annotation');
-                        annotation.dataset[annotationAttr] = '1';
-                        annotation.textContent = value;
-                        return annotation;
-                    };
-                    // 3) Position annotation relative to an element
-                    const positionAnnotation = (annotation, element) => {
-                        const rect = element.getBoundingClientRect();
-                        // Center the annotation on the element, adjusting for its size
-                        const x = rect.left + rect.width / 2 - 20 + window.scrollX;
-                        const y = rect.top + rect.height / 2 - 20 + window.scrollY;
-                        annotation.style.left = `${x}px`;
-                        annotation.style.top = `${y}px`;
-                    };
-                    // 4) Traverse DOM (including any nested shadow roots) to find interactable elements
-                    const processNode = (root) => {
-                        // Find elements with the interactable attribute
-                        const elements = root.querySelectorAll(`[${interactableAttr}]`);
-                        elements.forEach((element) => {
-                            const value = element.getAttribute(interactableAttr);
-                            if (value) {
-                                const annotation = createAnnotation(value);
-                                shadowRoot.appendChild(annotation);
-                                positionAnnotation(annotation, element);
-                            }
-                        });
-                        // Recursively process any child shadow roots
-                        root.querySelectorAll('*').forEach((el) => {
-                            if (el.shadowRoot) {
-                                processNode(el.shadowRoot);
-                            }
-                        });
-                    };
-                    // Start processing from the (frame) document root
-                    processNode(document);
-                }, [
-                    PlaywrightUtils.DONOBU_INTERACTABLE_ATTRIBUTE,
-                    PlaywrightUtils.convertToJsAttribute(PlaywrightUtils.DONOBU_ANNOTATION_ATTRIBUTE),
-                ]);
-            }
-        }
-        catch (error) {
-            if (PlaywrightUtils.isPageClosedError(error)) {
-                throw new PageClosedException_1.PageClosedException();
-            }
-            else {
-                throw error;
-            }
-        }
-    }
-    /**
-     * Removes all annotations with a {@link PlaywrightUtils.DONOBU_ANNOTATION_ATTRIBUTE}
-     * HTML attribute in the given page, including any containers in shadow DOM.
-     */
-    static async removeDonobuAnnotations(page) {
-        try {
-            const frames = page
-                .frames()
-                .filter((frame) => PlaywrightUtils.frameFilter(frame));
-            for (const frame of frames) {
-                await frame.evaluate(() => {
-                    const containerId = 'annotation-shadow-container';
-                    const container = document.getElementById(containerId);
-                    if (container) {
-                        container.remove();
-                    }
-                });
-            }
-        }
-        catch (error) {
-            if (PlaywrightUtils.isPageClosedError(error)) {
-                throw new PageClosedException_1.PageClosedException();
-            }
-            throw error;
-        }
-    }
-    /**
-     * Removes the {@link PlaywrightUtils.DONOBU_INTERACTABLE_ATTRIBUTE} HTML
-     * attribute for all elements in the given page. This attribute is normally
-     * added by the {@link PlaywrightUtils.attributeInteractableElements(Page)}
-     * function.
-     */
-    static async deattributeVisibleInteractableElements(page) {
-        try {
-            const frames = page
-                .frames()
-                .filter((frame) => PlaywrightUtils.frameFilter(frame));
-            for (const frame of frames) {
-                await frame.evaluate((attr) => {
-                    document.querySelectorAll(`[${attr}]`).forEach((element) => {
-                        element.removeAttribute(attr);
-                    });
-                }, PlaywrightUtils.DONOBU_INTERACTABLE_ATTRIBUTE);
-            }
-        }
-        catch (error) {
-            if (PlaywrightUtils.isPageClosedError(error)) {
-                throw new PageClosedException_1.PageClosedException();
-            }
-            else {
-                throw error;
-            }
-        }
-    }
-    static async parseUnambiguousSelector(elementHandle) {
-        try {
-            return await elementHandle.evaluate((element) => {
-                const escapeCssIdentifier = (ident) => {
-                    return ident.replace(/([!"#$%&'()*+,./:;<=>?@[\\]^{|}~])/g, '\\$1');
-                };
-                const getPath = (el) => {
-                    if (el.id)
-                        return '#' + escapeCssIdentifier(el.id);
-                    if (!el.parentNode || el.parentNode.nodeType === Node.DOCUMENT_NODE)
-                        return '';
-                    const siblings = Array.from(el.parentNode.children).filter((e) => e.tagName === el.tagName);
-                    const index = siblings.indexOf(el) + 1;
-                    const tag = el.tagName.toLowerCase();
-                    const parentPath = getPath(el.parentNode);
-                    return (parentPath +
-                        (parentPath ? ' > ' : '') +
-                        tag +
-                        (siblings.length > 1 ? ':nth-child(' + index + ')' : ''));
-                };
-                return getPath(element);
-            });
-        }
-        catch (error) {
-            if (PlaywrightUtils.isPageClosedError(error)) {
-                throw new PageClosedException_1.PageClosedException();
-            }
-            else {
-                throw error;
-            }
-        }
-    }
-    /**
-     * Converts an HTML attribute to a JavaScript attribute. For example,
-     * "data-foo-bar" is turned into "fooBar". Notice the dropping of the "data-"
-     * prefix, and the conversion from kebab-case to camelCase.
-     */
-    static convertToJsAttribute(htmlAttribute) {
-        if (htmlAttribute.startsWith('data-')) {
-            htmlAttribute = htmlAttribute.substring(5);
-        }
-        const parts = htmlAttribute.split('-');
-        const jsAttribute = parts[0] +
-            parts
-                .slice(1)
-                .map((part) => part.charAt(0).toUpperCase() + part.slice(1).toLowerCase())
-                .join('');
-        return jsAttribute;
+        await browserContext.addInitScript(RunAccessibilityTestTool_1.RunAccessibilityTestTool.INIT_SCRIPT);
+        await browserContext.addInitScript(PageInspector_1.PageInspector.INIT_SCRIPT);
+        await browserContext.addInitScript(PlaywrightUtils.DIALOG_PROMPT_TRACKER_INIT_SCRIPT);
+        await browserContext.addInitScript(PlaywrightUtils.SMART_SELECTOR_GENERATOR_INIT_SCRIPT);
     }
     /**
      * Returned true IFF the given error is a Playwright error regarding page closing,
@@ -802,294 +156,142 @@ class PlaywrightUtils {
             }
         }
     }
-    static frameFilter(frame) {
-        return (!frame.isDetached() &&
-            !frame.url().startsWith('about:') &&
-            !frame.url().startsWith('chrome:') &&
-            !frame.url().startsWith('edge:'));
-    }
     /**
-     * This function is injected into the page (or frame) context. It finds
-     * "interactable" elements, checks visibility, and sets a unique attribute.
-     * It returns the updated offset after labeling.
+     * Takes a screenshot of the webpage as if it were scrolled down by one viewport.
+     * If the page cannot be scrolled as a whole, tries to find and scroll individual
+     * scrollable elements on the page. After taking the screenshot, it reverts to
+     * the original scroll position.
+     *
+     * @param page The Playwright Page to take a screenshot of
+     * @returns An image buffer of the scrolled down view, or undefined if scrolling isn't possible
      */
-    static attributeElementsInContext(arg) {
-        let offset = arg[0];
-        const interactableAttribute = arg[1];
-        // --- Utility Functions ---
-        function isElementVisible(rect, style) {
-            return (rect.width > 0 &&
-                rect.height > 0 &&
-                style.display !== 'none' &&
-                style.visibility !== 'hidden');
-        }
-        function isElementMoreThanHalfInViewport(rect) {
-            const viewportWidth = window.innerWidth || document.documentElement.clientWidth;
-            const viewportHeight = window.innerHeight || document.documentElement.clientHeight;
-            const visibleWidth = Math.min(rect.right, viewportWidth) - Math.max(rect.left, 0);
-            const visibleHeight = Math.min(rect.bottom, viewportHeight) - Math.max(rect.top, 0);
-            const visibleArea = Math.max(0, visibleWidth) * Math.max(0, visibleHeight);
-            const elementArea = rect.width * rect.height;
-            return visibleArea >= elementArea / 2;
-        }
-        function isElementEnabled(element, style) {
-            // Check standard disabled attribute (for form controls like button, input, etc.)
-            if (element.hasAttribute('disabled')) {
-                return false;
-            }
-            // Check for ARIA attributes that indicate non-interactivity
-            if (element.getAttribute('aria-disabled') === 'true' ||
-                element.getAttribute('aria-hidden') === 'true') {
-                return false;
-            }
-            // Check for pointer-events: none which prevents interactions
-            if (style.pointerEvents === 'none') {
-                return false;
-            }
-            // Check for ARIA attributes indicating disabled state
-            if (element.getAttribute('aria-disabled') === 'true') {
-                return false;
-            }
-            // Check for inert attribute which makes elements non-interactive
-            if (element.hasAttribute('inert')) {
-                return false;
-            }
-            // If the element is in a form and the fieldset is disabled, it might be disabled as well
-            let parent = element.parentElement;
-            while (parent) {
-                if (parent.tagName.toLowerCase() === 'fieldset' &&
-                    parent.hasAttribute('disabled') &&
-                    element.tagName.toLowerCase() !== 'legend') {
-                    return false;
-                }
-                parent = parent.parentElement;
-            }
-            return true;
-        }
-        /**
-         * Generate a few test points on the element's bounding box. We only need
-         * a small offset (1px) from each corner, plus the center.
-         */
-        function getPointsToCheck(rect) {
-            const cornerOffset = 1;
-            return [
-                { x: rect.left + cornerOffset, y: rect.top + cornerOffset },
-                { x: rect.right - cornerOffset, y: rect.top + cornerOffset },
-                { x: rect.left + cornerOffset, y: rect.bottom - cornerOffset },
-                { x: rect.right - cornerOffset, y: rect.bottom - cornerOffset },
-                { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 },
-            ];
-        }
-        /**
-         * Like `elementFromPoint` but continues walking shadow roots if found.
-         */
-        function getDeepElementFromPoint(x, y) {
-            let el = document.elementFromPoint(x, y);
-            while (el && el.shadowRoot) {
-                const shadowEl = el.shadowRoot.elementFromPoint(x, y);
-                if (!shadowEl || shadowEl === el)
-                    break;
-                el = shadowEl;
-            }
-            return el;
-        }
-        function getAllInteractableElements(root) {
-            const selector = [
-                // ----- STANDARD HTML INTERACTIVE ELEMENTS -----
-                // Basic form controls and interactive elements
-                'button',
-                'button svg', // SVG inside buttons are also clickable
-                'input',
-                'textarea',
-                'a',
-                'select',
-                'summary', // Disclosure (details/summary) elements
-                'details',
-                'audio[controls]', // Media controls
-                'video[controls]',
-                // ----- ARIA ROLE-BASED INTERACTIVE ELEMENTS -----
-                // Elements with explicit interactive ARIA roles
-                '[role="button"]',
-                '[role="option"]',
-                '[role="link"]',
-                '[role="checkbox"]',
-                '[role="radio"]',
-                '[role="tab"]',
-                '[role="menuitem"]',
-                '[role="menuitemcheckbox"]',
-                '[role="menuitemradio"]',
-                '[role="combobox"]',
-                '[role="listbox"]',
-                '[role="searchbox"]',
-                '[role="spinbutton"]',
-                '[role="slider"]',
-                '[role="switch"]',
-                '[role="menu"]',
-                '[role="menubar"]',
-                '[role="treeitem"]',
-                // ----- INTERACTIVE ARIA ATTRIBUTES -----
-                // Elements with popup behavior
-                '[aria-haspopup]',
-                '[aria-controls]',
-                '[aria-expanded]', // Elements that can expand/collapse
-                '[aria-pressed]', // Toggle buttons
-                '[aria-selected]', // Selectable items
-                // ----- EDITABLE AND FOCUSABLE ELEMENTS -----
-                // Elements that can receive input
-                '[contenteditable="true"]',
-                // Elements that can be dragged
-                '[draggable="true"]',
-                // Elements with explicit focus capability
-                '[tabindex]:not([tabindex="-1"])',
-                // ----- CSS FRAMEWORK-SPECIFIC CLASSES -----
-                // Bootstrap & similar frameworks
-                '.btn', // Standard button class
-                '.dropdown-toggle', // Dropdown triggers
-                '.nav-link', // Navigation links
-                '.page-link', // Pagination links
-                '.card-header[data-toggle="collapse"]', // Collapsible headers
-                '.accordion-button', // Accordion toggles
-                '.close', // Close buttons
-                '.modal-close', // Modal close buttons
-                // Material Design & Angular Material
-                '.mdc-button',
-                '.mat-button',
-                '.mat-icon-button',
-                '.mat-fab',
-                '.mat-mini-fab',
-                '.mat-menu-item',
-                '.mat-tab-label',
-                // Foundation
-                '.button',
-                '.menu-item',
-                '.accordion-title',
-                // Tailwind UI & Headless UI components
-                '.tw-button',
-                '[x-on\\:click]', // Alpine.js click handlers
-                // Vue-based frameworks
-                '.v-btn', // Vuetify
-                '.el-button', // Element UI
-                // React-based frameworks
-                '.ant-btn', // Ant Design
-                '.chakra-button', // Chakra UI
-                '.mui-button', // Material-UI
-                // ----- CUSTOM INTERACTIVE PATTERNS -----
-                // Common custom interactive classes
-                '.clickable',
-                '.selectable',
-                '.interactive',
-                '.toggle',
-                '.expandable',
-                '.switch',
-                '.slider',
-                // Common dropdown/select libraries
-                '.select2-selection',
-                '.chosen-single',
-                '.vs__dropdown-toggle', // Vue Select
-                // Tabs and accordion components
-                '.tab',
-                '.tab-header',
-                '.tab-title',
-                '.accordion-header',
-                // Mobile-friendly interactive elements
-                '.swipe-item',
-                '.touch-target',
-                // ----- COMMON COMPONENT PATTERNS -----
-                // Cards and tiles that are often clickable
-                '.card[onclick]',
-                '.tile[onclick]',
-                '.card a', // Links inside cards
-                // Social media & e-commerce patterns
-                '.share-button',
-                '.like-button',
-                '.add-to-cart',
-                '.product-card',
-                // Notification and alert controls
-                '.alert .close',
-                '.toast .close',
-                '.notification-action',
-            ].join(', ');
-            let elements = Array.from(root.querySelectorAll(selector));
-            // Dive into shadow roots
-            root.querySelectorAll('*').forEach((el) => {
-                if (el.shadowRoot) {
-                    // Recurse
-                    elements = elements.concat(getAllInteractableElements(el.shadowRoot));
-                }
+    static async getScrolledDownScreenShot(page) {
+        try {
+            // Get the current scroll position and viewport height
+            const { scrollX, scrollY, viewportHeight } = await page.evaluate(() => {
+                return {
+                    scrollX: window.scrollX,
+                    scrollY: window.scrollY,
+                    viewportHeight: window.innerHeight,
+                };
             });
-            // Remove duplicates
-            return Array.from(new Set(elements));
-        }
-        // 1) Gather candidate elements (including any custom "clickable" from donobuGetClickableElements)
-        let interactableElements = getAllInteractableElements(document);
-        if (window.donobuGetClickableElements) {
-            interactableElements = interactableElements.concat(window
-                .donobuGetClickableElements()
-                // Exclude elements with a "donobu-" prefixed ID.
-                .filter((el) => !el.id || !el.id.startsWith('donobu-')));
-        }
-        // Use a Set to avoid duplicates
-        const uniqueElements = new Set(interactableElements);
-        // 2) Check each element for visibility + actual "interactability"
-        uniqueElements.forEach((element) => {
-            if (typeof element.getBoundingClientRect !== 'function') {
-                // Should be rare, but just in case
-                return;
-            }
-            const rect = element.getBoundingClientRect();
-            const style = window.getComputedStyle(element);
-            const visible = isElementVisible(rect, style) && isElementMoreThanHalfInViewport(rect);
-            // Check if the element is actually enabled/interactive
-            const enabled = isElementEnabled(element, style);
-            if (!visible || !enabled) {
-                return;
-            }
-            let annotated = false;
-            // 3) Test small set of points on the element to confirm it's truly topmost
-            for (const pt of getPointsToCheck(rect)) {
-                let elToCheck = getDeepElementFromPoint(pt.x, pt.y);
-                // Walk up the DOM tree from that point
-                while (elToCheck) {
-                    if (elToCheck === element) {
-                        // Found the actual element
-                        element.setAttribute(interactableAttribute, offset.toString());
-                        offset++;
-                        annotated = true;
-                        break;
-                    }
-                    else if (elToCheck.tagName?.toLowerCase() === 'label' &&
-                        elToCheck.htmlFor) {
-                        // If we found a label referencing an element, also mark that
-                        const forId = elToCheck.htmlFor;
-                        const associatedInput = document.getElementById(forId);
-                        if (associatedInput) {
-                            associatedInput.setAttribute(interactableAttribute, offset.toString());
-                            offset++;
+            // Step 1: Try to scroll the window down by one viewport
+            await page.evaluate((viewportHeight) => {
+                window.scrollBy(0, viewportHeight);
+            }, viewportHeight);
+            // Check if the page actually scrolled
+            const newScrollY = await page.evaluate(() => window.scrollY);
+            // If the window scrolled successfully, take the screenshot and return
+            if (newScrollY > scrollY) {
+                // Take the screenshot
+                const screenshot = await PlaywrightUtils.takeScreenshot(page);
+                // Scroll back to the original position
+                await page.evaluate(([x, y]) => {
+                    window.scrollTo(x, y);
+                }, [scrollX, scrollY]);
+                return screenshot;
+            }
+            // Step 2: If window didn't scroll, try to find scrollable elements
+            const didScrollElement = await page.evaluate(() => {
+                // Find all potentially scrollable elements
+                function findScrollableElements() {
+                    const allElements = document.querySelectorAll('*');
+                    const scrollableElements = [];
+                    allElements.forEach((el) => {
+                        const style = window.getComputedStyle(el);
+                        const overflowY = style.getPropertyValue('overflow-y');
+                        const overflowX = style.getPropertyValue('overflow-x');
+                        const overflow = style.getPropertyValue('overflow');
+                        // Check if the element has scrollable properties
+                        if (overflowY === 'auto' ||
+                            overflowY === 'scroll' ||
+                            overflowX === 'auto' ||
+                            overflowX === 'scroll' ||
+                            overflow === 'auto' ||
+                            overflow === 'scroll') {
+                            // Additional check to see if the element actually has content to scroll
+                            const scrollHeight = el.scrollHeight;
+                            const clientHeight = el.clientHeight;
+                            if (scrollHeight > clientHeight) {
+                                scrollableElements.push({
+                                    element: el,
+                                    area: el.clientWidth * el.clientHeight, // Calculate area for sorting
+                                    scrollTop: el.scrollTop, // Save original scroll position
+                                });
+                            }
                         }
-                        annotated = true;
+                    });
+                    return scrollableElements;
+                }
+                // Find and sort scrollable elements by size (largest first)
+                const scrollableElements = findScrollableElements().sort((a, b) => b.area - a.area);
+                let didScroll = false;
+                // Try to scroll each element until one successfully scrolls
+                for (const { element } of scrollableElements) {
+                    const originalScrollTop = element.scrollTop;
+                    // Try to scroll down by the element's client height
+                    element.scrollTop += element.clientHeight;
+                    // Check if scrolling was successful
+                    if (element.scrollTop > originalScrollTop) {
+                        didScroll = true;
+                        // Save the element in the window object so we can access it later
+                        // @ts-ignore
+                        window.__lastScrolledElement = element;
+                        // @ts-ignore
+                        window.__lastScrolledElementOriginalPosition = originalScrollTop;
                         break;
                     }
-                    elToCheck = elToCheck.parentElement;
                 }
-                if (annotated) {
-                    break;
+                return didScroll;
+            });
+            // If no element could be scrolled, return undefined
+            if (!didScrollElement) {
+                return undefined;
+            }
+            // Take a screenshot after scrolling the element
+            const screenshot = await PlaywrightUtils.takeScreenshot(page);
+            // Restore the original scroll position of the element
+            await page.evaluate(() => {
+                if (
+                // @ts-ignore
+                window.__lastScrolledElement &&
+                    // @ts-ignore
+                    window.__lastScrolledElementOriginalPosition !== undefined) {
+                    // @ts-ignore
+                    window.__lastScrolledElement.scrollTop = window.__lastScrolledElementOriginalPosition;
+                    // Clean up
+                    // @ts-ignore
+                    delete window.__lastScrolledElement;
+                    // @ts-ignore
+                    delete window.__lastScrolledElementOriginalPosition;
                 }
+            });
+            return screenshot;
+        }
+        catch (error) {
+            if (PlaywrightUtils.isPageClosedError(error)) {
+                throw new PageClosedException_1.PageClosedException();
             }
-        });
-        return offset;
+            else {
+                throw error;
+            }
+        }
     }
 }
 exports.PlaywrightUtils = PlaywrightUtils;
-PlaywrightUtils.ACCESSIBILITY_TEST_INIT_SCRIPT = MiscUtils_1.MiscUtils.getResourceFileAsString(path_1.default.join('axe.js'));
-PlaywrightUtils.CLICKABLE_ELEMENTS_TRACKER_INIT_SCRIPT = MiscUtils_1.MiscUtils.getResourceFileAsString(path_1.default.join('clickable-elements-tracker.js'));
+/**
+ * A JavaScript code snippet intended to run as an initialization
+ * script for `DonobuFlow` flows. This script helps flows handle/track
+ * prompts/confirmations. This is done specially since these browser actions
+ * pause the Javascript main thread and also cause issues with running various
+ * Playwright actions. See `DonobuFlow.onDialog(Dialog)` for details.
+ */
 PlaywrightUtils.DIALOG_PROMPT_TRACKER_INIT_SCRIPT = MiscUtils_1.MiscUtils.getResourceFileAsString(path_1.default.join('dialog-prompt-tracker.js'));
+/**
+ * Returns a JavaScript code snippet intended to run as an initialization
+ * script for `DonobuFlow` flows. This enables the {@link ReplayableInteraction}
+ * and {@link PageInteractionTracker.INIT_SCRIPT} to generate
+ * smart selectors.
+ */
 PlaywrightUtils.SMART_SELECTOR_GENERATOR_INIT_SCRIPT = MiscUtils_1.MiscUtils.getResourceFileAsString(path_1.default.join('smart-selector-generator.js'));
-PlaywrightUtils.PAGE_INTERACTIONS_TRACKER_INIT_SCRIPT = MiscUtils_1.MiscUtils.getResourceFileAsString(path_1.default.join('page-interactions-tracker.js'));
-PlaywrightUtils.DONOBU_CONTROL_PANEL_INIT_SCRIPT = MiscUtils_1.MiscUtils.getResourceFileAsString(path_1.default.join('control-panel.js'));
-// WARNING: If the control panel ID is changed here, you must also change the
-//          control-panel.js control panel ID value.
-PlaywrightUtils.DONOBU_CONTROL_PANEL_ELEMENT_ID = 'donobu-control-panel';
-PlaywrightUtils.DONOBU_CONTROL_PANEL_HEADLINE_ELEMENT_ID = 'donobu-control-panel-title-text';
-PlaywrightUtils.DONOBU_INTERACTABLE_ATTRIBUTE = 'data-donobu-interactable';
-PlaywrightUtils.DONOBU_ANNOTATION_ATTRIBUTE = 'data-donobu-annotation';
 //# sourceMappingURL=PlaywrightUtils.js.map