@probolabs/playwright 0.3.3 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (9) hide show
  1. package/dist/index.d.ts +9 -4
  2. package/dist/index.js +241 -64
  3. package/dist/index.js.map +1 -1
  4. package/dist/types/actions.d.ts.map +1 -1
  5. package/dist/types/api-client.d.ts.map +1 -1
  6. package/dist/types/highlight.d.ts.map +1 -1
  7. package/dist/types/index.d.ts.map +1 -1
  8. package/dist/types/utils.d.ts.map +1 -1
  9. package/package.json +2 -2
package/dist/index.js CHANGED
@@ -1,4 +1,4 @@
1
- const highlighterCode = "(function (global, factory) {\n typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports) :\n typeof define === 'function' && define.amd ? define(['exports'], factory) :\n (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global.ProboLabs = {}));\n})(this, (function (exports) { 'use strict';\n\n const ElementTag = {\n CLICKABLE: \"CLICKABLE\", // button, link, toggle switch, checkbox, radio, dropdowns, clickable divs\n FILLABLE: \"FILLABLE\", // input, textarea content_editable, date picker??\n SELECTABLE: \"SELECTABLE\", // select\n NON_INTERACTIVE_ELEMENT: 'NON_INTERACTIVE_ELEMENT',\n };\n\n class ElementInfo {\n constructor(element, index, {tag, type, text, html, xpath, css_selector, bounding_box}) {\n this.index = index.toString();\n this.tag = tag;\n this.type = type;\n this.text = text;\n this.html = html;\n this.xpath = xpath;\n this.css_selector = css_selector;\n this.bounding_box = bounding_box;\n this.element = element;\n this.depth = -1;\n }\n\n getSelector() {\n return this.xpath ? this.xpath : this.css_selector;\n }\n\n getDepth() {\n if (this.depth >= 0) {\n return this.depth;\n }\n \n this.depth = 0;\n let currentElement = this.element;\n \n while (currentElement.nodeType === Node.ELEMENT_NODE) { \n this.depth++;\n if (currentElement.assignedSlot) {\n currentElement = currentElement.assignedSlot;\n }\n else {\n currentElement = currentElement.parentNode;\n // Check if we're at a shadow root\n if (currentElement && currentElement.nodeType !== Node.ELEMENT_NODE && currentElement.getRootNode() instanceof ShadowRoot) {\n // Get the shadow root's host element\n currentElement = currentElement.getRootNode().host; \n }\n }\n }\n \n return this.depth;\n }\n }\n\n // import { realpath } from \"fs\";\n\n function getAllDocumentElementsIncludingShadow(selectors, root = document) {\n const elements = Array.from(root.querySelectorAll(selectors));\n\n root.querySelectorAll('*').forEach(el => {\n if (el.shadowRoot) {\n elements.push(...getAllDocumentElementsIncludingShadow(selectors, el.shadowRoot));\n }\n });\n return elements;\n }\n\n function getAllFrames(root = document) {\n const result = [root];\n const frames = getAllDocumentElementsIncludingShadow('frame, iframe', root); \n frames.forEach(frame => {\n try {\n const frameDocument = frame.contentDocument;\n if (frameDocument) {\n result.push(frameDocument);\n }\n } catch (e) {\n // Skip cross-origin frames\n console.warn('Could not access frame content:', e.message);\n }\n });\n\n return result;\n }\n\n function getAllElementsIncludingShadow(selectors, root = document) {\n const elements = [];\n\n getAllFrames(root).forEach(doc => {\n elements.push(...getAllDocumentElementsIncludingShadow(selectors, doc));\n });\n\n return elements;\n }\n\n /**\n * Deeply searches through DOM trees including Shadow DOM and frames/iframes\n * @param {string} selector - CSS selector to search for\n * @param {Document|Element} [root=document] - Starting point for the search\n * @param {Object} [options] - Search options\n * @param {boolean} [options.searchShadow=true] - Whether to search Shadow DOM\n * @param {boolean} [options.searchFrames=true] - Whether to search frames/iframes\n * @returns {Element[]} Array of found elements\n \n function getAllElementsIncludingShadow(selector, root = document, options = {}) {\n const {\n searchShadow = true,\n searchFrames = true\n } = options;\n\n const results = new Set();\n \n // Helper to check if an element is valid and not yet found\n const addIfValid = (element) => {\n if (element && !results.has(element)) {\n results.add(element);\n }\n };\n\n // Helper to process a single document or element\n function processNode(node) {\n // Search regular DOM\n node.querySelectorAll(selector).forEach(addIfValid);\n\n if (searchShadow) {\n // Search all shadow roots\n const treeWalker = document.createTreeWalker(\n node,\n NodeFilter.SHOW_ELEMENT,\n {\n acceptNode: (element) => {\n return element.shadowRoot ? \n NodeFilter.FILTER_ACCEPT : \n NodeFilter.FILTER_SKIP;\n }\n }\n );\n\n while (treeWalker.nextNode()) {\n const element = treeWalker.currentNode;\n if (element.shadowRoot) {\n // Search within shadow root\n element.shadowRoot.querySelectorAll(selector).forEach(addIfValid);\n // Recursively process the shadow root for nested shadow DOMs\n processNode(element.shadowRoot);\n }\n }\n }\n\n if (searchFrames) {\n // Search frames and iframes\n const frames = node.querySelectorAll('frame, iframe');\n frames.forEach(frame => {\n try {\n const frameDocument = frame.contentDocument;\n if (frameDocument) {\n processNode(frameDocument);\n }\n } catch (e) {\n // Skip cross-origin frames\n console.warn('Could not access frame content:', e.message);\n }\n });\n }\n }\n\n // Start processing from the root\n processNode(root);\n\n return Array.from(results);\n }\n */\n // <div x=1 y=2 role='combobox'> </div>\n function findDropdowns() {\n const dropdowns = [];\n \n // Native select elements\n dropdowns.push(...getAllElementsIncludingShadow('select'));\n \n // Elements with dropdown roles that don't have <input>..</input>\n const roleElements = getAllElementsIncludingShadow('[role=\"combobox\"], [role=\"listbox\"], [role=\"dropdown\"], [role=\"option\"], [role=\"menu\"], [role=\"menuitem\"]').filter(el => {\n return el.tagName.toLowerCase() !== 'input' || ![\"button\", \"checkbox\", \"radio\"].includes(el.getAttribute(\"type\"));\n });\n dropdowns.push(...roleElements);\n \n // Common dropdown class patterns\n const dropdownPattern = /.*(dropdown|select|combobox|menu).*/i;\n const elements = getAllElementsIncludingShadow('*');\n const dropdownClasses = Array.from(elements).filter(el => {\n const hasDropdownClass = dropdownPattern.test(el.className);\n const validTag = ['li', 'ul', 'span', 'div', 'p', 'a', 'button'].includes(el.tagName.toLowerCase());\n const style = window.getComputedStyle(el); \n const result = hasDropdownClass && validTag && (style.cursor === 'pointer' || el.tagName.toLowerCase() === 'a' || el.tagName.toLowerCase() === 'button');\n return result;\n });\n \n dropdowns.push(...dropdownClasses);\n \n // Elements with aria-haspopup attribute\n dropdowns.push(...getAllElementsIncludingShadow('[aria-haspopup=\"true\"], [aria-haspopup=\"listbox\"], [aria-haspopup=\"menu\"]'));\n\n // Improve navigation element detection\n // Semantic nav elements with list items\n dropdowns.push(...getAllElementsIncludingShadow('nav ul li, nav ol li'));\n \n // Navigation elements in common design patterns\n dropdowns.push(...getAllElementsIncludingShadow('header a, .header a, .nav a, .navigation a, .menu a, .sidebar a, aside a'));\n \n // Elements in primary navigation areas with common attributes\n dropdowns.push(...getAllElementsIncludingShadow('[role=\"navigation\"] a, [aria-label*=\"navigation\"] a, [aria-label*=\"menu\"] a'));\n\n return dropdowns;\n }\n\n function findClickables() {\n const clickables = [];\n \n const checkboxPattern = /checkbox/i;\n // Collect all clickable elements first\n const nativeLinks = [...getAllElementsIncludingShadow('a[href]')];\n const nativeButtons = [...getAllElementsIncludingShadow('button')];\n const inputButtons = [...getAllElementsIncludingShadow('input[type=\"button\"], input[type=\"submit\"], input[type=\"reset\"]')];\n const roleButtons = [...getAllElementsIncludingShadow('[role=\"button\"]')];\n // const tabbable = [...getAllElementsIncludingShadow('[tabindex=\"0\"]')];\n const clickHandlers = [...getAllElementsIncludingShadow('[onclick]')];\n const dropdowns = findDropdowns();\n const nativeCheckboxes = [...getAllElementsIncludingShadow('input[type=\"checkbox\"]')]; \n const fauxCheckboxes = getAllElementsIncludingShadow('*').filter(el => {\n if (checkboxPattern.test(el.className)) {\n const realCheckboxes = getAllElementsIncludingShadow('input[type=\"checkbox\"]', el);\n if (realCheckboxes.length === 1) {\n const boundingRect = realCheckboxes[0].getBoundingClientRect();\n return boundingRect.width <= 1 && boundingRect.height <= 1 \n }\n }\n return false;\n });\n const nativeRadios = [...getAllElementsIncludingShadow('input[type=\"radio\"]')];\n const toggles = findToggles();\n const pointerElements = findElementsWithPointer();\n // Add all elements at once\n clickables.push(\n ...nativeLinks,\n ...nativeButtons,\n ...inputButtons,\n ...roleButtons,\n // ...tabbable,\n ...clickHandlers,\n ...dropdowns,\n ...nativeCheckboxes,\n ...fauxCheckboxes,\n ...nativeRadios,\n ...toggles,\n ...pointerElements\n );\n\n // Only uniquify once at the end\n return clickables; // Let findElements handle the uniquification\n }\n\n function findToggles() {\n const toggles = [];\n const checkboxes = getAllElementsIncludingShadow('input[type=\"checkbox\"]');\n const togglePattern = /switch|toggle|slider/i;\n\n checkboxes.forEach(checkbox => {\n let isToggle = false;\n\n // Check the checkbox itself\n if (togglePattern.test(checkbox.className) || togglePattern.test(checkbox.getAttribute('role') || '')) {\n isToggle = true;\n }\n\n // Check parent elements (up to 3 levels)\n if (!isToggle) {\n let element = checkbox;\n for (let i = 0; i < 3; i++) {\n const parent = element.parentElement;\n if (!parent) break;\n\n const className = parent.className || '';\n const role = parent.getAttribute('role') || '';\n\n if (togglePattern.test(className) || togglePattern.test(role)) {\n isToggle = true;\n break;\n }\n element = parent;\n }\n }\n\n // Check next sibling\n if (!isToggle) {\n const nextSibling = checkbox.nextElementSibling;\n if (nextSibling) {\n const className = nextSibling.className || '';\n const role = nextSibling.getAttribute('role') || '';\n if (togglePattern.test(className) || togglePattern.test(role)) {\n isToggle = true;\n }\n }\n }\n\n if (isToggle) {\n toggles.push(checkbox);\n }\n });\n\n return toggles;\n }\n\n function findNonInteractiveElements() {\n // Get all elements in the document\n const all = Array.from(getAllElementsIncludingShadow('*'));\n \n // Filter elements based on Python implementation rules\n return all.filter(element => {\n if (!element.firstElementChild) {\n const tag = element.tagName.toLowerCase(); \n if (!['select', 'button', 'a'].includes(tag)) {\n const validTags = ['p', 'span', 'div', 'input', 'textarea'].includes(tag) || /^h\\d$/.test(tag) || /text/.test(tag);\n const boundingRect = element.getBoundingClientRect();\n return validTags && boundingRect.height > 1 && boundingRect.width > 1;\n }\n }\n return false;\n });\n }\n\n\n\n // export function findNonInteractiveElements() {\n // const all = [];\n // try {\n // const elements = getAllElementsIncludingShadow('*');\n // all.push(...elements);\n // } catch (e) {\n // console.warn('Error getting elements:', e);\n // }\n \n // console.debug('Total elements found:', all.length);\n \n // return all.filter(element => {\n // try {\n // const tag = element.tagName.toLowerCase(); \n\n // // Special handling for input elements\n // if (tag === 'input' || tag === 'textarea') {\n // const boundingRect = element.getBoundingClientRect();\n // const value = element.value || '';\n // const placeholder = element.placeholder || '';\n // return boundingRect.height > 1 && \n // boundingRect.width > 1 && \n // (value.trim() !== '' || placeholder.trim() !== '');\n // }\n\n \n // // Check if it's a valid tag for text content\n // const validTags = ['p', 'span', 'div', 'label', 'th', 'td', 'li', 'button', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'a', 'select'].includes(tag) || \n // /^h\\d$/.test(tag) || \n // /text/.test(tag);\n\n // const boundingRect = element.getBoundingClientRect();\n\n // // Get direct text content, excluding child element text\n // let directText = '';\n // for (const node of element.childNodes) {\n // // Only include text nodes (nodeType 3)\n // if (node.nodeType === 3) {\n // directText += node.textContent || '';\n // }\n // }\n \n // // If no direct text and it's a table cell or heading, check label content\n // if (!directText.trim() && (tag === 'th' || tag === 'td' || tag === 'h1')) {\n // const labels = element.getElementsByTagName('label');\n // for (const label of labels) {\n // directText += label.textContent || '';\n // }\n // }\n\n // // If still no text and it's a heading, get all text content\n // if (!directText.trim() && tag === 'h1') {\n // directText = element.textContent || '';\n // }\n\n // directText = directText.trim();\n\n // // Debug logging\n // if (directText) {\n // console.debugg('Text element found:', {\n // tag,\n // text: directText,\n // dimensions: boundingRect,\n // element\n // });\n // }\n\n // return validTags && \n // boundingRect.height > 1 && \n // boundingRect.width > 1 && \n // directText !== '';\n \n // } catch (e) {\n // console.warn('Error processing element:', e);\n // return false;\n // }\n // });\n // }\n\n\n\n\n\n function findElementsWithPointer() {\n const elements = [];\n const allElements = getAllElementsIncludingShadow('*');\n \n console.log('Checking elements with pointer style...');\n \n allElements.forEach(element => {\n // Skip SVG elements for now\n if (element instanceof SVGElement || element.tagName.toLowerCase() === 'svg') {\n return;\n }\n \n const style = window.getComputedStyle(element);\n if (style.cursor === 'pointer') {\n elements.push(element);\n }\n });\n \n console.log(`Found ${elements.length} elements with pointer cursor`);\n return elements;\n }\n\n function findCheckables() {\n const elements = [];\n\n elements.push(...getAllElementsIncludingShadow('input[type=\"checkbox\"]'));\n elements.push(...getAllElementsIncludingShadow('input[type=\"radio\"]'));\n const all_elements = getAllElementsIncludingShadow('label');\n const radioClasses = Array.from(all_elements).filter(el => {\n return /.*radio.*/i.test(el.className); \n });\n elements.push(...radioClasses);\n return elements;\n }\n\n function findFillables() {\n const elements = [];\n\n const inputs = [...getAllElementsIncludingShadow('input:not([type=\"radio\"]):not([type=\"checkbox\"])')];\n console.log('Found inputs:', inputs.length, inputs);\n elements.push(...inputs);\n \n const textareas = [...getAllElementsIncludingShadow('textarea')];\n console.log('Found textareas:', textareas.length);\n elements.push(...textareas);\n \n const editables = [...getAllElementsIncludingShadow('[contenteditable=\"true\"]')];\n console.log('Found editables:', editables.length);\n elements.push(...editables);\n\n return elements;\n }\n\n // Helper function to check if element is a form control\n function isFormControl(elementInfo) {\n return /^(input|select|textarea|button|label)$/i.test(elementInfo.tag);\n }\n\n const isDropdownItem = (elementInfo) => {\n const dropdownPatterns = [\n /dropdown[-_]?item/i, // matches: dropdown-item, dropdownitem, dropdown_item\n /menu[-_]?item/i, // matches: menu-item, menuitem, menu_item\n /dropdown[-_]?link/i, // matches: dropdown-link, dropdownlink, dropdown_link\n /list[-_]?item/i, // matches: list-item, listitem, list_item\n /select[-_]?item/i, // matches: select-item, selectitem, select_item \n ];\n\n const rolePatterns = [\n /menu[-_]?item/i, // matches: menuitem, menu-item\n /option/i, // matches: option\n /list[-_]?item/i, // matches: listitem, list-item\n /tree[-_]?item/i // matches: treeitem, tree-item\n ];\n\n const hasMatchingClass = elementInfo.element.className && \n dropdownPatterns.some(pattern => \n pattern.test(elementInfo.element.className)\n );\n\n const hasMatchingRole = elementInfo.element.getAttribute('role') && \n rolePatterns.some(pattern => \n pattern.test(elementInfo.element.getAttribute('role'))\n );\n\n return hasMatchingClass || hasMatchingRole;\n };\n\n /**\n * Finds the first element matching a CSS selector, traversing Shadow DOM if necessary\n * @param {string} selector - CSS selector to search for\n * @param {Element} [root=document] - Root element to start searching from\n * @returns {Element|null} - The first matching element or null if not found\n */\n function querySelectorShadow(selector, root = document) {\n // First try to find in light DOM\n let element = root.querySelector(selector);\n if (element) return element;\n \n // Get all elements with shadow root\n const shadowElements = Array.from(root.querySelectorAll('*'))\n .filter(el => el.shadowRoot);\n \n // Search through each shadow root until we find a match\n for (const el of shadowElements) {\n element = querySelectorShadow(selector, el.shadowRoot);\n if (element) return element;\n }\n \n return null;\n }\n\n const getElementByXPathOrCssSelector = (element_info) => {\n let element;\n\n if (element_info.xpath) { //try xpath if exists\n element = document.evaluate(\n element_info.xpath, \n document, \n null, \n XPathResult.FIRST_ORDERED_NODE_TYPE, \n null\n ).singleNodeValue;\n \n if (!element) {\n console.warn('Failed to find element with xpath:', element_info.xpath);\n }\n }\n else { //try CSS selector\n element = querySelectorShadow(element_info.css_selector);\n // console.log('found element by CSS elector: ', element);\n if (!element) {\n console.warn('Failed to find element with CSS selector:', element_info.css_selector);\n }\n }\n\n return element;\n };\n\n function generateXPath(element) {\n if (!element || element.getRootNode() instanceof ShadowRoot) return '';\n \n // If element has an id, use that (it's unique and shorter)\n if (element.id) {\n return `//*[@id=\"${element.id}\"]`;\n }\n \n const parts = [];\n let current = element;\n \n while (current && current.nodeType === Node.ELEMENT_NODE) {\n let index = 1;\n let sibling = current.previousSibling;\n \n while (sibling) {\n if (sibling.nodeType === Node.ELEMENT_NODE && sibling.tagName === current.tagName) {\n index++;\n }\n sibling = sibling.previousSibling;\n }\n \n const tagName = current.tagName.toLowerCase();\n parts.unshift(`${tagName}[${index}]`);\n current = current.parentNode;\n }\n \n return '/' + parts.join('/');\n }\n\n function isDecendent(parent, child) {\n let element = child;\n while (element.nodeType === Node.ELEMENT_NODE) { \n \n if (element.assignedSlot) {\n element = element.assignedSlot;\n }\n else {\n element = element.parentNode;\n // Check if we're at a shadow root\n if (element && element.nodeType !== Node.ELEMENT_NODE && element.getRootNode() instanceof ShadowRoot) {\n // Get the shadow root's host element\n element = element.getRootNode().host; \n }\n }\n if (element === parent)\n return true;\n }\n return false;\n }\n\n function generateCssPath(element) {\n if (!element) {\n console.error('ERROR: No element provided to generateCssPath returning empty string');\n return '';\n }\n const path = [];\n // console.group('Generating CSS path for:', element);\n while (element && element.nodeType === Node.ELEMENT_NODE) { \n let selector = element.nodeName.toLowerCase();\n // console.log('Element:', selector, element);\n \n // if (element.id) {\n // //escape special characters\n // const normalized_id = element.id.replace(/[:;.#()[\\]!@$%^&*]/g, '\\\\$&');\n // selector = `#${normalized_id}`;\n // path.unshift(selector);\n // break;\n // } \n \n let sibling = element;\n let nth = 1;\n while (sibling = sibling.previousElementSibling) {\n if (sibling.nodeName.toLowerCase() === selector) nth++;\n }\n sibling = element;\n while (sibling = sibling.nextElementSibling) {\n if (sibling.nodeName.toLowerCase() === selector) {\n break;\n }\n }\n selector += `:nth-of-type(${nth})`;\n \n \n path.unshift(selector);\n //console.log(` Current path: ${path.join(' > ')}`);\n\n if (element.assignedSlot) {\n element = element.assignedSlot;\n // console.log(' Moving to assigned slot');\n }\n else {\n element = element.parentNode;\n // console.log(' Moving to parent:', element);\n\n // Check if we're at a shadow root\n if (element && element.nodeType !== Node.ELEMENT_NODE && element.getRootNode() instanceof ShadowRoot) {\n console.log(' Found shadow root, moving to host');\n // Get the shadow root's host element\n element = element.getRootNode().host; \n }\n }\n }\n \n // console.log('Final selector:', path.join(' > '));\n // console.groupEnd();\n return path.join(' > ');\n }\n\n\n function cleanHTML(rawHTML) {\n const parser = new DOMParser();\n const doc = parser.parseFromString(rawHTML, \"text/html\");\n\n function cleanElement(element) {\n const allowedAttributes = new Set([\n \"role\",\n \"type\",\n \"class\",\n \"href\",\n \"alt\",\n \"title\",\n \"readonly\",\n \"checked\",\n \"enabled\",\n \"disabled\",\n ]);\n\n [...element.attributes].forEach(attr => {\n const name = attr.name.toLowerCase();\n const value = attr.value;\n\n const isTestAttribute = /^(testid|test-id|data-test-id)$/.test(name);\n const isDataAttribute = name.startsWith(\"data-\") && value;\n const isBooleanAttribute = [\"readonly\", \"checked\", \"enabled\", \"disabled\"].includes(name);\n\n if (!allowedAttributes.has(name) && !isDataAttribute && !isTestAttribute && !isBooleanAttribute) {\n element.removeAttribute(name);\n }\n });\n\n // Handle SVG content - more aggressive replacement\n if (element.tagName.toLowerCase() === \"svg\") {\n // Remove all attributes except class and role\n [...element.attributes].forEach(attr => {\n const name = attr.name.toLowerCase();\n if (name !== \"class\" && name !== \"role\") {\n element.removeAttribute(name);\n }\n });\n element.innerHTML = \"CONTENT REMOVED\";\n } else {\n // Recursively clean child elements\n Array.from(element.children).forEach(cleanElement);\n }\n\n // Only remove empty elements that aren't semantic or icon elements\n const keepEmptyElements = ['i', 'span', 'svg', 'button', 'input'];\n if (!keepEmptyElements.includes(element.tagName.toLowerCase()) && \n !element.children.length && \n !element.textContent.trim()) {\n element.remove();\n }\n }\n\n // Process all elements in the document body\n Array.from(doc.body.children).forEach(cleanElement);\n return doc.body.innerHTML;\n }\n\n function getElementInfo(element, index) {\n\n const xpath = generateXPath(element);\n const css_selector = generateCssPath(element);\n\n // Return element info with pre-calculated values\n return new ElementInfo(element, index, {\n tag: element.tagName.toLowerCase(),\n type: element.type || '',\n text: element.innerText, //getTextContent(element),\n html: cleanHTML(element.outerHTML),\n xpath: xpath,\n css_selector: css_selector,\n bounding_box: element.getBoundingClientRect()\n });\n }\n\n\n\n\n const filterZeroDimensions = (elementInfo) => {\n const rect = elementInfo.bounding_box;\n //single pixel elements are typically faux controls and should be filtered too\n const hasSize = rect.width > 1 && rect.height > 1;\n const style = window.getComputedStyle(elementInfo.element);\n const isVisible = style.display !== 'none' && style.visibility !== 'hidden';\n \n if (!hasSize || !isVisible) {\n // if (elementInfo.element.isConnected) {\n // console.log('Filtered out invisible/zero-size element:', {\n // tag: elementInfo.tag,\n // xpath: elementInfo.xpath,\n // element: elementInfo.element,\n // hasSize,\n // isVisible,\n // dimensions: rect\n // });\n // }\n return false;\n }\n return true;\n };\n\n\n\n function uniquifyElements(elements) {\n const seen = new Set();\n\n console.log(`Starting uniquification with ${elements.length} elements`);\n\n // Filter out testing infrastructure elements first\n const filteredInfrastructure = elements.filter(element_info => {\n // Skip the highlight-overlay element completely - it's part of the testing infrastructure\n if (element_info.element.id === 'highlight-overlay' || \n (element_info.css_selector && element_info.css_selector.includes('#highlight-overlay'))) {\n console.log('Filtered out testing infrastructure element:', element_info.css_selector);\n return false;\n }\n \n // Filter out UI framework container/manager elements\n const el = element_info.element;\n // UI framework container checks - generic detection for any framework\n if ((el.getAttribute('data-rendered-by') || \n el.getAttribute('data-reactroot') || \n el.getAttribute('ng-version') || \n el.getAttribute('data-component-id') ||\n el.getAttribute('data-root') ||\n el.getAttribute('data-framework')) && \n (el.className && \n typeof el.className === 'string' && \n (el.className.includes('Container') || \n el.className.includes('container') || \n el.className.includes('Manager') || \n el.className.includes('manager')))) {\n console.log('Filtered out UI framework container element:', element_info.css_selector);\n return false;\n }\n \n // Direct filter for framework container elements that shouldn't be interactive\n // Consolidating multiple container detection patterns into one efficient check\n const isFullViewport = element_info.bounding_box && \n element_info.bounding_box.x <= 5 && \n element_info.bounding_box.y <= 5 && \n element_info.bounding_box.width >= (window.innerWidth * 0.95) && \n element_info.bounding_box.height >= (window.innerHeight * 0.95);\n \n // Empty content check\n const isEmpty = !el.innerText || el.innerText.trim() === '';\n \n // Check if it's a framework container element\n if (element_info.element.tagName === 'DIV' && \n isFullViewport && \n isEmpty && \n (\n // Pattern matching for root containers\n (element_info.xpath && \n (element_info.xpath.match(/^\\/html\\[\\d+\\]\\/body\\[\\d+\\]\\/div\\[\\d+\\]\\/div\\[\\d+\\]$/) || \n element_info.xpath.match(/^\\/\\/\\*\\[@id='[^']+'\\]\\/div\\[\\d+\\]$/))) ||\n \n // Simple DOM structure\n (element_info.css_selector.split(' > ').length <= 4 && element_info.depth <= 5) ||\n \n // Empty or container-like classes\n (!el.className || el.className === '' || \n (typeof el.className === 'string' && \n (el.className.includes('overlay') || \n el.className.includes('container') || \n el.className.includes('wrapper'))))\n )) {\n console.log('Filtered out framework container element:', element_info.css_selector);\n return false;\n }\n \n return true;\n });\n\n // First filter out elements with zero dimensions\n const nonZeroElements = filteredInfrastructure.filter(filterZeroDimensions);\n // sort by CSS selector depth so parents are processed first\n nonZeroElements.sort((a, b) => a.getDepth() - b.getDepth());\n console.log(`After dimension filtering: ${nonZeroElements.length} elements remain (${elements.length - nonZeroElements.length} removed)`);\n \n const filteredByParent = nonZeroElements.filter(element_info => {\n\n const parent = findClosestParent(seen, element_info);\n const keep = parent == null || shouldKeepNestedElement(element_info, parent);\n // console.log(\"node \", element_info.index, \": keep=\", keep, \" parent=\", parent);\n // if (!keep && !element_info.xpath) {\n // console.log(\"Filtered out element \", element_info,\" because it's a nested element of \", parent);\n // }\n if (keep)\n seen.add(element_info.css_selector);\n\n return keep;\n });\n\n console.log(`After parent/child filtering: ${filteredByParent.length} elements remain (${nonZeroElements.length - filteredByParent.length} removed)`);\n\n // Final overlap filtering\n const filteredResults = filteredByParent.filter(element => {\n\n // Look for any element that came BEFORE this one in the array\n const hasEarlierOverlap = filteredByParent.some(other => {\n // Only check elements that came before (lower index)\n if (filteredByParent.indexOf(other) >= filteredByParent.indexOf(element)) {\n return false;\n }\n \n const isOverlapping = areElementsOverlapping(element, other); \n return isOverlapping;\n }); \n\n // Keep element if it has no earlier overlapping elements\n return !hasEarlierOverlap;\n });\n \n \n \n // Check for overlay removal\n console.log(`After filtering: ${filteredResults.length} (${filteredByParent.length - filteredResults.length} removed by overlap)`);\n \n const nonOverlaidElements = filteredResults.filter(element => {\n return !isOverlaid(element);\n });\n\n console.log(`Final elements after overlay removal: ${nonOverlaidElements.length} (${filteredResults.length - nonOverlaidElements.length} removed)`);\n \n return nonOverlaidElements;\n\n }\n\n\n\n const areElementsOverlapping = (element1, element2) => {\n if (element1.css_selector === element2.css_selector) {\n return true;\n }\n \n const box1 = element1.bounding_box;\n const box2 = element2.bounding_box;\n \n return box1.x === box2.x &&\n box1.y === box2.y &&\n box1.width === box2.width &&\n box1.height === box2.height;\n // element1.text === element2.text &&\n // element2.tag === 'a';\n };\n\n function findClosestParent(seen, element_info) { \n // //Use element child/parent queries\n // let parent = element_info.element.parentNode;\n // if (parent.getRootNode() instanceof ShadowRoot) {\n // // Get the shadow root's host element\n // parent = parent.getRootNode().host; \n // }\n\n // while (parent.nodeType === Node.ELEMENT_NODE) { \n // const css_selector = generateCssPath(parent);\n // if (seen.has(css_selector)) {\n // console.log(\"element \", element_info, \" closest parent is \", parent)\n // return parent; \n // }\n // parent = parent.parentNode;\n // if (parent.getRootNode() instanceof ShadowRoot) {\n // // Get the shadow root's host element\n // parent = parent.getRootNode().host; \n // }\n // }\n\n // Split the xpath into segments\n const segments = element_info.css_selector.split(' > ');\n \n // Try increasingly shorter paths until we find one in the seen set\n for (let i = segments.length - 1; i > 0; i--) {\n const parentPath = segments.slice(0, i).join(' > ');\n if (seen.has(parentPath)) {\n return parentPath;\n }\n }\n\n return null;\n }\n\n function shouldKeepNestedElement(elementInfo, parentPath) {\n let result = false;\n const parentSegments = parentPath.split(' > ');\n\n const isParentLink = /^a(:nth-of-type\\(\\d+\\))?$/.test(parentSegments[parentSegments.length - 1]);\n if (isParentLink) {\n return false; \n }\n // If this is a checkbox/radio input\n if (elementInfo.tag === 'input' && \n (elementInfo.type === 'checkbox' || elementInfo.type === 'radio')) {\n \n // Check if parent is a label by looking at the parent xpath's last segment\n \n const isParentLabel = /^label(:nth-of-type\\(\\d+\\))?$/.test(parentSegments[parentSegments.length - 1]);\n \n // If parent is a label, don't keep the input (we'll keep the label instead)\n if (isParentLabel) {\n return false;\n }\n }\n \n // Keep all other form controls and dropdown items\n if (isFormControl(elementInfo) || isDropdownItem(elementInfo)) {\n result = true;\n }\n \n // console.log(`shouldKeepNestedElement: ${elementInfo.tag} ${elementInfo.text} ${elementInfo.xpath} -> ${parentXPath} -> ${result}`);\n return result;\n }\n\n\n\n function isOverlaid(elementInfo) {\n const element = elementInfo.element;\n const boundingRect = element.getBoundingClientRect();\n \n\n \n \n // Create a diagnostic logging function that only logs when needed\n const diagnosticLog = (...args) => {\n };\n\n // Special handling for tooltips\n if (elementInfo.element.className && typeof elementInfo.element.className === 'string' && \n elementInfo.element.className.includes('tooltip')) {\n return false;\n }\n \n // Special handling for deep component hierarchies\n const cssSelector = elementInfo.css_selector;\n \n // Identify complex components with deep hierarchy (important for Shadow DOM components)\n const isComplexComponent = cssSelector.length > 500 || \n (cssSelector.includes('slot:nth-of-type') && \n cssSelector.split('>').length > 15);\n \n // Special handling for deep component buttons in any framework\n const isFrameworkButton = cssSelector.split('>').length > 10 && \n cssSelector.includes('button:nth-of-type');\n \n if (isComplexComponent || isFrameworkButton) {\n // Check if the element is visible on screen\n if (boundingRect.width > 0 && \n boundingRect.height > 0 && \n boundingRect.bottom > 0 && \n boundingRect.right > 0 &&\n boundingRect.top < window.innerHeight &&\n boundingRect.left < window.innerWidth) {\n \n // Get element at the center point to check if it's covered by a popup/modal\n const middleX = boundingRect.x + boundingRect.width/2;\n const middleY = boundingRect.y + boundingRect.height/2;\n const elementAtMiddle = element.ownerDocument.elementFromPoint(middleX, middleY);\n \n if (elementAtMiddle && \n elementAtMiddle !== element && \n !isDecendent(element, elementAtMiddle) && \n !isDecendent(elementAtMiddle, element)) {\n // Make sure it doesn't have CSS that would hide it\n const style = window.getComputedStyle(element);\n if (style.display !== 'none' && \n style.visibility !== 'hidden' && \n style.opacity !== '0') {\n diagnosticLog('Style properties:', { \n display: style.display, \n visibility: style.visibility, \n opacity: style.opacity \n });\n return false; // It's visible and not covered, so not overlaid\n }\n }\n \n // Check specifically if the element at middle is a popup/modal\n if (elementAtMiddle && isElementOrAncestorPopup(elementAtMiddle)) {\n return true; // It's under a popup, so it is overlaid\n }\n }\n }\n \n // Function to check if an element is likely a popup/modal/dialog\n function isLikelyPopupOrModal(el) {\n if (!el) return false;\n \n try {\n // First check if this is actually a navigation element - we don't want to\n // misidentify navigation elements as popups\n if (isLikelyNavElement(el)) {\n return false;\n }\n \n // Navigation elements can still be overlaid - remove the automatic exclusion\n \n const rect = el.getBoundingClientRect();\n const style = window.getComputedStyle(el);\n \n // Quick checks first to fail fast\n const position = style.position;\n if (position !== 'fixed' && position !== 'absolute') {\n return false;\n }\n \n // Check tag name and role - very fast checks\n const isDialogElement = el.tagName.toLowerCase() === 'dialog';\n const hasDialogRole = el.getAttribute('role') === 'dialog' || el.getAttribute('role') === 'alertdialog';\n \n if (isDialogElement || hasDialogRole) {\n return true;\n }\n \n // Class name checks are unreliable as they depend entirely on developer conventions\n // We rely on structural properties instead\n \n // Most expensive checks last\n const zIndex = parseInt(style.zIndex, 10);\n const hasHighZIndex = !isNaN(zIndex) && zIndex > 100;\n \n // Size check for modals - they typically cover a significant portion of the screen\n // Navigation elements at the top are usually not this large\n const isSizeOfModal = rect.width > (window.innerWidth * 0.5) && \n rect.height > (window.innerHeight * 0.3) &&\n // Additional check: modals are usually not at the very top of the page\n rect.top > 50;\n \n return (hasHighZIndex && position === 'fixed' && rect.top > 50) || \n (isSizeOfModal && position === 'fixed');\n } catch (e) {\n return false;\n }\n }\n \n // Renamed from isLikelyTopNavElement to isLikelyNavElement since it detects navigation anywhere\n function isLikelyNavElement(el) {\n if (!el) return false;\n \n try {\n // Don't restrict navigation detection based on top position\n \n // Check tag names associated with navigation\n const tagName = el.tagName.toLowerCase();\n if (tagName === 'nav' || tagName === 'header') {\n return true;\n }\n \n // Check for navigation roles\n const role = el.getAttribute('role');\n if (role === 'navigation') {\n return true;\n }\n \n // Check for navigation-related class names - strong indicator\n const className = typeof el.className === 'string' ? el.className.toLowerCase() : '';\n if (className.includes('navbar') ||\n className.includes('navigation') ||\n className.includes('header') ||\n className.includes('nav-') ||\n className.includes('-nav') ||\n className.includes('sidebar') ||\n className.includes('menu')) {\n return true;\n }\n \n // Check for tab items (common in navigation)\n if (el.tagName === 'SPAN' && \n (el.classList.contains('tab') || \n el.classList.contains('nav'))) {\n return true;\n }\n \n return false;\n } catch (e) {\n return false;\n }\n }\n \n // Function to check if an element or any of its ancestors are part of top navigation\n function isElementOrAncestorTopNav(el, maxDepth = 4) {\n if (!el) return false;\n \n // Check self first - most common case\n if (isLikelyNavElement(el)) {\n return true;\n }\n \n // Then check ancestors if needed\n let current = el.parentElement;\n let depth = 1; // Start at 1 since we already checked self\n \n while (current && depth < maxDepth) {\n if (isLikelyNavElement(current)) {\n return true;\n }\n current = current.parentElement;\n depth++;\n }\n \n return false;\n }\n \n // Function to check if an element or any of its ancestors are a popup/modal\n function isElementOrAncestorPopup(el, maxDepth = 4) {\n if (!el) return false;\n \n // Check self first - most common case\n if (isLikelyPopupOrModal(el)) {\n return true;\n }\n \n // Then check ancestors if needed\n let current = el.parentElement;\n let depth = 1; // Start at 1 since we already checked self\n \n while (current && depth < maxDepth) {\n if (isLikelyPopupOrModal(current)) {\n return true;\n }\n current = current.parentElement;\n depth++;\n }\n \n return false;\n }\n \n // Check if element is at the top of the viewport\n const isNearTopOfViewport = boundingRect.top >= 0 && boundingRect.top < 100;\n \n // For elements near the top of the viewport, we need a special case\n // that handles fixed position elements at the top of the viewport\n if (isNearTopOfViewport) {\n // Get element at middle point of the row\n const middleX = boundingRect.x + boundingRect.width/2;\n const middleY = boundingRect.y + boundingRect.height/2;\n const elementAtMiddle = element.ownerDocument.elementFromPoint(middleX, middleY);\n \n // If we found an element and it's not the row itself or a descendant/ancestor\n if (elementAtMiddle && \n elementAtMiddle !== element && \n !isDecendent(element, elementAtMiddle) && \n !isDecendent(elementAtMiddle, element)) {\n \n // First check if the overlaying element is part of a popup/modal\n // If so, the element IS overlaid (regardless of whether it's also under nav)\n if (isElementOrAncestorPopup(elementAtMiddle)) {\n /* Debug code - commented out\n if (isRowToDebug) {\n console.log('Element is under a popup/modal');\n console.log('Final overlay decision: OVERLAID');\n console.log('-------------------------------------------');\n }\n */\n return true;\n }\n \n // Additional check specifically for the problematic span elements in debug logs\n if (elementAtMiddle.tagName === 'SPAN' && \n elementAtMiddle.className && \n typeof elementAtMiddle.className === 'string' && \n (elementAtMiddle.className.includes('tab') || \n elementAtMiddle.className.includes('nav'))) {\n /* Debug code - commented out\n if (isRowToDebug) {\n console.log('Element is under a navigation tab element');\n console.log('Final overlay decision: NOT OVERLAID');\n console.log('-------------------------------------------');\n }\n */\n return false;\n }\n \n // Check if the overlaying element is part of top navigation\n if (isElementOrAncestorTopNav(elementAtMiddle)) {\n /* Debug code - commented out\n if (isRowToDebug) {\n console.log('Element is under a top navigation element');\n console.log('Final overlay decision: NOT OVERLAID');\n console.log('-------------------------------------------');\n }\n */\n return false;\n }\n }\n }\n \n // Check if element is mostly or completely outside the viewport\n const viewportHeight = window.innerHeight;\n const viewportWidth = window.innerWidth;\n const elementHeight = boundingRect.height;\n const elementWidth = boundingRect.width;\n \n // Calculate how much of the element is visible in the viewport\n const visibleTop = Math.max(0, boundingRect.top);\n const visibleBottom = Math.min(viewportHeight, boundingRect.bottom);\n const visibleLeft = Math.max(0, boundingRect.left);\n const visibleRight = Math.min(viewportWidth, boundingRect.right);\n \n // Calculate visible area\n const visibleHeight = Math.max(0, visibleBottom - visibleTop);\n const visibleWidth = Math.max(0, visibleRight - visibleLeft);\n const visibleArea = visibleHeight * visibleWidth;\n const totalArea = elementHeight * elementWidth;\n \n // Check if less than half of the element is visible in the viewport\n const isPartiallyOffScreen = visibleArea < (totalArea / 2);\n \n // If the element is mostly offscreen, don't consider it overlaid - this avoids\n // false positives from elements near viewport boundaries\n if (isPartiallyOffScreen) {\n return false;\n }\n \n // Track if we find any popup elements covering our element\n let isUnderPopup = false;\n \n // Check multiple points to improve cross-browser consistency\n const points = [\n { x: boundingRect.x + boundingRect.width/2, y: boundingRect.y + boundingRect.height/2 }, // Center\n { x: boundingRect.x + 5, y: boundingRect.y + 5 }, // Top-left\n { x: boundingRect.x + boundingRect.width - 5, y: boundingRect.y + 5 }, // Top-right\n { x: boundingRect.x + 5, y: boundingRect.y + boundingRect.height - 5 }, // Bottom-left\n { x: boundingRect.x + boundingRect.width - 5, y: boundingRect.y + boundingRect.height - 5 } // Bottom-right\n ];\n \n // We'll only consider points that are inside the viewport\n let validPoints = 0;\n let overlaidPoints = 0;\n \n for (const point of points) {\n // Skip points outside viewport\n if (point.x < 0 || point.x >= viewportWidth || point.y < 0 || point.y >= viewportHeight) {\n continue;\n }\n \n validPoints++;\n const elementAtPoint = element.ownerDocument.elementFromPoint(point.x, point.y);\n \n if (isComplexComponent || isFrameworkButton) {\n diagnosticLog('Element at point:', elementAtPoint ? {\n tagName: elementAtPoint.tagName,\n className: elementAtPoint.className,\n id: elementAtPoint.id\n } : 'None found');\n }\n \n // Define helper functions for scroll detection outside the point loop\n function calculateVisibleAreaRatio(elementRect, containerRect) {\n // Calculate intersection\n const intersectionRect = {\n top: Math.max(elementRect.top, containerRect.top),\n left: Math.max(elementRect.left, containerRect.left),\n bottom: Math.min(elementRect.bottom, containerRect.bottom),\n right: Math.min(elementRect.right, containerRect.right)\n };\n \n // Check if there is any intersection at all\n if (intersectionRect.top >= intersectionRect.bottom || \n intersectionRect.left >= intersectionRect.right) {\n return 0; // No intersection, element is completely outside\n }\n \n // Calculate areas\n const intersectionArea = (intersectionRect.right - intersectionRect.left) * \n (intersectionRect.bottom - intersectionRect.top);\n const elementArea = (elementRect.right - elementRect.left) * \n (elementRect.bottom - elementRect.top);\n \n // Return ratio of visible area\n return elementArea > 0 ? intersectionArea / elementArea : 0;\n }\n \n function isInScrollableContainerButNotFullyVisible(el) {\n if (!el) return false;\n \n const rect = el.getBoundingClientRect();\n \n // First check if the element itself is sufficiently visible in the viewport\n const viewportVisibilityRatio = calculateVisibleAreaRatio(rect, {\n top: 0,\n left: 0,\n bottom: window.innerHeight,\n right: window.innerWidth\n });\n \n // If element is less than 80% visible in viewport, consider it overlaid\n if (viewportVisibilityRatio < 0.8) {\n if (isComplexComponent || isFrameworkButton) {\n diagnosticLog('Element is not sufficiently visible in viewport:', {\n visibilityRatio: viewportVisibilityRatio,\n elementRect: {\n top: rect.top,\n right: rect.right,\n bottom: rect.bottom,\n left: rect.left\n }\n });\n }\n return true;\n }\n \n // Then check all ancestor containers\n let parent = el.parentElement;\n while (parent) {\n const parentStyle = window.getComputedStyle(parent);\n \n // Reliable way to detect if an element has scrollbars or is scrollable\n const hasScrollHeight = parent.scrollHeight > parent.clientHeight;\n const hasScrollWidth = parent.scrollWidth > parent.clientWidth;\n \n // Check actual style properties\n const hasOverflowY = parentStyle.overflowY === 'auto' || \n parentStyle.overflowY === 'scroll' || \n parentStyle.overflowY === 'overlay';\n const hasOverflowX = parentStyle.overflowX === 'auto' || \n parentStyle.overflowX === 'scroll' || \n parentStyle.overflowX === 'overlay';\n \n // Check common class names and attributes for scrollable containers across frameworks\n const hasScrollClasses = parent.classList.contains('scroll') || \n parent.classList.contains('scrollable') ||\n parent.classList.contains('overflow') ||\n parent.classList.contains('overflow-auto') ||\n parent.classList.contains('overflow-scroll') ||\n parent.getAttribute('data-scrollable') === 'true';\n \n // Check for height/max-height constraints that often indicate scrolling content\n const hasHeightConstraint = parentStyle.maxHeight && \n parentStyle.maxHeight !== 'none' && \n parentStyle.maxHeight !== 'auto';\n \n // An element is scrollable if it has:\n // 1. Actual scrollbars in use (most reliable check) OR\n // 2. Overflow styles allowing scrolling AND content that would require scrolling\n const isScrollable = (hasScrollHeight && hasOverflowY) || \n (hasScrollWidth && hasOverflowX) ||\n (hasScrollClasses && (hasScrollHeight || hasScrollWidth)) ||\n (hasHeightConstraint && hasScrollHeight);\n \n if (isScrollable) {\n const parentRect = parent.getBoundingClientRect();\n \n // Calculate how much of the element is visible within the container\n const visibilityRatio = calculateVisibleAreaRatio(rect, parentRect);\n \n // Only consider elements as overlaid if they're completely invisible (0% visible)\n // This ensures elements partially visible at the bottom of scrollable containers are still detected\n if (visibilityRatio === 0) {\n diagnosticLog('Element is in scrollable container but completely invisible:', {\n visibilityRatio: visibilityRatio,\n elementRect: {\n top: rect.top,\n right: rect.right,\n bottom: rect.bottom,\n left: rect.left\n },\n containerRect: {\n top: parentRect.top,\n right: parentRect.right,\n bottom: parentRect.bottom,\n left: parentRect.left\n },\n containerClass: parent.className,\n hasScrollHeight,\n hasScrollWidth,\n hasOverflowY,\n hasOverflowX\n });\n return true;\n }\n }\n \n parent = parent.parentElement;\n }\n \n return false;\n }\n \n // Check if element is in a scrollable container but not fully visible\n let isInScrollableContainerOnly = false;\n \n // For all elements, check if in scrollable container but not fully visible\n isInScrollableContainerOnly = isInScrollableContainerButNotFullyVisible(element);\n \n // Special handling for elements in scrollable containers\n if (isInScrollableContainerOnly) {\n // Special handling for very deep nested selectors (which are often in scrollable containers)\n // These are typically elements in tables, lists, and grids with scroll\n if (elementInfo.css_selector && \n (elementInfo.css_selector.split('>').length > 10 || \n (elementInfo.element.className && \n typeof elementInfo.element.className === 'string' && \n (elementInfo.element.className.includes('widget') || \n elementInfo.element.className.includes('list-item') ||\n elementInfo.element.className.includes('item') ||\n elementInfo.element.className.includes('card') ||\n elementInfo.element.className.includes('result'))))) {\n \n diagnosticLog('Allowing deeply nested element in scrollable container:', \n elementInfo.css_selector.substring(0, 100) + '...');\n // Don't consider it overlaid, this is likely a valid item in a scrollable list/table/widget\n return false;\n }\n return true; // Element needs scrolling to be fully visible\n }\n \n // If no element found at this point, or if element is self/ancestor/descendant, \n // this point is not overlaid\n if (!elementAtPoint ||\n elementAtPoint === element || \n isDecendent(element, elementAtPoint) || \n isDecendent(elementAtPoint, element)) {\n continue;\n }\n \n // Check if this point is covered by a popup/modal - this takes precedence\n if (isElementOrAncestorPopup(elementAtPoint)) {\n isUnderPopup = true;\n overlaidPoints++;\n continue;\n }\n \n // Special handling for complex components like those with Shadow DOM\n if ((isComplexComponent || isFrameworkButton) && !isUnderPopup) {\n // Only count as overlaid if it's clearly not part of the component's internal structure\n // For complex components, elementFromPoint often returns internal implementation details\n // that shouldn't count as \"overlaying\" the component itself\n \n // Skip points that might be part of the same component system\n if (elementAtPoint.tagName && elementAtPoint.tagName.toLowerCase().includes('-')) {\n // Custom elements (containing dash in the name) are likely part of the component framework\n continue;\n }\n \n // Only count clear UI overlays like dialogs or tooltips as actual overlays\n const elementTag = elementAtPoint.tagName.toLowerCase();\n const isOverlayingElement = elementTag === 'dialog' || \n elementAtPoint.getAttribute('role') === 'dialog' ||\n elementAtPoint.getAttribute('role') === 'tooltip' ||\n (elementAtPoint.className && \n typeof elementAtPoint.className === 'string' && \n (elementAtPoint.className.includes('overlay') || \n elementAtPoint.className.includes('modal') ||\n elementAtPoint.className.includes('popup')));\n \n if (!isOverlayingElement) {\n continue; // Don't count this point as overlaid for complex components\n }\n }\n \n // Special check for debug rows and elements from topbar\n /* Debug code - commented out\n if (isRowToDebug && elementAtPoint.tagName === 'SPAN' && \n elementAtPoint.className && \n typeof elementAtPoint.className === 'string' && \n (elementAtPoint.className.includes('tab') || \n elementAtPoint.className.includes('nav') ||\n elementAtPoint.className.includes('topbar'))) {\n continue;\n }\n */\n \n // If this point is covered by a top navigation element, don't count it as overlaid\n if (isNearTopOfViewport && isElementOrAncestorTopNav(elementAtPoint)) {\n continue;\n }\n \n // This point is overlaid by something else\n overlaidPoints++;\n if (isComplexComponent || isFrameworkButton) {\n diagnosticLog('Point is overlaid by element:', elementAtPoint.tagName);\n }\n }\n \n // If we don't have any valid points (all outside viewport), element isn't overlaid\n if (validPoints === 0) {\n /* Debug code - commented out\n if (isRowToDebug) {\n console.log('No valid points inside viewport');\n console.log('Final overlay decision: NOT OVERLAID');\n console.log('-------------------------------------------');\n }\n */\n return false;\n }\n \n // If we detected the element is under a popup with significant coverage, it's overlaid\n if (isUnderPopup && overlaidPoints > (validPoints / 2)) {\n /* Debug code - commented out\n if (isRowToDebug) {\n console.log('Element is under a popup/modal with significant coverage');\n console.log('Final overlay decision: OVERLAID');\n console.log('-------------------------------------------');\n }\n */\n return true;\n }\n \n // Calculate percentage of points that are overlaid\n const overlaidPercentage = overlaidPoints / validPoints;\n \n // For complex components, we want to be especially certain they're truly overlaid\n // rather than just having framework-specific DOM complexities\n let overlayThreshold = 0.5; // Default threshold - 50% of points must be overlaid\n \n // Complex components require stronger evidence of overlay\n if ((isComplexComponent || isFrameworkButton) && !isUnderPopup) {\n // For complex components like those with Shadow DOM, \n // we need stronger evidence they're actually overlaid\n overlayThreshold = 0.7; // 70% of points must be overlaid for complex components\n }\n \n // More robust threshold - require sufficient points to be overlaid based on component type\n const isElementOverlaid = overlaidPercentage > overlayThreshold;\n \n return isElementOverlaid;\n }\n\n const highlight = {\n execute: async function(elementTypes, handleScroll=false) {\n const elements = await findElements(elementTypes);\n highlightElements(elements);\n return elements;\n },\n\n unexecute: function(handleScroll=false) {\n unhighlightElements(handleScroll);\n },\n\n generateJSON: async function() {\n const json = {};\n await Promise.all(Object.values(ElementTag).map(async elementType => {\n const elements = await findElements(elementType);\n json[elementType] = elements;\n }));\n\n // Serialize the JSON object\n const jsonString = JSON.stringify(json, null, 4); // Pretty print with 4 spaces\n\n console.log(`JSON: ${jsonString}`);\n return jsonString;\n },\n\n getElementInfo\n };\n\n\n function unhighlightElements(handleScroll=false) {\n const documents = getAllFrames();\n documents.forEach(doc => {\n const overlay = doc.getElementById('highlight-overlay');\n if (overlay) {\n if (handleScroll) {\n // Remove event listeners\n doc.removeEventListener('scroll', overlay.scrollHandler, true);\n doc.removeEventListener('resize', overlay.resizeHandler);\n }\n overlay.remove();\n }\n });\n }\n\n\n\n\n async function findElements(elementTypes, verbose=true) {\n const typesArray = Array.isArray(elementTypes) ? elementTypes : [elementTypes];\n console.log('Starting element search for types:', typesArray);\n\n const elements = [];\n typesArray.forEach(elementType => {\n if (elementType === ElementTag.FILLABLE) {\n elements.push(...findFillables());\n }\n if (elementType === ElementTag.SELECTABLE) {\n elements.push(...findDropdowns());\n }\n if (elementType === ElementTag.CLICKABLE) {\n elements.push(...findClickables());\n elements.push(...findToggles());\n elements.push(...findCheckables());\n }\n if (elementType === ElementTag.NON_INTERACTIVE_ELEMENT) {\n elements.push(...findNonInteractiveElements());\n }\n });\n\n // console.log('Before uniquify:', elements.length);\n const elementsWithInfo = elements.map((element, index) => \n getElementInfo(element, index)\n );\n \n const uniqueElements = uniquifyElements(elementsWithInfo);\n console.log(`Found ${uniqueElements.length} elements:`);\n \n // More comprehensive visibility check\n const visibleElements = uniqueElements.filter(elementInfo => {\n const el = elementInfo.element;\n const style = getComputedStyle(el);\n \n // Check various style properties that affect visibility\n if (style.display === 'none' || \n style.visibility === 'hidden' || \n parseFloat(style.opacity) === 0) {\n return false;\n }\n \n // Check if element has non-zero dimensions\n const rect = el.getBoundingClientRect();\n if (rect.width === 0 || rect.height === 0) {\n return false;\n }\n \n // Check if element is within viewport\n if (rect.bottom < 0 || \n rect.top > window.innerHeight || \n rect.right < 0 || \n rect.left > window.innerWidth) {\n // Element is outside viewport, but still might be valid \n // if user scrolls to it, so we'll include it\n return true;\n }\n \n return true;\n });\n \n console.log(`Out of which ${visibleElements.length} elements are visible:`);\n if (verbose) {\n visibleElements.forEach(info => {\n console.log(`Element ${info.index}:`, info);\n });\n }\n \n return visibleElements;\n }\n\n // elements is an array of objects with index, xpath\n function highlightElements(elements) {\n // console.log('Starting highlight for elements:', elements);\n \n // Create overlay if it doesn't exist and store it in a dictionary\n const documents = getAllFrames(); \n let overlays = {};\n documents.forEach(doc => {\n let overlay = doc.getElementById('highlight-overlay');\n if (!overlay) {\n overlay = doc.createElement('div');\n overlay.id = 'highlight-overlay';\n overlay.style.cssText = `\n position: fixed;\n top: 0;\n left: 0;\n width: 100%;\n height: 100%;\n pointer-events: none;\n z-index: 10000;\n `;\n doc.body.appendChild(overlay);\n }\n overlays[doc.documentURI] = overlay;\n });\n \n\n const updateHighlights = (doc = null) => {\n if (doc) {\n overlays[doc.documentURI].innerHTML = '';\n } else {\n Object.values(overlays).forEach(overlay => { overlay.innerHTML = ''; });\n } \n elements.forEach(elementInfo => {\n // console.log('updateHighlights-Processing elementInfo:', elementInfo);\n let element = elementInfo.element; //getElementByXPathOrCssSelector(elementInfo);\n if (!element) {\n element = getElementByXPathOrCssSelector(elementInfo);\n if (!element)\n return;\n }\n \n //if highlights requested for a specific doc, skip unrelated elements\n if (doc && element.ownerDocument !== doc) {\n console.log(\"skipped element \", element, \" since it doesn't belong to document \", doc);\n return;\n }\n\n const rect = element.getBoundingClientRect();\n // console.log('Element rect:', elementInfo.tag, rect);\n \n if (rect.width === 0 || rect.height === 0) {\n console.warn('Element has zero dimensions:', elementInfo);\n return;\n }\n \n // Create border highlight (red rectangle)\n // use ownerDocument to support iframes/frames\n const highlight = element.ownerDocument.createElement('div');\n highlight.style.cssText = `\n position: fixed;\n left: ${rect.x}px;\n top: ${rect.y}px;\n width: ${rect.width}px;\n height: ${rect.height}px;\n border: 1px solid rgb(255, 0, 0);\n transition: all 0.2s ease-in-out;\n `;\n\n // Create index label container - now positioned to the right and slightly up\n const labelContainer = element.ownerDocument.createElement('div');\n labelContainer.style.cssText = `\n position: absolute;\n right: -10px; /* Offset to the right */\n top: -10px; /* Offset upwards */\n padding: 4px;\n background-color: rgba(255, 255, 0, 0.6);\n display: flex;\n align-items: center;\n justify-content: center;\n `;\n\n const text = element.ownerDocument.createElement('span');\n text.style.cssText = `\n color: rgb(0, 0, 0, 0.8);\n font-family: 'Courier New', Courier, monospace;\n font-size: 12px;\n font-weight: bold;\n line-height: 1;\n `;\n text.textContent = elementInfo.index;\n \n labelContainer.appendChild(text);\n highlight.appendChild(labelContainer); \n overlays[element.ownerDocument.documentURI].appendChild(highlight);\n });\n };\n\n // Initial highlight\n updateHighlights();\n\n documents.forEach(doc => {\n // Update highlights on scroll and resize\n console.log('registering scroll and resize handlers for document: ', doc);\n const scrollHandler = () => {\n requestAnimationFrame(() => updateHighlights(doc));\n };\n const resizeHandler = () => {\n updateHighlights(doc);\n };\n doc.addEventListener('scroll', scrollHandler, true);\n doc.addEventListener('resize', resizeHandler);\n // Store event handlers for cleanup\n overlays[doc.documentURI].scrollHandler = scrollHandler;\n overlays[doc.documentURI].resizeHandler = resizeHandler;\n }); \n }\n\n // function unexecute() {\n // unhighlightElements();\n // }\n\n // Make it available globally for both Extension and Playwright\n if (typeof window !== 'undefined') {\n window.ProboLabs = {\n ElementTag,\n highlight,\n unhighlightElements,\n findElements,\n highlightElements,\n getElementInfo\n };\n }\n\n exports.findElements = findElements;\n exports.getElementInfo = getElementInfo;\n exports.highlight = highlight;\n exports.highlightElements = highlightElements;\n exports.unhighlightElements = unhighlightElements;\n\n Object.defineProperty(exports, '__esModule', { value: true });\n\n}));\n//# sourceMappingURL=probolabs.umd.js.map\n";
1
+ const highlighterCode = "(function (global, factory) {\n typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports) :\n typeof define === 'function' && define.amd ? define(['exports'], factory) :\n (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global.ProboLabs = {}));\n})(this, (function (exports) { 'use strict';\n\n const ElementTag = {\n CLICKABLE: \"CLICKABLE\", // button, link, toggle switch, checkbox, radio, dropdowns, clickable divs\n FILLABLE: \"FILLABLE\", // input, textarea content_editable, date picker??\n SELECTABLE: \"SELECTABLE\", // select\n NON_INTERACTIVE_ELEMENT: 'NON_INTERACTIVE_ELEMENT',\n };\n\n class ElementInfo {\n constructor(element, index, {tag, type, text, html, xpath, css_selector, bounding_box, iframe_selector}) {\n this.index = index.toString();\n this.tag = tag;\n this.type = type;\n this.text = text;\n this.html = html;\n this.xpath = xpath;\n this.css_selector = css_selector;\n this.bounding_box = bounding_box;\n this.iframe_selector = iframe_selector;\n this.element = element;\n this.depth = -1;\n }\n\n getSelector() {\n return this.xpath ? this.xpath : this.css_selector;\n }\n\n getDepth() {\n if (this.depth >= 0) {\n return this.depth;\n }\n \n this.depth = 0;\n let currentElement = this.element;\n \n while (currentElement.nodeType === Node.ELEMENT_NODE) { \n this.depth++;\n if (currentElement.assignedSlot) {\n currentElement = currentElement.assignedSlot;\n }\n else {\n currentElement = currentElement.parentNode;\n // Check if we're at a shadow root\n if (currentElement && currentElement.nodeType !== Node.ELEMENT_NODE && currentElement.getRootNode() instanceof ShadowRoot) {\n // Get the shadow root's host element\n currentElement = currentElement.getRootNode().host; \n }\n }\n }\n \n return this.depth;\n }\n }\n\n // import { realpath } from \"fs\";\n\n function getAllDocumentElementsIncludingShadow(selectors, root = document) {\n const elements = Array.from(root.querySelectorAll(selectors));\n\n root.querySelectorAll('*').forEach(el => {\n if (el.shadowRoot) {\n elements.push(...getAllDocumentElementsIncludingShadow(selectors, el.shadowRoot));\n }\n });\n return elements;\n }\n\n function getAllFrames(root = document) {\n const result = [root];\n const frames = getAllDocumentElementsIncludingShadow('frame, iframe', root); \n frames.forEach(frame => {\n try {\n const frameDocument = frame.contentDocument;\n if (frameDocument) {\n result.push(frameDocument);\n }\n } catch (e) {\n // Skip cross-origin frames\n console.warn('Could not access frame content:', e.message);\n }\n });\n\n return result;\n }\n\n function getAllElementsIncludingShadow(selectors, root = document) {\n const elements = [];\n\n getAllFrames(root).forEach(doc => {\n elements.push(...getAllDocumentElementsIncludingShadow(selectors, doc));\n });\n\n return elements;\n }\n\n /**\n * Deeply searches through DOM trees including Shadow DOM and frames/iframes\n * @param {string} selector - CSS selector to search for\n * @param {Document|Element} [root=document] - Starting point for the search\n * @param {Object} [options] - Search options\n * @param {boolean} [options.searchShadow=true] - Whether to search Shadow DOM\n * @param {boolean} [options.searchFrames=true] - Whether to search frames/iframes\n * @returns {Element[]} Array of found elements\n \n function getAllElementsIncludingShadow(selector, root = document, options = {}) {\n const {\n searchShadow = true,\n searchFrames = true\n } = options;\n\n const results = new Set();\n \n // Helper to check if an element is valid and not yet found\n const addIfValid = (element) => {\n if (element && !results.has(element)) {\n results.add(element);\n }\n };\n\n // Helper to process a single document or element\n function processNode(node) {\n // Search regular DOM\n node.querySelectorAll(selector).forEach(addIfValid);\n\n if (searchShadow) {\n // Search all shadow roots\n const treeWalker = document.createTreeWalker(\n node,\n NodeFilter.SHOW_ELEMENT,\n {\n acceptNode: (element) => {\n return element.shadowRoot ? \n NodeFilter.FILTER_ACCEPT : \n NodeFilter.FILTER_SKIP;\n }\n }\n );\n\n while (treeWalker.nextNode()) {\n const element = treeWalker.currentNode;\n if (element.shadowRoot) {\n // Search within shadow root\n element.shadowRoot.querySelectorAll(selector).forEach(addIfValid);\n // Recursively process the shadow root for nested shadow DOMs\n processNode(element.shadowRoot);\n }\n }\n }\n\n if (searchFrames) {\n // Search frames and iframes\n const frames = node.querySelectorAll('frame, iframe');\n frames.forEach(frame => {\n try {\n const frameDocument = frame.contentDocument;\n if (frameDocument) {\n processNode(frameDocument);\n }\n } catch (e) {\n // Skip cross-origin frames\n console.warn('Could not access frame content:', e.message);\n }\n });\n }\n }\n\n // Start processing from the root\n processNode(root);\n\n return Array.from(results);\n }\n */\n // <div x=1 y=2 role='combobox'> </div>\n function findDropdowns() {\n const dropdowns = [];\n \n // Native select elements\n dropdowns.push(...getAllElementsIncludingShadow('select'));\n \n // Elements with dropdown roles that don't have <input>..</input>\n const roleElements = getAllElementsIncludingShadow('[role=\"combobox\"], [role=\"listbox\"], [role=\"dropdown\"], [role=\"option\"], [role=\"menu\"], [role=\"menuitem\"]').filter(el => {\n return el.tagName.toLowerCase() !== 'input' || ![\"button\", \"checkbox\", \"radio\"].includes(el.getAttribute(\"type\"));\n });\n dropdowns.push(...roleElements);\n \n // Common dropdown class patterns\n const dropdownPattern = /.*(dropdown|select|combobox|menu).*/i;\n const elements = getAllElementsIncludingShadow('*');\n const dropdownClasses = Array.from(elements).filter(el => {\n const hasDropdownClass = dropdownPattern.test(el.className);\n const validTag = ['li', 'ul', 'span', 'div', 'p', 'a', 'button'].includes(el.tagName.toLowerCase());\n const style = window.getComputedStyle(el); \n const result = hasDropdownClass && validTag && (style.cursor === 'pointer' || el.tagName.toLowerCase() === 'a' || el.tagName.toLowerCase() === 'button');\n return result;\n });\n \n dropdowns.push(...dropdownClasses);\n \n // Elements with aria-haspopup attribute\n dropdowns.push(...getAllElementsIncludingShadow('[aria-haspopup=\"true\"], [aria-haspopup=\"listbox\"], [aria-haspopup=\"menu\"]'));\n\n // Improve navigation element detection\n // Semantic nav elements with list items\n dropdowns.push(...getAllElementsIncludingShadow('nav ul li, nav ol li'));\n \n // Navigation elements in common design patterns\n dropdowns.push(...getAllElementsIncludingShadow('header a, .header a, .nav a, .navigation a, .menu a, .sidebar a, aside a'));\n \n // Elements in primary navigation areas with common attributes\n dropdowns.push(...getAllElementsIncludingShadow('[role=\"navigation\"] a, [aria-label*=\"navigation\"] a, [aria-label*=\"menu\"] a'));\n\n return dropdowns;\n }\n\n function findClickables() {\n const clickables = [];\n \n const checkboxPattern = /checkbox/i;\n // Collect all clickable elements first\n const nativeLinks = [...getAllElementsIncludingShadow('a[href]')];\n const nativeButtons = [...getAllElementsIncludingShadow('button')];\n const inputButtons = [...getAllElementsIncludingShadow('input[type=\"button\"], input[type=\"submit\"], input[type=\"reset\"]')];\n const roleButtons = [...getAllElementsIncludingShadow('[role=\"button\"]')];\n // const tabbable = [...getAllElementsIncludingShadow('[tabindex=\"0\"]')];\n const clickHandlers = [...getAllElementsIncludingShadow('[onclick]')];\n const dropdowns = findDropdowns();\n const nativeCheckboxes = [...getAllElementsIncludingShadow('input[type=\"checkbox\"]')]; \n const fauxCheckboxes = getAllElementsIncludingShadow('*').filter(el => {\n if (checkboxPattern.test(el.className)) {\n const realCheckboxes = getAllElementsIncludingShadow('input[type=\"checkbox\"]', el);\n if (realCheckboxes.length === 1) {\n const boundingRect = realCheckboxes[0].getBoundingClientRect();\n return boundingRect.width <= 1 && boundingRect.height <= 1 \n }\n }\n return false;\n });\n const nativeRadios = [...getAllElementsIncludingShadow('input[type=\"radio\"]')];\n const toggles = findToggles();\n const pointerElements = findElementsWithPointer();\n // Add all elements at once\n clickables.push(\n ...nativeLinks,\n ...nativeButtons,\n ...inputButtons,\n ...roleButtons,\n // ...tabbable,\n ...clickHandlers,\n ...dropdowns,\n ...nativeCheckboxes,\n ...fauxCheckboxes,\n ...nativeRadios,\n ...toggles,\n ...pointerElements\n );\n\n // Only uniquify once at the end\n return clickables; // Let findElements handle the uniquification\n }\n\n function findToggles() {\n const toggles = [];\n const checkboxes = getAllElementsIncludingShadow('input[type=\"checkbox\"]');\n const togglePattern = /switch|toggle|slider/i;\n\n checkboxes.forEach(checkbox => {\n let isToggle = false;\n\n // Check the checkbox itself\n if (togglePattern.test(checkbox.className) || togglePattern.test(checkbox.getAttribute('role') || '')) {\n isToggle = true;\n }\n\n // Check parent elements (up to 3 levels)\n if (!isToggle) {\n let element = checkbox;\n for (let i = 0; i < 3; i++) {\n const parent = element.parentElement;\n if (!parent) break;\n\n const className = parent.className || '';\n const role = parent.getAttribute('role') || '';\n\n if (togglePattern.test(className) || togglePattern.test(role)) {\n isToggle = true;\n break;\n }\n element = parent;\n }\n }\n\n // Check next sibling\n if (!isToggle) {\n const nextSibling = checkbox.nextElementSibling;\n if (nextSibling) {\n const className = nextSibling.className || '';\n const role = nextSibling.getAttribute('role') || '';\n if (togglePattern.test(className) || togglePattern.test(role)) {\n isToggle = true;\n }\n }\n }\n\n if (isToggle) {\n toggles.push(checkbox);\n }\n });\n\n return toggles;\n }\n\n function findNonInteractiveElements() {\n // Get all elements in the document\n const all = Array.from(getAllElementsIncludingShadow('*'));\n \n // Filter elements based on Python implementation rules\n return all.filter(element => {\n if (!element.firstElementChild) {\n const tag = element.tagName.toLowerCase(); \n if (!['select', 'button', 'a'].includes(tag)) {\n const validTags = ['p', 'span', 'div', 'input', 'textarea'].includes(tag) || /^h\\d$/.test(tag) || /text/.test(tag);\n const boundingRect = element.getBoundingClientRect();\n return validTags && boundingRect.height > 1 && boundingRect.width > 1;\n }\n }\n return false;\n });\n }\n\n\n\n // export function findNonInteractiveElements() {\n // const all = [];\n // try {\n // const elements = getAllElementsIncludingShadow('*');\n // all.push(...elements);\n // } catch (e) {\n // console.warn('Error getting elements:', e);\n // }\n \n // console.debug('Total elements found:', all.length);\n \n // return all.filter(element => {\n // try {\n // const tag = element.tagName.toLowerCase(); \n\n // // Special handling for input elements\n // if (tag === 'input' || tag === 'textarea') {\n // const boundingRect = element.getBoundingClientRect();\n // const value = element.value || '';\n // const placeholder = element.placeholder || '';\n // return boundingRect.height > 1 && \n // boundingRect.width > 1 && \n // (value.trim() !== '' || placeholder.trim() !== '');\n // }\n\n \n // // Check if it's a valid tag for text content\n // const validTags = ['p', 'span', 'div', 'label', 'th', 'td', 'li', 'button', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'a', 'select'].includes(tag) || \n // /^h\\d$/.test(tag) || \n // /text/.test(tag);\n\n // const boundingRect = element.getBoundingClientRect();\n\n // // Get direct text content, excluding child element text\n // let directText = '';\n // for (const node of element.childNodes) {\n // // Only include text nodes (nodeType 3)\n // if (node.nodeType === 3) {\n // directText += node.textContent || '';\n // }\n // }\n \n // // If no direct text and it's a table cell or heading, check label content\n // if (!directText.trim() && (tag === 'th' || tag === 'td' || tag === 'h1')) {\n // const labels = element.getElementsByTagName('label');\n // for (const label of labels) {\n // directText += label.textContent || '';\n // }\n // }\n\n // // If still no text and it's a heading, get all text content\n // if (!directText.trim() && tag === 'h1') {\n // directText = element.textContent || '';\n // }\n\n // directText = directText.trim();\n\n // // Debug logging\n // if (directText) {\n // console.debugg('Text element found:', {\n // tag,\n // text: directText,\n // dimensions: boundingRect,\n // element\n // });\n // }\n\n // return validTags && \n // boundingRect.height > 1 && \n // boundingRect.width > 1 && \n // directText !== '';\n \n // } catch (e) {\n // console.warn('Error processing element:', e);\n // return false;\n // }\n // });\n // }\n\n\n\n\n\n function findElementsWithPointer() {\n const elements = [];\n const allElements = getAllElementsIncludingShadow('*');\n \n console.log('Checking elements with pointer style...');\n \n allElements.forEach(element => {\n // Skip SVG elements for now\n if (element instanceof SVGElement || element.tagName.toLowerCase() === 'svg') {\n return;\n }\n \n const style = window.getComputedStyle(element);\n if (style.cursor === 'pointer') {\n elements.push(element);\n }\n });\n \n console.log(`Found ${elements.length} elements with pointer cursor`);\n return elements;\n }\n\n function findCheckables() {\n const elements = [];\n\n elements.push(...getAllElementsIncludingShadow('input[type=\"checkbox\"]'));\n elements.push(...getAllElementsIncludingShadow('input[type=\"radio\"]'));\n const all_elements = getAllElementsIncludingShadow('label');\n const radioClasses = Array.from(all_elements).filter(el => {\n return /.*radio.*/i.test(el.className); \n });\n elements.push(...radioClasses);\n return elements;\n }\n\n function findFillables() {\n const elements = [];\n\n const inputs = [...getAllElementsIncludingShadow('input:not([type=\"radio\"]):not([type=\"checkbox\"])')];\n console.log('Found inputs:', inputs.length, inputs);\n elements.push(...inputs);\n \n const textareas = [...getAllElementsIncludingShadow('textarea')];\n console.log('Found textareas:', textareas.length);\n elements.push(...textareas);\n \n const editables = [...getAllElementsIncludingShadow('[contenteditable=\"true\"]')];\n console.log('Found editables:', editables.length);\n elements.push(...editables);\n\n return elements;\n }\n\n // Helper function to check if element is a form control\n function isFormControl(elementInfo) {\n return /^(input|select|textarea|button|label)$/i.test(elementInfo.tag);\n }\n\n const isDropdownItem = (elementInfo) => {\n const dropdownPatterns = [\n /dropdown[-_]?item/i, // matches: dropdown-item, dropdownitem, dropdown_item\n /menu[-_]?item/i, // matches: menu-item, menuitem, menu_item\n /dropdown[-_]?link/i, // matches: dropdown-link, dropdownlink, dropdown_link\n /list[-_]?item/i, // matches: list-item, listitem, list_item\n /select[-_]?item/i, // matches: select-item, selectitem, select_item \n ];\n\n const rolePatterns = [\n /menu[-_]?item/i, // matches: menuitem, menu-item\n /option/i, // matches: option\n /list[-_]?item/i, // matches: listitem, list-item\n /tree[-_]?item/i // matches: treeitem, tree-item\n ];\n\n const hasMatchingClass = elementInfo.element.className && \n dropdownPatterns.some(pattern => \n pattern.test(elementInfo.element.className)\n );\n\n const hasMatchingRole = elementInfo.element.getAttribute('role') && \n rolePatterns.some(pattern => \n pattern.test(elementInfo.element.getAttribute('role'))\n );\n\n return hasMatchingClass || hasMatchingRole;\n };\n\n /**\n * Finds the first element matching a CSS selector, traversing Shadow DOM if necessary\n * @param {string} selector - CSS selector to search for\n * @param {Element} [root=document] - Root element to start searching from\n * @returns {Element|null} - The first matching element or null if not found\n */\n function querySelectorShadow(selector, root = document) {\n // First try to find in light DOM\n let element = root.querySelector(selector);\n if (element) return element;\n \n // Get all elements with shadow root\n const shadowElements = Array.from(root.querySelectorAll('*'))\n .filter(el => el.shadowRoot);\n \n // Search through each shadow root until we find a match\n for (const el of shadowElements) {\n element = querySelectorShadow(selector, el.shadowRoot);\n if (element) return element;\n }\n \n return null;\n }\n\n const getElementByXPathOrCssSelector = (element_info) => {\n let element;\n\n if (element_info.xpath) { //try xpath if exists\n element = document.evaluate(\n element_info.xpath, \n document, \n null, \n XPathResult.FIRST_ORDERED_NODE_TYPE, \n null\n ).singleNodeValue;\n \n if (!element) {\n console.warn('Failed to find element with xpath:', element_info.xpath);\n }\n }\n else { //try CSS selector\n if (element_info.iframe_selector) {\n const iframe = querySelectorShadow(element_info.iframe_selector);\n element = querySelectorShadow(element_info.css_selector, iframe); \n }\n else\n element = querySelectorShadow(element_info.css_selector);\n // console.log('found element by CSS elector: ', element);\n if (!element) {\n console.warn('Failed to find element with CSS selector:', element_info.css_selector);\n }\n }\n\n return element;\n };\n\n function generateXPath(element) {\n if (!element || element.getRootNode() instanceof ShadowRoot) return '';\n \n // If element has an id, use that (it's unique and shorter)\n if (element.id) {\n return `//*[@id=\"${element.id}\"]`;\n }\n \n const parts = [];\n let current = element;\n \n while (current && current.nodeType === Node.ELEMENT_NODE) {\n let index = 1;\n let sibling = current.previousSibling;\n \n while (sibling) {\n if (sibling.nodeType === Node.ELEMENT_NODE && sibling.tagName === current.tagName) {\n index++;\n }\n sibling = sibling.previousSibling;\n }\n \n const tagName = current.tagName.toLowerCase();\n parts.unshift(`${tagName}[${index}]`);\n current = current.parentNode;\n }\n \n return '/' + parts.join('/');\n }\n\n function isDecendent(parent, child) {\n let element = child;\n while (element.nodeType === Node.ELEMENT_NODE) { \n \n if (element.assignedSlot) {\n element = element.assignedSlot;\n }\n else {\n element = element.parentNode;\n // Check if we're at a shadow root\n if (element && element.nodeType !== Node.ELEMENT_NODE && element.getRootNode() instanceof ShadowRoot) {\n // Get the shadow root's host element\n element = element.getRootNode().host; \n }\n }\n if (element === parent)\n return true;\n }\n return false;\n }\n\n function generateCssPath(element) {\n if (!element) {\n console.error('ERROR: No element provided to generateCssPath returning empty string');\n return '';\n }\n const path = [];\n // console.group('Generating CSS path for:', element);\n while (element && element.nodeType === Node.ELEMENT_NODE) { \n let selector = element.nodeName.toLowerCase();\n // console.log('Element:', selector, element);\n \n // if (element.id) {\n // //escape special characters\n // const normalized_id = element.id.replace(/[:;.#()[\\]!@$%^&*]/g, '\\\\$&');\n // selector = `#${normalized_id}`;\n // path.unshift(selector);\n // break;\n // } \n \n let sibling = element;\n let nth = 1;\n while (sibling = sibling.previousElementSibling) {\n if (sibling.nodeName.toLowerCase() === selector) nth++;\n }\n sibling = element;\n while (sibling = sibling.nextElementSibling) {\n if (sibling.nodeName.toLowerCase() === selector) {\n break;\n }\n }\n selector += `:nth-of-type(${nth})`;\n \n \n path.unshift(selector);\n //console.log(` Current path: ${path.join(' > ')}`);\n\n if (element.assignedSlot) {\n element = element.assignedSlot;\n // console.log(' Moving to assigned slot');\n }\n else {\n element = element.parentNode;\n // console.log(' Moving to parent:', element);\n\n // Check if we're at a shadow root\n if (element && element.nodeType !== Node.ELEMENT_NODE && element.getRootNode() instanceof ShadowRoot) {\n console.log(' Found shadow root, moving to host');\n // Get the shadow root's host element\n element = element.getRootNode().host; \n }\n }\n }\n \n // console.log('Final selector:', path.join(' > '));\n // console.groupEnd();\n return path.join(' > ');\n }\n\n\n function cleanHTML(rawHTML) {\n const parser = new DOMParser();\n const doc = parser.parseFromString(rawHTML, \"text/html\");\n\n function cleanElement(element) {\n const allowedAttributes = new Set([\n \"role\",\n \"type\",\n \"class\",\n \"href\",\n \"alt\",\n \"title\",\n \"readonly\",\n \"checked\",\n \"enabled\",\n \"disabled\",\n ]);\n\n [...element.attributes].forEach(attr => {\n const name = attr.name.toLowerCase();\n const value = attr.value;\n\n const isTestAttribute = /^(testid|test-id|data-test-id)$/.test(name);\n const isDataAttribute = name.startsWith(\"data-\") && value;\n const isBooleanAttribute = [\"readonly\", \"checked\", \"enabled\", \"disabled\"].includes(name);\n\n if (!allowedAttributes.has(name) && !isDataAttribute && !isTestAttribute && !isBooleanAttribute) {\n element.removeAttribute(name);\n }\n });\n\n // Handle SVG content - more aggressive replacement\n if (element.tagName.toLowerCase() === \"svg\") {\n // Remove all attributes except class and role\n [...element.attributes].forEach(attr => {\n const name = attr.name.toLowerCase();\n if (name !== \"class\" && name !== \"role\") {\n element.removeAttribute(name);\n }\n });\n element.innerHTML = \"CONTENT REMOVED\";\n } else {\n // Recursively clean child elements\n Array.from(element.children).forEach(cleanElement);\n }\n\n // Only remove empty elements that aren't semantic or icon elements\n const keepEmptyElements = ['i', 'span', 'svg', 'button', 'input'];\n if (!keepEmptyElements.includes(element.tagName.toLowerCase()) && \n !element.children.length && \n !element.textContent.trim()) {\n element.remove();\n }\n }\n\n // Process all elements in the document body\n Array.from(doc.body.children).forEach(cleanElement);\n return doc.body.innerHTML;\n }\n\n function getContainingIframe(element) {\n // If not in an iframe, return null\n if (element.ownerDocument.defaultView === window.top) {\n return null;\n }\n \n // Try to find the iframe in the parent document that contains our element\n try {\n const parentDocument = element.ownerDocument.defaultView.parent.document;\n const iframes = parentDocument.querySelectorAll('iframe');\n \n for (const iframe of iframes) {\n if (iframe.contentWindow === element.ownerDocument.defaultView) {\n return iframe;\n }\n }\n } catch (e) {\n // Cross-origin restriction\n return \"Cross-origin iframe - cannot access details\";\n }\n \n return null;\n }\n\n function getElementInfo(element, index) {\n\n const xpath = generateXPath(element);\n const css_selector = generateCssPath(element);\n\n const iframe = getContainingIframe(element);\n const iframe_selector = iframe ? generateCssPath(iframe) : \"\";\n\n // Return element info with pre-calculated values\n return new ElementInfo(element, index, {\n tag: element.tagName.toLowerCase(),\n type: element.type || '',\n text: element.innerText, //getTextContent(element),\n html: cleanHTML(element.outerHTML),\n xpath: xpath,\n css_selector: css_selector,\n bounding_box: element.getBoundingClientRect(),\n iframe_selector: iframe_selector\n });\n }\n\n\n\n\n const filterZeroDimensions = (elementInfo) => {\n const rect = elementInfo.bounding_box;\n //single pixel elements are typically faux controls and should be filtered too\n const hasSize = rect.width > 1 && rect.height > 1;\n const style = window.getComputedStyle(elementInfo.element);\n const isVisible = style.display !== 'none' && style.visibility !== 'hidden';\n \n if (!hasSize || !isVisible) {\n // if (elementInfo.element.isConnected) {\n // console.log('Filtered out invisible/zero-size element:', {\n // tag: elementInfo.tag,\n // xpath: elementInfo.xpath,\n // element: elementInfo.element,\n // hasSize,\n // isVisible,\n // dimensions: rect\n // });\n // }\n return false;\n }\n return true;\n };\n\n\n\n function uniquifyElements(elements) {\n const seen = new Set();\n\n console.log(`Starting uniquification with ${elements.length} elements`);\n\n // Filter out testing infrastructure elements first\n const filteredInfrastructure = elements.filter(element_info => {\n // Skip the highlight-overlay element completely - it's part of the testing infrastructure\n if (element_info.element.id === 'highlight-overlay' || \n (element_info.css_selector && element_info.css_selector.includes('#highlight-overlay'))) {\n console.log('Filtered out testing infrastructure element:', element_info.css_selector);\n return false;\n }\n \n // Filter out UI framework container/manager elements\n const el = element_info.element;\n // UI framework container checks - generic detection for any framework\n if ((el.getAttribute('data-rendered-by') || \n el.getAttribute('data-reactroot') || \n el.getAttribute('ng-version') || \n el.getAttribute('data-component-id') ||\n el.getAttribute('data-root') ||\n el.getAttribute('data-framework')) && \n (el.className && \n typeof el.className === 'string' && \n (el.className.includes('Container') || \n el.className.includes('container') || \n el.className.includes('Manager') || \n el.className.includes('manager')))) {\n console.log('Filtered out UI framework container element:', element_info.css_selector);\n return false;\n }\n \n // Direct filter for framework container elements that shouldn't be interactive\n // Consolidating multiple container detection patterns into one efficient check\n const isFullViewport = element_info.bounding_box && \n element_info.bounding_box.x <= 5 && \n element_info.bounding_box.y <= 5 && \n element_info.bounding_box.width >= (window.innerWidth * 0.95) && \n element_info.bounding_box.height >= (window.innerHeight * 0.95);\n \n // Empty content check\n const isEmpty = !el.innerText || el.innerText.trim() === '';\n \n // Check if it's a framework container element\n if (element_info.element.tagName === 'DIV' && \n isFullViewport && \n isEmpty && \n (\n // Pattern matching for root containers\n (element_info.xpath && \n (element_info.xpath.match(/^\\/html\\[\\d+\\]\\/body\\[\\d+\\]\\/div\\[\\d+\\]\\/div\\[\\d+\\]$/) || \n element_info.xpath.match(/^\\/\\/\\*\\[@id='[^']+'\\]\\/div\\[\\d+\\]$/))) ||\n \n // Simple DOM structure\n (element_info.css_selector.split(' > ').length <= 4 && element_info.depth <= 5) ||\n \n // Empty or container-like classes\n (!el.className || el.className === '' || \n (typeof el.className === 'string' && \n (el.className.includes('overlay') || \n el.className.includes('container') || \n el.className.includes('wrapper'))))\n )) {\n console.log('Filtered out framework container element:', element_info.css_selector);\n return false;\n }\n \n return true;\n });\n\n // First filter out elements with zero dimensions\n const nonZeroElements = filteredInfrastructure.filter(filterZeroDimensions);\n // sort by CSS selector depth so parents are processed first\n nonZeroElements.sort((a, b) => a.getDepth() - b.getDepth());\n console.log(`After dimension filtering: ${nonZeroElements.length} elements remain (${elements.length - nonZeroElements.length} removed)`);\n \n const filteredByParent = nonZeroElements.filter(element_info => {\n\n const parent = findClosestParent(seen, element_info);\n const keep = parent == null || shouldKeepNestedElement(element_info, parent);\n // console.log(\"node \", element_info.index, \": keep=\", keep, \" parent=\", parent);\n // if (!keep && !element_info.xpath) {\n // console.log(\"Filtered out element \", element_info,\" because it's a nested element of \", parent);\n // }\n if (keep)\n seen.add(element_info.css_selector);\n\n return keep;\n });\n\n console.log(`After parent/child filtering: ${filteredByParent.length} elements remain (${nonZeroElements.length - filteredByParent.length} removed)`);\n\n // Final overlap filtering\n const filteredResults = filteredByParent.filter(element => {\n\n // Look for any element that came BEFORE this one in the array\n const hasEarlierOverlap = filteredByParent.some(other => {\n // Only check elements that came before (lower index)\n if (filteredByParent.indexOf(other) >= filteredByParent.indexOf(element)) {\n return false;\n }\n \n const isOverlapping = areElementsOverlapping(element, other); \n return isOverlapping;\n }); \n\n // Keep element if it has no earlier overlapping elements\n return !hasEarlierOverlap;\n });\n \n \n \n // Check for overlay removal\n console.log(`After filtering: ${filteredResults.length} (${filteredByParent.length - filteredResults.length} removed by overlap)`);\n \n const nonOverlaidElements = filteredResults.filter(element => {\n return !isOverlaid(element);\n });\n\n console.log(`Final elements after overlay removal: ${nonOverlaidElements.length} (${filteredResults.length - nonOverlaidElements.length} removed)`);\n \n return nonOverlaidElements;\n\n }\n\n\n\n const areElementsOverlapping = (element1, element2) => {\n if (element1.css_selector === element2.css_selector) {\n return true;\n }\n \n const box1 = element1.bounding_box;\n const box2 = element2.bounding_box;\n \n return box1.x === box2.x &&\n box1.y === box2.y &&\n box1.width === box2.width &&\n box1.height === box2.height;\n // element1.text === element2.text &&\n // element2.tag === 'a';\n };\n\n function findClosestParent(seen, element_info) { \n // //Use element child/parent queries\n // let parent = element_info.element.parentNode;\n // if (parent.getRootNode() instanceof ShadowRoot) {\n // // Get the shadow root's host element\n // parent = parent.getRootNode().host; \n // }\n\n // while (parent.nodeType === Node.ELEMENT_NODE) { \n // const css_selector = generateCssPath(parent);\n // if (seen.has(css_selector)) {\n // console.log(\"element \", element_info, \" closest parent is \", parent)\n // return parent; \n // }\n // parent = parent.parentNode;\n // if (parent.getRootNode() instanceof ShadowRoot) {\n // // Get the shadow root's host element\n // parent = parent.getRootNode().host; \n // }\n // }\n\n // Split the xpath into segments\n const segments = element_info.css_selector.split(' > ');\n \n // Try increasingly shorter paths until we find one in the seen set\n for (let i = segments.length - 1; i > 0; i--) {\n const parentPath = segments.slice(0, i).join(' > ');\n if (seen.has(parentPath)) {\n return parentPath;\n }\n }\n\n return null;\n }\n\n function shouldKeepNestedElement(elementInfo, parentPath) {\n let result = false;\n const parentSegments = parentPath.split(' > ');\n\n const isParentLink = /^a(:nth-of-type\\(\\d+\\))?$/.test(parentSegments[parentSegments.length - 1]);\n if (isParentLink) {\n return false; \n }\n // If this is a checkbox/radio input\n if (elementInfo.tag === 'input' && \n (elementInfo.type === 'checkbox' || elementInfo.type === 'radio')) {\n \n // Check if parent is a label by looking at the parent xpath's last segment\n \n const isParentLabel = /^label(:nth-of-type\\(\\d+\\))?$/.test(parentSegments[parentSegments.length - 1]);\n \n // If parent is a label, don't keep the input (we'll keep the label instead)\n if (isParentLabel) {\n return false;\n }\n }\n \n // Keep all other form controls and dropdown items\n if (isFormControl(elementInfo) || isDropdownItem(elementInfo)) {\n result = true;\n }\n \n // console.log(`shouldKeepNestedElement: ${elementInfo.tag} ${elementInfo.text} ${elementInfo.xpath} -> ${parentXPath} -> ${result}`);\n return result;\n }\n\n\n\n function isOverlaid(elementInfo) {\n const element = elementInfo.element;\n const boundingRect = element.getBoundingClientRect();\n \n\n \n \n // Create a diagnostic logging function that only logs when needed\n const diagnosticLog = (...args) => {\n };\n\n // Special handling for tooltips\n if (elementInfo.element.className && typeof elementInfo.element.className === 'string' && \n elementInfo.element.className.includes('tooltip')) {\n return false;\n }\n \n // Special handling for deep component hierarchies\n const cssSelector = elementInfo.css_selector;\n \n // Identify complex components with deep hierarchy (important for Shadow DOM components)\n const isComplexComponent = cssSelector.length > 500 || \n (cssSelector.includes('slot:nth-of-type') && \n cssSelector.split('>').length > 15);\n \n // Special handling for deep component buttons in any framework\n const isFrameworkButton = cssSelector.split('>').length > 10 && \n cssSelector.includes('button:nth-of-type');\n \n if (isComplexComponent || isFrameworkButton) {\n // Check if the element is visible on screen\n if (boundingRect.width > 0 && \n boundingRect.height > 0 && \n boundingRect.bottom > 0 && \n boundingRect.right > 0 &&\n boundingRect.top < window.innerHeight &&\n boundingRect.left < window.innerWidth) {\n \n // Get element at the center point to check if it's covered by a popup/modal\n const middleX = boundingRect.x + boundingRect.width/2;\n const middleY = boundingRect.y + boundingRect.height/2;\n const elementAtMiddle = element.ownerDocument.elementFromPoint(middleX, middleY);\n \n if (elementAtMiddle && \n elementAtMiddle !== element && \n !isDecendent(element, elementAtMiddle) && \n !isDecendent(elementAtMiddle, element)) {\n // Make sure it doesn't have CSS that would hide it\n const style = window.getComputedStyle(element);\n if (style.display !== 'none' && \n style.visibility !== 'hidden' && \n style.opacity !== '0') {\n diagnosticLog('Style properties:', { \n display: style.display, \n visibility: style.visibility, \n opacity: style.opacity \n });\n return false; // It's visible and not covered, so not overlaid\n }\n }\n \n // Check specifically if the element at middle is a popup/modal\n if (elementAtMiddle && isElementOrAncestorPopup(elementAtMiddle)) {\n return true; // It's under a popup, so it is overlaid\n }\n }\n }\n \n // Function to check if an element is likely a popup/modal/dialog\n function isLikelyPopupOrModal(el) {\n if (!el) return false;\n \n try {\n // First check if this is actually a navigation element - we don't want to\n // misidentify navigation elements as popups\n if (isLikelyNavElement(el)) {\n return false;\n }\n \n // Navigation elements can still be overlaid - remove the automatic exclusion\n \n const rect = el.getBoundingClientRect();\n const style = window.getComputedStyle(el);\n \n // Quick checks first to fail fast\n const position = style.position;\n if (position !== 'fixed' && position !== 'absolute') {\n return false;\n }\n \n // Check tag name and role - very fast checks\n const isDialogElement = el.tagName.toLowerCase() === 'dialog';\n const hasDialogRole = el.getAttribute('role') === 'dialog' || el.getAttribute('role') === 'alertdialog';\n \n if (isDialogElement || hasDialogRole) {\n return true;\n }\n \n // Class name checks are unreliable as they depend entirely on developer conventions\n // We rely on structural properties instead\n \n // Most expensive checks last\n const zIndex = parseInt(style.zIndex, 10);\n const hasHighZIndex = !isNaN(zIndex) && zIndex > 100;\n \n // Size check for modals - they typically cover a significant portion of the screen\n // Navigation elements at the top are usually not this large\n const isSizeOfModal = rect.width > (window.innerWidth * 0.5) && \n rect.height > (window.innerHeight * 0.3) &&\n // Additional check: modals are usually not at the very top of the page\n rect.top > 50;\n \n return (hasHighZIndex && position === 'fixed' && rect.top > 50) || \n (isSizeOfModal && position === 'fixed');\n } catch (e) {\n return false;\n }\n }\n \n // Renamed from isLikelyTopNavElement to isLikelyNavElement since it detects navigation anywhere\n function isLikelyNavElement(el) {\n if (!el) return false;\n \n try {\n // Don't restrict navigation detection based on top position\n \n // Check tag names associated with navigation\n const tagName = el.tagName.toLowerCase();\n if (tagName === 'nav' || tagName === 'header') {\n return true;\n }\n \n // Check for navigation roles\n const role = el.getAttribute('role');\n if (role === 'navigation') {\n return true;\n }\n \n // Check for navigation-related class names - strong indicator\n const className = typeof el.className === 'string' ? el.className.toLowerCase() : '';\n if (className.includes('navbar') ||\n className.includes('navigation') ||\n className.includes('header') ||\n className.includes('nav-') ||\n className.includes('-nav') ||\n className.includes('sidebar') ||\n className.includes('menu')) {\n return true;\n }\n \n // Check for tab items (common in navigation)\n if (el.tagName === 'SPAN' && \n (el.classList.contains('tab') || \n el.classList.contains('nav'))) {\n return true;\n }\n \n return false;\n } catch (e) {\n return false;\n }\n }\n \n // Function to check if an element or any of its ancestors are part of top navigation\n function isElementOrAncestorTopNav(el, maxDepth = 4) {\n if (!el) return false;\n \n // Check self first - most common case\n if (isLikelyNavElement(el)) {\n return true;\n }\n \n // Then check ancestors if needed\n let current = el.parentElement;\n let depth = 1; // Start at 1 since we already checked self\n \n while (current && depth < maxDepth) {\n if (isLikelyNavElement(current)) {\n return true;\n }\n current = current.parentElement;\n depth++;\n }\n \n return false;\n }\n \n // Function to check if an element or any of its ancestors are a popup/modal\n function isElementOrAncestorPopup(el, maxDepth = 4) {\n if (!el) return false;\n \n // Check self first - most common case\n if (isLikelyPopupOrModal(el)) {\n return true;\n }\n \n // Then check ancestors if needed\n let current = el.parentElement;\n let depth = 1; // Start at 1 since we already checked self\n \n while (current && depth < maxDepth) {\n if (isLikelyPopupOrModal(current)) {\n return true;\n }\n current = current.parentElement;\n depth++;\n }\n \n return false;\n }\n \n // Check if element is at the top of the viewport\n const isNearTopOfViewport = boundingRect.top >= 0 && boundingRect.top < 100;\n \n // For elements near the top of the viewport, we need a special case\n // that handles fixed position elements at the top of the viewport\n if (isNearTopOfViewport) {\n // Get element at middle point of the row\n const middleX = boundingRect.x + boundingRect.width/2;\n const middleY = boundingRect.y + boundingRect.height/2;\n const elementAtMiddle = element.ownerDocument.elementFromPoint(middleX, middleY);\n \n // If we found an element and it's not the row itself or a descendant/ancestor\n if (elementAtMiddle && \n elementAtMiddle !== element && \n !isDecendent(element, elementAtMiddle) && \n !isDecendent(elementAtMiddle, element)) {\n \n // First check if the overlaying element is part of a popup/modal\n // If so, the element IS overlaid (regardless of whether it's also under nav)\n if (isElementOrAncestorPopup(elementAtMiddle)) {\n /* Debug code - commented out\n if (isRowToDebug) {\n console.log('Element is under a popup/modal');\n console.log('Final overlay decision: OVERLAID');\n console.log('-------------------------------------------');\n }\n */\n return true;\n }\n \n // Additional check specifically for the problematic span elements in debug logs\n if (elementAtMiddle.tagName === 'SPAN' && \n elementAtMiddle.className && \n typeof elementAtMiddle.className === 'string' && \n (elementAtMiddle.className.includes('tab') || \n elementAtMiddle.className.includes('nav'))) {\n /* Debug code - commented out\n if (isRowToDebug) {\n console.log('Element is under a navigation tab element');\n console.log('Final overlay decision: NOT OVERLAID');\n console.log('-------------------------------------------');\n }\n */\n return false;\n }\n \n // Check if the overlaying element is part of top navigation\n if (isElementOrAncestorTopNav(elementAtMiddle)) {\n /* Debug code - commented out\n if (isRowToDebug) {\n console.log('Element is under a top navigation element');\n console.log('Final overlay decision: NOT OVERLAID');\n console.log('-------------------------------------------');\n }\n */\n return false;\n }\n }\n }\n \n // Check if element is mostly or completely outside the viewport\n const viewportHeight = window.innerHeight;\n const viewportWidth = window.innerWidth;\n const elementHeight = boundingRect.height;\n const elementWidth = boundingRect.width;\n \n // Calculate how much of the element is visible in the viewport\n const visibleTop = Math.max(0, boundingRect.top);\n const visibleBottom = Math.min(viewportHeight, boundingRect.bottom);\n const visibleLeft = Math.max(0, boundingRect.left);\n const visibleRight = Math.min(viewportWidth, boundingRect.right);\n \n // Calculate visible area\n const visibleHeight = Math.max(0, visibleBottom - visibleTop);\n const visibleWidth = Math.max(0, visibleRight - visibleLeft);\n const visibleArea = visibleHeight * visibleWidth;\n const totalArea = elementHeight * elementWidth;\n \n // Check if less than half of the element is visible in the viewport\n const isPartiallyOffScreen = visibleArea < (totalArea / 2);\n \n // If the element is mostly offscreen, don't consider it overlaid - this avoids\n // false positives from elements near viewport boundaries\n if (isPartiallyOffScreen) {\n return false;\n }\n \n // Track if we find any popup elements covering our element\n let isUnderPopup = false;\n \n // Check multiple points to improve cross-browser consistency\n const points = [\n { x: boundingRect.x + boundingRect.width/2, y: boundingRect.y + boundingRect.height/2 }, // Center\n { x: boundingRect.x + 5, y: boundingRect.y + 5 }, // Top-left\n { x: boundingRect.x + boundingRect.width - 5, y: boundingRect.y + 5 }, // Top-right\n { x: boundingRect.x + 5, y: boundingRect.y + boundingRect.height - 5 }, // Bottom-left\n { x: boundingRect.x + boundingRect.width - 5, y: boundingRect.y + boundingRect.height - 5 } // Bottom-right\n ];\n \n // We'll only consider points that are inside the viewport\n let validPoints = 0;\n let overlaidPoints = 0;\n \n for (const point of points) {\n // Skip points outside viewport\n if (point.x < 0 || point.x >= viewportWidth || point.y < 0 || point.y >= viewportHeight) {\n continue;\n }\n \n validPoints++;\n const elementAtPoint = element.ownerDocument.elementFromPoint(point.x, point.y);\n \n if (isComplexComponent || isFrameworkButton) {\n diagnosticLog('Element at point:', elementAtPoint ? {\n tagName: elementAtPoint.tagName,\n className: elementAtPoint.className,\n id: elementAtPoint.id\n } : 'None found');\n }\n \n // Define helper functions for scroll detection outside the point loop\n function calculateVisibleAreaRatio(elementRect, containerRect) {\n // Calculate intersection\n const intersectionRect = {\n top: Math.max(elementRect.top, containerRect.top),\n left: Math.max(elementRect.left, containerRect.left),\n bottom: Math.min(elementRect.bottom, containerRect.bottom),\n right: Math.min(elementRect.right, containerRect.right)\n };\n \n // Check if there is any intersection at all\n if (intersectionRect.top >= intersectionRect.bottom || \n intersectionRect.left >= intersectionRect.right) {\n return 0; // No intersection, element is completely outside\n }\n \n // Calculate areas\n const intersectionArea = (intersectionRect.right - intersectionRect.left) * \n (intersectionRect.bottom - intersectionRect.top);\n const elementArea = (elementRect.right - elementRect.left) * \n (elementRect.bottom - elementRect.top);\n \n // Return ratio of visible area\n return elementArea > 0 ? intersectionArea / elementArea : 0;\n }\n \n function isInScrollableContainerButNotFullyVisible(el) {\n if (!el) return false;\n \n const rect = el.getBoundingClientRect();\n \n // First check if the element itself is sufficiently visible in the viewport\n const viewportVisibilityRatio = calculateVisibleAreaRatio(rect, {\n top: 0,\n left: 0,\n bottom: window.innerHeight,\n right: window.innerWidth\n });\n \n // If element is less than 80% visible in viewport, consider it overlaid\n if (viewportVisibilityRatio < 0.8) {\n if (isComplexComponent || isFrameworkButton) {\n diagnosticLog('Element is not sufficiently visible in viewport:', {\n visibilityRatio: viewportVisibilityRatio,\n elementRect: {\n top: rect.top,\n right: rect.right,\n bottom: rect.bottom,\n left: rect.left\n }\n });\n }\n return true;\n }\n \n // Then check all ancestor containers\n let parent = el.parentElement;\n while (parent) {\n const parentStyle = window.getComputedStyle(parent);\n \n // Reliable way to detect if an element has scrollbars or is scrollable\n const hasScrollHeight = parent.scrollHeight > parent.clientHeight;\n const hasScrollWidth = parent.scrollWidth > parent.clientWidth;\n \n // Check actual style properties\n const hasOverflowY = parentStyle.overflowY === 'auto' || \n parentStyle.overflowY === 'scroll' || \n parentStyle.overflowY === 'overlay';\n const hasOverflowX = parentStyle.overflowX === 'auto' || \n parentStyle.overflowX === 'scroll' || \n parentStyle.overflowX === 'overlay';\n \n // Check common class names and attributes for scrollable containers across frameworks\n const hasScrollClasses = parent.classList.contains('scroll') || \n parent.classList.contains('scrollable') ||\n parent.classList.contains('overflow') ||\n parent.classList.contains('overflow-auto') ||\n parent.classList.contains('overflow-scroll') ||\n parent.getAttribute('data-scrollable') === 'true';\n \n // Check for height/max-height constraints that often indicate scrolling content\n const hasHeightConstraint = parentStyle.maxHeight && \n parentStyle.maxHeight !== 'none' && \n parentStyle.maxHeight !== 'auto';\n \n // An element is scrollable if it has:\n // 1. Actual scrollbars in use (most reliable check) OR\n // 2. Overflow styles allowing scrolling AND content that would require scrolling\n const isScrollable = (hasScrollHeight && hasOverflowY) || \n (hasScrollWidth && hasOverflowX) ||\n (hasScrollClasses && (hasScrollHeight || hasScrollWidth)) ||\n (hasHeightConstraint && hasScrollHeight);\n \n if (isScrollable) {\n const parentRect = parent.getBoundingClientRect();\n \n // Calculate how much of the element is visible within the container\n const visibilityRatio = calculateVisibleAreaRatio(rect, parentRect);\n \n // Only consider elements as overlaid if they're completely invisible (0% visible)\n // This ensures elements partially visible at the bottom of scrollable containers are still detected\n if (visibilityRatio === 0) {\n diagnosticLog('Element is in scrollable container but completely invisible:', {\n visibilityRatio: visibilityRatio,\n elementRect: {\n top: rect.top,\n right: rect.right,\n bottom: rect.bottom,\n left: rect.left\n },\n containerRect: {\n top: parentRect.top,\n right: parentRect.right,\n bottom: parentRect.bottom,\n left: parentRect.left\n },\n containerClass: parent.className,\n hasScrollHeight,\n hasScrollWidth,\n hasOverflowY,\n hasOverflowX\n });\n return true;\n }\n }\n \n parent = parent.parentElement;\n }\n \n return false;\n }\n \n // Check if element is in a scrollable container but not fully visible\n let isInScrollableContainerOnly = false;\n \n // For all elements, check if in scrollable container but not fully visible\n isInScrollableContainerOnly = isInScrollableContainerButNotFullyVisible(element);\n \n // Special handling for elements in scrollable containers\n if (isInScrollableContainerOnly) {\n // Special handling for very deep nested selectors (which are often in scrollable containers)\n // These are typically elements in tables, lists, and grids with scroll\n if (elementInfo.css_selector && \n (elementInfo.css_selector.split('>').length > 10 || \n (elementInfo.element.className && \n typeof elementInfo.element.className === 'string' && \n (elementInfo.element.className.includes('widget') || \n elementInfo.element.className.includes('list-item') ||\n elementInfo.element.className.includes('item') ||\n elementInfo.element.className.includes('card') ||\n elementInfo.element.className.includes('result'))))) {\n \n diagnosticLog('Allowing deeply nested element in scrollable container:', \n elementInfo.css_selector.substring(0, 100) + '...');\n // Don't consider it overlaid, this is likely a valid item in a scrollable list/table/widget\n return false;\n }\n return true; // Element needs scrolling to be fully visible\n }\n \n // If no element found at this point, or if element is self/ancestor/descendant, \n // this point is not overlaid\n if (!elementAtPoint ||\n elementAtPoint === element || \n isDecendent(element, elementAtPoint) || \n isDecendent(elementAtPoint, element)) {\n continue;\n }\n \n // Check if this point is covered by a popup/modal - this takes precedence\n if (isElementOrAncestorPopup(elementAtPoint)) {\n isUnderPopup = true;\n overlaidPoints++;\n continue;\n }\n \n // Special handling for complex components like those with Shadow DOM\n if ((isComplexComponent || isFrameworkButton) && !isUnderPopup) {\n // Only count as overlaid if it's clearly not part of the component's internal structure\n // For complex components, elementFromPoint often returns internal implementation details\n // that shouldn't count as \"overlaying\" the component itself\n \n // Skip points that might be part of the same component system\n if (elementAtPoint.tagName && elementAtPoint.tagName.toLowerCase().includes('-')) {\n // Custom elements (containing dash in the name) are likely part of the component framework\n continue;\n }\n \n // Only count clear UI overlays like dialogs or tooltips as actual overlays\n const elementTag = elementAtPoint.tagName.toLowerCase();\n const isOverlayingElement = elementTag === 'dialog' || \n elementAtPoint.getAttribute('role') === 'dialog' ||\n elementAtPoint.getAttribute('role') === 'tooltip' ||\n (elementAtPoint.className && \n typeof elementAtPoint.className === 'string' && \n (elementAtPoint.className.includes('overlay') || \n elementAtPoint.className.includes('modal') ||\n elementAtPoint.className.includes('popup')));\n \n if (!isOverlayingElement) {\n continue; // Don't count this point as overlaid for complex components\n }\n }\n \n // Special check for debug rows and elements from topbar\n /* Debug code - commented out\n if (isRowToDebug && elementAtPoint.tagName === 'SPAN' && \n elementAtPoint.className && \n typeof elementAtPoint.className === 'string' && \n (elementAtPoint.className.includes('tab') || \n elementAtPoint.className.includes('nav') ||\n elementAtPoint.className.includes('topbar'))) {\n continue;\n }\n */\n \n // If this point is covered by a top navigation element, don't count it as overlaid\n if (isNearTopOfViewport && isElementOrAncestorTopNav(elementAtPoint)) {\n continue;\n }\n \n // This point is overlaid by something else\n overlaidPoints++;\n if (isComplexComponent || isFrameworkButton) {\n diagnosticLog('Point is overlaid by element:', elementAtPoint.tagName);\n }\n }\n \n // If we don't have any valid points (all outside viewport), element isn't overlaid\n if (validPoints === 0) {\n /* Debug code - commented out\n if (isRowToDebug) {\n console.log('No valid points inside viewport');\n console.log('Final overlay decision: NOT OVERLAID');\n console.log('-------------------------------------------');\n }\n */\n return false;\n }\n \n // If we detected the element is under a popup with significant coverage, it's overlaid\n if (isUnderPopup && overlaidPoints > (validPoints / 2)) {\n /* Debug code - commented out\n if (isRowToDebug) {\n console.log('Element is under a popup/modal with significant coverage');\n console.log('Final overlay decision: OVERLAID');\n console.log('-------------------------------------------');\n }\n */\n return true;\n }\n \n // Calculate percentage of points that are overlaid\n const overlaidPercentage = overlaidPoints / validPoints;\n \n // For complex components, we want to be especially certain they're truly overlaid\n // rather than just having framework-specific DOM complexities\n let overlayThreshold = 0.5; // Default threshold - 50% of points must be overlaid\n \n // Complex components require stronger evidence of overlay\n if ((isComplexComponent || isFrameworkButton) && !isUnderPopup) {\n // For complex components like those with Shadow DOM, \n // we need stronger evidence they're actually overlaid\n overlayThreshold = 0.7; // 70% of points must be overlaid for complex components\n }\n \n // More robust threshold - require sufficient points to be overlaid based on component type\n const isElementOverlaid = overlaidPercentage > overlayThreshold;\n \n return isElementOverlaid;\n }\n\n const highlight = {\n execute: async function(elementTypes, handleScroll=false) {\n const elements = await findElements(elementTypes);\n highlightElements(elements);\n return elements;\n },\n\n unexecute: function(handleScroll=false) {\n unhighlightElements(handleScroll);\n },\n\n generateJSON: async function() {\n const json = {};\n await Promise.all(Object.values(ElementTag).map(async elementType => {\n const elements = await findElements(elementType);\n json[elementType] = elements;\n }));\n\n // Serialize the JSON object\n const jsonString = JSON.stringify(json, null, 4); // Pretty print with 4 spaces\n\n console.log(`JSON: ${jsonString}`);\n return jsonString;\n },\n\n getElementInfo\n };\n\n\n function unhighlightElements(handleScroll=false) {\n const documents = getAllFrames();\n documents.forEach(doc => {\n const overlay = doc.getElementById('highlight-overlay');\n if (overlay) {\n if (handleScroll) {\n // Remove event listeners\n doc.removeEventListener('scroll', overlay.scrollHandler, true);\n doc.removeEventListener('resize', overlay.resizeHandler);\n }\n overlay.remove();\n }\n });\n }\n\n\n\n\n async function findElements(elementTypes, verbose=true) {\n const typesArray = Array.isArray(elementTypes) ? elementTypes : [elementTypes];\n console.log('Starting element search for types:', typesArray);\n\n const elements = [];\n typesArray.forEach(elementType => {\n if (elementType === ElementTag.FILLABLE) {\n elements.push(...findFillables());\n }\n if (elementType === ElementTag.SELECTABLE) {\n elements.push(...findDropdowns());\n }\n if (elementType === ElementTag.CLICKABLE) {\n elements.push(...findClickables());\n elements.push(...findToggles());\n elements.push(...findCheckables());\n }\n if (elementType === ElementTag.NON_INTERACTIVE_ELEMENT) {\n elements.push(...findNonInteractiveElements());\n }\n });\n\n // console.log('Before uniquify:', elements.length);\n const elementsWithInfo = elements.map((element, index) => \n getElementInfo(element, index)\n );\n \n const uniqueElements = uniquifyElements(elementsWithInfo);\n console.log(`Found ${uniqueElements.length} elements:`);\n \n // More comprehensive visibility check\n const visibleElements = uniqueElements.filter(elementInfo => {\n const el = elementInfo.element;\n const style = getComputedStyle(el);\n \n // Check various style properties that affect visibility\n if (style.display === 'none' || \n style.visibility === 'hidden' || \n parseFloat(style.opacity) === 0) {\n return false;\n }\n \n // Check if element has non-zero dimensions\n const rect = el.getBoundingClientRect();\n if (rect.width === 0 || rect.height === 0) {\n return false;\n }\n \n // Check if element is within viewport\n if (rect.bottom < 0 || \n rect.top > window.innerHeight || \n rect.right < 0 || \n rect.left > window.innerWidth) {\n // Element is outside viewport, but still might be valid \n // if user scrolls to it, so we'll include it\n return true;\n }\n \n return true;\n });\n \n console.log(`Out of which ${visibleElements.length} elements are visible:`);\n if (verbose) {\n visibleElements.forEach(info => {\n console.log(`Element ${info.index}:`, info);\n });\n }\n \n return visibleElements;\n }\n\n // elements is an array of objects with index, xpath\n function highlightElements(elements) {\n // console.log('Starting highlight for elements:', elements);\n \n // Create overlay if it doesn't exist and store it in a dictionary\n const documents = getAllFrames(); \n let overlays = {};\n documents.forEach(doc => {\n let overlay = doc.getElementById('highlight-overlay');\n if (!overlay) {\n overlay = doc.createElement('div');\n overlay.id = 'highlight-overlay';\n overlay.style.cssText = `\n position: fixed;\n top: 0;\n left: 0;\n width: 100%;\n height: 100%;\n pointer-events: none;\n z-index: 10000;\n `;\n doc.body.appendChild(overlay);\n }\n overlays[doc.documentURI] = overlay;\n });\n \n\n const updateHighlights = (doc = null) => {\n if (doc) {\n overlays[doc.documentURI].innerHTML = '';\n } else {\n Object.values(overlays).forEach(overlay => { overlay.innerHTML = ''; });\n } \n elements.forEach(elementInfo => {\n // console.log('updateHighlights-Processing elementInfo:', elementInfo);\n let element = elementInfo.element; //getElementByXPathOrCssSelector(elementInfo);\n if (!element) {\n element = getElementByXPathOrCssSelector(elementInfo);\n if (!element)\n return;\n }\n \n //if highlights requested for a specific doc, skip unrelated elements\n if (doc && element.ownerDocument !== doc) {\n console.log(\"skipped element \", element, \" since it doesn't belong to document \", doc);\n return;\n }\n\n const rect = element.getBoundingClientRect();\n // console.log('Element rect:', elementInfo.tag, rect);\n \n if (rect.width === 0 || rect.height === 0) {\n console.warn('Element has zero dimensions:', elementInfo);\n return;\n }\n \n // Create border highlight (red rectangle)\n // use ownerDocument to support iframes/frames\n const highlight = element.ownerDocument.createElement('div');\n highlight.style.cssText = `\n position: fixed;\n left: ${rect.x}px;\n top: ${rect.y}px;\n width: ${rect.width}px;\n height: ${rect.height}px;\n border: 1px solid rgb(255, 0, 0);\n transition: all 0.2s ease-in-out;\n `;\n\n // Create index label container - now positioned to the right and slightly up\n const labelContainer = element.ownerDocument.createElement('div');\n labelContainer.style.cssText = `\n position: absolute;\n right: -10px; /* Offset to the right */\n top: -10px; /* Offset upwards */\n padding: 4px;\n background-color: rgba(255, 255, 0, 0.6);\n display: flex;\n align-items: center;\n justify-content: center;\n `;\n\n const text = element.ownerDocument.createElement('span');\n text.style.cssText = `\n color: rgb(0, 0, 0, 0.8);\n font-family: 'Courier New', Courier, monospace;\n font-size: 12px;\n font-weight: bold;\n line-height: 1;\n `;\n text.textContent = elementInfo.index;\n \n labelContainer.appendChild(text);\n highlight.appendChild(labelContainer); \n overlays[element.ownerDocument.documentURI].appendChild(highlight);\n });\n };\n\n // Initial highlight\n updateHighlights();\n\n documents.forEach(doc => {\n // Update highlights on scroll and resize\n console.log('registering scroll and resize handlers for document: ', doc);\n const scrollHandler = () => {\n requestAnimationFrame(() => updateHighlights(doc));\n };\n const resizeHandler = () => {\n updateHighlights(doc);\n };\n doc.addEventListener('scroll', scrollHandler, true);\n doc.addEventListener('resize', resizeHandler);\n // Store event handlers for cleanup\n overlays[doc.documentURI].scrollHandler = scrollHandler;\n overlays[doc.documentURI].resizeHandler = resizeHandler;\n }); \n }\n\n // function unexecute() {\n // unhighlightElements();\n // }\n\n // Make it available globally for both Extension and Playwright\n if (typeof window !== 'undefined') {\n window.ProboLabs = {\n ElementTag,\n highlight,\n unhighlightElements,\n findElements,\n highlightElements,\n getElementInfo\n };\n }\n\n exports.findElements = findElements;\n exports.getElementInfo = getElementInfo;\n exports.highlight = highlight;\n exports.highlightElements = highlightElements;\n exports.unhighlightElements = unhighlightElements;\n\n Object.defineProperty(exports, '__esModule', { value: true });\n\n}));\n//# sourceMappingURL=probolabs.umd.js.map\n";
2
2
  const ElementTag = {
3
3
  CLICKABLE: "CLICKABLE", // button, link, toggle switch, checkbox, radio, dropdowns, clickable divs
4
4
  FILLABLE: "FILLABLE", // input, textarea content_editable, date picker??
@@ -83,6 +83,7 @@ function cleanupElementInfo(elementInfo) {
83
83
  html: elementInfo.html,
84
84
  xpath: elementInfo.xpath,
85
85
  css_selector: elementInfo.css_selector,
86
+ iframe_selector: elementInfo.iframe_selector,
86
87
  bounding_box: elementInfo.bounding_box,
87
88
  depth: depth
88
89
  };
@@ -126,7 +127,7 @@ const PlaywrightAction = {
126
127
  /**
127
128
  * Handle potential navigation exactly like Python's handle_potential_navigation
128
129
  */
129
- async function handlePotentialNavigation(page, selector = null, options = {}) {
130
+ async function handlePotentialNavigation(page, locator = null, options = {}) {
130
131
  const { initialTimeout = 5000, navigationTimeout = 7000, globalTimeout = 15000, } = options;
131
132
  const startTime = Date.now();
132
133
  let navigationCount = 0;
@@ -141,9 +142,9 @@ async function handlePotentialNavigation(page, selector = null, options = {}) {
141
142
  proboLogger.debug(`DEBUG_NAV[0.000s]: Starting navigation detection`);
142
143
  page.on("framenavigated", onFrameNav);
143
144
  try {
144
- if (selector) {
145
- proboLogger.debug(`DEBUG_NAV: Executing click(${selector})`);
146
- await page.click(selector);
145
+ if (locator) {
146
+ proboLogger.debug(`DEBUG_NAV: Executing click()`);
147
+ await locator.click({ noWaitAfter: false });
147
148
  }
148
149
  // wait for any initial nav to fire
149
150
  await page.waitForTimeout(initialTimeout);
@@ -168,7 +169,7 @@ async function handlePotentialNavigation(page, selector = null, options = {}) {
168
169
  await page.waitForLoadState("load", { timeout: globalTimeout });
169
170
  proboLogger.debug(`DEBUG_NAV: waiting for networkidle`);
170
171
  try {
171
- // shorter idle‐wait so we don’t hang the full globalTimeout here
172
+ // shorter idle‐wait so we don't hang the full globalTimeout here
172
173
  await page.waitForLoadState("networkidle", {
173
174
  timeout: navigationTimeout,
174
175
  });
@@ -193,7 +194,7 @@ async function handlePotentialNavigation(page, selector = null, options = {}) {
193
194
  */
194
195
  async function scrollToBottomRight(page) {
195
196
  const startTime = performance.now();
196
- proboLogger.debug(`DEBUG: Starting scroll to bottom-right`);
197
+ proboLogger.debug(`Starting scroll to bottom-right`);
197
198
  let lastHeight = await page.evaluate(() => document.documentElement.scrollHeight);
198
199
  let lastWidth = await page.evaluate(() => document.documentElement.scrollWidth);
199
200
  while (true) {
@@ -208,26 +209,26 @@ async function scrollToBottomRight(page) {
208
209
  await page.waitForTimeout(50);
209
210
  smoothingSteps++;
210
211
  }
211
- proboLogger.debug(`DEBUG: performed ${smoothingSteps} smoothing steps while scrolling`);
212
+ proboLogger.debug(`performed ${smoothingSteps} smoothing steps while scrolling`);
212
213
  const newHeight = await page.evaluate(() => document.documentElement.scrollHeight);
213
214
  const newWidth = await page.evaluate(() => document.documentElement.scrollWidth);
214
215
  if (newHeight === lastHeight && newWidth === lastWidth)
215
216
  break;
216
- proboLogger.debug(`DEBUG: page dimensions updated, repeating scroll`);
217
+ proboLogger.debug(`page dimensions updated, repeating scroll`);
217
218
  lastHeight = newHeight;
218
219
  lastWidth = newWidth;
219
220
  }
220
221
  await page.waitForTimeout(200);
221
222
  await page.evaluate('window.scrollTo(0, 0)');
222
223
  await page.waitForTimeout(50);
223
- proboLogger.debug(`DEBUG: Scroll completed in ${(performance.now() - startTime).toFixed(3)}ms`);
224
+ proboLogger.debug(`Scroll completed in ${(performance.now() - startTime).toFixed(3)}ms`);
224
225
  }
225
226
  /**
226
227
  * Wait for DOM mutations to settle using MutationObserver logic
227
228
  */
228
229
  async function waitForMutationsToSettle(page, mutationTimeout = 1500, initTimeout = 2000) {
229
230
  const startTime = Date.now();
230
- proboLogger.debug(`DEBUG: Starting mutation settlement (initTimeout=${initTimeout}, mutationTimeout=${mutationTimeout})`);
231
+ proboLogger.debug(`Starting mutation settlement (initTimeout=${initTimeout}, mutationTimeout=${mutationTimeout})`);
231
232
  const result = await page.evaluate(async ({ mutationTimeout, initTimeout }) => {
232
233
  async function blockUntilStable(targetNode, options = { childList: true, subtree: true }) {
233
234
  return new Promise((resolve) => {
@@ -259,47 +260,47 @@ async function waitForMutationsToSettle(page, mutationTimeout = 1500, initTimeou
259
260
  }, { mutationTimeout, initTimeout });
260
261
  const total = ((Date.now() - startTime) / 1000).toFixed(2);
261
262
  proboLogger.debug(result
262
- ? `DEBUG: Mutations settled. Took ${total}s`
263
- : `DEBUG: No mutations observed. Took ${total}s`);
263
+ ? `Mutations settled. Took ${total}s`
264
+ : `No mutations observed. Took ${total}s`);
264
265
  return result;
265
266
  }
266
267
  /**
267
268
  * Get element text value: allTextContents + innerText fallback
268
269
  */
269
- async function getElementValue(page, selector) {
270
- const texts = await page.locator(selector).allTextContents();
270
+ async function getElementValue(page, locator) {
271
+ const texts = await locator.allTextContents();
271
272
  let allText = texts.join('').trim();
272
273
  if (!allText) {
273
- allText = await page.locator(selector).evaluate((el) => el.innerText);
274
+ allText = await locator.evaluate((el) => el.innerText);
274
275
  }
275
- proboLogger.debug(`DEBUG: getElementValue for ${selector}: [${allText}]`);
276
+ proboLogger.debug(`getElementValue: [${allText}]`);
276
277
  return allText;
277
278
  }
278
279
  /**
279
280
  * Select dropdown option: native <select>, <option>, child select, or ARIA listbox
280
281
  */
281
- async function selectDropdownOption(page, selector, value) {
282
- const locator = page.locator(selector);
283
- const tagName = await locator.evaluate((el) => el.tagName.toLowerCase());
284
- const role = await locator.getAttribute('role');
282
+ async function selectDropdownOption(page, locator, value) {
283
+ const tagName = await (locator === null || locator === void 0 ? void 0 : locator.evaluate((el) => el.tagName.toLowerCase()));
284
+ const role = await (locator === null || locator === void 0 ? void 0 : locator.getAttribute('role'));
285
285
  if (tagName === 'option' || role === 'option') {
286
286
  proboLogger.debug('selectDropdownOption: option role detected');
287
- await locator.click();
287
+ await (locator === null || locator === void 0 ? void 0 : locator.click());
288
288
  }
289
289
  else if (tagName === 'select') {
290
290
  proboLogger.debug('selectDropdownOption: simple select tag detected');
291
291
  try {
292
- await page.selectOption(selector, value);
292
+ await (locator === null || locator === void 0 ? void 0 : locator.selectOption(value));
293
293
  }
294
294
  catch (_a) {
295
295
  proboLogger.debug('selectDropdownOption: manual change event fallback');
296
- await page.evaluate(({ sel, val }) => {
297
- const el = document.querySelector(sel);
296
+ const handle = locator ? await locator.elementHandle() : null;
297
+ await page.evaluate(({ h, val }) => {
298
+ const el = h;
298
299
  if (el) {
299
300
  el.value = Array.isArray(val) ? val[0] : val;
300
301
  el.dispatchEvent(new Event('change', { bubbles: true }));
301
302
  }
302
- }, { sel: selector, val: value });
303
+ }, { h: handle, val: value });
303
304
  }
304
305
  }
305
306
  else {
@@ -345,27 +346,115 @@ async function selectDropdownOption(page, selector, value) {
345
346
  /**
346
347
  * Execute a given Playwright action, mirroring Python's _perform_action
347
348
  */
348
- async function executePlaywrightAction(page, action, value, element_css_selector) {
349
- proboLogger.info(`DEBUG: performing Action: ${action} Value: ${value}`);
349
+ async function executePlaywrightAction(page, action, value, iframe_selector, element_css_selector) {
350
+ proboLogger.info(`performing Action: ${action} Value: ${value}`);
351
+ try {
352
+ if (action === PlaywrightAction.VISIT_BASE_URL || action === PlaywrightAction.VISIT_URL) {
353
+ await page.goto(value, { waitUntil: 'load' });
354
+ await handlePotentialNavigation(page, null);
355
+ }
356
+ else {
357
+ let locator = undefined;
358
+ if (iframe_selector)
359
+ locator = page.frameLocator(iframe_selector).locator(element_css_selector);
360
+ else
361
+ locator = page.locator(element_css_selector);
362
+ switch (action) {
363
+ case PlaywrightAction.CLICK:
364
+ await handlePotentialNavigation(page, locator);
365
+ break;
366
+ case PlaywrightAction.FILL_IN:
367
+ await locator.click();
368
+ await locator.fill(value);
369
+ break;
370
+ case PlaywrightAction.TYPE_KEYS:
371
+ await (locator === null || locator === void 0 ? void 0 : locator.pressSequentially(value));
372
+ break;
373
+ case PlaywrightAction.SELECT_DROPDOWN:
374
+ await selectDropdownOption(page, locator, value);
375
+ break;
376
+ case PlaywrightAction.SELECT_MULTIPLE_DROPDOWN:
377
+ let optsArr;
378
+ if (value.startsWith('[')) {
379
+ try {
380
+ optsArr = JSON.parse(value);
381
+ }
382
+ catch (_a) {
383
+ optsArr = value.slice(1, -1).split(',').map(o => o.trim());
384
+ }
385
+ }
386
+ else {
387
+ optsArr = value.split(',').map(o => o.trim());
388
+ }
389
+ await (locator === null || locator === void 0 ? void 0 : locator.selectOption(optsArr));
390
+ break;
391
+ case PlaywrightAction.CHECK_CHECKBOX:
392
+ await (locator === null || locator === void 0 ? void 0 : locator.setChecked(value.toLowerCase() === 'true'));
393
+ break;
394
+ case PlaywrightAction.SELECT_RADIO:
395
+ case PlaywrightAction.TOGGLE_SWITCH:
396
+ await (locator === null || locator === void 0 ? void 0 : locator.click());
397
+ break;
398
+ case PlaywrightAction.VALIDATE_EXACT_VALUE:
399
+ const actualExact = await getElementValue(page, locator);
400
+ proboLogger.debug(`actual value is [${actualExact}]`);
401
+ if (actualExact !== value) {
402
+ proboLogger.info(`Validation *FAIL* expected '${value}' but got '${actualExact}'`);
403
+ return false;
404
+ }
405
+ proboLogger.info('Validation *PASS*');
406
+ break;
407
+ case PlaywrightAction.VALIDATE_CONTAINS_VALUE:
408
+ const actualContains = await getElementValue(page, locator);
409
+ proboLogger.debug(`actual value is [${actualContains}]`);
410
+ if (!actualContains.includes(value)) {
411
+ proboLogger.info(`Validation *FAIL* expected '${value}' to be contained in '${actualContains}'`);
412
+ return false;
413
+ }
414
+ proboLogger.info('Validation *PASS*');
415
+ break;
416
+ case PlaywrightAction.VALIDATE_URL:
417
+ const currUrl = page.url();
418
+ if (currUrl !== value) {
419
+ proboLogger.info(`Validation *FAIL* expected url '${value}' while is '${currUrl}'`);
420
+ return false;
421
+ }
422
+ proboLogger.info('Validation *PASS*');
423
+ break;
424
+ default:
425
+ throw new Error(`Unknown action: ${action}`);
426
+ }
427
+ }
428
+ return true;
429
+ }
430
+ catch (e) {
431
+ proboLogger.debug(`***ERROR failed to execute action ${action}: ${e}`);
432
+ throw e;
433
+ }
434
+ }
435
+ /**
436
+ * Execute a given Playwright action using native Playwright functions where possible
437
+ */
438
+ async function executeCachedPlaywrightAction(page, action, value, element_css_selector) {
439
+ proboLogger.log(`performing Cached Action: ${action} Value: ${value} on locator: ${element_css_selector}`);
350
440
  try {
351
441
  switch (action) {
352
442
  case PlaywrightAction.VISIT_BASE_URL:
353
443
  case PlaywrightAction.VISIT_URL:
354
- await page.goto(value, { waitUntil: 'load' });
355
- await handlePotentialNavigation(page, null);
444
+ await page.goto(value, { waitUntil: 'networkidle' });
356
445
  break;
357
446
  case PlaywrightAction.CLICK:
358
- await handlePotentialNavigation(page, element_css_selector);
447
+ await page.click(element_css_selector);
448
+ await handlePotentialNavigation(page);
359
449
  break;
360
450
  case PlaywrightAction.FILL_IN:
361
- await page.click(element_css_selector);
362
451
  await page.fill(element_css_selector, value);
363
452
  break;
364
453
  case PlaywrightAction.TYPE_KEYS:
365
- await page.locator(element_css_selector).pressSequentially(value);
454
+ await page.type(element_css_selector, value);
366
455
  break;
367
456
  case PlaywrightAction.SELECT_DROPDOWN:
368
- await selectDropdownOption(page, element_css_selector, value);
457
+ await page.selectOption(element_css_selector, value);
369
458
  break;
370
459
  case PlaywrightAction.SELECT_MULTIPLE_DROPDOWN:
371
460
  let optsArr;
@@ -387,33 +476,35 @@ async function executePlaywrightAction(page, action, value, element_css_selector
387
476
  break;
388
477
  case PlaywrightAction.SELECT_RADIO:
389
478
  case PlaywrightAction.TOGGLE_SWITCH:
390
- await page.click(element_css_selector, { force: true });
479
+ await page.click(element_css_selector);
391
480
  break;
392
481
  case PlaywrightAction.VALIDATE_EXACT_VALUE:
393
- const actualExact = await getElementValue(page, element_css_selector);
394
- proboLogger.debug(`DEBUG: actual value is [${actualExact}]`);
395
- if (actualExact !== value) {
396
- proboLogger.info(`DEBUG: Validation *FAIL* expected '${value}' but got '${actualExact}'`);
482
+ const actualExact = await page.locator(element_css_selector).textContent();
483
+ const trimmedExact = actualExact ? actualExact.trim() : '';
484
+ proboLogger.debug(`actual value is [${trimmedExact}]`);
485
+ if (trimmedExact !== value) {
486
+ proboLogger.info(`Validation *FAIL* expected '${value}' but got '${trimmedExact}'`);
397
487
  return false;
398
488
  }
399
- proboLogger.info('DEBUG: Validation *PASS*');
489
+ proboLogger.info('Validation *PASS*');
400
490
  break;
401
491
  case PlaywrightAction.VALIDATE_CONTAINS_VALUE:
402
- const actualContains = await getElementValue(page, element_css_selector);
403
- proboLogger.debug(`DEBUG: actual value is [${actualContains}]`);
404
- if (!actualContains.includes(value)) {
405
- proboLogger.info(`DEBUG: Validation *FAIL* expected '${value}' to be contained in '${actualContains}'`);
492
+ const actualContains = await page.locator(element_css_selector).textContent();
493
+ const trimmedContains = actualContains ? actualContains.trim() : '';
494
+ proboLogger.debug(`actual value is [${trimmedContains}]`);
495
+ if (!trimmedContains.includes(value)) {
496
+ proboLogger.info(`Validation *FAIL* expected '${value}' to be contained in '${trimmedContains}'`);
406
497
  return false;
407
498
  }
408
- proboLogger.info('DEBUG: Validation *PASS*');
499
+ proboLogger.info('Validation *PASS*');
409
500
  break;
410
501
  case PlaywrightAction.VALIDATE_URL:
411
502
  const currUrl = page.url();
412
503
  if (currUrl !== value) {
413
- proboLogger.info(`DEBUG: Validation *FAIL* expected url '${value}' while is '${currUrl}'`);
504
+ proboLogger.info(`Validation *FAIL* expected url '${value}' while is '${currUrl}'`);
414
505
  return false;
415
506
  }
416
- proboLogger.info('DEBUG: Validation *PASS*');
507
+ proboLogger.info('Validation *PASS*');
417
508
  break;
418
509
  default:
419
510
  throw new Error(`Unknown action: ${action}`);
@@ -421,7 +512,7 @@ async function executePlaywrightAction(page, action, value, element_css_selector
421
512
  return true;
422
513
  }
423
514
  catch (e) {
424
- proboLogger.debug(`***ERROR failed to execute action ${action}: ${e}`);
515
+ proboLogger.debug(`***ERROR failed to execute cached action ${action}: ${e}`);
425
516
  throw e;
426
517
  }
427
518
  }
@@ -470,6 +561,10 @@ class Highlighter {
470
561
  //proboLogger.debug('Browser: Found elements:', elements);
471
562
  return elements;
472
563
  }, elementTag);
564
+ // for (let i = 0; i < result.length; i++) {
565
+ // result[i].element = '';
566
+ // };
567
+ // console.log('highlighted elements: ', result);
473
568
  return result;
474
569
  }
475
570
  async unhighlightElements(page) {
@@ -480,10 +575,10 @@ class Highlighter {
480
575
  (_b = (_a = window === null || window === void 0 ? void 0 : window.ProboLabs) === null || _a === void 0 ? void 0 : _a.highlight) === null || _b === void 0 ? void 0 : _b.unexecute();
481
576
  });
482
577
  }
483
- async highlightElement(page, element_css_selector, element_index) {
578
+ async highlightElement(page, element_css_selector, iframe_selector, element_index) {
484
579
  await this.ensureHighlighterScript(page);
485
- proboLogger.debug('Highlighting element with:', { element_css_selector, element_index });
486
- await page.evaluate(({ css_selector, index }) => {
580
+ proboLogger.debug('Highlighting element with:', { element_css_selector, iframe_selector, element_index });
581
+ await page.evaluate(({ css_selector, iframe_selector, index }) => {
487
582
  const proboLabs = window.ProboLabs;
488
583
  if (!proboLabs) {
489
584
  proboLogger.warn('ProboLabs not initialized');
@@ -492,12 +587,14 @@ class Highlighter {
492
587
  // Create ElementInfo object for the element
493
588
  const elementInfo = {
494
589
  css_selector: css_selector,
590
+ iframe_selector: iframe_selector,
495
591
  index: index
496
592
  };
497
593
  // Call highlightElements directly
498
594
  proboLabs.highlightElements([elementInfo]);
499
595
  }, {
500
596
  css_selector: element_css_selector,
597
+ iframe_selector: iframe_selector,
501
598
  index: element_index
502
599
  });
503
600
  }
@@ -987,6 +1084,7 @@ class ApiClient {
987
1084
  return headers;
988
1085
  }
989
1086
  async createStep(options) {
1087
+ proboLogger.debug('creating step ', options.stepPrompt);
990
1088
  return pRetry(async () => {
991
1089
  const response = await fetch(`${this.apiUrl}/step-runners/`, {
992
1090
  method: 'POST',
@@ -1006,13 +1104,14 @@ class ApiClient {
1006
1104
  retries: this.maxRetries,
1007
1105
  minTimeout: this.initialBackoff,
1008
1106
  onFailedAttempt: error => {
1009
- console.log(`[probolib-LOG] API call failed, attempt ${error.attemptNumber} of ${error.retriesLeft + error.attemptNumber}...`);
1107
+ proboLogger.warn(`API call failed, attempt ${error.attemptNumber} of ${error.retriesLeft + error.attemptNumber}...`);
1010
1108
  }
1011
1109
  });
1012
1110
  }
1013
1111
  async resolveNextInstruction(stepId, instruction) {
1112
+ proboLogger.debug(`resolving next instruction: ${instruction}`);
1014
1113
  return pRetry(async () => {
1015
- console.debug(`[probolib-LOG] API client: Resolving next instruction for step ${stepId}`);
1114
+ proboLogger.debug(`API client: Resolving next instruction for step ${stepId}`);
1016
1115
  const cleanInstruction = cleanupInstructionElements(instruction);
1017
1116
  const response = await fetch(`${this.apiUrl}/step-runners/${stepId}/run/`, {
1018
1117
  method: 'POST',
@@ -1025,7 +1124,7 @@ class ApiClient {
1025
1124
  retries: this.maxRetries,
1026
1125
  minTimeout: this.initialBackoff,
1027
1126
  onFailedAttempt: error => {
1028
- console.log(`[probolib-LOG] API call failed, attempt ${error.attemptNumber} of ${error.retriesLeft + error.attemptNumber}...`);
1127
+ proboLogger.warn(`API call failed, attempt ${error.attemptNumber} of ${error.retriesLeft + error.attemptNumber}...`);
1029
1128
  }
1030
1129
  });
1031
1130
  }
@@ -1044,7 +1143,41 @@ class ApiClient {
1044
1143
  retries: this.maxRetries,
1045
1144
  minTimeout: this.initialBackoff,
1046
1145
  onFailedAttempt: error => {
1047
- console.log(`[probolib-LOG] API call failed, attempt ${error.attemptNumber} of ${error.retriesLeft + error.attemptNumber}...`);
1146
+ proboLogger.warn(`API call failed, attempt ${error.attemptNumber} of ${error.retriesLeft + error.attemptNumber}...`);
1147
+ }
1148
+ });
1149
+ }
1150
+ async findStepByPrompt(prompt, scenarioName) {
1151
+ proboLogger.debug(`Finding step by prompt: ${prompt} and scenario: ${scenarioName}`);
1152
+ return pRetry(async () => {
1153
+ const response = await fetch(`${this.apiUrl}/step-runners/find-step-by-prompt/`, {
1154
+ method: 'POST',
1155
+ headers: this.getHeaders(),
1156
+ body: JSON.stringify({
1157
+ prompt: prompt,
1158
+ scenario_name: scenarioName
1159
+ }),
1160
+ });
1161
+ try {
1162
+ const data = await this.handleResponse(response);
1163
+ return {
1164
+ step: data.step,
1165
+ total_count: data.total_count
1166
+ };
1167
+ }
1168
+ catch (error) {
1169
+ // If we get a 404, the step doesn't exist
1170
+ if (error instanceof ApiError && error.status === 404) {
1171
+ return null;
1172
+ }
1173
+ // For any other error, rethrow
1174
+ throw error;
1175
+ }
1176
+ }, {
1177
+ retries: this.maxRetries,
1178
+ minTimeout: this.initialBackoff,
1179
+ onFailedAttempt: error => {
1180
+ proboLogger.warn(`API call failed, attempt ${error.attemptNumber} of ${error.retriesLeft + error.attemptNumber}...`);
1048
1181
  }
1049
1182
  });
1050
1183
  }
@@ -1067,11 +1200,20 @@ class Probo {
1067
1200
  proboLogger.setLogLevel(debugLevel);
1068
1201
  proboLogger.log(`Initializing: scenarioName: ${scenarioName}, apiUrl: ${apiUrl}, enableConsoleLogs: ${enableConsoleLogs}, debugLevel: ${debugLevel}`);
1069
1202
  }
1070
- async runStep(page, stepPrompt, stepIdFromServer, // Make optional
1071
- useCache = true) {
1072
- proboLogger.log(`runStep: ${stepIdFromServer ? '#' + stepIdFromServer + ' - ' : ''}${stepPrompt}, pageUrl: ${page.url()}`);
1203
+ async runStep(page, stepPrompt, options = { useCache: true, stepIdFromServer: undefined }) {
1204
+ proboLogger.log(`runStep: ${options.stepIdFromServer ? '#' + options.stepIdFromServer + ' - ' : ''}${stepPrompt}, pageUrl: ${page.url()}`);
1073
1205
  this.setupConsoleLogs(page);
1074
- const stepId = await this._handleStepCreation(page, stepPrompt, stepIdFromServer, useCache);
1206
+ // First check if the step exists in the database
1207
+ let stepId;
1208
+ if (options.useCache) {
1209
+ const isCachedStep = await this._handleCachedStep(page, stepPrompt);
1210
+ if (isCachedStep) {
1211
+ proboLogger.debug('performed cached step!');
1212
+ return true;
1213
+ }
1214
+ }
1215
+ proboLogger.debug(`Cache disabled or step not found, creating new step`);
1216
+ stepId = await this._handleStepCreation(page, stepPrompt, options.stepIdFromServer, options.useCache);
1075
1217
  proboLogger.debug('Step ID:', stepId);
1076
1218
  let instruction = null;
1077
1219
  // Main execution loop
@@ -1125,6 +1267,36 @@ class Probo {
1125
1267
  }
1126
1268
  }
1127
1269
  }
1270
+ async _handleCachedStep(page, stepPrompt) {
1271
+ proboLogger.debug(`Checking if step exists in database: ${stepPrompt}`);
1272
+ const result = await this.apiClient.findStepByPrompt(stepPrompt, this.scenarioName);
1273
+ if (result) {
1274
+ proboLogger.log(`Found existing step with ID: ${result.step.id} going to perform action: ${result.step.action} with value: ${result.step.action_value}`);
1275
+ proboLogger.debug(`Step in the DB: #${result.step.id} status: ${result.step.status} action: ${result.step.action} action_value: ${result.step.action_value} locator: ${result.step.element_css_selector}`);
1276
+ if (result.step.status !== 'EXECUTED') {
1277
+ proboLogger.debug(`Step ${result.step.id} is not executed, returning false`);
1278
+ return false;
1279
+ }
1280
+ proboLogger.debug(`Step ${result.step.id} is in status executed, performing action directly with Playwright`);
1281
+ const element_css_selector = result.step.element_css_selector;
1282
+ result.step.iframe_selector;
1283
+ if (element_css_selector) {
1284
+ try {
1285
+ await page.waitForSelector(element_css_selector, { timeout: 2000 });
1286
+ }
1287
+ catch (error) {
1288
+ proboLogger.error(`Error waiting for locator for step ${result.step.id} going to ignore the caching mechanism`);
1289
+ return false;
1290
+ }
1291
+ }
1292
+ await executeCachedPlaywrightAction(page, result.step.action, result.step.action_value, element_css_selector);
1293
+ return true;
1294
+ }
1295
+ else {
1296
+ proboLogger.debug(`Step not found in database, continuing with the normal flow`);
1297
+ return false;
1298
+ }
1299
+ }
1128
1300
  async _handleStepCreation(page, stepPrompt, stepIdFromServer, useCache) {
1129
1301
  proboLogger.debug(`Taking initial screenshot from the page ${page.url()}`);
1130
1302
  // not sure if this is needed
@@ -1171,12 +1343,15 @@ class Probo {
1171
1343
  async unhighlightElements(page) {
1172
1344
  return this.highlighter.unhighlightElements(page);
1173
1345
  }
1174
- async highlightElement(page, element_css_selector, element_index) {
1175
- return this.highlighter.highlightElement(page, element_css_selector, element_index);
1346
+ async highlightElement(page, element_css_selector, iframe_selector, element_index) {
1347
+ return this.highlighter.highlightElement(page, element_css_selector, iframe_selector, element_index);
1176
1348
  }
1177
1349
  async screenshot(page) {
1350
+ proboLogger.debug(`taking screenshot of current page: ${page.url()}`);
1351
+ // await page.evaluate(() => document.fonts?.ready.catch(() => {}));
1178
1352
  const screenshot_bytes = await page.screenshot({ fullPage: true, animations: 'disabled' });
1179
1353
  // make an api call to upload the screenshot to cloudinary
1354
+ proboLogger.debug('uploading image data to cloudinary');
1180
1355
  const screenshot_url = await this.apiClient.uploadScreenshot(screenshot_bytes);
1181
1356
  return screenshot_url;
1182
1357
  }
@@ -1185,15 +1360,16 @@ class Probo {
1185
1360
  const action = nextInstruction.args.action;
1186
1361
  const value = nextInstruction.args.value;
1187
1362
  const element_css_selector = nextInstruction.args.element_css_selector;
1363
+ const iframe_selector = nextInstruction.args.iframe_selector;
1188
1364
  const element_index = nextInstruction.args.element_index;
1189
1365
  if (action !== PlaywrightAction.VISIT_URL) {
1190
1366
  await this.unhighlightElements(page);
1191
1367
  proboLogger.debug('Unhighlighted elements');
1192
- await this.highlightElement(page, element_css_selector, element_index);
1368
+ await this.highlightElement(page, element_css_selector, iframe_selector, element_index);
1193
1369
  proboLogger.debug('Highlighted element');
1194
1370
  }
1195
1371
  const pre_action_screenshot_url = await this.screenshot(page);
1196
- const step_status = await executePlaywrightAction(page, action, value, element_css_selector);
1372
+ const step_status = await executePlaywrightAction(page, action, value, iframe_selector, element_css_selector);
1197
1373
  await this.unhighlightElements(page);
1198
1374
  proboLogger.debug('UnHighlighted element');
1199
1375
  await waitForMutationsToSettle(page);
@@ -1218,7 +1394,8 @@ class Probo {
1218
1394
  args: {
1219
1395
  action: action,
1220
1396
  value: value,
1221
- element_css_selector: element_css_selector
1397
+ element_css_selector: element_css_selector,
1398
+ iframe_selector: iframe_selector
1222
1399
  },
1223
1400
  result: {
1224
1401
  pre_action_screenshot_url: pre_action_screenshot_url,