@probolabs/playwright 1.0.17 → 1.0.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +227 -227
- package/dist/.tsbuildinfo +1 -0
- package/dist/fixtures.cjs +272 -0
- package/dist/fixtures.cjs.map +1 -0
- package/dist/fixtures.js +178 -9
- package/dist/fixtures.js.map +1 -1
- package/dist/index.cjs +4935 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.ts +114 -58
- package/dist/index.js +505 -275
- package/dist/index.js.map +1 -1
- package/dist/types/fixtures.d.ts.map +1 -1
- package/dist/types/highlight.d.ts.map +1 -1
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/replay-utils.d.ts.map +1 -1
- package/loaded_extensions/README.md +23 -23
- package/package.json +6 -5
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
const highlighterCode = "(function (global, factory) {\n typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports) :\n typeof define === 'function' && define.amd ? define(['exports'], factory) :\n (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global.ProboLabs = {}));\n})(this, (function (exports) { 'use strict';\n\n const ElementTag = {\n CLICKABLE: \"CLICKABLE\", // button, link, toggle switch, checkbox, radio, dropdowns, clickable divs\n FILLABLE: \"FILLABLE\", // input, textarea content_editable, date picker??\n SELECTABLE: \"SELECTABLE\", // select\n NON_INTERACTIVE_ELEMENT: 'NON_INTERACTIVE_ELEMENT',\n };\n\n class ElementInfo {\n constructor(element, index, {tag, type, text, html, xpath, css_selector, bounding_box, iframe_selector, short_css_selector, short_iframe_selector}) {\n this.index = index.toString();\n this.tag = tag;\n this.type = type;\n this.text = text;\n this.html = html;\n this.xpath = xpath;\n this.css_selector = css_selector;\n this.bounding_box = bounding_box;\n this.iframe_selector = iframe_selector;\n this.element = element;\n this.depth = -1;\n this.short_css_selector = short_css_selector;\n this.short_iframe_selector = short_iframe_selector;\n }\n\n getSelector() {\n return this.xpath ? this.xpath : this.css_selector;\n }\n\n getDepth() {\n if (this.depth >= 0) {\n return this.depth;\n }\n \n this.depth = 0;\n let currentElement = this.element;\n \n while (currentElement.nodeType === Node.ELEMENT_NODE) { \n this.depth++;\n currentElement = getParentNode(currentElement);\n }\n \n return this.depth;\n }\n }\n\n function getParentNode(element) {\n if (!element || element.nodeType !== Node.ELEMENT_NODE) return null;\n \n let parent = null;\n // SF is using slots and shadow DOM heavily\n // However, there might be slots in the light DOM which shouldn't be traversed\n if (element.assignedSlot && element.getRootNode() instanceof ShadowRoot)\n parent = element.assignedSlot;\n else \n parent = element.parentNode;\n \n // Check if we're at a shadow root\n if (parent && parent.nodeType !== Node.ELEMENT_NODE && parent.getRootNode() instanceof ShadowRoot) \n parent = parent.getRootNode().host; \n\n return parent;\n }\n\n // License: MIT\n // Author: Anton Medvedev <anton@medv.io>\n // Source: https://github.com/antonmedv/finder\n const acceptedAttrNames = new Set(['role', 'name', 'aria-label', 'rel', 'href']);\n /** Check if attribute name and value are word-like. */\n function attr(name, value) {\n let nameIsOk = acceptedAttrNames.has(name);\n nameIsOk ||= name.startsWith('data-') && wordLike(name);\n let valueIsOk = wordLike(value) && value.length < 100;\n valueIsOk ||= value.startsWith('#') && wordLike(value.slice(1));\n return nameIsOk && valueIsOk;\n }\n /** Check if id name is word-like. */\n function idName(name) {\n return wordLike(name);\n }\n /** Check if class name is word-like. */\n function className(name) {\n return wordLike(name);\n }\n /** Check if tag name is word-like. */\n function tagName(name) {\n return true;\n }\n /** Finds unique CSS selectors for the given element. */\n function finder(input, options) {\n if (input.nodeType !== Node.ELEMENT_NODE) {\n throw new Error(`Can't generate CSS selector for non-element node type.`);\n }\n if (input.tagName.toLowerCase() === 'html') {\n return 'html';\n }\n const defaults = {\n root: document.body,\n idName: idName,\n className: className,\n tagName: tagName,\n attr: attr,\n timeoutMs: 1000,\n seedMinLength: 3,\n optimizedMinLength: 2,\n maxNumberOfPathChecks: Infinity,\n };\n const startTime = new Date();\n const config = { ...defaults, ...options };\n const rootDocument = findRootDocument(config.root, defaults);\n let foundPath;\n let count = 0;\n for (const candidate of search(input, config, rootDocument)) {\n const elapsedTimeMs = new Date().getTime() - startTime.getTime();\n if (elapsedTimeMs > config.timeoutMs ||\n count >= config.maxNumberOfPathChecks) {\n const fPath = fallback(input, rootDocument);\n if (!fPath) {\n throw new Error(`Timeout: Can't find a unique selector after ${config.timeoutMs}ms`);\n }\n return selector(fPath);\n }\n count++;\n if (unique(candidate, rootDocument)) {\n foundPath = candidate;\n break;\n }\n }\n if (!foundPath) {\n throw new Error(`Selector was not found.`);\n }\n const optimized = [\n ...optimize(foundPath, input, config, rootDocument, startTime),\n ];\n optimized.sort(byPenalty);\n if (optimized.length > 0) {\n return selector(optimized[0]);\n }\n return selector(foundPath);\n }\n function* search(input, config, rootDocument) {\n const stack = [];\n let paths = [];\n let current = input;\n let i = 0;\n while (current && current !== rootDocument) {\n const level = tie(current, config);\n for (const node of level) {\n node.level = i;\n }\n stack.push(level);\n current = current.parentElement;\n i++;\n paths.push(...combinations(stack));\n if (i >= config.seedMinLength) {\n paths.sort(byPenalty);\n for (const candidate of paths) {\n yield candidate;\n }\n paths = [];\n }\n }\n paths.sort(byPenalty);\n for (const candidate of paths) {\n yield candidate;\n }\n }\n function wordLike(name) {\n if (/^[a-z\\-]{3,}$/i.test(name)) {\n const words = name.split(/-|[A-Z]/);\n for (const word of words) {\n if (word.length <= 2) {\n return false;\n }\n if (/[^aeiou]{4,}/i.test(word)) {\n return false;\n }\n }\n return true;\n }\n return false;\n }\n function tie(element, config) {\n const level = [];\n const elementId = element.getAttribute('id');\n if (elementId && config.idName(elementId)) {\n level.push({\n name: '#' + CSS.escape(elementId),\n penalty: 0,\n });\n }\n for (let i = 0; i < element.classList.length; i++) {\n const name = element.classList[i];\n if (config.className(name)) {\n level.push({\n name: '.' + CSS.escape(name),\n penalty: 1,\n });\n }\n }\n for (let i = 0; i < element.attributes.length; i++) {\n const attr = element.attributes[i];\n if (config.attr(attr.name, attr.value)) {\n level.push({\n name: `[${CSS.escape(attr.name)}=\"${CSS.escape(attr.value)}\"]`,\n penalty: 2,\n });\n }\n }\n const tagName = element.tagName.toLowerCase();\n if (config.tagName(tagName)) {\n level.push({\n name: tagName,\n penalty: 5,\n });\n const index = indexOf(element, tagName);\n if (index !== undefined) {\n level.push({\n name: nthOfType(tagName, index),\n penalty: 10,\n });\n }\n }\n const nth = indexOf(element);\n if (nth !== undefined) {\n level.push({\n name: nthChild(tagName, nth),\n penalty: 50,\n });\n }\n return level;\n }\n function selector(path) {\n let node = path[0];\n let query = node.name;\n for (let i = 1; i < path.length; i++) {\n const level = path[i].level || 0;\n if (node.level === level - 1) {\n query = `${path[i].name} > ${query}`;\n }\n else {\n query = `${path[i].name} ${query}`;\n }\n node = path[i];\n }\n return query;\n }\n function penalty(path) {\n return path.map((node) => node.penalty).reduce((acc, i) => acc + i, 0);\n }\n function byPenalty(a, b) {\n return penalty(a) - penalty(b);\n }\n function indexOf(input, tagName) {\n const parent = input.parentNode;\n if (!parent) {\n return undefined;\n }\n let child = parent.firstChild;\n if (!child) {\n return undefined;\n }\n let i = 0;\n while (child) {\n if (child.nodeType === Node.ELEMENT_NODE &&\n (tagName === undefined ||\n child.tagName.toLowerCase() === tagName)) {\n i++;\n }\n if (child === input) {\n break;\n }\n child = child.nextSibling;\n }\n return i;\n }\n function fallback(input, rootDocument) {\n let i = 0;\n let current = input;\n const path = [];\n while (current && current !== rootDocument) {\n const tagName = current.tagName.toLowerCase();\n const index = indexOf(current, tagName);\n if (index === undefined) {\n return;\n }\n path.push({\n name: nthOfType(tagName, index),\n penalty: NaN,\n level: i,\n });\n current = current.parentElement;\n i++;\n }\n if (unique(path, rootDocument)) {\n return path;\n }\n }\n function nthChild(tagName, index) {\n if (tagName === 'html') {\n return 'html';\n }\n return `${tagName}:nth-child(${index})`;\n }\n function nthOfType(tagName, index) {\n if (tagName === 'html') {\n return 'html';\n }\n return `${tagName}:nth-of-type(${index})`;\n }\n function* combinations(stack, path = []) {\n if (stack.length > 0) {\n for (let node of stack[0]) {\n yield* combinations(stack.slice(1, stack.length), path.concat(node));\n }\n }\n else {\n yield path;\n }\n }\n function findRootDocument(rootNode, defaults) {\n if (rootNode.nodeType === Node.DOCUMENT_NODE) {\n return rootNode;\n }\n if (rootNode === defaults.root) {\n return rootNode.ownerDocument;\n }\n return rootNode;\n }\n function unique(path, rootDocument) {\n const css = selector(path);\n switch (rootDocument.querySelectorAll(css).length) {\n case 0:\n throw new Error(`Can't select any node with this selector: ${css}`);\n case 1:\n return true;\n default:\n return false;\n }\n }\n function* optimize(path, input, config, rootDocument, startTime) {\n if (path.length > 2 && path.length > config.optimizedMinLength) {\n for (let i = 1; i < path.length - 1; i++) {\n const elapsedTimeMs = new Date().getTime() - startTime.getTime();\n if (elapsedTimeMs > config.timeoutMs) {\n return;\n }\n const newPath = [...path];\n newPath.splice(i, 1);\n if (unique(newPath, rootDocument) &&\n rootDocument.querySelector(selector(newPath)) === input) {\n yield newPath;\n yield* optimize(newPath, input, config, rootDocument, startTime);\n }\n }\n }\n }\n\n // import { realpath } from \"fs\";\n\n function getAllDocumentElementsIncludingShadow(selectors, root = document) {\n const elements = Array.from(root.querySelectorAll(selectors));\n\n root.querySelectorAll('*').forEach(el => {\n if (el.shadowRoot) {\n elements.push(...getAllDocumentElementsIncludingShadow(selectors, el.shadowRoot));\n }\n });\n return elements;\n }\n\n function getAllFrames(root = document) {\n const result = [root];\n const frames = getAllDocumentElementsIncludingShadow('frame, iframe', root); \n frames.forEach(frame => {\n try {\n const frameDocument = frame.contentDocument || frame.contentWindow.document;\n if (frameDocument) {\n result.push(frameDocument);\n }\n } catch (e) {\n // Skip cross-origin frames\n console.warn('Could not access frame content:', e.message);\n }\n });\n\n return result;\n }\n\n function getAllElementsIncludingShadow(selectors, root = document) {\n const elements = [];\n\n getAllFrames(root).forEach(doc => {\n elements.push(...getAllDocumentElementsIncludingShadow(selectors, doc));\n });\n\n return elements;\n }\n\n /**\n * Deeply searches through DOM trees including Shadow DOM and frames/iframes\n * @param {string} selector - CSS selector to search for\n * @param {Document|Element} [root=document] - Starting point for the search\n * @param {Object} [options] - Search options\n * @param {boolean} [options.searchShadow=true] - Whether to search Shadow DOM\n * @param {boolean} [options.searchFrames=true] - Whether to search frames/iframes\n * @returns {Element[]} Array of found elements\n \n function getAllElementsIncludingShadow(selector, root = document, options = {}) {\n const {\n searchShadow = true,\n searchFrames = true\n } = options;\n\n const results = new Set();\n \n // Helper to check if an element is valid and not yet found\n const addIfValid = (element) => {\n if (element && !results.has(element)) {\n results.add(element);\n }\n };\n\n // Helper to process a single document or element\n function processNode(node) {\n // Search regular DOM\n node.querySelectorAll(selector).forEach(addIfValid);\n\n if (searchShadow) {\n // Search all shadow roots\n const treeWalker = document.createTreeWalker(\n node,\n NodeFilter.SHOW_ELEMENT,\n {\n acceptNode: (element) => {\n return element.shadowRoot ? \n NodeFilter.FILTER_ACCEPT : \n NodeFilter.FILTER_SKIP;\n }\n }\n );\n\n while (treeWalker.nextNode()) {\n const element = treeWalker.currentNode;\n if (element.shadowRoot) {\n // Search within shadow root\n element.shadowRoot.querySelectorAll(selector).forEach(addIfValid);\n // Recursively process the shadow root for nested shadow DOMs\n processNode(element.shadowRoot);\n }\n }\n }\n\n if (searchFrames) {\n // Search frames and iframes\n const frames = node.querySelectorAll('frame, iframe');\n frames.forEach(frame => {\n try {\n const frameDocument = frame.contentDocument;\n if (frameDocument) {\n processNode(frameDocument);\n }\n } catch (e) {\n // Skip cross-origin frames\n console.warn('Could not access frame content:', e.message);\n }\n });\n }\n }\n\n // Start processing from the root\n processNode(root);\n\n return Array.from(results);\n }\n */\n // <div x=1 y=2 role='combobox'> </div>\n function findDropdowns() {\n const dropdowns = [];\n \n // Native select elements\n dropdowns.push(...getAllElementsIncludingShadow('select'));\n \n // Elements with dropdown roles that don't have <input>..</input>\n const roleElements = getAllElementsIncludingShadow('[role=\"combobox\"], [role=\"listbox\"], [role=\"dropdown\"], [role=\"option\"], [role=\"menu\"], [role=\"menuitem\"]').filter(el => {\n return el.tagName.toLowerCase() !== 'input' || ![\"button\", \"checkbox\", \"radio\"].includes(el.getAttribute(\"type\"));\n });\n dropdowns.push(...roleElements);\n \n // Common dropdown class patterns\n const dropdownPattern = /.*(dropdown|select|combobox|menu).*/i;\n const elements = getAllElementsIncludingShadow('*');\n const dropdownClasses = Array.from(elements).filter(el => {\n const hasDropdownClass = dropdownPattern.test(el.className);\n const validTag = ['li', 'ul', 'span', 'div', 'p', 'a', 'button'].includes(el.tagName.toLowerCase());\n const style = window.getComputedStyle(el); \n const result = hasDropdownClass && validTag && (style.cursor === 'pointer' || el.tagName.toLowerCase() === 'a' || el.tagName.toLowerCase() === 'button');\n return result;\n });\n \n dropdowns.push(...dropdownClasses);\n \n // Elements with aria-haspopup attribute\n dropdowns.push(...getAllElementsIncludingShadow('[aria-haspopup=\"true\"], [aria-haspopup=\"listbox\"], [aria-haspopup=\"menu\"]'));\n\n // Improve navigation element detection\n // Semantic nav elements with list items\n dropdowns.push(...getAllElementsIncludingShadow('nav ul li, nav ol li'));\n \n // Navigation elements in common design patterns\n dropdowns.push(...getAllElementsIncludingShadow('header a, .header a, .nav a, .navigation a, .menu a, .sidebar a, aside a'));\n \n // Elements in primary navigation areas with common attributes\n dropdowns.push(...getAllElementsIncludingShadow('[role=\"navigation\"] a, [aria-label*=\"navigation\"] a, [aria-label*=\"menu\"] a'));\n\n return dropdowns;\n }\n\n function findClickables() {\n const clickables = [];\n \n const checkboxPattern = /checkbox/i;\n // Collect all clickable elements first\n const nativeLinks = [...getAllElementsIncludingShadow('a')];\n const nativeButtons = [...getAllElementsIncludingShadow('button')];\n const inputButtons = [...getAllElementsIncludingShadow('input[type=\"button\"], input[type=\"submit\"], input[type=\"reset\"]')];\n const roleButtons = [...getAllElementsIncludingShadow('[role=\"button\"]')];\n // const tabbable = [...getAllElementsIncludingShadow('[tabindex=\"0\"]')];\n const clickHandlers = [...getAllElementsIncludingShadow('[onclick]')];\n const dropdowns = findDropdowns();\n const nativeCheckboxes = [...getAllElementsIncludingShadow('input[type=\"checkbox\"]')]; \n const fauxCheckboxes = getAllElementsIncludingShadow('*').filter(el => {\n if (checkboxPattern.test(el.className)) {\n const realCheckboxes = getAllElementsIncludingShadow('input[type=\"checkbox\"]', el);\n if (realCheckboxes.length === 1) {\n const boundingRect = realCheckboxes[0].getBoundingClientRect();\n return boundingRect.width <= 1 && boundingRect.height <= 1 \n }\n }\n return false;\n });\n const nativeRadios = [...getAllElementsIncludingShadow('input[type=\"radio\"]')];\n const toggles = findToggles();\n const pointerElements = findElementsWithPointer();\n // Add all elements at once\n clickables.push(\n ...nativeLinks,\n ...nativeButtons,\n ...inputButtons,\n ...roleButtons,\n // ...tabbable,\n ...clickHandlers,\n ...dropdowns,\n ...nativeCheckboxes,\n ...fauxCheckboxes,\n ...nativeRadios,\n ...toggles,\n ...pointerElements\n );\n\n // Only uniquify once at the end\n return clickables; // Let findElements handle the uniquification\n }\n\n function findToggles() {\n const toggles = [];\n const checkboxes = getAllElementsIncludingShadow('input[type=\"checkbox\"]');\n const togglePattern = /switch|toggle|slider/i;\n\n checkboxes.forEach(checkbox => {\n let isToggle = false;\n\n // Check the checkbox itself\n if (togglePattern.test(checkbox.className) || togglePattern.test(checkbox.getAttribute('role') || '')) {\n isToggle = true;\n }\n\n // Check parent elements (up to 3 levels)\n if (!isToggle) {\n let element = checkbox;\n for (let i = 0; i < 3; i++) {\n const parent = element.parentElement;\n if (!parent) break;\n\n const className = parent.className || '';\n const role = parent.getAttribute('role') || '';\n\n if (togglePattern.test(className) || togglePattern.test(role)) {\n isToggle = true;\n break;\n }\n element = parent;\n }\n }\n\n // Check next sibling\n if (!isToggle) {\n const nextSibling = checkbox.nextElementSibling;\n if (nextSibling) {\n const className = nextSibling.className || '';\n const role = nextSibling.getAttribute('role') || '';\n if (togglePattern.test(className) || togglePattern.test(role)) {\n isToggle = true;\n }\n }\n }\n\n if (isToggle) {\n toggles.push(checkbox);\n }\n });\n\n return toggles;\n }\n\n function findNonInteractiveElements() {\n // Get all elements in the document\n const all = Array.from(getAllElementsIncludingShadow('*'));\n \n // Filter elements based on Python implementation rules\n return all.filter(element => {\n if (!element.firstElementChild) {\n const tag = element.tagName.toLowerCase(); \n if (!['select', 'button', 'a'].includes(tag)) {\n const validTags = ['p', 'span', 'div', 'input', 'textarea','td','th'].includes(tag) || /^h\\d$/.test(tag) || /text/.test(tag);\n const boundingRect = element.getBoundingClientRect();\n return validTags && boundingRect.height > 1 && boundingRect.width > 1;\n }\n }\n return false;\n });\n }\n\n\n\n // export function findNonInteractiveElements() {\n // const all = [];\n // try {\n // const elements = getAllElementsIncludingShadow('*');\n // all.push(...elements);\n // } catch (e) {\n // console.warn('Error getting elements:', e);\n // }\n \n // console.debug('Total elements found:', all.length);\n \n // return all.filter(element => {\n // try {\n // const tag = element.tagName.toLowerCase(); \n\n // // Special handling for input elements\n // if (tag === 'input' || tag === 'textarea') {\n // const boundingRect = element.getBoundingClientRect();\n // const value = element.value || '';\n // const placeholder = element.placeholder || '';\n // return boundingRect.height > 1 && \n // boundingRect.width > 1 && \n // (value.trim() !== '' || placeholder.trim() !== '');\n // }\n\n \n // // Check if it's a valid tag for text content\n // const validTags = ['p', 'span', 'div', 'label', 'th', 'td', 'li', 'button', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'a', 'select'].includes(tag) || \n // /^h\\d$/.test(tag) || \n // /text/.test(tag);\n\n // const boundingRect = element.getBoundingClientRect();\n\n // // Get direct text content, excluding child element text\n // let directText = '';\n // for (const node of element.childNodes) {\n // // Only include text nodes (nodeType 3)\n // if (node.nodeType === 3) {\n // directText += node.textContent || '';\n // }\n // }\n \n // // If no direct text and it's a table cell or heading, check label content\n // if (!directText.trim() && (tag === 'th' || tag === 'td' || tag === 'h1')) {\n // const labels = element.getElementsByTagName('label');\n // for (const label of labels) {\n // directText += label.textContent || '';\n // }\n // }\n\n // // If still no text and it's a heading, get all text content\n // if (!directText.trim() && tag === 'h1') {\n // directText = element.textContent || '';\n // }\n\n // directText = directText.trim();\n\n // // Debug logging\n // if (directText) {\n // console.debugg('Text element found:', {\n // tag,\n // text: directText,\n // dimensions: boundingRect,\n // element\n // });\n // }\n\n // return validTags && \n // boundingRect.height > 1 && \n // boundingRect.width > 1 && \n // directText !== '';\n \n // } catch (e) {\n // console.warn('Error processing element:', e);\n // return false;\n // }\n // });\n // }\n\n\n\n\n\n function findElementsWithPointer() {\n const elements = [];\n const allElements = getAllElementsIncludingShadow('*');\n \n console.log('Checking elements with pointer style...');\n \n allElements.forEach(element => {\n // Skip SVG elements for now\n if (element instanceof SVGElement || element.tagName.toLowerCase() === 'svg') {\n return;\n }\n \n const style = window.getComputedStyle(element);\n if (style.cursor === 'pointer') {\n elements.push(element);\n }\n });\n \n console.log(`Found ${elements.length} elements with pointer cursor`);\n return elements;\n }\n\n function findCheckables() {\n const elements = [];\n\n elements.push(...getAllElementsIncludingShadow('input[type=\"checkbox\"]'));\n elements.push(...getAllElementsIncludingShadow('input[type=\"radio\"]'));\n const all_elements = getAllElementsIncludingShadow('label');\n const radioClasses = Array.from(all_elements).filter(el => {\n return /.*radio.*/i.test(el.className); \n });\n elements.push(...radioClasses);\n return elements;\n }\n\n function findFillables() {\n const elements = [];\n\n const inputs = [...getAllElementsIncludingShadow('input:not([type=\"radio\"]):not([type=\"checkbox\"])')];\n console.log('Found inputs:', inputs.length, inputs);\n elements.push(...inputs);\n \n const textareas = [...getAllElementsIncludingShadow('textarea')];\n console.log('Found textareas:', textareas.length);\n elements.push(...textareas);\n \n const editables = [...getAllElementsIncludingShadow('[contenteditable=\"true\"]')];\n console.log('Found editables:', editables.length);\n elements.push(...editables);\n\n return elements;\n }\n\n // Helper function to check if element is a form control\n function isFormControl(elementInfo) {\n return /^(input|select|textarea|button|label)$/i.test(elementInfo.tag);\n }\n\n const isDropdownItem = (elementInfo) => {\n const dropdownPatterns = [\n /dropdown[-_]?item/i, // matches: dropdown-item, dropdownitem, dropdown_item\n /menu[-_]?item/i, // matches: menu-item, menuitem, menu_item\n /dropdown[-_]?link/i, // matches: dropdown-link, dropdownlink, dropdown_link\n /list[-_]?item/i, // matches: list-item, listitem, list_item\n /select[-_]?item/i, // matches: select-item, selectitem, select_item \n ];\n\n const rolePatterns = [\n /menu[-_]?item/i, // matches: menuitem, menu-item\n /option/i, // matches: option\n /list[-_]?item/i, // matches: listitem, list-item\n /tree[-_]?item/i // matches: treeitem, tree-item\n ];\n\n const hasMatchingClass = elementInfo.element.className && \n dropdownPatterns.some(pattern => \n pattern.test(elementInfo.element.className)\n );\n\n const hasMatchingRole = elementInfo.element.getAttribute('role') && \n rolePatterns.some(pattern => \n pattern.test(elementInfo.element.getAttribute('role'))\n );\n\n return hasMatchingClass || hasMatchingRole;\n };\n\n /**\n * Finds the first element matching a CSS selector, traversing Shadow DOM if necessary\n * @param {string} selector - CSS selector to search for\n * @param {Element} [root=document] - Root element to start searching from\n * @returns {Element|null} - The first matching element or null if not found\n */\n function querySelectorShadow(selector, root = document) {\n // First try to find in light DOM\n let element = root.querySelector(selector);\n if (element) return element;\n \n // Get all elements with shadow root\n const shadowElements = Array.from(root.querySelectorAll('*'))\n .filter(el => el.shadowRoot);\n \n // Search through each shadow root until we find a match\n for (const el of shadowElements) {\n element = querySelectorShadow(selector, el.shadowRoot);\n if (element) return element;\n }\n \n return null;\n }\n\n const getElementByXPathOrCssSelector = (element_info) => {\n console.log('getElementByXPathOrCssSelector:', element_info);\n\n findElement(document, element_info.iframe_selector, element_info.css_selector);\n };\n\n const findElement = (root, iframeSelector, cssSelector) => {\n let element;\n \n if (iframeSelector) { \n const frames = getAllDocumentElementsIncludingShadow('iframe', root);\n \n // Iterate over all frames and compare their CSS selectors\n for (const frame of frames) {\n const selector = generateCssPath(frame);\n if (selector === iframeSelector) {\n const frameDocument = frame.contentDocument || frame.contentWindow.document;\n element = querySelectorShadow(cssSelector, frameDocument);\n console.log('found element ', element);\n break;\n } \n } }\n else\n element = querySelectorShadow(cssSelector, root);\n \n if (!element) {\n console.warn('Failed to find element with CSS selector:', cssSelector);\n }\n\n return element;\n };\n\n\n function isDecendent(parent, child) {\n let element = child;\n while (element && element !== parent && element.nodeType === Node.ELEMENT_NODE) { \n element = getParentNode(element); \n }\n return element === parent;\n }\n\n function generateXPath(element) {\n return '/'+extractElementPath(element).map(item => `${item.tagName}${item.onlyChild ? '' : `[${item.index}]`}`).join('/');\n }\n\n function generateCssPath(element) {\n return extractElementPath(element).map(item => `${item.tagName}:nth-of-type(${item.index})`).join(' > ');\n }\n\n function extractElementPath(element) {\n if (!element) {\n console.error('ERROR: No element provided to generatePath');\n return [];\n }\n const path = [];\n // traversing up the DOM tree\n while (element && element.nodeType === Node.ELEMENT_NODE) { \n let tagName = element.nodeName.toLowerCase();\n \n let sibling = element;\n let index = 1;\n \n while (sibling = sibling.previousElementSibling) {\n if (sibling.nodeName.toLowerCase() === tagName) index++;\n }\n sibling = element;\n \n let onlyChild = (index === 1);\n while (onlyChild && (sibling = sibling.nextElementSibling)) {\n if (sibling.nodeName.toLowerCase() === tagName) onlyChild = false;\n }\n \n // add a tuple with tagName, index (nth), and onlyChild \n path.unshift({\n tagName: tagName,\n index: index,\n onlyChild: onlyChild \n }); \n\n element = getParentNode(element);\n }\n \n return path;\n }\n\n function cleanHTML(rawHTML) {\n const parser = new DOMParser();\n const doc = parser.parseFromString(rawHTML, \"text/html\");\n\n function cleanElement(element) {\n const allowedAttributes = new Set([\n \"role\",\n \"type\",\n \"class\",\n \"href\",\n \"alt\",\n \"title\",\n \"readonly\",\n \"checked\",\n \"enabled\",\n \"disabled\",\n ]);\n\n [...element.attributes].forEach(attr => {\n const name = attr.name.toLowerCase();\n const value = attr.value;\n\n const isTestAttribute = /^(testid|test-id|data-test-id)$/.test(name);\n const isDataAttribute = name.startsWith(\"data-\") && value;\n const isBooleanAttribute = [\"readonly\", \"checked\", \"enabled\", \"disabled\"].includes(name);\n\n if (!allowedAttributes.has(name) && !isDataAttribute && !isTestAttribute && !isBooleanAttribute) {\n element.removeAttribute(name);\n }\n });\n\n // Handle SVG content - more aggressive replacement\n if (element.tagName.toLowerCase() === \"svg\") {\n // Remove all attributes except class and role\n [...element.attributes].forEach(attr => {\n const name = attr.name.toLowerCase();\n if (name !== \"class\" && name !== \"role\") {\n element.removeAttribute(name);\n }\n });\n element.innerHTML = \"CONTENT REMOVED\";\n } else {\n // Recursively clean child elements\n Array.from(element.children).forEach(cleanElement);\n }\n\n // Only remove empty elements that aren't semantic or icon elements\n const keepEmptyElements = ['i', 'span', 'svg', 'button', 'input'];\n if (!keepEmptyElements.includes(element.tagName.toLowerCase()) && \n !element.children.length && \n !element.textContent.trim()) {\n element.remove();\n }\n }\n\n // Process all elements in the document body\n Array.from(doc.body.children).forEach(cleanElement);\n return doc.body.innerHTML;\n }\n\n function getContainingIframe(element) {\n // If not in an iframe, return null\n if (element.ownerDocument.defaultView === window.top) {\n return null;\n }\n \n // Try to find the iframe in the parent document that contains our element\n try {\n const parentDocument = element.ownerDocument.defaultView.parent.document;\n const iframes = parentDocument.querySelectorAll('iframe');\n \n for (const iframe of iframes) {\n if (iframe.contentWindow === element.ownerDocument.defaultView) {\n return iframe;\n }\n }\n } catch (e) {\n // Cross-origin restriction\n return \"Cross-origin iframe - cannot access details\";\n }\n \n return null;\n }\n\n function getElementInfo(element, index) {\n // Get text content with spaces between elements\n /* function getTextContent(element) {\n const walker = document.createTreeWalker(\n element,\n NodeFilter.SHOW_TEXT,\n null,\n false\n );\n\n let text = '';\n let node;\n\n while (node = walker.nextNode()) {\n const trimmedText = node.textContent.trim();\n if (trimmedText) {\n // Add space if there's already text\n if (text) {\n text += ' ';\n }\n text += trimmedText;\n }\n }\n\n return text;\n } */\n \n const xpath = generateXPath(element);\n const css_selector = generateCssPath(element);\n //disabled since it's blocking event handling in recorder\n const short_css_selector = ''; //getRobustSelector(element);\n\n const iframe = getContainingIframe(element); \n const iframe_selector = iframe ? generateCssPath(iframe) : \"\";\n //disabled since it's blocking event handling in recorder\n const short_iframe_selector = ''; //iframe ? getRobustSelector(iframe) : \"\";\n\n // Return element info with pre-calculated values\n return new ElementInfo(element, index, {\n tag: element.tagName.toLowerCase(),\n type: element.type || '',\n text: element.innerText || element.placeholder || '', //getTextContent(element),\n html: cleanHTML(element.outerHTML),\n xpath: xpath,\n css_selector: css_selector,\n bounding_box: element.getBoundingClientRect(),\n iframe_selector: iframe_selector,\n short_css_selector: short_css_selector,\n short_iframe_selector: short_iframe_selector\n });\n }\n\n function getAriaLabelledByText(elementInfo, includeHidden=true) {\n if (!elementInfo.element.hasAttribute('aria-labelledby')) return '';\n\n const ids = elementInfo.element.getAttribute('aria-labelledby').split(/\\s+/);\n let labelText = '';\n\n //locate root (document or iFrame document if element is contained in an iframe)\n let root = document;\n if (elementInfo.iframe_selector) { \n const frames = getAllDocumentElementsIncludingShadow('iframe', document);\n \n // Iterate over all frames and compare their CSS selectors\n for (const frame of frames) {\n const selector = generateCssPath(frame);\n if (selector === elementInfo.iframe_selector) {\n root = frame.contentDocument || frame.contentWindow.document; \n break;\n }\n } \n }\n\n ids.forEach(id => {\n const el = querySelectorShadow(`#${CSS.escape(id)}`, root);\n if (el) {\n if (includeHidden || el.offsetParent !== null || getComputedStyle(el).display !== 'none') {\n labelText += el.textContent.trim() + ' ';\n }\n }\n });\n\n return labelText.trim();\n }\n\n\n\n const filterZeroDimensions = (elementInfo) => {\n const rect = elementInfo.bounding_box;\n //single pixel elements are typically faux controls and should be filtered too\n const hasSize = rect.width > 1 && rect.height > 1;\n const style = window.getComputedStyle(elementInfo.element);\n const isVisible = style.display !== 'none' && style.visibility !== 'hidden';\n \n if (!hasSize || !isVisible) {\n \n return false;\n }\n return true;\n };\n\n\n\n function uniquifyElements(elements) {\n const seen = new Set();\n\n console.log(`Starting uniquification with ${elements.length} elements`);\n\n // Filter out testing infrastructure elements first\n const filteredInfrastructure = elements.filter(element_info => {\n // Skip the highlight-overlay element completely - it's part of the testing infrastructure\n if (element_info.element.id === 'highlight-overlay' || \n (element_info.css_selector && element_info.css_selector.includes('#highlight-overlay'))) {\n console.log('Filtered out testing infrastructure element:', element_info.css_selector);\n return false;\n }\n \n // Filter out UI framework container/manager elements\n const el = element_info.element;\n // UI framework container checks - generic detection for any framework\n if ((el.getAttribute('data-rendered-by') || \n el.getAttribute('data-reactroot') || \n el.getAttribute('ng-version') || \n el.getAttribute('data-component-id') ||\n el.getAttribute('data-root') ||\n el.getAttribute('data-framework')) && \n (el.className && \n typeof el.className === 'string' && \n (el.className.includes('Container') || \n el.className.includes('container') || \n el.className.includes('Manager') || \n el.className.includes('manager')))) {\n console.log('Filtered out UI framework container element:', element_info.css_selector);\n return false;\n }\n \n // Direct filter for framework container elements that shouldn't be interactive\n // Consolidating multiple container detection patterns into one efficient check\n const isFullViewport = element_info.bounding_box && \n element_info.bounding_box.x <= 5 && \n element_info.bounding_box.y <= 5 && \n element_info.bounding_box.width >= (window.innerWidth * 0.95) && \n element_info.bounding_box.height >= (window.innerHeight * 0.95);\n \n // Empty content check\n const isEmpty = !el.innerText || el.innerText.trim() === '';\n \n // Check if it's a framework container element\n if (element_info.element.tagName === 'DIV' && \n isFullViewport && \n isEmpty && \n (\n // Pattern matching for root containers\n (element_info.xpath && \n (element_info.xpath.match(/^\\/html\\[\\d+\\]\\/body\\[\\d+\\]\\/div\\[\\d+\\]\\/div\\[\\d+\\]$/) || \n element_info.xpath.match(/^\\/\\/\\*\\[@id='[^']+'\\]\\/div\\[\\d+\\]$/))) ||\n \n // Simple DOM structure\n (element_info.css_selector.split(' > ').length <= 4 && element_info.depth <= 5) ||\n \n // Empty or container-like classes\n (!el.className || el.className === '' || \n (typeof el.className === 'string' && \n (el.className.includes('overlay') || \n el.className.includes('container') || \n el.className.includes('wrapper'))))\n )) {\n console.log('Filtered out framework container element:', element_info.css_selector);\n return false;\n }\n \n return true;\n });\n\n // First filter out elements with zero dimensions\n const nonZeroElements = filteredInfrastructure.filter(filterZeroDimensions);\n // sort by CSS selector depth so parents are processed first\n nonZeroElements.sort((a, b) => a.getDepth() - b.getDepth());\n console.log(`After dimension filtering: ${nonZeroElements.length} elements remain (${elements.length - nonZeroElements.length} removed)`);\n \n const filteredByParent = nonZeroElements.filter(element_info => {\n\n const parent = findClosestParent(seen, element_info);\n const keep = parent == null || shouldKeepNestedElement(element_info, parent);\n // console.log(\"node \", element_info.index, \": keep=\", keep, \" parent=\", parent);\n // if (!keep && !element_info.xpath) {\n // console.log(\"Filtered out element \", element_info,\" because it's a nested element of \", parent);\n // }\n if (keep)\n seen.add(element_info.css_selector);\n\n return keep;\n });\n\n console.log(`After parent/child filtering: ${filteredByParent.length} elements remain (${nonZeroElements.length - filteredByParent.length} removed)`);\n\n // Final overlap filtering\n const filteredResults = filteredByParent.filter(element => {\n\n // Look for any element that came BEFORE this one in the array\n const hasEarlierOverlap = filteredByParent.some(other => {\n // Only check elements that came before (lower index)\n if (filteredByParent.indexOf(other) >= filteredByParent.indexOf(element)) {\n return false;\n }\n \n const isOverlapping = areElementsOverlapping(element, other); \n return isOverlapping;\n }); \n\n // Keep element if it has no earlier overlapping elements\n return !hasEarlierOverlap;\n });\n \n \n \n // Check for overlay removal\n console.log(`After filtering: ${filteredResults.length} (${filteredByParent.length - filteredResults.length} removed by overlap)`);\n \n const nonOverlaidElements = filteredResults.filter(element => {\n return !isOverlaid(element);\n });\n\n console.log(`Final elements after overlay removal: ${nonOverlaidElements.length} (${filteredResults.length - nonOverlaidElements.length} removed)`);\n \n return nonOverlaidElements;\n\n }\n\n\n\n const areElementsOverlapping = (element1, element2) => {\n if (element1.css_selector === element2.css_selector) {\n return true;\n }\n \n const box1 = element1.bounding_box;\n const box2 = element2.bounding_box;\n \n return box1.x === box2.x &&\n box1.y === box2.y &&\n box1.width === box2.width &&\n box1.height === box2.height;\n // element1.text === element2.text &&\n // element2.tag === 'a';\n };\n\n function findClosestParent(seen, element_info) { \n \n // Split the xpath into segments\n const segments = element_info.css_selector.split(' > ');\n \n // Try increasingly shorter paths until we find one in the seen set\n for (let i = segments.length - 1; i > 0; i--) {\n const parentPath = segments.slice(0, i).join(' > ');\n if (seen.has(parentPath)) {\n return parentPath;\n }\n }\n\n return null;\n }\n\n function shouldKeepNestedElement(elementInfo, parentPath) {\n let result = false;\n const parentSegments = parentPath.split(' > ');\n\n const isParentLink = /^a(:nth-of-type\\(\\d+\\))?$/.test(parentSegments[parentSegments.length - 1]);\n if (isParentLink) {\n return false; \n }\n // If this is a checkbox/radio input\n if (elementInfo.tag === 'input' && \n (elementInfo.type === 'checkbox' || elementInfo.type === 'radio')) {\n \n // Check if parent is a label by looking at the parent xpath's last segment\n \n const isParentLabel = /^label(:nth-of-type\\(\\d+\\))?$/.test(parentSegments[parentSegments.length - 1]);\n \n // If parent is a label, don't keep the input (we'll keep the label instead)\n if (isParentLabel) {\n return false;\n }\n }\n \n // Keep all other form controls and dropdown items\n if (isFormControl(elementInfo) || isDropdownItem(elementInfo)) {\n result = true;\n }\n\n if(isTableCell(elementInfo)) {\n result = true;\n }\n \n \n // console.log(`shouldKeepNestedElement: ${elementInfo.tag} ${elementInfo.text} ${elementInfo.xpath} -> ${parentXPath} -> ${result}`);\n return result;\n }\n\n\n function isTableCell(elementInfo) {\n const element = elementInfo.element;\n if(!element || !(element instanceof HTMLElement)) {\n return false;\n }\n const validTags = new Set(['td', 'th']);\n const validRoles = new Set(['cell', 'gridcell', 'columnheader', 'rowheader']);\n \n const tag = element.tagName.toLowerCase();\n const role = element.getAttribute('role')?.toLowerCase();\n\n if (validTags.has(tag) || (role && validRoles.has(role))) {\n return true;\n }\n return false;\n \n }\n\n function isOverlaid(elementInfo) {\n const element = elementInfo.element;\n const boundingRect = elementInfo.bounding_box;\n \n\n \n \n // Create a diagnostic logging function that only logs when needed\n const diagnosticLog = (...args) => {\n { // set to true for debugging\n console.log('[OVERLAY-DEBUG]', ...args);\n }\n };\n\n // Special handling for tooltips\n if (elementInfo.element.className && typeof elementInfo.element.className === 'string' && \n elementInfo.element.className.includes('tooltip')) {\n diagnosticLog('Element is a tooltip, not considering it overlaid');\n return false;\n }\n \n \n \n // Get element at the center point to check if it's covered by a popup/modal\n const middleX = boundingRect.x + boundingRect.width/2;\n const middleY = boundingRect.y + boundingRect.height/2;\n const elementAtMiddle = element.ownerDocument.elementFromPoint(middleX, middleY);\n \n if (elementAtMiddle && \n elementAtMiddle !== element && \n !isDecendent(element, elementAtMiddle) && \n !isDecendent(elementAtMiddle, element)) {\n\n \n return true;\n }\n \n \n return false;\n \n }\n\n\n\n /**\n * Get the “best” short, unique, and robust CSS selector for an element.\n * \n * @param {Element} element\n * @returns {string} A selector guaranteed to find exactly that element in its context\n */\n function getRobustSelector(element) {\n // 1. Figure out the real “root” (iframe doc, shadow root, or main doc)\n const root = (() => {\n const rootNode = element.getRootNode();\n if (rootNode instanceof ShadowRoot) {\n return rootNode;\n }\n return element.ownerDocument;\n })();\n\n // 2. Options to bias toward stable attrs and away from auto-generated classes\n const options = {\n root,\n // only use data-*, id or aria-label by default\n attr(name, value) {\n if (name === 'id' || name.startsWith('data-') || name === 'aria-label') {\n return true;\n }\n return false;\n },\n // skip framework junk\n filter(name, value) {\n if (name.startsWith('ng-') || name.startsWith('_ngcontent') || /^p-/.test(name)) {\n return false;\n }\n return true;\n },\n // let finder try really short seeds\n seedMinLength: 1,\n optimizedMinLength: 1,\n };\n\n let selector;\n try {\n selector = finder(element, options);\n // 3. Verify it really works in the context\n const found = root.querySelectorAll(selector);\n if (found.length !== 1 || found[0] !== element) {\n throw new Error('not unique or not found');\n }\n return selector;\n } catch (err) {\n // 4. Fallback: full path (you already have this utility)\n console.warn('[getRobustSelector] finder failed, falling back to full path:', err);\n return generateCssPath(element); // you’d import or define this elsewhere\n }\n }\n\n /**\n * Checks if an element is scrollable (has scrollable content)\n * \n * @param element - The element to check\n * @returns boolean indicating if the element is scrollable\n */\n function isScrollableContainer(element) {\n if (!element) return false;\n \n const style = window.getComputedStyle(element);\n \n // Reliable way to detect if an element has scrollbars or is scrollable\n const hasScrollHeight = element.scrollHeight > element.clientHeight;\n const hasScrollWidth = element.scrollWidth > element.clientWidth;\n \n // Check actual style properties\n const hasOverflowY = style.overflowY === 'auto' || \n style.overflowY === 'scroll' || \n style.overflowY === 'overlay';\n const hasOverflowX = style.overflowX === 'auto' || \n style.overflowX === 'scroll' || \n style.overflowX === 'overlay';\n \n // Check common class names and attributes for scrollable containers across frameworks\n const hasScrollClasses = element.classList.contains('scroll') || \n element.classList.contains('scrollable') ||\n element.classList.contains('overflow') ||\n element.classList.contains('overflow-auto') ||\n element.classList.contains('overflow-scroll') ||\n element.getAttribute('data-scrollable') === 'true';\n \n // Check for height/max-height constraints that often indicate scrolling content\n const hasHeightConstraint = style.maxHeight && \n style.maxHeight !== 'none' && \n style.maxHeight !== 'auto';\n \n // An element is scrollable if it has:\n // 1. Actual scrollbars in use (most reliable check) OR\n // 2. Overflow styles allowing scrolling AND content that would require scrolling\n return (hasScrollHeight && hasOverflowY) || \n (hasScrollWidth && hasOverflowX) ||\n (hasScrollClasses && (hasScrollHeight || hasScrollWidth)) ||\n (hasHeightConstraint && hasScrollHeight);\n }\n\n /**\n * Detects scrollable containers that are ancestors of the target element\n * \n * This function traverses up the DOM tree from the target element and identifies\n * all scrollable containers (elements that have scrollable content).\n * \n * @param target - The target element to start the search from\n * @returns Array of objects with selector and scroll properties\n */\n function detectScrollableContainers(target) {\n const scrollableContainers = [];\n \n if (!target) {\n return scrollableContainers;\n }\n \n console.log('🔍 [detectScrollableContainers] Starting detection for target:', target.tagName, target.id, target.className);\n \n // Detect if target is inside an iframe\n const iframe = getContainingIframe(target);\n const iframe_selector = iframe ? generateCssPath(iframe) : \"\";\n \n console.log('🔍 [detectScrollableContainers] Iframe context:', iframe ? 'inside iframe' : 'main document', 'selector:', iframe_selector);\n \n // Start from the target element and traverse up the DOM tree\n let currentElement = target;\n let depth = 0;\n const MAX_DEPTH = 10; // Limit traversal depth to avoid infinite loops\n \n while (currentElement && currentElement.nodeType === Node.ELEMENT_NODE && depth < MAX_DEPTH) { \n // Check if the current element is scrollable\n if (isScrollableContainer(currentElement)) {\n console.log('🔍 [detectScrollableContainers] Found scrollable container at depth', depth, ':', currentElement.tagName, currentElement.id, currentElement.className);\n \n const container = {\n containerEl: currentElement,\n selector: generateCssPath(currentElement),\n iframe_selector: iframe_selector,\n scrollTop: currentElement.scrollTop,\n scrollLeft: currentElement.scrollLeft,\n scrollHeight: currentElement.scrollHeight,\n scrollWidth: currentElement.scrollWidth,\n clientHeight: currentElement.clientHeight,\n clientWidth: currentElement.clientWidth\n };\n \n scrollableContainers.push(container);\n }\n \n // Move to parent element\n currentElement = getParentNode(currentElement);\n \n depth++;\n }\n \n console.log('🔍 [detectScrollableContainers] Detection complete. Found', scrollableContainers.length, 'scrollable containers');\n return scrollableContainers;\n }\n\n class DOMSerializer {\n constructor(options = {}) {\n this.options = {\n includeStyles: true,\n includeScripts: false, // Security consideration\n includeFrames: true,\n includeShadowDOM: true,\n maxDepth: 50,\n ...options\n };\n this.serializedFrames = new Map();\n this.shadowRoots = new Map();\n }\n \n /**\n * Serialize a complete document or element\n */\n serialize(rootElement = document) {\n try {\n const serialized = {\n type: 'document',\n doctype: this.serializeDoctype(rootElement),\n documentElement: this.serializeElement(rootElement.documentElement || rootElement),\n frames: [],\n timestamp: Date.now(),\n url: rootElement.URL || window.location?.href,\n metadata: {\n title: rootElement.title,\n charset: rootElement.characterSet,\n contentType: rootElement.contentType\n }\n };\n \n // Serialize frames and iframes if enabled\n if (this.options.includeFrames) {\n serialized.frames = this.serializeFrames(rootElement);\n }\n \n return serialized;\n } catch (error) {\n console.error('Serialization error:', error);\n throw new Error(`DOM serialization failed: ${error.message}`);\n }\n }\n \n /**\n * Serialize document type declaration\n */\n serializeDoctype(doc) {\n if (!doc.doctype) return null;\n \n return {\n name: doc.doctype.name,\n publicId: doc.doctype.publicId,\n systemId: doc.doctype.systemId\n };\n }\n \n /**\n * Serialize an individual element and its children\n */\n serializeElement(element, depth = 0) {\n if (depth > this.options.maxDepth) {\n return { type: 'text', content: '<!-- Max depth exceeded -->' };\n }\n \n const nodeType = element.nodeType;\n \n switch (nodeType) {\n case Node.ELEMENT_NODE:\n return this.serializeElementNode(element, depth);\n case Node.TEXT_NODE:\n return this.serializeTextNode(element);\n case Node.COMMENT_NODE:\n return this.serializeCommentNode(element);\n case Node.DOCUMENT_FRAGMENT_NODE:\n return this.serializeDocumentFragment(element, depth);\n default:\n return null;\n }\n }\n \n /**\n * Serialize element node with attributes and children\n */\n serializeElementNode(element, depth) {\n const tagName = element.tagName.toLowerCase();\n \n // Skip script tags for security unless explicitly enabled\n if (tagName === 'script' && !this.options.includeScripts) {\n return { type: 'comment', content: '<!-- Script tag removed for security -->' };\n }\n \n const serialized = {\n type: 'element',\n tagName: tagName,\n attributes: this.serializeAttributes(element),\n children: [],\n shadowRoot: null\n };\n \n // Handle Shadow DOM\n if (this.options.includeShadowDOM && element.shadowRoot) {\n serialized.shadowRoot = this.serializeShadowRoot(element.shadowRoot, depth + 1);\n }\n \n // Handle special elements\n if (tagName === 'iframe' || tagName === 'frame') {\n serialized.frameData = this.serializeFrameElement(element);\n }\n \n // Serialize children\n for (const child of element.childNodes) {\n const serializedChild = this.serializeElement(child, depth + 1);\n if (serializedChild) {\n serialized.children.push(serializedChild);\n }\n }\n \n // Include computed styles if enabled\n if (this.options.includeStyles && element.nodeType === Node.ELEMENT_NODE) {\n serialized.computedStyle = this.serializeComputedStyle(element);\n }\n \n return serialized;\n }\n \n /**\n * Serialize element attributes\n */\n serializeAttributes(element) {\n const attributes = {};\n \n if (element.attributes) {\n for (const attr of element.attributes) {\n attributes[attr.name] = attr.value;\n }\n }\n \n return attributes;\n }\n \n /**\n * Serialize computed styles\n */\n serializeComputedStyle(element) {\n try {\n const computedStyle = window.getComputedStyle(element);\n const styles = {};\n \n // Only serialize non-default values to reduce size\n const importantStyles = [\n 'display', 'position', 'width', 'height', 'margin', 'padding',\n 'border', 'background', 'color', 'font-family', 'font-size',\n 'text-align', 'visibility', 'z-index', 'transform'\n ];\n \n for (const prop of importantStyles) {\n const value = computedStyle.getPropertyValue(prop);\n if (value && value !== 'initial' && value !== 'normal') {\n styles[prop] = value;\n }\n }\n \n return styles;\n } catch (error) {\n return {};\n }\n }\n \n /**\n * Serialize text node\n */\n serializeTextNode(node) {\n return {\n type: 'text',\n content: node.textContent\n };\n }\n \n /**\n * Serialize comment node\n */\n serializeCommentNode(node) {\n return {\n type: 'comment',\n content: node.textContent\n };\n }\n \n /**\n * Serialize document fragment\n */\n serializeDocumentFragment(fragment, depth) {\n const serialized = {\n type: 'fragment',\n children: []\n };\n \n for (const child of fragment.childNodes) {\n const serializedChild = this.serializeElement(child, depth + 1);\n if (serializedChild) {\n serialized.children.push(serializedChild);\n }\n }\n \n return serialized;\n }\n \n /**\n * Serialize Shadow DOM\n */\n serializeShadowRoot(shadowRoot, depth) {\n const serialized = {\n type: 'shadowRoot',\n mode: shadowRoot.mode,\n children: []\n };\n \n for (const child of shadowRoot.childNodes) {\n const serializedChild = this.serializeElement(child, depth + 1);\n if (serializedChild) {\n serialized.children.push(serializedChild);\n }\n }\n \n return serialized;\n }\n \n /**\n * Serialize frame/iframe elements\n */\n serializeFrameElement(frameElement) {\n const frameData = {\n src: frameElement.src,\n name: frameElement.name,\n id: frameElement.id,\n sandbox: frameElement.sandbox?.toString() || '',\n allowfullscreen: frameElement.allowFullscreen\n };\n \n // Try to access frame content (may fail due to CORS)\n try {\n const frameDoc = frameElement.contentDocument;\n if (frameDoc && this.options.includeFrames) {\n frameData.content = this.serialize(frameDoc);\n }\n } catch (error) {\n frameData.accessError = 'Cross-origin frame content not accessible';\n }\n \n return frameData;\n }\n \n /**\n * Serialize all frames in document\n */\n serializeFrames(doc) {\n const frames = [];\n const frameElements = doc.querySelectorAll('iframe, frame');\n \n for (const frameElement of frameElements) {\n try {\n const frameDoc = frameElement.contentDocument;\n if (frameDoc) {\n frames.push({\n element: this.serializeElement(frameElement),\n content: this.serialize(frameDoc)\n });\n }\n } catch (error) {\n frames.push({\n element: this.serializeElement(frameElement),\n error: 'Frame content not accessible'\n });\n }\n }\n \n return frames;\n }\n \n /**\n * Deserialize serialized DOM data back to DOM nodes\n */\n deserialize(serializedData, targetDocument = document) {\n try {\n if (serializedData.type === 'document') {\n return this.deserializeDocument(serializedData, targetDocument);\n } else {\n return this.deserializeElement(serializedData, targetDocument);\n }\n } catch (error) {\n console.error('Deserialization error:', error);\n throw new Error(`DOM deserialization failed: ${error.message}`);\n }\n }\n \n /**\n * Deserialize complete document\n */\n deserializeDocument(serializedDoc, targetDoc) {\n // Create new document if needed\n const doc = targetDoc || document.implementation.createHTMLDocument();\n \n // Set doctype if present\n if (serializedDoc.doctype) {\n const doctype = document.implementation.createDocumentType(\n serializedDoc.doctype.name,\n serializedDoc.doctype.publicId,\n serializedDoc.doctype.systemId\n );\n doc.replaceChild(doctype, doc.doctype);\n }\n \n // Deserialize document element\n if (serializedDoc.documentElement) {\n const newDocElement = this.deserializeElement(serializedDoc.documentElement, doc);\n doc.replaceChild(newDocElement, doc.documentElement);\n }\n \n // Handle metadata\n if (serializedDoc.metadata) {\n doc.title = serializedDoc.metadata.title || '';\n }\n \n return doc;\n }\n \n /**\n * Deserialize individual element\n */\n deserializeElement(serializedNode, doc) {\n switch (serializedNode.type) {\n case 'element':\n return this.deserializeElementNode(serializedNode, doc);\n case 'text':\n return doc.createTextNode(serializedNode.content);\n case 'comment':\n return doc.createComment(serializedNode.content);\n case 'fragment':\n return this.deserializeDocumentFragment(serializedNode, doc);\n case 'shadowRoot':\n // Shadow roots are handled during element creation\n return null;\n default:\n return null;\n }\n }\n \n /**\n * Deserialize element node\n */\n deserializeElementNode(serializedElement, doc) {\n const element = doc.createElement(serializedElement.tagName);\n \n // Set attributes\n if (serializedElement.attributes) {\n for (const [name, value] of Object.entries(serializedElement.attributes)) {\n try {\n element.setAttribute(name, value);\n } catch (error) {\n console.warn(`Failed to set attribute ${name}:`, error);\n }\n }\n }\n \n // Apply computed styles if available\n if (serializedElement.computedStyle && this.options.includeStyles) {\n for (const [prop, value] of Object.entries(serializedElement.computedStyle)) {\n try {\n element.style.setProperty(prop, value);\n } catch (error) {\n console.warn(`Failed to set style ${prop}:`, error);\n }\n }\n }\n \n // Create shadow root if present\n if (serializedElement.shadowRoot && element.attachShadow) {\n try {\n const shadowRoot = element.attachShadow({ \n mode: serializedElement.shadowRoot.mode || 'open' \n });\n \n // Deserialize shadow root children\n for (const child of serializedElement.shadowRoot.children) {\n const childElement = this.deserializeElement(child, doc);\n if (childElement) {\n shadowRoot.appendChild(childElement);\n }\n }\n } catch (error) {\n console.warn('Failed to create shadow root:', error);\n }\n }\n \n // Deserialize children\n if (serializedElement.children) {\n for (const child of serializedElement.children) {\n const childElement = this.deserializeElement(child, doc);\n if (childElement) {\n element.appendChild(childElement);\n }\n }\n }\n \n // Handle frame content\n if (serializedElement.frameData && serializedElement.frameData.content) {\n // Frame content deserialization would happen after the frame loads\n element.addEventListener('load', () => {\n try {\n const frameDoc = element.contentDocument;\n if (frameDoc) {\n this.deserializeDocument(serializedElement.frameData.content, frameDoc);\n }\n } catch (error) {\n console.warn('Failed to deserialize frame content:', error);\n }\n });\n }\n \n return element;\n }\n \n /**\n * Deserialize document fragment\n */\n deserializeDocumentFragment(serializedFragment, doc) {\n const fragment = doc.createDocumentFragment();\n \n if (serializedFragment.children) {\n for (const child of serializedFragment.children) {\n const childElement = this.deserializeElement(child, doc);\n if (childElement) {\n fragment.appendChild(childElement);\n }\n }\n }\n \n return fragment;\n }\n }\n \n // Usage example and utility functions\n class DOMUtils {\n /**\n * Create serializer with common presets\n */\n static createSerializer(preset = 'default') {\n const presets = {\n default: {\n includeStyles: true,\n includeScripts: false,\n includeFrames: true,\n includeShadowDOM: true\n },\n minimal: {\n includeStyles: false,\n includeScripts: false,\n includeFrames: false,\n includeShadowDOM: false\n },\n complete: {\n includeStyles: true,\n includeScripts: true,\n includeFrames: true,\n includeShadowDOM: true\n },\n secure: {\n includeStyles: true,\n includeScripts: false,\n includeFrames: false,\n includeShadowDOM: true\n }\n };\n \n return new DOMSerializer(presets[preset] || presets.default);\n }\n \n /**\n * Serialize DOM to JSON string\n */\n static serializeToJSON(element, options) {\n const serializer = new DOMSerializer(options);\n const serialized = serializer.serialize(element);\n return JSON.stringify(serialized, null, 2);\n }\n \n /**\n * Deserialize from JSON string\n */\n static deserializeFromJSON(jsonString, targetDocument) {\n const serialized = JSON.parse(jsonString);\n const serializer = new DOMSerializer();\n return serializer.deserialize(serialized, targetDocument);\n }\n \n /**\n * Clone DOM with full fidelity including Shadow DOM\n */\n static deepClone(element, options) {\n const serializer = new DOMSerializer(options);\n const serialized = serializer.serialize(element);\n return serializer.deserialize(serialized, element.ownerDocument);\n }\n \n /**\n * Compare two DOM structures\n */\n static compare(element1, element2, options) {\n const serializer = new DOMSerializer(options);\n const serialized1 = serializer.serialize(element1);\n const serialized2 = serializer.serialize(element2);\n \n return JSON.stringify(serialized1) === JSON.stringify(serialized2);\n }\n }\n \n /*\n // Export for use\n if (typeof module !== 'undefined' && module.exports) {\n module.exports = { DOMSerializer, DOMUtils };\n } else if (typeof window !== 'undefined') {\n window.DOMSerializer = DOMSerializer;\n window.DOMUtils = DOMUtils;\n }\n */\n\n /* Usage Examples:\n \n // Basic serialization\n const serializer = new DOMSerializer();\n const serialized = serializer.serialize(document);\n console.log(JSON.stringify(serialized, null, 2));\n \n // Deserialize back to DOM\n const clonedDoc = serializer.deserialize(serialized);\n \n // Using presets\n const minimalSerializer = DOMUtils.createSerializer('minimal');\n const secureSerializer = DOMUtils.createSerializer('secure');\n \n // Serialize specific element with Shadow DOM\n const customElement = document.querySelector('my-custom-element');\n const serializedElement = serializer.serialize(customElement);\n \n // JSON utilities\n const jsonString = DOMUtils.serializeToJSON(document.body);\n const restored = DOMUtils.deserializeFromJSON(jsonString);\n \n // Deep clone with Shadow DOM support\n const clone = DOMUtils.deepClone(document.body, { includeShadowDOM: true });\n \n */\n\n function serializeNodeToJSON(nodeElement) {\n return DOMUtils.serializeToJSON(nodeElement, {includeStyles: false});\n }\n\n function deserializeNodeFromJSON(jsonString) {\n return DOMUtils.deserializeFromJSON(jsonString);\n }\n\n /**\n * Checks if a point is inside a bounding box\n * \n * @param point The point to check\n * @param box The bounding box\n * @returns boolean indicating if the point is inside the box\n */\n function isPointInsideBox(point, box) {\n return point.x >= box.x &&\n point.x <= box.x + box.width &&\n point.y >= box.y &&\n point.y <= box.y + box.height;\n }\n\n /**\n * Calculates the overlap area between two bounding boxes\n * \n * @param box1 First bounding box\n * @param box2 Second bounding box\n * @returns The overlap area\n */\n function calculateOverlap(box1, box2) {\n const xOverlap = Math.max(0,\n Math.min(box1.x + box1.width, box2.x + box2.width) -\n Math.max(box1.x, box2.x)\n );\n const yOverlap = Math.max(0,\n Math.min(box1.y + box1.height, box2.y + box2.height) -\n Math.max(box1.y, box2.y)\n );\n return xOverlap * yOverlap;\n }\n\n /**\n * Finds an exact match between candidate elements and the actual interaction element\n * \n * @param candidate_elements Array of candidate element infos\n * @param actualInteractionElementInfo The actual interaction element info\n * @returns The matching candidate element info, or null if no match is found\n */\n function findExactMatch(candidate_elements, actualInteractionElementInfo) {\n if (!actualInteractionElementInfo.element) {\n return null;\n }\n\n const exactMatch = candidate_elements.find(elementInfo => \n elementInfo.element && elementInfo.element === actualInteractionElementInfo.element\n );\n \n if (exactMatch) {\n console.log('✅ Found exact element match:', {\n matchedElement: exactMatch.element?.tagName,\n matchedElementClass: exactMatch.element?.className,\n index: exactMatch.index\n });\n return exactMatch;\n }\n \n return null;\n }\n\n /**\n * Finds a match by traversing up the parent elements\n * \n * @param candidate_elements Array of candidate element infos\n * @param actualInteractionElementInfo The actual interaction element info\n * @returns The matching candidate element info, or null if no match is found\n */\n function findParentMatch(candidate_elements, actualInteractionElementInfo) {\n if (!actualInteractionElementInfo.element) {\n return null;\n }\n\n let element = actualInteractionElementInfo.element;\n while (element.parentElement) {\n element = element.parentElement;\n const parentMatch = candidate_elements.find(candidate => \n candidate.element && candidate.element === element\n );\n \n if (parentMatch) {\n console.log('✅ Found parent element match:', {\n matchedElement: parentMatch.element?.tagName,\n matchedElementClass: parentMatch.element?.className,\n index: parentMatch.index,\n depth: element.tagName\n });\n return parentMatch;\n }\n \n // Stop if we hit another candidate element\n if (candidate_elements.some(candidate => \n candidate.element && candidate.element === element\n )) {\n console.log('⚠️ Stopped parent search - hit another candidate element:', element.tagName);\n break;\n }\n }\n \n return null;\n }\n\n /**\n * Finds a match based on spatial relationships between elements\n * \n * @param candidate_elements Array of candidate element infos\n * @param actualInteractionElementInfo The actual interaction element info\n * @returns The matching candidate element info, or null if no match is found\n */\n function findSpatialMatch(candidate_elements, actualInteractionElementInfo) {\n if (!actualInteractionElementInfo.element || !actualInteractionElementInfo.bounding_box) {\n return null;\n }\n\n const actualBox = actualInteractionElementInfo.bounding_box;\n let bestMatch = null;\n let bestScore = 0;\n\n for (const candidateInfo of candidate_elements) {\n if (!candidateInfo.bounding_box) continue;\n \n const candidateBox = candidateInfo.bounding_box;\n let score = 0;\n\n // Check if actual element is contained within candidate\n if (isPointInsideBox({ x: actualBox.x, y: actualBox.y }, candidateBox) &&\n isPointInsideBox({ x: actualBox.x + actualBox.width, y: actualBox.y + actualBox.height }, candidateBox)) {\n score += 100; // High score for containment\n }\n\n // Calculate overlap area as a factor\n const overlap = calculateOverlap(actualBox, candidateBox);\n score += overlap;\n\n // Consider proximity if no containment\n if (score === 0) {\n const distance = Math.sqrt(\n Math.pow((actualBox.x + actualBox.width/2) - (candidateBox.x + candidateBox.width/2), 2) +\n Math.pow((actualBox.y + actualBox.height/2) - (candidateBox.y + candidateBox.height/2), 2)\n );\n // Convert distance to a score (closer = higher score)\n score = 1000 / (distance + 1);\n }\n\n if (score > bestScore) {\n bestScore = score;\n bestMatch = candidateInfo;\n console.log('📏 New best spatial match:', {\n element: candidateInfo.element?.tagName,\n class: candidateInfo.element?.className,\n index: candidateInfo.index,\n score: score\n });\n }\n }\n\n if (bestMatch) {\n console.log('✅ Final spatial match selected:', {\n element: bestMatch.element?.tagName,\n class: bestMatch.element?.className,\n index: bestMatch.index,\n finalScore: bestScore\n });\n return bestMatch;\n }\n\n return null;\n }\n\n /**\n * Finds a matching candidate element for an actual interaction element\n * \n * @param candidate_elements Array of candidate element infos\n * @param actualInteractionElementInfo The actual interaction element info\n * @returns The matching candidate element info, or null if no match is found\n */\n function findMatchingCandidateElementInfo(candidate_elements, actualInteractionElementInfo) {\n if (!actualInteractionElementInfo.element || !actualInteractionElementInfo.bounding_box) {\n console.error('❌ Missing required properties in actualInteractionElementInfo');\n return null;\n }\n\n console.log('🔍 Starting element matching for:', {\n clickedElement: actualInteractionElementInfo.element.tagName,\n clickedElementClass: actualInteractionElementInfo.element.className,\n totalCandidates: candidate_elements.length\n });\n\n // First try exact element match\n const exactMatch = findExactMatch(candidate_elements, actualInteractionElementInfo);\n if (exactMatch) {\n return exactMatch;\n }\n console.log('❌ No exact element match found, trying parent matching...');\n\n // Try finding closest clickable parent\n const parentMatch = findParentMatch(candidate_elements, actualInteractionElementInfo);\n if (parentMatch) {\n return parentMatch;\n }\n console.log('❌ No parent match found, falling back to spatial matching...');\n\n // If no exact or parent match, look for spatial relationships\n const spatialMatch = findSpatialMatch(candidate_elements, actualInteractionElementInfo);\n if (spatialMatch) {\n return spatialMatch;\n }\n\n console.error('❌ No matching element found for actual interaction element:', actualInteractionElementInfo);\n return null;\n }\n\n const highlight = {\n execute: async function(elementTypes, handleScroll=false) {\n const elements = await findElements(elementTypes);\n highlightElements(elements, handleScroll);\n return elements;\n },\n\n unexecute: function(handleScroll=false) {\n unhighlightElements(handleScroll);\n },\n\n generateJSON: async function() {\n const json = {};\n\n // Capture viewport dimensions\n const viewportData = {\n width: window.innerWidth,\n height: window.innerHeight,\n documentWidth: document.documentElement.clientWidth,\n documentHeight: document.documentElement.clientHeight,\n timestamp: new Date().toISOString()\n };\n\n // Add viewport data to the JSON output\n json.viewport = viewportData;\n\n\n await Promise.all(Object.values(ElementTag).map(async elementType => {\n const elements = await findElements(elementType);\n json[elementType] = elements;\n }));\n\n // Serialize the JSON object\n const jsonString = JSON.stringify(json, null, 4); // Pretty print with 4 spaces\n\n console.log(`JSON: ${jsonString}`);\n return jsonString;\n },\n\n getElementInfo\n };\n\n\n function unhighlightElements(handleScroll=false) {\n const documents = getAllFrames();\n documents.forEach(doc => {\n const overlay = doc.getElementById('highlight-overlay');\n if (overlay) {\n if (handleScroll) {\n // Remove event listeners\n doc.removeEventListener('scroll', overlay.scrollHandler, true);\n doc.removeEventListener('resize', overlay.resizeHandler);\n }\n overlay.remove();\n }\n });\n }\n\n\n\n\n async function findElements(elementTypes, verbose=true) {\n const typesArray = Array.isArray(elementTypes) ? elementTypes : [elementTypes];\n console.log('Starting element search for types:', typesArray);\n\n const elements = [];\n typesArray.forEach(elementType => {\n if (elementType === ElementTag.FILLABLE) {\n elements.push(...findFillables());\n }\n if (elementType === ElementTag.SELECTABLE) {\n elements.push(...findDropdowns());\n }\n if (elementType === ElementTag.CLICKABLE) {\n elements.push(...findClickables());\n elements.push(...findToggles());\n elements.push(...findCheckables());\n }\n if (elementType === ElementTag.NON_INTERACTIVE_ELEMENT) {\n elements.push(...findNonInteractiveElements());\n }\n });\n\n // console.log('Before uniquify:', elements.length);\n const elementsWithInfo = elements.map((element, index) => \n getElementInfo(element, index)\n );\n\n \n \n const uniqueElements = uniquifyElements(elementsWithInfo);\n console.log(`Found ${uniqueElements.length} elements:`);\n \n // More comprehensive visibility check\n const visibleElements = uniqueElements.filter(elementInfo => {\n const el = elementInfo.element;\n const style = getComputedStyle(el);\n \n // Check various style properties that affect visibility\n if (style.display === 'none' || \n style.visibility === 'hidden') {\n return false;\n }\n \n // Check if element has non-zero dimensions\n const rect = el.getBoundingClientRect();\n if (rect.width === 0 || rect.height === 0) {\n return false;\n }\n \n // Check if element is within viewport\n if (rect.bottom < 0 || \n rect.top > window.innerHeight || \n rect.right < 0 || \n rect.left > window.innerWidth) {\n // Element is outside viewport, but still might be valid \n // if user scrolls to it, so we'll include it\n return true;\n }\n \n return true;\n });\n \n console.log(`Out of which ${visibleElements.length} elements are visible:`);\n if (verbose) {\n visibleElements.forEach(info => {\n console.log(`Element ${info.index}:`, info);\n });\n }\n \n return visibleElements;\n }\n\n // elements is an array of objects with index, xpath\n function highlightElements(elements, handleScroll=false) {\n // console.log('[highlightElements] called with', elements.length, 'elements');\n // Create overlay if it doesn't exist and store it in a dictionary\n const documents = getAllFrames(); \n let overlays = {};\n documents.forEach(doc => {\n let overlay = doc.getElementById('highlight-overlay');\n if (!overlay) {\n overlay = doc.createElement('div');\n overlay.id = 'highlight-overlay';\n overlay.style.cssText = `\n position: fixed;\n top: 0;\n left: 0;\n width: 100%;\n height: 100%;\n pointer-events: none;\n z-index: 2147483647;\n `;\n doc.body.appendChild(overlay);\n // console.log('[highlightElements] Created overlay in document:', doc);\n }\n overlays[doc.documentURI] = overlay;\n });\n \n\n const updateHighlights = (doc = null) => {\n if (doc) {\n overlays[doc.documentURI].innerHTML = '';\n } else {\n Object.values(overlays).forEach(overlay => { overlay.innerHTML = ''; });\n } \n elements.forEach((elementInfo, idx) => {\n //console.log(`[highlightElements] Processing element ${idx}:`, elementInfo.tag, elementInfo.css_selector, elementInfo.bounding_box);\n let element = elementInfo.element; //getElementByXPathOrCssSelector(elementInfo);\n if (!element) {\n element = getElementByXPathOrCssSelector(elementInfo);\n if (!element) {\n console.warn('[highlightElements] Could not find element for:', elementInfo);\n return;\n }\n }\n //if highlights requested for a specific doc, skip unrelated elements\n if (doc && element.ownerDocument !== doc) {\n console.log(\"[highlightElements] Skipped element since it doesn't belong to document\", doc);\n return;\n }\n const rect = element.getBoundingClientRect();\n if (rect.width === 0 || rect.height === 0) {\n console.warn('[highlightElements] Element has zero dimensions:', elementInfo);\n return;\n }\n // Create border highlight (red rectangle)\n // use ownerDocument to support iframes/frames\n const highlight = element.ownerDocument.createElement('div');\n highlight.style.cssText = `\n position: fixed;\n left: ${rect.x}px;\n top: ${rect.y}px;\n width: ${rect.width}px;\n height: ${rect.height}px;\n border: 1px solid rgb(255, 0, 0);\n transition: all 0.2s ease-in-out;\n `;\n // Create index label container - now positioned to the right and slightly up\n const labelContainer = element.ownerDocument.createElement('div');\n labelContainer.style.cssText = `\n position: absolute;\n right: -10px; /* Offset to the right */\n top: -10px; /* Offset upwards */\n padding: 4px;\n background-color: rgba(255, 255, 0, 0.6);\n display: flex;\n align-items: center;\n justify-content: center;\n `;\n const text = element.ownerDocument.createElement('span');\n text.style.cssText = `\n color: rgb(0, 0, 0, 0.8);\n font-family: 'Courier New', Courier, monospace;\n font-size: 12px;\n font-weight: bold;\n line-height: 1;\n `;\n text.textContent = elementInfo.index;\n labelContainer.appendChild(text);\n highlight.appendChild(labelContainer); \n overlays[element.ownerDocument.documentURI].appendChild(highlight);\n \n });\n };\n\n // Initial highlight\n updateHighlights();\n\n if (handleScroll) {\n documents.forEach(doc => {\n // Update highlights on scroll and resize\n console.log('registering scroll and resize handlers for document: ', doc);\n const scrollHandler = () => {\n requestAnimationFrame(() => updateHighlights(doc));\n };\n const resizeHandler = () => {\n updateHighlights(doc);\n };\n doc.addEventListener('scroll', scrollHandler, true);\n doc.addEventListener('resize', resizeHandler);\n // Store event handlers for cleanup\n overlays[doc.documentURI].scrollHandler = scrollHandler;\n overlays[doc.documentURI].resizeHandler = resizeHandler;\n }); \n }\n }\n\n // function unexecute() {\n // unhighlightElements();\n // }\n\n // Make it available globally for both Extension and Playwright\n if (typeof window !== 'undefined') {\n function stripElementRefs(elementInfo) {\n if (!elementInfo) return null;\n const { element, ...rest } = elementInfo;\n return rest;\n }\n\n window.ProboLabs = window.ProboLabs || {};\n\n // --- Caching State ---\n window.ProboLabs.candidates = [];\n window.ProboLabs.actual = null;\n window.ProboLabs.matchingCandidate = null;\n\n // --- Methods ---\n /**\n * Find and cache candidate elements of a given type (e.g., 'CLICKABLE').\n * NOTE: This function is async and must be awaited from Playwright/Node.\n */\n window.ProboLabs.findAndCacheCandidateElements = async function(elementType) {\n //console.log('[ProboLabs] findAndCacheCandidateElements called with:', elementType);\n const found = await findElements(elementType);\n window.ProboLabs.candidates = found;\n // console.log('[ProboLabs] candidates set to:', found, 'type:', typeof found, 'isArray:', Array.isArray(found));\n return found.length;\n };\n\n window.ProboLabs.findAndCacheActualElement = function(cssSelector, iframeSelector, isHover=false) {\n // console.log('[ProboLabs] findAndCacheActualElement called with:', cssSelector, iframeSelector);\n let el = findElement(document, iframeSelector, cssSelector);\n if(isHover) {\n const visibleElement = findClosestVisibleElement(el);\n if (visibleElement) {\n el = visibleElement;\n }\n }\n if (!el) {\n window.ProboLabs.actual = null;\n // console.log('[ProboLabs] actual set to null');\n return false;\n }\n window.ProboLabs.actual = getElementInfo(el, -1);\n // console.log('[ProboLabs] actual set to:', window.ProboLabs.actual);\n return true;\n };\n\n window.ProboLabs.findAndCacheMatchingCandidate = function() {\n // console.log('[ProboLabs] findAndCacheMatchingCandidate called');\n if (!window.ProboLabs.candidates.length || !window.ProboLabs.actual) {\n window.ProboLabs.matchingCandidate = null;\n // console.log('[ProboLabs] matchingCandidate set to null');\n return false;\n }\n window.ProboLabs.matchingCandidate = findMatchingCandidateElementInfo(window.ProboLabs.candidates, window.ProboLabs.actual);\n // console.log('[ProboLabs] matchingCandidate set to:', window.ProboLabs.matchingCandidate);\n return !!window.ProboLabs.matchingCandidate;\n };\n\n window.ProboLabs.highlightCachedElements = function(which) {\n let elements = [];\n if (which === 'candidates') elements = window.ProboLabs.candidates;\n if (which === 'actual' && window.ProboLabs.actual) elements = [window.ProboLabs.actual];\n if (which === 'matching' && window.ProboLabs.matchingCandidate) elements = [window.ProboLabs.matchingCandidate];\n console.log(`[ProboLabs] highlightCachedElements ${which} with ${elements.length} elements`);\n highlightElements(elements);\n };\n\n window.ProboLabs.unhighlight = function() {\n // console.log('[ProboLabs] unhighlight called');\n unhighlightElements();\n };\n\n window.ProboLabs.reset = function() {\n console.log('[ProboLabs] reset called');\n window.ProboLabs.candidates = [];\n window.ProboLabs.actual = null;\n window.ProboLabs.matchingCandidate = null;\n unhighlightElements();\n };\n\n window.ProboLabs.getCandidates = function() {\n // console.log('[ProboLabs] getCandidates called. candidates:', window.ProboLabs.candidates, 'type:', typeof window.ProboLabs.candidates, 'isArray:', Array.isArray(window.ProboLabs.candidates));\n const arr = Array.isArray(window.ProboLabs.candidates) ? window.ProboLabs.candidates : [];\n return arr.map(stripElementRefs);\n };\n window.ProboLabs.getActual = function() {\n return stripElementRefs(window.ProboLabs.actual);\n };\n window.ProboLabs.getMatchingCandidate = function() {\n return stripElementRefs(window.ProboLabs.matchingCandidate);\n };\n\n // Retain existing API for backward compatibility\n window.ProboLabs.ElementTag = ElementTag;\n window.ProboLabs.highlightElements = highlightElements;\n window.ProboLabs.unhighlightElements = unhighlightElements;\n window.ProboLabs.findElements = findElements;\n window.ProboLabs.getElementInfo = getElementInfo;\n window.ProboLabs.highlight = window.ProboLabs.highlight;\n window.ProboLabs.unhighlight = window.ProboLabs.unhighlight;\n\n // --- Utility Functions ---\n function findClosestVisibleElement(element) {\n let current = element;\n while (current) {\n const style = window.getComputedStyle(current);\n if (\n style &&\n style.display !== 'none' &&\n style.visibility !== 'hidden' &&\n current.offsetWidth > 0 &&\n current.offsetHeight > 0\n ) {\n return current;\n }\n if (!current.parentElement || current === document.body) break;\n current = current.parentElement;\n }\n return null;\n }\n }\n\n exports.ElementInfo = ElementInfo;\n exports.ElementTag = ElementTag;\n exports.deserializeNodeFromJSON = deserializeNodeFromJSON;\n exports.detectScrollableContainers = detectScrollableContainers;\n exports.findElement = findElement;\n exports.findElements = findElements;\n exports.generateCssPath = generateCssPath;\n exports.getAriaLabelledByText = getAriaLabelledByText;\n exports.getContainingIframe = getContainingIframe;\n exports.getElementInfo = getElementInfo;\n exports.getParentNode = getParentNode;\n exports.getRobustSelector = getRobustSelector;\n exports.highlight = highlight;\n exports.highlightElements = highlightElements;\n exports.isScrollableContainer = isScrollableContainer;\n exports.serializeNodeToJSON = serializeNodeToJSON;\n exports.unhighlightElements = unhighlightElements;\n\n}));\n//# sourceMappingURL=probolabs.umd.js.map\n";
|
|
1
|
+
const highlighterCode = "(function (global, factory) {\n typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports) :\n typeof define === 'function' && define.amd ? define(['exports'], factory) :\n (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global.ProboLabs = {}));\n})(this, (function (exports) { 'use strict';\n\n /**\n * Element tag constants for different types of interactive elements\n */\n const ElementTag = {\n CLICKABLE: \"CLICKABLE\",\n FILLABLE: \"FILLABLE\",\n SELECTABLE: \"SELECTABLE\",\n NON_INTERACTIVE_ELEMENT: 'NON_INTERACTIVE_ELEMENT',\n };\n\n var ApplyAIStatus;\n (function (ApplyAIStatus) {\n ApplyAIStatus[\"PREPARE_START\"] = \"PREPARE_START\";\n ApplyAIStatus[\"PREPARE_SUCCESS\"] = \"PREPARE_SUCCESS\";\n ApplyAIStatus[\"PREPARE_ERROR\"] = \"PREPARE_ERROR\";\n ApplyAIStatus[\"SEND_START\"] = \"SEND_START\";\n ApplyAIStatus[\"SEND_SUCCESS\"] = \"SEND_SUCCESS\";\n ApplyAIStatus[\"SEND_ERROR\"] = \"SEND_ERROR\";\n ApplyAIStatus[\"APPLY_AI_ERROR\"] = \"APPLY_AI_ERROR\";\n ApplyAIStatus[\"APPLY_AI_CANCELLED\"] = \"APPLY_AI_CANCELLED\";\n ApplyAIStatus[\"SUMMARY_COMPLETED\"] = \"SUMMARY_COMPLETED\";\n ApplyAIStatus[\"SUMMARY_ERROR\"] = \"SUMMARY_ERROR\";\n })(ApplyAIStatus || (ApplyAIStatus = {}));\n var ReplayStatus;\n (function (ReplayStatus) {\n ReplayStatus[\"REPLAY_START\"] = \"REPLAY_START\";\n ReplayStatus[\"REPLAY_SUCCESS\"] = \"REPLAY_SUCCESS\";\n ReplayStatus[\"REPLAY_ERROR\"] = \"REPLAY_ERROR\";\n ReplayStatus[\"REPLAY_CANCELLED\"] = \"REPLAY_CANCELLED\";\n })(ReplayStatus || (ReplayStatus = {}));\n\n // Action constants\n var PlaywrightAction;\n (function (PlaywrightAction) {\n PlaywrightAction[\"VISIT_BASE_URL\"] = \"VISIT_BASE_URL\";\n PlaywrightAction[\"VISIT_URL\"] = \"VISIT_URL\";\n PlaywrightAction[\"CLICK\"] = \"CLICK\";\n PlaywrightAction[\"FILL_IN\"] = \"FILL_IN\";\n PlaywrightAction[\"SELECT_DROPDOWN\"] = \"SELECT_DROPDOWN\";\n PlaywrightAction[\"SELECT_MULTIPLE_DROPDOWN\"] = \"SELECT_MULTIPLE_DROPDOWN\";\n PlaywrightAction[\"CHECK_CHECKBOX\"] = \"CHECK_CHECKBOX\";\n PlaywrightAction[\"SELECT_RADIO\"] = \"SELECT_RADIO\";\n PlaywrightAction[\"TOGGLE_SWITCH\"] = \"TOGGLE_SWITCH\";\n PlaywrightAction[\"SET_SLIDER\"] = \"SET_SLIDER\";\n PlaywrightAction[\"TYPE_KEYS\"] = \"TYPE_KEYS\";\n PlaywrightAction[\"HOVER\"] = \"HOVER\";\n PlaywrightAction[\"ASSERT_EXACT_VALUE\"] = \"ASSERT_EXACT_VALUE\";\n PlaywrightAction[\"ASSERT_CONTAINS_VALUE\"] = \"ASSERT_CONTAINS_VALUE\";\n PlaywrightAction[\"ASSERT_URL\"] = \"ASSERT_URL\";\n PlaywrightAction[\"SCROLL_TO_ELEMENT\"] = \"SCROLL_TO_ELEMENT\";\n PlaywrightAction[\"EXTRACT_VALUE\"] = \"EXTRACT_VALUE\";\n PlaywrightAction[\"ASK_AI\"] = \"ASK_AI\";\n PlaywrightAction[\"EXECUTE_SCRIPT\"] = \"EXECUTE_SCRIPT\";\n PlaywrightAction[\"UPLOAD_FILES\"] = \"UPLOAD_FILES\";\n PlaywrightAction[\"WAIT_FOR\"] = \"WAIT_FOR\";\n PlaywrightAction[\"WAIT_FOR_OTP\"] = \"WAIT_FOR_OTP\";\n PlaywrightAction[\"GEN_TOTP\"] = \"GEN_TOTP\";\n })(PlaywrightAction || (PlaywrightAction = {}));\n\n /**\n * Checks if any string in the array matches the given regular expression pattern.\n *\n * @param array - Array of strings to test.\n * @param pattern - Regular expression to test against each string.\n * @returns true if at least one string matches the pattern, false otherwise.\n */\n function testArray(array, pattern) {\n return array.some(item => pattern.test(item));\n }\n /**\n * Determines if an element is clickable\n *\n * @param element The DOM element to check\n * @returns boolean indicating if the element is fillable\n */\n function isClickableElement(element) {\n if (!element)\n return false;\n let depth = 0;\n while (depth < 5 && element && element.nodeType === Node.ELEMENT_NODE) {\n if (isClickableElementHelper(element) === IsClickable.YES)\n return true;\n if (isClickableElementHelper(element) === IsClickable.NO)\n return false;\n // if maybe, continue searching up to 5 levels up the DOM tree\n element = element.parentNode;\n depth++;\n }\n return false;\n }\n // clickable element detection result\n var IsClickable;\n (function (IsClickable) {\n IsClickable[\"YES\"] = \"YES\";\n IsClickable[\"NO\"] = \"NO\";\n IsClickable[\"MAYBE\"] = \"MAYBE\";\n })(IsClickable || (IsClickable = {}));\n function isClickableElementHelper(element) {\n var _a, _b;\n if (!element)\n return IsClickable.NO;\n //check for tag name\n const tagName = element.tagName.toLowerCase();\n const clickableTags = [\n 'a', 'button',\n ];\n if (clickableTags.includes(tagName))\n return IsClickable.YES;\n //check for clickable <input>\n const inputType = (_a = element.type) === null || _a === void 0 ? void 0 : _a.toLowerCase();\n const clickableTypes = [\n 'button', 'submit', 'reset', 'checkbox', 'radio',\n ];\n const ariaAutocompleteValues = [\n 'list', 'both',\n ];\n if (tagName === 'input') {\n if (clickableTypes.includes(inputType) || ariaAutocompleteValues.includes((_b = element.getAttribute('aria-autocomplete')) !== null && _b !== void 0 ? _b : ''))\n return IsClickable.YES;\n if (['date', 'number', 'range'].includes(inputType))\n return IsClickable.NO; //don't record the click as a change event will be generated for elements that generate an input change event\n }\n // check for cursor type\n const style = window.getComputedStyle(element);\n if (style.cursor === 'pointer')\n return IsClickable.YES;\n // check for attributes\n const clickableRoles = [\n 'button', 'combobox', 'listbox', 'dropdown', 'option', 'menu', 'menuitem',\n 'navigation', 'checkbox', 'switch', 'toggle', 'slider', 'textbox', 'listitem',\n 'presentation',\n ];\n const ariaPopupValues = [\n 'true', 'listbox', 'menu',\n ];\n if (element.hasAttribute('onclick') ||\n clickableRoles.includes(element.getAttribute('role') || '') ||\n ariaPopupValues.includes(element.getAttribute('aria-haspopup') || ''))\n return IsClickable.YES;\n // check for tabindex (means element is focusable and therefore clickable)\n if (parseInt(element.getAttribute('tabindex') || '-1') >= 0)\n return IsClickable.YES;\n // extract class names\n const classNames = Array.from(element.classList);\n // check for checkbox/radio-like class name - TODO: check if can be removed\n const checkboxPattern = /checkbox|switch|toggle|slider/i;\n if (testArray(classNames, checkboxPattern))\n return IsClickable.YES;\n // check for Material UI class names\n const muiClickableClassPattern = /MuiButton|MuiIconButton|MuiChip|MuiMenuItem|MuiListItem|MuiInputBase|MuiOutlinedInput|MuiSelect|MuiAutocomplete|MuiToggleButton|MuiBackdrop-root|MuiBackdrop-invisible/;\n if (testArray(classNames, muiClickableClassPattern))\n return IsClickable.YES;\n // check for SalesForce class names\n const sfClassPattern = /slds-button|slds-dropdown|slds-combobox|slds-picklist|slds-tabs|slds-pill|slds-action|slds-row-action|slds-context-bar|slds-input|slds-rich-text-area|slds-radio|slds-checkbox|slds-toggle|slds-link|slds-accordion|slds-tree/;\n if (testArray(classNames, sfClassPattern))\n return IsClickable.YES;\n // check for chart dots\n const chartClickableClassPattern = /recharts-dot/;\n if (testArray(classNames, chartClickableClassPattern))\n return IsClickable.YES;\n // check for React component classes\n const reactClickableClassPattern = /react-select|ant-select|rc-select|react-dropdown|react-autocomplete|react-datepicker|react-modal|react-tooltip|react-popover|react-menu|react-tabs|react-accordion|react-collapse|react-toggle|react-switch|react-checkbox|react-radio|react-button|react-link|react-card|react-list-item|react-menu-item|react-option|react-tab|react-panel|react-drawer|react-sidebar|react-nav|react-breadcrumb|react-pagination|react-stepper|react-wizard|react-carousel|react-slider|react-range|react-progress|react-badge|react-chip|react-tag|react-avatar|react-icon|react-fab|react-speed-dial|react-floating|react-sticky|react-affix|react-backdrop|react-overlay|react-portal|react-transition|react-animate|react-spring|react-framer|react-gesture|react-drag|react-drop|react-sortable|react-resizable|react-split|react-grid|react-table|react-datagrid|react-tree|react-treeview|react-file|react-upload|react-cropper|react-image|react-gallery|react-lightbox|react-player|react-video|react-audio|react-chart|react-graph|react-diagram|react-flow|react-d3|react-plotly|react-vega|react-vis|react-nivo|react-recharts|react-victory|react-echarts|react-highcharts|react-google-charts|react-fusioncharts|react-apexcharts|react-chartjs|react-chartkick|react-sparklines|react-trend|react-smooth|react-animated|react-lottie|react-spring|react-framer-motion|react-pose|react-motion|react-transition-group|react-router|react-navigation/i;\n if (testArray(classNames, reactClickableClassPattern))\n return IsClickable.YES;\n //check for cloudinary class names\n const cloudinaryClickableClassPattern = /cld-combobox|cld-upload-button|cld-controls|cld-player|cld-tab|cld-menu-item|cld-close|cld-play|cld-pause|cld-fullscreen|cld-browse|cld-cancel|cld-retry/;\n if (testArray(classNames, cloudinaryClickableClassPattern))\n return IsClickable.YES;\n return IsClickable.MAYBE;\n }\n function getParentNode(element) {\n if (!element || element.nodeType !== Node.ELEMENT_NODE)\n return null;\n let parent = null;\n // SF is using slots and shadow DOM heavily\n // However, there might be slots in the light DOM which shouldn't be traversed\n if (element.assignedSlot && element.getRootNode() instanceof ShadowRoot)\n parent = element.assignedSlot;\n else\n parent = element.parentNode;\n // Check if we're at a shadow root\n if (parent && parent.nodeType !== Node.ELEMENT_NODE && parent.getRootNode() instanceof ShadowRoot)\n parent = parent.getRootNode().host;\n return parent;\n }\n function getElementDepth(element) {\n let depth = 0;\n let currentElement = element;\n while ((currentElement === null || currentElement === void 0 ? void 0 : currentElement.nodeType) === Node.ELEMENT_NODE) {\n depth++;\n currentElement = getParentNode(currentElement);\n }\n return depth;\n }\n\n // WebSocketsMessageType enum for WebSocket and event message types shared across the app\n var WebSocketsMessageType;\n (function (WebSocketsMessageType) {\n WebSocketsMessageType[\"INTERACTION_APPLY_AI_PREPARE_START\"] = \"INTERACTION_APPLY_AI_PREPARE_START\";\n WebSocketsMessageType[\"INTERACTION_APPLY_AI_PREPARE_SUCCESS\"] = \"INTERACTION_APPLY_AI_PREPARE_SUCCESS\";\n WebSocketsMessageType[\"INTERACTION_APPLY_AI_PREPARE_ERROR\"] = \"INTERACTION_APPLY_AI_PREPARE_ERROR\";\n WebSocketsMessageType[\"INTERACTION_APPLY_AI_SEND_TO_LLM_START\"] = \"INTERACTION_APPLY_AI_SEND_TO_LLM_START\";\n WebSocketsMessageType[\"INTERACTION_APPLY_AI_SEND_TO_LLM_SUCCESS\"] = \"INTERACTION_APPLY_AI_SEND_TO_LLM_SUCCESS\";\n WebSocketsMessageType[\"INTERACTION_APPLY_AI_SEND_TO_LLM_ERROR\"] = \"INTERACTION_APPLY_AI_SEND_TO_LLM_ERROR\";\n WebSocketsMessageType[\"INTERACTION_REPLAY_START\"] = \"INTERACTION_REPLAY_START\";\n WebSocketsMessageType[\"INTERACTION_REPLAY_SUCCESS\"] = \"INTERACTION_REPLAY_SUCCESS\";\n WebSocketsMessageType[\"INTERACTION_REPLAY_ERROR\"] = \"INTERACTION_REPLAY_ERROR\";\n WebSocketsMessageType[\"INTERACTION_REPLAY_CANCELLED\"] = \"INTERACTION_REPLAY_CANCELLED\";\n WebSocketsMessageType[\"INTERACTION_APPLY_AI_CANCELLED\"] = \"INTERACTION_APPLY_AI_CANCELLED\";\n WebSocketsMessageType[\"INTERACTION_APPLY_AI_ERROR\"] = \"INTERACTION_APPLY_AI_ERROR\";\n WebSocketsMessageType[\"INTERACTION_STEP_CREATED\"] = \"INTERACTION_STEP_CREATED\";\n WebSocketsMessageType[\"INTERACTION_APPLY_AI_SUMMARY_COMPLETED\"] = \"INTERACTION_APPLY_AI_SUMMARY_COMPLETED\";\n WebSocketsMessageType[\"INTERACTION_APPLY_AI_SUMMARY_ERROR\"] = \"INTERACTION_APPLY_AI_SUMMARY_ERROR\";\n WebSocketsMessageType[\"OTP_RETRIEVED\"] = \"OTP_RETRIEVED\";\n })(WebSocketsMessageType || (WebSocketsMessageType = {}));\n\n /**\n * Logging levels for Probo\n */\n var ProboLogLevel;\n (function (ProboLogLevel) {\n ProboLogLevel[\"DEBUG\"] = \"DEBUG\";\n ProboLogLevel[\"INFO\"] = \"INFO\";\n ProboLogLevel[\"LOG\"] = \"LOG\";\n ProboLogLevel[\"WARN\"] = \"WARN\";\n ProboLogLevel[\"ERROR\"] = \"ERROR\";\n })(ProboLogLevel || (ProboLogLevel = {}));\n const logLevelOrder = {\n [ProboLogLevel.DEBUG]: 0,\n [ProboLogLevel.INFO]: 1,\n [ProboLogLevel.LOG]: 2,\n [ProboLogLevel.WARN]: 3,\n [ProboLogLevel.ERROR]: 4,\n };\n class ProboLogger {\n constructor(prefix, level = ProboLogLevel.INFO) {\n this.prefix = prefix;\n this.level = level;\n }\n setLogLevel(level) {\n console.log(`[${this.prefix}] Setting log level to: ${level} (was: ${this.level})`);\n this.level = level;\n }\n shouldLog(level) {\n return logLevelOrder[level] >= logLevelOrder[this.level];\n }\n preamble(level) {\n const now = new Date();\n const hours = String(now.getHours()).padStart(2, '0');\n const minutes = String(now.getMinutes()).padStart(2, '0');\n const seconds = String(now.getSeconds()).padStart(2, '0');\n const milliseconds = String(now.getMilliseconds()).padStart(3, '0');\n return `[${hours}:${minutes}:${seconds}.${milliseconds}] [${this.prefix}] [${level}]`;\n }\n debug(...args) { if (this.shouldLog(ProboLogLevel.DEBUG))\n console.debug(this.preamble(ProboLogLevel.DEBUG), ...args); }\n info(...args) { if (this.shouldLog(ProboLogLevel.INFO))\n console.info(this.preamble(ProboLogLevel.INFO), ...args); }\n log(...args) { if (this.shouldLog(ProboLogLevel.LOG))\n console.log(this.preamble(ProboLogLevel.LOG), ...args); }\n warn(...args) { if (this.shouldLog(ProboLogLevel.WARN))\n console.warn(this.preamble(ProboLogLevel.WARN), ...args); }\n error(...args) { if (this.shouldLog(ProboLogLevel.ERROR))\n console.error(this.preamble(ProboLogLevel.ERROR), ...args); }\n }\n function doubleQuoteString(str) {\n if (!str)\n return '';\n return `\"${str.replace(/\"/g, '\\\\\"')}\"`;\n }\n\n new ProboLogger('apiclient');\n\n /**\n * Available AI models for LLM operations\n */\n var AIModel;\n (function (AIModel) {\n AIModel[\"AZURE_GPT4\"] = \"azure-gpt4\";\n AIModel[\"AZURE_GPT4_MINI\"] = \"azure-gpt4-mini\";\n AIModel[\"GEMINI_1_5_FLASH\"] = \"gemini-1.5-flash\";\n AIModel[\"GEMINI_2_5_FLASH\"] = \"gemini-2.5-flash\";\n AIModel[\"GPT4\"] = \"gpt4\";\n AIModel[\"GPT4_MINI\"] = \"gpt4-mini\";\n AIModel[\"CLAUDE_3_5\"] = \"claude-3.5\";\n AIModel[\"CLAUDE_SONNET_4_5\"] = \"claude-sonnet-4.5\";\n AIModel[\"CLAUDE_HAIKU_4_5\"] = \"claude-haiku-4.5\";\n AIModel[\"CLAUDE_OPUS_4_1\"] = \"claude-opus-4.1\";\n AIModel[\"GROK_2\"] = \"grok-2\";\n AIModel[\"LLAMA_4_SCOUT\"] = \"llama-4-scout\";\n AIModel[\"DEEPSEEK_V3\"] = \"deepseek-v3\";\n })(AIModel || (AIModel = {}));\n\n const highlighterLogger = new ProboLogger('highlighter');\r\n\r\n // Get text content of an element (recursively including child elements)\r\n function getElementText(element) {\r\n if (!element) return '';\r\n return element.textContent || '';\r\n }\r\n\r\n // Get placeholder text of an element\r\n function getElementPlaceholder(element) {\r\n if (!element) return '';\r\n return element.placeholder || '';\r\n }\r\n\r\n // Get CSS selector of an element (not recursively, just for a single element)\r\n function getElementCssSelector(element) {\r\n if (!(element instanceof Element)) return '';\r\n \r\n const tag = element.tagName.toLowerCase();\r\n let selector = tag;\r\n \r\n // Find nth-of-type for elements without ID\r\n let nth = 1;\r\n let sibling = element.previousElementSibling;\r\n while (sibling) {\r\n if (sibling.nodeName.toLowerCase() === tag) nth++;\r\n sibling = sibling.previousElementSibling;\r\n }\r\n if (nth > 1 || element.nextElementSibling) selector += `:nth-of-type(${nth})`;\r\n \r\n return selector;\r\n }\r\n\r\n // Helper function to check if element is a form control\r\n function isFormControl(elementInfo) {\r\n return /^(input|select|textarea|button|label)$/i.test(elementInfo.tag);\r\n }\r\n\r\n // Helper function to check if element is a dropdown item\r\n const isDropdownItem = (elementInfo) => {\r\n const dropdownPatterns = [\r\n /dropdown[-_]?item/i, // matches: dropdown-item, dropdownitem, dropdown_item\r\n /menu[-_]?item/i, // matches: menu-item, menuitem, menu_item\r\n /dropdown[-_]?link/i, // matches: dropdown-link, dropdownlink, dropdown_link\r\n /list[-_]?item/i, // matches: list-item, listitem, list_item\r\n /select[-_]?item/i, // matches: select-item, selectitem, select_item \r\n ];\r\n\r\n const rolePatterns = [\r\n /menu[-_]?item/i, // matches: menuitem, menu-item\r\n /option/i, // matches: option\r\n /list[-_]?item/i, // matches: listitem, list-item\r\n /tree[-_]?item/i // matches: treeitem, tree-item\r\n ];\r\n\r\n const hasMatchingClass = elementInfo.element.className && \r\n dropdownPatterns.some(pattern => \r\n pattern.test(elementInfo.element.className)\r\n );\r\n\r\n const hasMatchingRole = elementInfo.element.getAttribute('role') && \r\n rolePatterns.some(pattern => \r\n pattern.test(elementInfo.element.getAttribute('role'))\r\n );\r\n\r\n return hasMatchingClass || hasMatchingRole;\r\n };\r\n\r\n /**\r\n * Element Query Utilities\r\n */\r\n\r\n function getAllDocumentElementsIncludingShadow(selectors, root = document) {\r\n const elements = Array.from(root.querySelectorAll(selectors));\r\n\r\n root.querySelectorAll('*').forEach(el => {\r\n if (el.shadowRoot) {\r\n elements.push(...getAllDocumentElementsIncludingShadow(selectors, el.shadowRoot));\r\n }\r\n });\r\n return elements;\r\n }\r\n\r\n function getAllFrames(root = document) {\r\n const result = [root];\r\n const frames = getAllDocumentElementsIncludingShadow('frame, iframe', root); \r\n frames.forEach(frame => {\r\n try {\r\n const frameDocument = frame.contentDocument || frame.contentWindow.document;\r\n if (frameDocument) {\r\n result.push(frameDocument);\r\n }\r\n } catch (e) {\r\n // Skip cross-origin frames\r\n highlighterLogger.warn('Could not access frame content:', e.message);\r\n }\r\n });\r\n\r\n return result;\r\n }\r\n\r\n function getAllElementsIncludingShadow(selectors, root = document) {\r\n const elements = [];\r\n\r\n getAllFrames(root).forEach(doc => {\r\n elements.push(...getAllDocumentElementsIncludingShadow(selectors, doc));\r\n });\r\n\r\n return elements;\r\n }\r\n\r\n /**\r\n * Finds the first element matching a CSS selector, traversing Shadow DOM if necessary\r\n * @param {string} selector - CSS selector to search for\r\n * @param {Element} [root=document] - Root element to start searching from\r\n * @returns {Element|null} - The first matching element or null if not found\r\n */\r\n function robustQuerySelector(selector, root = document, all = false) {\r\n // First try to find in light DOM\r\n let elements = all ? root.querySelectorAll(selector) : root.querySelector(selector);\r\n if (elements) return elements;\r\n \r\n // Get all shadow roots\r\n const shadowElements = Array.from(root.querySelectorAll('*'))\r\n .filter(el => el.shadowRoot);\r\n \r\n // Search through each shadow root until we find a match\r\n for (const el of shadowElements) {\r\n elements = robustQuerySelector(selector, el.shadowRoot, all);\r\n if (elements) return elements;\r\n }\r\n \r\n return all ? [] : null;\r\n }\r\n\r\n\r\n function isDecendent(parent, child) {\r\n let element = child;\r\n while (element && element !== parent && element.nodeType === Node.ELEMENT_NODE) { \r\n element = getParentNode(element); \r\n }\r\n return element === parent;\r\n }\r\n\r\n function generateXPath(element) {\r\n return '/'+extractElementPath(element).map(item => `${item.tagName}${item.onlyChild ? '' : `[${item.index}]`}`).join('/');\r\n }\r\n\r\n function generateCssPath(element) {\r\n return extractElementPath(element).map(item => `${item.tagName}:nth-of-type(${item.index})`).join(' > ');\r\n }\r\n\r\n // unified path generation for xpath and css selector\r\n function extractElementPath(element) {\r\n if (!element) {\r\n highlighterLogger.error('ERROR: No element provided to generatePath');\r\n return [];\r\n }\r\n const path = [];\r\n // traversing up the DOM tree\r\n while (element && element.nodeType === Node.ELEMENT_NODE) { \r\n let tagName = element.nodeName.toLowerCase();\r\n \r\n let sibling = element;\r\n let index = 1;\r\n \r\n while (sibling = sibling.previousElementSibling) {\r\n if (sibling.nodeName.toLowerCase() === tagName) index++;\r\n }\r\n sibling = element;\r\n \r\n let onlyChild = (index === 1);\r\n while (onlyChild && (sibling = sibling.nextElementSibling)) {\r\n if (sibling.nodeName.toLowerCase() === tagName) onlyChild = false;\r\n }\r\n \r\n // add a tuple with tagName, index (nth), and onlyChild \r\n path.unshift({\r\n tagName: tagName,\r\n index: index,\r\n onlyChild: onlyChild \r\n }); \r\n\r\n element = getParentNode(element);\r\n }\r\n \r\n return path;\r\n }\r\n\r\n function cleanHTML(rawHTML) {\r\n const parser = new DOMParser();\r\n const doc = parser.parseFromString(rawHTML, \"text/html\");\r\n\r\n function cleanElement(element) {\r\n const allowedAttributes = new Set([\r\n \"role\",\r\n \"type\",\r\n \"class\",\r\n \"href\",\r\n \"alt\",\r\n \"title\",\r\n \"readonly\",\r\n \"checked\",\r\n \"enabled\",\r\n \"disabled\",\r\n ]);\r\n\r\n [...element.attributes].forEach(attr => {\r\n const name = attr.name.toLowerCase();\r\n const value = attr.value;\r\n\r\n const isTestAttribute = /^(testid|test-id|data-test-id)$/.test(name);\r\n const isDataAttribute = name.startsWith(\"data-\") && value;\r\n const isBooleanAttribute = [\"readonly\", \"checked\", \"enabled\", \"disabled\"].includes(name);\r\n\r\n if (!allowedAttributes.has(name) && !isDataAttribute && !isTestAttribute && !isBooleanAttribute) {\r\n element.removeAttribute(name);\r\n }\r\n });\r\n\r\n // Handle SVG content - more aggressive replacement\r\n if (element.tagName.toLowerCase() === \"svg\") {\r\n // Remove all attributes except class and role\r\n [...element.attributes].forEach(attr => {\r\n const name = attr.name.toLowerCase();\r\n if (name !== \"class\" && name !== \"role\") {\r\n element.removeAttribute(name);\r\n }\r\n });\r\n element.innerHTML = \"CONTENT REMOVED\";\r\n } else {\r\n // Recursively clean child elements\r\n Array.from(element.children).forEach(cleanElement);\r\n }\r\n\r\n // Only remove empty elements that aren't semantic or icon elements\r\n const keepEmptyElements = ['i', 'span', 'svg', 'button', 'input'];\r\n if (!keepEmptyElements.includes(element.tagName.toLowerCase()) && \r\n !element.children.length && \r\n !element.textContent.trim()) {\r\n element.remove();\r\n }\r\n }\r\n\r\n // Process all elements in the document body\r\n Array.from(doc.body.children).forEach(cleanElement);\r\n return doc.body.innerHTML;\r\n }\r\n\r\n function getContainingIFrame(element) {\r\n // If not in an iframe, return null\r\n if (element.ownerDocument.defaultView === window.top) {\r\n return null;\r\n }\r\n \r\n // Try to find the iframe in the parent document that contains our element\r\n try {\r\n const parentDocument = element.ownerDocument.defaultView.parent.document;\r\n const iframes = parentDocument.querySelectorAll('iframe');\r\n \r\n for (const iframe of iframes) {\r\n if (iframe.contentWindow === element.ownerDocument.defaultView) {\r\n return iframe;\r\n }\r\n }\r\n } catch (e) {\r\n // Cross-origin restriction\r\n return \"Cross-origin iframe - cannot access details\";\r\n }\r\n \r\n return null;\r\n }\r\n\r\n function getAriaLabelledByText(elementInfo, includeHidden=true) {\r\n if (!elementInfo.element.hasAttribute('aria-labelledby')) return '';\r\n\r\n const ids = elementInfo.element.getAttribute('aria-labelledby').split(/\\s+/);\r\n let labelText = '';\r\n\r\n //locate root (document or iFrame document if element is contained in an iframe)\r\n let root = document;\r\n if (elementInfo.iframe_selector) { \r\n const frames = getAllDocumentElementsIncludingShadow('iframe', document);\r\n \r\n // Iterate over all frames and compare their CSS selectors\r\n for (const frame of frames) {\r\n const selector = generateCssPath(frame);\r\n if (selector === elementInfo.iframe_selector) {\r\n root = frame.contentDocument || frame.contentWindow.document; \r\n break;\r\n }\r\n } \r\n }\r\n\r\n ids.forEach(id => {\r\n const el = robustQuerySelector(`#${CSS.escape(id)}`, root);\r\n if (el) {\r\n if (includeHidden || el.offsetParent !== null || getComputedStyle(el).display !== 'none') {\r\n labelText += el.textContent.trim() + ' ';\r\n }\r\n }\r\n });\r\n\r\n return labelText.trim();\r\n }\r\n\r\n\r\n\r\n const filterZeroDimensions = (elementInfo) => {\r\n const rect = elementInfo.bounding_box;\r\n //single pixel elements are typically faux controls and should be filtered too\r\n const hasSize = rect.width > 1 && rect.height > 1;\r\n const style = window.getComputedStyle(elementInfo.element);\r\n const isVisible = style.display !== 'none' && style.visibility !== 'hidden';\r\n \r\n if (!hasSize || !isVisible) {\r\n \r\n return false;\r\n }\r\n return true;\r\n };\r\n\r\n\r\n\r\n function uniquifyElements(elements) {\r\n const seen = new Set();\r\n\r\n highlighterLogger.debug(`Starting uniquification with ${elements.length} elements`);\r\n\r\n // Filter out testing infrastructure elements first\r\n const filteredInfrastructure = elements.filter(element_info => {\r\n // Skip the highlight-overlay element completely - it's part of the testing infrastructure\r\n if (element_info.element.id === 'highlight-overlay' || \r\n (element_info.css_selector && element_info.css_selector.includes('#highlight-overlay'))) {\r\n highlighterLogger.debug('Filtered out testing infrastructure element:', element_info.css_selector);\r\n return false;\r\n }\r\n \r\n // Filter out UI framework container/manager elements\r\n const el = element_info.element;\r\n // UI framework container checks - generic detection for any framework\r\n if ((el.getAttribute('data-rendered-by') || \r\n el.getAttribute('data-reactroot') || \r\n el.getAttribute('ng-version') || \r\n el.getAttribute('data-component-id') ||\r\n el.getAttribute('data-root') ||\r\n el.getAttribute('data-framework')) && \r\n (el.className && \r\n typeof el.className === 'string' && \r\n (el.className.includes('Container') || \r\n el.className.includes('container') || \r\n el.className.includes('Manager') || \r\n el.className.includes('manager')))) {\r\n highlighterLogger.debug('Filtered out UI framework container element:', element_info.css_selector);\r\n return false;\r\n }\r\n \r\n // Direct filter for framework container elements that shouldn't be interactive\r\n // Consolidating multiple container detection patterns into one efficient check\r\n const isFullViewport = element_info.bounding_box && \r\n element_info.bounding_box.x <= 5 && \r\n element_info.bounding_box.y <= 5 && \r\n element_info.bounding_box.width >= (window.innerWidth * 0.95) && \r\n element_info.bounding_box.height >= (window.innerHeight * 0.95);\r\n \r\n // Empty content check\r\n const isEmpty = !el.innerText || el.innerText.trim() === '';\r\n \r\n // Check if it's a framework container element\r\n if (element_info.element.tagName === 'DIV' && \r\n isFullViewport && \r\n isEmpty && \r\n (\r\n // Pattern matching for root containers\r\n (element_info.xpath && \r\n (element_info.xpath.match(/^\\/html\\[\\d+\\]\\/body\\[\\d+\\]\\/div\\[\\d+\\]\\/div\\[\\d+\\]$/) || \r\n element_info.xpath.match(/^\\/\\/\\*\\[@id='[^']+'\\]\\/div\\[\\d+\\]$/))) ||\r\n \r\n // Simple DOM structure\r\n (element_info.css_selector.split(' > ').length <= 4 && element_info.depth <= 5) ||\r\n \r\n // Empty or container-like classes\r\n (!el.className || el.className === '' || \r\n (typeof el.className === 'string' && \r\n (el.className.includes('overlay') || \r\n el.className.includes('container') || \r\n el.className.includes('wrapper'))))\r\n )) {\r\n highlighterLogger.debug('Filtered out framework container element:', element_info.css_selector);\r\n return false;\r\n }\r\n \r\n return true;\r\n });\r\n\r\n // First filter out elements with zero dimensions\r\n const nonZeroElements = filteredInfrastructure.filter(filterZeroDimensions);\r\n // sort by CSS selector depth so parents are processed first\r\n nonZeroElements.sort((a, b) => a.depth - b.depth);\r\n highlighterLogger.debug(`After dimension filtering: ${nonZeroElements.length} elements remain (${elements.length - nonZeroElements.length} removed)`);\r\n \r\n const filteredByParent = nonZeroElements.filter(element_info => {\r\n\r\n const parent = findClosestParent(seen, element_info);\r\n const keep = parent == null || shouldKeepNestedElement(element_info, parent);\r\n // console.log(\"node \", element_info.index, \": keep=\", keep, \" parent=\", parent);\r\n // if (!keep && !element_info.xpath) {\r\n // console.log(\"Filtered out element \", element_info,\" because it's a nested element of \", parent);\r\n // }\r\n if (keep)\r\n seen.add(element_info.css_selector);\r\n\r\n return keep;\r\n });\r\n\r\n highlighterLogger.debug(`After parent/child filtering: ${filteredByParent.length} elements remain (${nonZeroElements.length - filteredByParent.length} removed)`);\r\n\r\n // Final overlap filtering\r\n const filteredResults = filteredByParent.filter(element => {\r\n\r\n // Look for any element that came BEFORE this one in the array\r\n const hasEarlierOverlap = filteredByParent.some(other => {\r\n // Only check elements that came before (lower index)\r\n if (filteredByParent.indexOf(other) >= filteredByParent.indexOf(element)) {\r\n return false;\r\n }\r\n \r\n const isOverlapping = areElementsOverlapping(element, other); \r\n return isOverlapping;\r\n }); \r\n\r\n // Keep element if it has no earlier overlapping elements\r\n return !hasEarlierOverlap;\r\n });\r\n \r\n \r\n \r\n // Check for overlay removal\r\n highlighterLogger.debug(`After filtering: ${filteredResults.length} (${filteredByParent.length - filteredResults.length} removed by overlap)`);\r\n \r\n const nonOverlaidElements = filteredResults.filter(element => {\r\n return !isOverlaid(element);\r\n });\r\n\r\n highlighterLogger.debug(`Final elements after overlay removal: ${nonOverlaidElements.length} (${filteredResults.length - nonOverlaidElements.length} removed)`);\r\n \r\n return nonOverlaidElements;\r\n\r\n }\r\n\r\n\r\n\r\n const areElementsOverlapping = (element1, element2) => {\r\n if (element1.css_selector === element2.css_selector) {\r\n return true;\r\n }\r\n \r\n const box1 = element1.bounding_box;\r\n const box2 = element2.bounding_box;\r\n \r\n return box1.x === box2.x &&\r\n box1.y === box2.y &&\r\n box1.width === box2.width &&\r\n box1.height === box2.height;\r\n // element1.text === element2.text &&\r\n // element2.tag === 'a';\r\n };\r\n\r\n function findClosestParent(seen, element_info) { \r\n \r\n // Split the xpath into segments\r\n const segments = element_info.css_selector.split(' > ');\r\n \r\n // Try increasingly shorter paths until we find one in the seen set\r\n for (let i = segments.length - 1; i > 0; i--) {\r\n const parentPath = segments.slice(0, i).join(' > ');\r\n if (seen.has(parentPath)) {\r\n return parentPath;\r\n }\r\n }\r\n\r\n return null;\r\n }\r\n\r\n function shouldKeepNestedElement(elementInfo, parentPath) {\r\n let result = false;\r\n const parentSegments = parentPath.split(' > ');\r\n\r\n const isParentLink = /^a(:nth-of-type\\(\\d+\\))?$/.test(parentSegments[parentSegments.length - 1]);\r\n if (isParentLink) {\r\n return false; \r\n }\r\n // If this is a checkbox/radio input\r\n if (elementInfo.tag === 'input' && \r\n (elementInfo.type === 'checkbox' || elementInfo.type === 'radio')) {\r\n \r\n // Check if parent is a label by looking at the parent xpath's last segment\r\n \r\n const isParentLabel = /^label(:nth-of-type\\(\\d+\\))?$/.test(parentSegments[parentSegments.length - 1]);\r\n \r\n // If parent is a label, don't keep the input (we'll keep the label instead)\r\n if (isParentLabel) {\r\n return false;\r\n }\r\n }\r\n \r\n // Keep all other form controls and dropdown items\r\n if (isFormControl(elementInfo) || isDropdownItem(elementInfo)) {\r\n result = true;\r\n }\r\n\r\n if(isTableCell(elementInfo)) {\r\n result = true;\r\n }\r\n \r\n \r\n // console.log(`shouldKeepNestedElement: ${elementInfo.tag} ${elementInfo.text} ${elementInfo.xpath} -> ${parentXPath} -> ${result}`);\r\n return result;\r\n }\r\n\r\n\r\n function isTableCell(elementInfo) {\r\n const element = elementInfo.element;\r\n if(!element || !(element instanceof HTMLElement)) {\r\n return false;\r\n }\r\n const validTags = new Set(['td', 'th']);\r\n const validRoles = new Set(['cell', 'gridcell', 'columnheader', 'rowheader']);\r\n \r\n const tag = element.tagName.toLowerCase();\r\n const role = element.getAttribute('role')?.toLowerCase();\r\n\r\n if (validTags.has(tag) || (role && validRoles.has(role))) {\r\n return true;\r\n }\r\n return false;\r\n \r\n }\r\n\r\n function isOverlaid(elementInfo) {\r\n const element = elementInfo.element;\r\n const boundingRect = elementInfo.bounding_box;\r\n \r\n\r\n \r\n \r\n // Create a diagnostic logging function that only logs when needed\r\n const diagnosticLog = (...args) => {\r\n { // set to true for debugging\r\n highlighterLogger.debug('[OVERLAY-DEBUG]', ...args);\r\n }\r\n };\r\n\r\n // Special handling for tooltips\r\n if (elementInfo.element.className && typeof elementInfo.element.className === 'string' && \r\n elementInfo.element.className.includes('tooltip')) {\r\n diagnosticLog('Element is a tooltip, not considering it overlaid');\r\n return false;\r\n }\r\n \r\n \r\n \r\n // Get element at the center point to check if it's covered by a popup/modal\r\n const middleX = boundingRect.x + boundingRect.width/2;\r\n const middleY = boundingRect.y + boundingRect.height/2;\r\n const elementAtMiddle = element.ownerDocument.elementFromPoint(middleX, middleY);\r\n \r\n if (elementAtMiddle && \r\n elementAtMiddle !== element && \r\n !isDecendent(element, elementAtMiddle) && \r\n !isDecendent(elementAtMiddle, element)) {\r\n\r\n \r\n return true;\r\n }\r\n \r\n \r\n return false;\r\n \r\n }\r\n\r\n /**\r\n * Checks if an element is scrollable (has scrollable content)\r\n * \r\n * @param element - The element to check\r\n * @returns boolean indicating if the element is scrollable\r\n */\r\n function isScrollableContainer(element) {\r\n if (!element) return false;\r\n \r\n const style = window.getComputedStyle(element);\r\n \r\n // Reliable way to detect if an element has scrollbars or is scrollable\r\n const hasScrollHeight = element.scrollHeight > element.clientHeight;\r\n const hasScrollWidth = element.scrollWidth > element.clientWidth;\r\n \r\n // Check actual style properties\r\n const hasOverflowY = style.overflowY === 'auto' || \r\n style.overflowY === 'scroll' || \r\n style.overflowY === 'overlay';\r\n const hasOverflowX = style.overflowX === 'auto' || \r\n style.overflowX === 'scroll' || \r\n style.overflowX === 'overlay';\r\n \r\n // Check common class names and attributes for scrollable containers across frameworks\r\n const hasScrollClasses = element.classList.contains('scroll') || \r\n element.classList.contains('scrollable') ||\r\n element.classList.contains('overflow') ||\r\n element.classList.contains('overflow-auto') ||\r\n element.classList.contains('overflow-scroll') ||\r\n element.getAttribute('data-scrollable') === 'true';\r\n \r\n // Check for height/max-height constraints that often indicate scrolling content\r\n const hasHeightConstraint = style.maxHeight && \r\n style.maxHeight !== 'none' && \r\n style.maxHeight !== 'auto';\r\n \r\n // An element is scrollable if it has:\r\n // 1. Actual scrollbars in use (most reliable check) OR\r\n // 2. Overflow styles allowing scrolling AND content that would require scrolling\r\n return (hasScrollHeight && hasOverflowY) || \r\n (hasScrollWidth && hasOverflowX) ||\r\n (hasScrollClasses && (hasScrollHeight || hasScrollWidth)) ||\r\n (hasHeightConstraint && hasScrollHeight);\r\n }\r\n\r\n /**\r\n * Detects scrollable containers that are ancestors of the target element\r\n * \r\n * This function traverses up the DOM tree from the target element and identifies\r\n * all scrollable containers (elements that have scrollable content).\r\n * \r\n * @param target - The target element to start the search from\r\n * @returns Array of objects with selector and scroll properties\r\n */\r\n function detectScrollableContainers(target) {\r\n const scrollableContainers = [];\r\n \r\n if (!target) {\r\n return scrollableContainers;\r\n }\r\n \r\n highlighterLogger.debug('🔍 [detectScrollableContainers] Starting detection for target:', target.tagName, target.id, target.className);\r\n \r\n // Detect if target is inside an iframe\r\n const iframe = getContainingIFrame(target);\r\n const iframe_selector = iframe ? generateCssPath(iframe) : \"\";\r\n \r\n highlighterLogger.debug('🔍 [detectScrollableContainers] Iframe context:', iframe ? 'inside iframe' : 'main document', 'selector:', iframe_selector);\r\n \r\n // Start from the target element and traverse up the DOM tree\r\n let currentElement = target;\r\n let depth = 0;\r\n const MAX_DEPTH = 10; // Limit traversal depth to avoid infinite loops\r\n \r\n while (currentElement && currentElement.nodeType === Node.ELEMENT_NODE && depth < MAX_DEPTH) { \r\n // Check if the current element is scrollable\r\n if (isScrollableContainer(currentElement)) {\r\n highlighterLogger.debug('🔍 [detectScrollableContainers] Found scrollable container at depth', depth, ':', currentElement.tagName, currentElement.id, currentElement.className);\r\n \r\n const container = {\r\n containerEl: currentElement,\r\n selector: generateCssPath(currentElement),\r\n iframe_selector: iframe_selector,\r\n scrollTop: currentElement.scrollTop,\r\n scrollLeft: currentElement.scrollLeft,\r\n scrollHeight: currentElement.scrollHeight,\r\n scrollWidth: currentElement.scrollWidth,\r\n clientHeight: currentElement.clientHeight,\r\n clientWidth: currentElement.clientWidth\r\n };\r\n \r\n scrollableContainers.push(container);\r\n }\r\n \r\n // Move to parent element\r\n currentElement = getParentNode(currentElement);\r\n \r\n depth++;\r\n }\r\n \r\n highlighterLogger.debug('🔍 [detectScrollableContainers] Detection complete. Found', scrollableContainers.length, 'scrollable containers');\r\n return scrollableContainers;\r\n }\n\n class DOMSerializer {\r\n constructor(options = {}) {\r\n this.options = {\r\n includeStyles: true,\r\n includeScripts: false, // Security consideration\r\n includeFrames: true,\r\n includeShadowDOM: true,\r\n maxDepth: 50,\r\n ...options\r\n };\r\n this.serializedFrames = new Map();\r\n this.shadowRoots = new Map();\r\n }\r\n \r\n /**\r\n * Serialize a complete document or element\r\n */\r\n serialize(rootElement = document) {\r\n try {\r\n const serialized = {\r\n type: 'document',\r\n doctype: this.serializeDoctype(rootElement),\r\n documentElement: this.serializeElement(rootElement.documentElement || rootElement),\r\n frames: [],\r\n timestamp: Date.now(),\r\n url: rootElement.URL || window.location?.href,\r\n metadata: {\r\n title: rootElement.title,\r\n charset: rootElement.characterSet,\r\n contentType: rootElement.contentType\r\n }\r\n };\r\n \r\n // Serialize frames and iframes if enabled\r\n if (this.options.includeFrames) {\r\n serialized.frames = this.serializeFrames(rootElement);\r\n }\r\n \r\n return serialized;\r\n } catch (error) {\r\n console.error('Serialization error:', error);\r\n throw new Error(`DOM serialization failed: ${error.message}`);\r\n }\r\n }\r\n \r\n /**\r\n * Serialize document type declaration\r\n */\r\n serializeDoctype(doc) {\r\n if (!doc.doctype) return null;\r\n \r\n return {\r\n name: doc.doctype.name,\r\n publicId: doc.doctype.publicId,\r\n systemId: doc.doctype.systemId\r\n };\r\n }\r\n \r\n /**\r\n * Serialize an individual element and its children\r\n */\r\n serializeElement(element, depth = 0) {\r\n if (depth > this.options.maxDepth) {\r\n return { type: 'text', content: '<!-- Max depth exceeded -->' };\r\n }\r\n \r\n const nodeType = element.nodeType;\r\n \r\n switch (nodeType) {\r\n case Node.ELEMENT_NODE:\r\n return this.serializeElementNode(element, depth);\r\n case Node.TEXT_NODE:\r\n return this.serializeTextNode(element);\r\n case Node.COMMENT_NODE:\r\n return this.serializeCommentNode(element);\r\n case Node.DOCUMENT_FRAGMENT_NODE:\r\n return this.serializeDocumentFragment(element, depth);\r\n default:\r\n return null;\r\n }\r\n }\r\n \r\n /**\r\n * Serialize element node with attributes and children\r\n */\r\n serializeElementNode(element, depth) {\r\n const tagName = element.tagName.toLowerCase();\r\n \r\n // Skip script tags for security unless explicitly enabled\r\n if (tagName === 'script' && !this.options.includeScripts) {\r\n return { type: 'comment', content: '<!-- Script tag removed for security -->' };\r\n }\r\n \r\n const serialized = {\r\n type: 'element',\r\n tagName: tagName,\r\n attributes: this.serializeAttributes(element),\r\n children: [],\r\n shadowRoot: null\r\n };\r\n \r\n // Handle Shadow DOM\r\n if (this.options.includeShadowDOM && element.shadowRoot) {\r\n serialized.shadowRoot = this.serializeShadowRoot(element.shadowRoot, depth + 1);\r\n }\r\n \r\n // Handle special elements\r\n if (tagName === 'iframe' || tagName === 'frame') {\r\n serialized.frameData = this.serializeFrameElement(element);\r\n }\r\n \r\n // Serialize children\r\n for (const child of element.childNodes) {\r\n const serializedChild = this.serializeElement(child, depth + 1);\r\n if (serializedChild) {\r\n serialized.children.push(serializedChild);\r\n }\r\n }\r\n \r\n // Include computed styles if enabled\r\n if (this.options.includeStyles && element.nodeType === Node.ELEMENT_NODE) {\r\n serialized.computedStyle = this.serializeComputedStyle(element);\r\n }\r\n \r\n return serialized;\r\n }\r\n \r\n /**\r\n * Serialize element attributes\r\n */\r\n serializeAttributes(element) {\r\n const attributes = {};\r\n \r\n if (element.attributes) {\r\n for (const attr of element.attributes) {\r\n attributes[attr.name] = attr.value;\r\n }\r\n }\r\n \r\n return attributes;\r\n }\r\n \r\n /**\r\n * Serialize computed styles\r\n */\r\n serializeComputedStyle(element) {\r\n try {\r\n const computedStyle = window.getComputedStyle(element);\r\n const styles = {};\r\n \r\n // Only serialize non-default values to reduce size\r\n const importantStyles = [\r\n 'display', 'position', 'width', 'height', 'margin', 'padding',\r\n 'border', 'background', 'color', 'font-family', 'font-size',\r\n 'text-align', 'visibility', 'z-index', 'transform'\r\n ];\r\n \r\n for (const prop of importantStyles) {\r\n const value = computedStyle.getPropertyValue(prop);\r\n if (value && value !== 'initial' && value !== 'normal') {\r\n styles[prop] = value;\r\n }\r\n }\r\n \r\n return styles;\r\n } catch (error) {\r\n return {};\r\n }\r\n }\r\n \r\n /**\r\n * Serialize text node\r\n */\r\n serializeTextNode(node) {\r\n return {\r\n type: 'text',\r\n content: node.textContent\r\n };\r\n }\r\n \r\n /**\r\n * Serialize comment node\r\n */\r\n serializeCommentNode(node) {\r\n return {\r\n type: 'comment',\r\n content: node.textContent\r\n };\r\n }\r\n \r\n /**\r\n * Serialize document fragment\r\n */\r\n serializeDocumentFragment(fragment, depth) {\r\n const serialized = {\r\n type: 'fragment',\r\n children: []\r\n };\r\n \r\n for (const child of fragment.childNodes) {\r\n const serializedChild = this.serializeElement(child, depth + 1);\r\n if (serializedChild) {\r\n serialized.children.push(serializedChild);\r\n }\r\n }\r\n \r\n return serialized;\r\n }\r\n \r\n /**\r\n * Serialize Shadow DOM\r\n */\r\n serializeShadowRoot(shadowRoot, depth) {\r\n const serialized = {\r\n type: 'shadowRoot',\r\n mode: shadowRoot.mode,\r\n children: []\r\n };\r\n \r\n for (const child of shadowRoot.childNodes) {\r\n const serializedChild = this.serializeElement(child, depth + 1);\r\n if (serializedChild) {\r\n serialized.children.push(serializedChild);\r\n }\r\n }\r\n \r\n return serialized;\r\n }\r\n \r\n /**\r\n * Serialize frame/iframe elements\r\n */\r\n serializeFrameElement(frameElement) {\r\n const frameData = {\r\n src: frameElement.src,\r\n name: frameElement.name,\r\n id: frameElement.id,\r\n sandbox: frameElement.sandbox?.toString() || '',\r\n allowfullscreen: frameElement.allowFullscreen\r\n };\r\n \r\n // Try to access frame content (may fail due to CORS)\r\n try {\r\n const frameDoc = frameElement.contentDocument;\r\n if (frameDoc && this.options.includeFrames) {\r\n frameData.content = this.serialize(frameDoc);\r\n }\r\n } catch (error) {\r\n frameData.accessError = 'Cross-origin frame content not accessible';\r\n }\r\n \r\n return frameData;\r\n }\r\n \r\n /**\r\n * Serialize all frames in document\r\n */\r\n serializeFrames(doc) {\r\n const frames = [];\r\n const frameElements = doc.querySelectorAll('iframe, frame');\r\n \r\n for (const frameElement of frameElements) {\r\n try {\r\n const frameDoc = frameElement.contentDocument;\r\n if (frameDoc) {\r\n frames.push({\r\n element: this.serializeElement(frameElement),\r\n content: this.serialize(frameDoc)\r\n });\r\n }\r\n } catch (error) {\r\n frames.push({\r\n element: this.serializeElement(frameElement),\r\n error: 'Frame content not accessible'\r\n });\r\n }\r\n }\r\n \r\n return frames;\r\n }\r\n \r\n /**\r\n * Deserialize serialized DOM data back to DOM nodes\r\n */\r\n deserialize(serializedData, targetDocument = document) {\r\n try {\r\n if (serializedData.type === 'document') {\r\n return this.deserializeDocument(serializedData, targetDocument);\r\n } else {\r\n return this.deserializeElement(serializedData, targetDocument);\r\n }\r\n } catch (error) {\r\n console.error('Deserialization error:', error);\r\n throw new Error(`DOM deserialization failed: ${error.message}`);\r\n }\r\n }\r\n \r\n /**\r\n * Deserialize complete document\r\n */\r\n deserializeDocument(serializedDoc, targetDoc) {\r\n // Create new document if needed\r\n const doc = targetDoc || document.implementation.createHTMLDocument();\r\n \r\n // Set doctype if present\r\n if (serializedDoc.doctype) {\r\n const doctype = document.implementation.createDocumentType(\r\n serializedDoc.doctype.name,\r\n serializedDoc.doctype.publicId,\r\n serializedDoc.doctype.systemId\r\n );\r\n doc.replaceChild(doctype, doc.doctype);\r\n }\r\n \r\n // Deserialize document element\r\n if (serializedDoc.documentElement) {\r\n const newDocElement = this.deserializeElement(serializedDoc.documentElement, doc);\r\n doc.replaceChild(newDocElement, doc.documentElement);\r\n }\r\n \r\n // Handle metadata\r\n if (serializedDoc.metadata) {\r\n doc.title = serializedDoc.metadata.title || '';\r\n }\r\n \r\n return doc;\r\n }\r\n \r\n /**\r\n * Deserialize individual element\r\n */\r\n deserializeElement(serializedNode, doc) {\r\n switch (serializedNode.type) {\r\n case 'element':\r\n return this.deserializeElementNode(serializedNode, doc);\r\n case 'text':\r\n return doc.createTextNode(serializedNode.content);\r\n case 'comment':\r\n return doc.createComment(serializedNode.content);\r\n case 'fragment':\r\n return this.deserializeDocumentFragment(serializedNode, doc);\r\n case 'shadowRoot':\r\n // Shadow roots are handled during element creation\r\n return null;\r\n default:\r\n return null;\r\n }\r\n }\r\n \r\n /**\r\n * Deserialize element node\r\n */\r\n deserializeElementNode(serializedElement, doc) {\r\n const element = doc.createElement(serializedElement.tagName);\r\n \r\n // Set attributes\r\n if (serializedElement.attributes) {\r\n for (const [name, value] of Object.entries(serializedElement.attributes)) {\r\n try {\r\n element.setAttribute(name, value);\r\n } catch (error) {\r\n console.warn(`Failed to set attribute ${name}:`, error);\r\n }\r\n }\r\n }\r\n \r\n // Apply computed styles if available\r\n if (serializedElement.computedStyle && this.options.includeStyles) {\r\n for (const [prop, value] of Object.entries(serializedElement.computedStyle)) {\r\n try {\r\n element.style.setProperty(prop, value);\r\n } catch (error) {\r\n console.warn(`Failed to set style ${prop}:`, error);\r\n }\r\n }\r\n }\r\n \r\n // Create shadow root if present\r\n if (serializedElement.shadowRoot && element.attachShadow) {\r\n try {\r\n const shadowRoot = element.attachShadow({ \r\n mode: serializedElement.shadowRoot.mode || 'open' \r\n });\r\n \r\n // Deserialize shadow root children\r\n for (const child of serializedElement.shadowRoot.children) {\r\n const childElement = this.deserializeElement(child, doc);\r\n if (childElement) {\r\n shadowRoot.appendChild(childElement);\r\n }\r\n }\r\n } catch (error) {\r\n console.warn('Failed to create shadow root:', error);\r\n }\r\n }\r\n \r\n // Deserialize children\r\n if (serializedElement.children) {\r\n for (const child of serializedElement.children) {\r\n const childElement = this.deserializeElement(child, doc);\r\n if (childElement) {\r\n element.appendChild(childElement);\r\n }\r\n }\r\n }\r\n \r\n // Handle frame content\r\n if (serializedElement.frameData && serializedElement.frameData.content) {\r\n // Frame content deserialization would happen after the frame loads\r\n element.addEventListener('load', () => {\r\n try {\r\n const frameDoc = element.contentDocument;\r\n if (frameDoc) {\r\n this.deserializeDocument(serializedElement.frameData.content, frameDoc);\r\n }\r\n } catch (error) {\r\n console.warn('Failed to deserialize frame content:', error);\r\n }\r\n });\r\n }\r\n \r\n return element;\r\n }\r\n \r\n /**\r\n * Deserialize document fragment\r\n */\r\n deserializeDocumentFragment(serializedFragment, doc) {\r\n const fragment = doc.createDocumentFragment();\r\n \r\n if (serializedFragment.children) {\r\n for (const child of serializedFragment.children) {\r\n const childElement = this.deserializeElement(child, doc);\r\n if (childElement) {\r\n fragment.appendChild(childElement);\r\n }\r\n }\r\n }\r\n \r\n return fragment;\r\n }\r\n }\r\n \r\n // Usage example and utility functions\r\n class DOMUtils {\r\n /**\r\n * Create serializer with common presets\r\n */\r\n static createSerializer(preset = 'default') {\r\n const presets = {\r\n default: {\r\n includeStyles: true,\r\n includeScripts: false,\r\n includeFrames: true,\r\n includeShadowDOM: true\r\n },\r\n minimal: {\r\n includeStyles: false,\r\n includeScripts: false,\r\n includeFrames: false,\r\n includeShadowDOM: false\r\n },\r\n complete: {\r\n includeStyles: true,\r\n includeScripts: true,\r\n includeFrames: true,\r\n includeShadowDOM: true\r\n },\r\n secure: {\r\n includeStyles: true,\r\n includeScripts: false,\r\n includeFrames: false,\r\n includeShadowDOM: true\r\n }\r\n };\r\n \r\n return new DOMSerializer(presets[preset] || presets.default);\r\n }\r\n \r\n /**\r\n * Serialize DOM to JSON string\r\n */\r\n static serializeToJSON(element, options) {\r\n const serializer = new DOMSerializer(options);\r\n const serialized = serializer.serialize(element);\r\n return JSON.stringify(serialized, null, 2);\r\n }\r\n \r\n /**\r\n * Deserialize from JSON string\r\n */\r\n static deserializeFromJSON(jsonString, targetDocument) {\r\n const serialized = JSON.parse(jsonString);\r\n const serializer = new DOMSerializer();\r\n return serializer.deserialize(serialized, targetDocument);\r\n }\r\n \r\n /**\r\n * Clone DOM with full fidelity including Shadow DOM\r\n */\r\n static deepClone(element, options) {\r\n const serializer = new DOMSerializer(options);\r\n const serialized = serializer.serialize(element);\r\n return serializer.deserialize(serialized, element.ownerDocument);\r\n }\r\n \r\n /**\r\n * Compare two DOM structures\r\n */\r\n static compare(element1, element2, options) {\r\n const serializer = new DOMSerializer(options);\r\n const serialized1 = serializer.serialize(element1);\r\n const serialized2 = serializer.serialize(element2);\r\n \r\n return JSON.stringify(serialized1) === JSON.stringify(serialized2);\r\n }\r\n }\r\n \r\n /*\r\n // Export for use\r\n if (typeof module !== 'undefined' && module.exports) {\r\n module.exports = { DOMSerializer, DOMUtils };\r\n } else if (typeof window !== 'undefined') {\r\n window.DOMSerializer = DOMSerializer;\r\n window.DOMUtils = DOMUtils;\r\n }\r\n */\r\n\r\n /* Usage Examples:\r\n \r\n // Basic serialization\r\n const serializer = new DOMSerializer();\r\n const serialized = serializer.serialize(document);\r\n console.log(JSON.stringify(serialized, null, 2));\r\n \r\n // Deserialize back to DOM\r\n const clonedDoc = serializer.deserialize(serialized);\r\n \r\n // Using presets\r\n const minimalSerializer = DOMUtils.createSerializer('minimal');\r\n const secureSerializer = DOMUtils.createSerializer('secure');\r\n \r\n // Serialize specific element with Shadow DOM\r\n const customElement = document.querySelector('my-custom-element');\r\n const serializedElement = serializer.serialize(customElement);\r\n \r\n // JSON utilities\r\n const jsonString = DOMUtils.serializeToJSON(document.body);\r\n const restored = DOMUtils.deserializeFromJSON(jsonString);\r\n \r\n // Deep clone with Shadow DOM support\r\n const clone = DOMUtils.deepClone(document.body, { includeShadowDOM: true });\r\n \r\n */\r\n\r\n function serializeNodeToJSON(nodeElement) {\r\n return DOMUtils.serializeToJSON(nodeElement, {includeStyles: false});\r\n }\r\n\r\n function deserializeNodeFromJSON(jsonString) {\r\n return DOMUtils.deserializeFromJSON(jsonString);\r\n }\n\n function findDropdowns() {\r\n const dropdowns = [];\r\n \r\n // Native select elements\r\n dropdowns.push(...getAllElementsIncludingShadow('select'));\r\n \r\n // Elements with dropdown roles that don't have <input>..</input>\r\n const roleElements = getAllElementsIncludingShadow('[role=\"combobox\"], [role=\"listbox\"], [role=\"dropdown\"], [role=\"option\"], [role=\"menu\"], [role=\"menuitem\"]').filter(el => {\r\n return el.tagName.toLowerCase() !== 'input' || ![\"button\", \"checkbox\", \"radio\"].includes(el.getAttribute(\"type\"));\r\n });\r\n dropdowns.push(...roleElements);\r\n \r\n // Common dropdown class patterns\r\n const dropdownPattern = /.*(dropdown|select|combobox|menu).*/i;\r\n const elements = getAllElementsIncludingShadow('*');\r\n const dropdownClasses = Array.from(elements).filter(el => {\r\n const hasDropdownClass = dropdownPattern.test(el.className);\r\n const validTag = ['li', 'ul', 'span', 'div', 'p', 'a', 'button'].includes(el.tagName.toLowerCase());\r\n const style = window.getComputedStyle(el); \r\n const result = hasDropdownClass && validTag && (style.cursor === 'pointer' || el.tagName.toLowerCase() === 'a' || el.tagName.toLowerCase() === 'button');\r\n return result;\r\n });\r\n \r\n dropdowns.push(...dropdownClasses);\r\n \r\n // Elements with aria-haspopup attribute\r\n dropdowns.push(...getAllElementsIncludingShadow('[aria-haspopup=\"true\"], [aria-haspopup=\"listbox\"], [aria-haspopup=\"menu\"]'));\r\n\r\n // Improve navigation element detection\r\n // Semantic nav elements with list items\r\n dropdowns.push(...getAllElementsIncludingShadow('nav ul li, nav ol li'));\r\n \r\n // Navigation elements in common design patterns\r\n dropdowns.push(...getAllElementsIncludingShadow('header a, .header a, .nav a, .navigation a, .menu a, .sidebar a, aside a'));\r\n \r\n // Elements in primary navigation areas with common attributes\r\n dropdowns.push(...getAllElementsIncludingShadow('[role=\"navigation\"] a, [aria-label*=\"navigation\"] a, [aria-label*=\"menu\"] a'));\r\n\r\n return dropdowns;\r\n }\r\n\r\n function findClickables() {\r\n const clickables = [];\r\n \r\n const checkboxPattern = /checkbox/i;\r\n // Collect all clickable elements first\r\n const nativeLinks = [...getAllElementsIncludingShadow('a')];\r\n const nativeButtons = [...getAllElementsIncludingShadow('button')];\r\n const inputButtons = [...getAllElementsIncludingShadow('input[type=\"button\"], input[type=\"submit\"], input[type=\"reset\"]')];\r\n const roleButtons = [...getAllElementsIncludingShadow('[role=\"button\"]')];\r\n // const tabbable = [...getAllElementsIncludingShadow('[tabindex=\"0\"]')];\r\n const clickHandlers = [...getAllElementsIncludingShadow('[onclick]')];\r\n const dropdowns = findDropdowns();\r\n const nativeCheckboxes = [...getAllElementsIncludingShadow('input[type=\"checkbox\"]')]; \r\n const fauxCheckboxes = getAllElementsIncludingShadow('*').filter(el => {\r\n if (checkboxPattern.test(el.className)) {\r\n const realCheckboxes = getAllElementsIncludingShadow('input[type=\"checkbox\"]', el);\r\n if (realCheckboxes.length === 1) {\r\n const boundingRect = realCheckboxes[0].getBoundingClientRect();\r\n return boundingRect.width <= 1 && boundingRect.height <= 1 \r\n }\r\n }\r\n return false;\r\n });\r\n const nativeRadios = [...getAllElementsIncludingShadow('input[type=\"radio\"]')];\r\n const toggles = findToggles();\r\n const pointerElements = findElementsWithPointer();\r\n // Add all elements at once\r\n clickables.push(\r\n ...nativeLinks,\r\n ...nativeButtons,\r\n ...inputButtons,\r\n ...roleButtons,\r\n // ...tabbable,\r\n ...clickHandlers,\r\n ...dropdowns,\r\n ...nativeCheckboxes,\r\n ...fauxCheckboxes,\r\n ...nativeRadios,\r\n ...toggles,\r\n ...pointerElements\r\n );\r\n\r\n // Only uniquify once at the end\r\n return clickables; // Let findElements handle the uniquification\r\n }\r\n\r\n function findToggles() {\r\n const toggles = [];\r\n const checkboxes = getAllElementsIncludingShadow('input[type=\"checkbox\"]');\r\n const togglePattern = /switch|toggle|slider/i;\r\n\r\n checkboxes.forEach(checkbox => {\r\n let isToggle = false;\r\n\r\n // Check the checkbox itself\r\n if (togglePattern.test(checkbox.className) || togglePattern.test(checkbox.getAttribute('role') || '')) {\r\n isToggle = true;\r\n }\r\n\r\n // Check parent elements (up to 3 levels)\r\n if (!isToggle) {\r\n let element = checkbox;\r\n for (let i = 0; i < 3; i++) {\r\n const parent = element.parentElement;\r\n if (!parent) break;\r\n\r\n const className = parent.className || '';\r\n const role = parent.getAttribute('role') || '';\r\n\r\n if (togglePattern.test(className) || togglePattern.test(role)) {\r\n isToggle = true;\r\n break;\r\n }\r\n element = parent;\r\n }\r\n }\r\n\r\n // Check next sibling\r\n if (!isToggle) {\r\n const nextSibling = checkbox.nextElementSibling;\r\n if (nextSibling) {\r\n const className = nextSibling.className || '';\r\n const role = nextSibling.getAttribute('role') || '';\r\n if (togglePattern.test(className) || togglePattern.test(role)) {\r\n isToggle = true;\r\n }\r\n }\r\n }\r\n\r\n if (isToggle) {\r\n toggles.push(checkbox);\r\n }\r\n });\r\n\r\n return toggles;\r\n }\r\n\r\n function findNonInteractiveElements() {\r\n // Get all elements in the document\r\n const all = Array.from(getAllElementsIncludingShadow('*'));\r\n \r\n // Filter elements based on Python implementation rules\r\n return all.filter(element => {\r\n if (!element.firstElementChild) {\r\n const tag = element.tagName.toLowerCase(); \r\n if (!['select', 'button', 'a'].includes(tag)) {\r\n const validTags = ['p', 'span', 'div', 'input', 'textarea','td','th'].includes(tag) || /^h\\d$/.test(tag) || /text/.test(tag);\r\n const boundingRect = element.getBoundingClientRect();\r\n return validTags && boundingRect.height > 1 && boundingRect.width > 1;\r\n }\r\n }\r\n return false;\r\n });\r\n }\r\n\r\n\r\n\r\n function findElementsWithPointer() {\r\n const elements = [];\r\n const allElements = getAllElementsIncludingShadow('*');\r\n \r\n highlighterLogger.debug('Checking elements with pointer style...');\r\n \r\n allElements.forEach(element => {\r\n // Skip SVG elements for now\r\n if (element instanceof SVGElement || element.tagName.toLowerCase() === 'svg') {\r\n return;\r\n }\r\n \r\n const style = window.getComputedStyle(element);\r\n if (style.cursor === 'pointer') {\r\n elements.push(element);\r\n }\r\n });\r\n \r\n highlighterLogger.debug(`Found ${elements.length} elements with pointer cursor`);\r\n return elements;\r\n }\r\n\r\n function findCheckables() {\r\n const elements = [];\r\n\r\n elements.push(...getAllElementsIncludingShadow('input[type=\"checkbox\"]'));\r\n elements.push(...getAllElementsIncludingShadow('input[type=\"radio\"]'));\r\n const all_elements = getAllElementsIncludingShadow('label');\r\n const radioClasses = Array.from(all_elements).filter(el => {\r\n return /.*radio.*/i.test(el.className); \r\n });\r\n elements.push(...radioClasses);\r\n return elements;\r\n }\r\n\r\n function findFillables() {\r\n const elements = [];\r\n\r\n const inputs = [...getAllElementsIncludingShadow('input:not([type=\"radio\"]):not([type=\"checkbox\"])')];\r\n highlighterLogger.debug('Found inputs:', inputs.length, inputs);\r\n elements.push(...inputs);\r\n \r\n const textareas = [...getAllElementsIncludingShadow('textarea')];\r\n highlighterLogger.debug('Found textareas:', textareas.length);\r\n elements.push(...textareas);\r\n \r\n const editables = [...getAllElementsIncludingShadow('[contenteditable=\"true\"]')];\r\n highlighterLogger.debug('Found editables:', editables.length);\r\n elements.push(...editables);\r\n\r\n return elements;\r\n }\n\n // maximum number of characters in a logical name\r\n const MAX_LOGICAL_NAME_LENGTH = 25;\r\n \r\n /**\r\n * Query elements by CSS selector with :has-text(\"...\") pseudo-selector\r\n * @param {string} selector - CSS selector with optional :has-text(\"...\") pseudo-selector\r\n * @param {Element|Document} context - Context element to query within (defaults to document)\r\n * @returns {Element[]} Array of matched elements\r\n * @example\r\n * // Find all buttons with text \"Submit\"\r\n * querySelectorAllByText('button:has-text(\"Submit\")')\r\n * \r\n * // Find element with specific role and text\r\n * querySelectorAllByText('[role=\"button\"]:has-text(\"Click me\")')\r\n * \r\n * // Complex selector with descendants\r\n * querySelectorAllByText('button:has-text(\"foo\") > div > ul')\r\n * \r\n * // Multiple components\r\n * querySelectorAllByText('.container > button:has-text(\"Submit\")')\r\n * \r\n * // Multiple text filters\r\n * querySelectorAllByText('button:has-text(\"foo\") > div:has-text(\"bar\") > span')\r\n */\r\n function querySelectorAllByText(selector, context = document) {\r\n \t// Local helper function to compare two strings by normalizing whitespace and ignoring case\r\n \t// mimic the behavior of Playwright's :has-text() selector modifier\r\n \tfunction pwHasText(str1, str2) {\r\n \t\tif (!str1 || !str2) return false;\r\n \t\t\r\n \t\t// Normalize whitespace by replacing multiple spaces with single space and trim\r\n \t\tconst normalize = (str) => cleanText(str.toLowerCase());\r\n \t\t\r\n \t\tconst normalized1 = normalize(str1);\r\n \t\tconst normalized2 = normalize(str2);\r\n \t\t\r\n \t\treturn normalized1.includes(normalized2);\r\n \t}\r\n \t\r\n \t// Check if selector contains :has-text() pseudo-selector\r\n \tif (!selector.includes(':has-text(')) {\r\n \t\t// No :has-text pseudo-selector, use regular querySelectorAll\r\n \t\treturn Array.from(robustQuerySelector(selector, context, true));\r\n \t}\r\n \t\r\n \t// Split selector by combinators while preserving them\r\n \t// Matches: >, +, ~, or space (descendant combinator)\r\n \tconst parts = [];\r\n \tlet currentPart = '';\r\n \tlet inBracket = false;\r\n \tlet inTextPseudo = false;\r\n \tlet inTextQuotes = false;\r\n \tlet i = 0;\r\n \t\r\n \twhile (i < selector.length) {\r\n \t\tconst char = selector[i];\r\n \t\t\r\n \t\t// Track :has-text(\"...\") to avoid splitting inside it\r\n \t\tif (selector.substring(i, i + 10) === ':has-text(') {\r\n \t\t\tinTextPseudo = true;\r\n \t\t\tinTextQuotes = false;\r\n \t\t}\r\n \t\tif (inTextPseudo && char === '\"' && !inTextQuotes) {\r\n \t\t\tinTextQuotes = true;\r\n \t\t} else if (inTextPseudo && char === '\"' && inTextQuotes) {\r\n \t\t\t// Check if this quote is escaped\r\n \t\t\tif (i > 0 && selector[i - 1] === '\\\\') ; else {\r\n \t\t\t\tinTextQuotes = false;\r\n \t\t\t}\r\n \t\t}\r\n \t\tif (inTextPseudo && char === ')' && !inTextQuotes) {\r\n \t\t\tinTextPseudo = false;\r\n \t\t\tcurrentPart += char;\r\n \t\t\ti++;\r\n \t\t\tcontinue;\r\n \t\t}\r\n \t\t\r\n \t\t// Track brackets to avoid splitting on combinators inside attribute selectors\r\n \t\tif (char === '[') inBracket = true;\r\n \t\tif (char === ']') inBracket = false;\r\n \t\t\r\n \t\t// Check for combinators (not inside brackets or :has-text())\r\n \t\tif (!inBracket && !inTextPseudo && (char === '>' || char === '+' || char === '~')) {\r\n \t\t\tif (currentPart.trim()) parts.push({ selector: currentPart.trim(), combinator: null });\r\n \t\t\tparts.push({ selector: null, combinator: char.trim() });\r\n \t\t\tcurrentPart = '';\r\n \t\t\ti++;\r\n \t\t\tcontinue;\r\n \t\t}\r\n \t\t\r\n \t\t// Check for space combinator (descendant)\r\n \t\tif (!inBracket && !inTextPseudo && char === ' ') {\r\n \t\t\t// Skip multiple spaces\r\n \t\t\twhile (i < selector.length && selector[i] === ' ') i++;\r\n \t\t\t// Check if next char is a combinator (>, +, ~)\r\n \t\t\tif (i < selector.length && !'><+~'.includes(selector[i])) {\r\n \t\t\t\tif (currentPart.trim()) {\r\n \t\t\t\t\tparts.push({ selector: currentPart.trim(), combinator: null });\r\n \t\t\t\t\tparts.push({ selector: null, combinator: ' ' });\r\n \t\t\t\t\tcurrentPart = '';\r\n \t\t\t\t}\r\n \t\t\t}\r\n \t\t\tcontinue;\r\n \t\t}\r\n \t\t\r\n \t\tcurrentPart += char;\r\n \t\ti++;\r\n \t}\r\n \tif (currentPart.trim()) parts.push({ selector: currentPart.trim(), combinator: null });\r\n \t\r\n \t// Filter out combinator-only entries and rebuild as selector+combinator pairs\r\n \tconst selectorParts = [];\r\n \tfor (let j = 0; j < parts.length; j++) {\r\n \t\tif (parts[j].selector) {\r\n \t\t\tconst combinator = (j + 1 < parts.length && parts[j + 1].combinator) ? parts[j + 1].combinator : null;\r\n \t\t\t// Extract text value if present\r\n \t\t\tconst textMatch = parts[j].selector.match(/:has-text\\(\"((?:[^\"\\\\]|\\\\.)*)\"\\)/);\r\n \t\t\tconst textValue = textMatch ? textMatch[1] : null;\r\n \t\t\tconst selectorWithoutText = textMatch ? parts[j].selector.replace(/:has-text\\(\"((?:[^\"\\\\]|\\\\.)*)\"\\)/, '') : parts[j].selector;\r\n \t\t\t\r\n \t\t\tselectorParts.push({ \r\n \t\t\t\tselector: parts[j].selector,\r\n \t\t\t\tselectorWithoutText,\r\n \t\t\t\ttextValue,\r\n \t\t\t\tcombinator \r\n \t\t\t});\r\n \t\t}\r\n \t}\r\n \t\r\n \t// Process selector parts sequentially, applying text filters as we go\r\n \tlet currentElements = [context];\r\n \t\r\n \tfor (let partIndex = 0; partIndex < selectorParts.length; partIndex++) {\r\n \t\tconst part = selectorParts[partIndex];\r\n \t\tconst nextElements = [];\r\n \t\t\r\n \t\tfor (const ctx of currentElements) {\r\n \t\t\tlet candidates;\r\n \t\t\t\r\n \t\t\tif (partIndex === 0 && ctx === context) {\r\n \t\t\t\t// First part, query from document/context\r\n \t\t\t\tconst querySelector = part.selectorWithoutText.trim() || '*';\r\n \t\t\t\tcandidates = Array.from(robustQuerySelector(querySelector, ctx, true));\r\n \t\t\t\thighlighterLogger.debug(`querySelectorAllByText: first part, querySelector=${querySelector}, candidates=`, candidates);\r\n \t\t\t} \r\n \t\t\telse {\r\n \t\t\t\t// Subsequent parts, use combinator\r\n \t\t\t\tconst prevPart = selectorParts[partIndex - 1];\r\n \t\t\t\tconst combinator = prevPart.combinator;\r\n \t\t\t\tconst querySelector = part.selectorWithoutText.trim() || '*';\r\n \t\t\t\t\r\n \t\t\t\tif (combinator === '>') {\r\n \t\t\t\t\t// Direct children\r\n \t\t\t\t\tcandidates = Array.from(ctx.children).filter(child => {\r\n \t\t\t\t\t\treturn child.matches(querySelector);\r\n \t\t\t\t\t});\r\n \t\t\t\t} else if (combinator === '+') {\r\n \t\t\t\t\t// Next sibling\r\n \t\t\t\t\tconst sibling = ctx.nextElementSibling;\r\n \t\t\t\t\tcandidates = (sibling && sibling.matches(querySelector)) ? [sibling] : [];\r\n \t\t\t\t} else if (combinator === '~') {\r\n \t\t\t\t\t// Following siblings\r\n \t\t\t\t\tcandidates = [];\r\n \t\t\t\t\tlet sibling = ctx.nextElementSibling;\r\n \t\t\t\t\twhile (sibling) {\r\n \t\t\t\t\t\tif (sibling.matches(querySelector)) {\r\n \t\t\t\t\t\t\tcandidates.push(sibling);\r\n \t\t\t\t\t\t}\r\n \t\t\t\t\t\tsibling = sibling.nextElementSibling;\r\n \t\t\t\t\t}\r\n \t\t\t\t} \r\n \t\t\t\telse {\r\n \t\t\t\t\t// Descendant (space)\r\n \t\t\t\t\tcandidates = Array.from(robustQuerySelector(querySelector, ctx, true));\r\n \t\t\t\t}\r\n \t\t\t}\r\n \t\t\t\r\n \t\t\t// Apply text filter if present\r\n \t\t\tif (part.textValue !== null) {\r\n \t\t\t\tcandidates = candidates.filter(el => pwHasText(getElementText(el), part.textValue));\r\n \t\t\t}\r\n \t\t\t\r\n \t\t\tnextElements.push(...candidates);\r\n \t\t}\r\n \t\t\r\n \t\tcurrentElements = nextElements;\r\n \t\t\r\n \t\t// If no matches, bail early\r\n \t\tif (currentElements.length === 0) {\r\n \t\t\treturn [];\r\n \t\t}\r\n \t}\r\n \t\r\n \treturn currentElements;\r\n }\r\n\r\n const cleanText = (text) => {\r\n \treturn text.replace(/\\s+/g, ' ').trim();\r\n };\r\n\r\n // Helper function to get name selector\r\n const getNameSelector = (element) => {\r\n \tconst text = cleanText(getElementText(element));\r\n \tconst title = element.getAttribute('title');\r\n \tconst name = element.getAttribute('name');\r\n \tconst ariaLabel = element.getAttribute('aria-label');\r\n \tif (!text && !title && !name && !ariaLabel) throw new Error('No name attribute provided');\r\n \tif (text) return `:has-text(${doubleQuoteString(text)})`;\r\n \tif (title) return `[title=${doubleQuoteString(title)}]`;\r\n \tif (ariaLabel) return `[aria-label=${doubleQuoteString(ariaLabel)}]`;\r\n \treturn `[name=${doubleQuoteString(name)}]`; \r\n };\r\n\r\n const getLogicalName = (element) => {\r\n \tconst text = cleanText(getElementText(element));\r\n \tconst title = element.getAttribute('title');\r\n \tconst name = element.getAttribute('name');\r\n \tconst ariaLabel = element.getAttribute('aria-label');\r\n \tconst role = element.getAttribute('role');\r\n \tconst logicalName = text || title || name || ariaLabel;\r\n \treturn {\r\n \t\thasLogicalName: logicalName && logicalName.length < MAX_LOGICAL_NAME_LENGTH && !role?.startsWith('row'),\r\n \t\tlogicalName: logicalName\r\n \t};\r\n };\r\n\r\n\r\n /**\r\n * Individual strategy functions\r\n */\r\n\r\n function tryRoleStrategy(element) { \r\n \tconst role = element.getAttribute('role');\r\n \tconst tag = element.tagName.toLowerCase();\r\n \tconst { hasLogicalName, logicalName } = getLogicalName(element);\r\n \tif (role && hasLogicalName) {\r\n \t\treturn { \r\n \t\t\tstrategy: 'role', \r\n \t\t\tselector: `[role=${doubleQuoteString(role)}]${getNameSelector(element)}`\r\n \t\t};\r\n \t}\r\n \t\r\n \t// Tag-based role (button, a, input, etc.) + name\r\n \tif (['button', 'a', 'input', 'textarea', 'select'].includes(tag) && hasLogicalName) {\r\n \t\treturn { \r\n \t\t\tstrategy: 'role', \r\n \t\t\tselector: `${tag}${getNameSelector(element)}`\r\n \t\t};\r\n \t}\r\n \t\r\n \treturn null;\r\n }\r\n\r\n function tryPlaceholderStrategy(element) {\r\n \tconst placeholder = element.getAttribute('placeholder');\r\n \tif (placeholder) {\r\n \t\treturn { \r\n \t\t\tstrategy: 'placeholder', \r\n \t\t\tselector: `[placeholder=${doubleQuoteString(placeholder)}]` \r\n \t\t};\r\n \t}\r\n \treturn null;\r\n }\r\n\r\n function tryLabelStrategy(element) {\r\n \tconst tag = element.tagName.toLowerCase();\r\n \tif (['input', 'textarea', 'select'].includes(tag)) {\r\n \t\tconst id = element.id;\r\n \t\tif (id) {\r\n \t\t\tconst labels = robustQuerySelector(`label[for=${doubleQuoteString(id)}]`, document, true);\r\n \t\t\tif (labels.length === 1) { //make sure id is unique\r\n \t\t\t\tconst labelText = cleanText(getElementText(labels[0]));\r\n \t\t\t\tif (labelText) {\r\n \t\t\t\t\tconst labelSelector = `label:has-text(${doubleQuoteString(labelText)})`;\r\n \t\t\t\t\ttry {\r\n \t\t\t\t\t\tconst els = querySelectorAllByText(labelSelector, document);\r\n \t\t\t\t\t\tif (els.length === 1) {\r\n \t\t\t\t\t\t\treturn { \r\n \t\t\t\t\t\t\t\tstrategy: 'label for',\r\n \t\t\t\t\t\t\t\tlabel_selector: labelSelector\r\n \t\t\t\t\t\t\t};\r\n \t\t\t\t\t\t}\r\n \t\t\t\t\t} catch (error) {\r\n \t\t\t\t\t\thighlighterLogger.error(`internal error: while querying label selector ${labelSelector} for element ${element.outerHTML}`, error);\r\n \t\t\t\t\t\tthrow error;\r\n \t\t\t\t\t}\r\n \t\t\t\t}\r\n \t\t\t}\r\n \t\t}\r\n \t}\r\n \treturn null;\r\n }\r\n\r\n function tryAriaLabelledByStrategy(element) {\r\n \tconst arialabeledBy = element.getAttribute('aria-labelledby');\r\n \tconst tag = element.tagName.toLowerCase();\r\n \t\r\n \tif (arialabeledBy) {\r\n \t\tconst labels = robustQuerySelector(`#${CSS.escape(arialabeledBy)}`, document, true);\r\n \t\tif (labels.length === 1) {\r\n \t\t\tconst labelText = cleanText(getElementText(labels[0]));\r\n \t\t\tif (labelText) { //verify the label text is unique\r\n \t\t\t\tconst labelSelector = `${tag}:has-text(${doubleQuoteString(labelText)})`;\r\n \t\t\t\ttry {\r\n \t\t\t\t\tconst els = querySelectorAllByText(labelSelector, document);\r\n \t\t\t\t\tif (els.length === 1) {\r\n \t\t\t\t\t\treturn { \r\n \t\t\t\t\t\t\tstrategy: 'label by', \r\n \t\t\t\t\t\t\tlabel_selector: labelSelector\r\n \t\t\t\t\t\t};\r\n \t\t\t\t\t}\r\n \t\t\t\t} catch (error) {\r\n \t\t\t\t\thighlighterLogger.error(`internal error: while querying aria-labelledby selector ${labelSelector} for element ${element.outerHTML}`, error);\r\n \t\t\t\t\tthrow error;\r\n \t\t\t\t}\r\n \t\t\t}\r\n \t\t}\r\n \t}\r\n \treturn null;\r\n }\r\n\r\n // Only use text strategy if element is clickable/interactive \r\n function tryNameStrategy(element) {\t\r\n \tconst { hasLogicalName } = getLogicalName(element);\r\n \tconst classes = Array.from(element.classList).map(cls => CSS.escape(cls));\r\n \t\r\n \t// Only use name strategy if element is clickable/interactive \r\n \tif (isClickableElement(element) && hasLogicalName && classes.length > 0) {\r\n \t\treturn { \r\n \t\t\tstrategy: 'name', \r\n \t\t\tselector: `.${classes.join('.')}${getNameSelector(element)}` \r\n \t\t\t// selector: `${element.tagName.toLowerCase()}${getNameSelector(element)}`\r\n \t\t};\r\n \t}\r\n \treturn null;\r\n }\r\n\r\n function tryDataTestIdStrategy(element) {\r\n \tconst dataTestId = element.getAttribute('data-testid');\r\n \tif (dataTestId) {\r\n \t\treturn { \r\n \t\t\tstrategy: 'data-testid',\t\t\t\r\n \t\t\tselector: `[data-testid=${doubleQuoteString(dataTestId)}]` \r\n \t\t};\r\n \t}\r\n \treturn null;\r\n }\r\n\r\n function tryIdStrategy(element) {\r\n \tconst id = element.getAttribute('id');\r\n \tif (id && !/:|\\s|\\\\/.test(id)) { //avoid using id if it contain unusual characters (react creates ids like this: \":r:42\")\r\n \t\treturn { \r\n \t\t\tstrategy: 'id', \r\n \t\t\tselector: `#${CSS.escape(id)}` \r\n \t\t};\r\n \t}\r\n \treturn null;\r\n }\r\n\r\n // Try all strategies on an element in order of precedence\r\n function attemptStrategiesOnElement(element) {\r\n \tconst strategies = [\r\n \t\ttryRoleStrategy, \t\t\r\n \t\ttryAriaLabelledByStrategy,\r\n \t\ttryNameStrategy,\r\n \t\ttryLabelStrategy,\r\n \t\ttryDataTestIdStrategy,\r\n \t\ttryPlaceholderStrategy,\r\n \t\ttryIdStrategy,\t\t\r\n \t\t// tryClassStrategy\r\n \t];\r\n \t\r\n \tfor (const strategy of strategies) {\r\n \t\tconst result = strategy(element); \r\n \t\tif (result) { \r\n \t\t\treturn result;\r\n \t\t}\r\n \t}\r\n \treturn null;\r\n }\r\n\r\n /**\r\n * Generate a smart selector for an element\r\n * @param {Element} element - The element to generate a locator for\r\n * @param {string} childPath - The child path to generate a locator for\r\n * @returns {Object} The selector for the element\r\n */\r\n function generateSmartSelector(element, childPath = '') {\r\n \t// Terminate recursion\r\n \tif (!element || element.nodeType !== Node.ELEMENT_NODE) {\r\n \t\treturn {\r\n \t\t\tstrategy: 'css',\r\n \t\t\tselector: childPath\r\n \t\t};\r\n \t}\r\n \t\r\n \t// Try strategies 1-6 on current element\r\n \tconst result = attemptStrategiesOnElement(element);\r\n \tconst is_label = result?.strategy === 'label for' || result?.strategy === 'label by';\r\n\r\n \tif (result) { // found a strategy\t\t\t\r\n \t\t// If we have a child path, concatenate parent strategy result with child CSS path \r\n \t\tlet extendedResult = result; \r\n \t\tif (childPath) {\r\n \t\t\tconst selector = is_label ? `${childPath}` : `${result?.selector} > ${childPath}`;\r\n \t\t\textendedResult = {\r\n \t\t\t\t...extendedResult,\r\n \t\t\t\tselector: selector,\r\n \t\t\t\tstrategy: `${extendedResult.strategy} (concatenated with child CSS path)`\r\n \t\t\t};\r\n \t\t} \t\t\r\n \t\t// check if the combined selector is unique and if not add the index to the strategy\r\n \t\tif (!is_label) { //label element is already checked for uniqueness\r\n \t\t\ttry {\r\n \t\t\t\tconst matchedElements = querySelectorAllByText(result?.selector); \r\n \t\t\t\thighlighterLogger.debug(`querySelectorAllByText(${result?.selector}) matched ${matchedElements.length} elements`);\r\n \t\t\t\thighlighterLogger.debug(matchedElements);\r\n \t\t\t\tif (matchedElements.length > 1) {\r\n \t\t\t\t\tconst index = matchedElements.findIndex(el => el === element);\r\n \t\t\t\t\tif (index === -1) {\r\n \t\t\t\t\t\thighlighterLogger.error('internal error: Element not found in matched elements', element, result?.selector, matchedElements);\r\n \t\t\t\t\t\treturn null;\r\n \t\t\t\t\t}\r\n \t\t\t\t\textendedResult = {\r\n \t\t\t\t\t\t...extendedResult,\r\n \t\t\t\t\t\tstrategy: `${extendedResult.strategy} (by index)`,\r\n \t\t\t\t\t\tindex,\r\n \t\t\t\t\t}; \r\n \t\t\t\t}\r\n \t\t\t} catch (error) {\r\n \t\t\t\thighlighterLogger.error(`internal error: while checking if selector ${result?.selector} is unique for element ${element.outerHTML}`, error);\r\n \t\t\t\tthrow error;\r\n \t\t\t}\t\t\t\r\n \t\t}\r\n \t\treturn extendedResult;\r\n \t}\r\n \t\r\n \t// Get CSS selector for current element and build child path \r\n \tlet newChildPath;\r\n \t// didn't find strategy\r\n \tconst elementSelector = getElementCssSelector(element);\r\n \tnewChildPath = childPath ? `${elementSelector} > ${childPath}` : elementSelector;\r\n \tconst matchedElements = robustQuerySelector(newChildPath, document, true);\r\n \tif (matchedElements.length === 1) {\r\n \t\treturn {\r\n \t\t\tstrategy: 'css',\t\t\t\t\r\n \t\t\tselector: newChildPath\r\n \t\t};\r\n \t}\r\n\r\n \t// Recursively try parent element\r\n \treturn generateSmartSelector(getParentNode(element), newChildPath);\r\n }\r\n\r\n const findElementByInfo = (elementInfo, useSmartSelectors = false) => {\r\n \thighlighterLogger.debug('findElementByInfo:', elementInfo);\r\n \tif (useSmartSelectors) {\r\n \t return findElementBySelectors(elementInfo.smart_iframe_selector, elementInfo.smart_selector, true);\r\n \t}\r\n \treturn findElementBySelectors(elementInfo.iframe_selector, elementInfo.css_selector, false);\r\n };\r\n\r\n const compareSelectors = (selector1, selector2, useSmartSelectors = false) => {\r\n \tif (useSmartSelectors) {\r\n \t\treturn JSON.stringify(selector1) === JSON.stringify(selector2);\r\n \t}\r\n \treturn selector1 === selector2;\r\n };\r\n\r\n const findElementBySelectors = (iframeSelector, elementSelector, useSmartSelectors = false) => {\r\n \tlet element;\r\n \t\r\n \t// first find the iframe that contains the element\r\n \tif (iframeSelector) { \r\n \t const frames = getAllDocumentElementsIncludingShadow('iframe', document);\r\n \t \r\n \t // Iterate over all frames and compare their CSS selectors\r\n \t for (const frame of frames) {\r\n \t\tconst selector = useSmartSelectors ? generateSmartSelector(frame) : generateCssPath(frame);\r\n \t\tif (compareSelectors(selector, iframeSelector, useSmartSelectors)) {\r\n \t\t const frameDocument = frame.contentDocument || frame.contentWindow.document;\r\n \t\t element = useSmartSelectors ? findElementBySmartSelector(elementSelector, frameDocument) : robustQuerySelector(elementSelector, frameDocument);\t\t \r\n \t\t break;\r\n \t\t} \r\n \t }\t}\r\n \telse {\t\t\r\n \t\telement = useSmartSelectors ? findElementBySmartSelector(elementSelector, document) : robustQuerySelector(elementSelector, document);\t\t\r\n \t}\r\n \t \r\n \tif (element) {\r\n \t\thighlighterLogger.debug('findElementBySelectors: found element ', element);\r\n \t}\r\n \telse {\r\n \t highlighterLogger.warn('findElementBySelectors: failed to find element with CSS selector:', JSON.stringify(elementSelector));\r\n \t}\r\n \r\n \treturn element;\r\n };\r\n \r\n function findElementBySmartSelector(smartSelector, root = document) {\t\r\n \thighlighterLogger.debug(`findElementBySmartSelector: called with smartSelector=${JSON.stringify(smartSelector)}`);\r\n \tlet combinedSelector;\r\n \tconst mainStrategy = smartSelector.strategy.split(' ')[0]; //extract the main strategy from the full strategy string\r\n \tswitch(mainStrategy) {\r\n case 'role': \r\n case 'placeholder':\r\n case 'name':\r\n case 'data-testid':\r\n case 'id':\r\n case 'css':\r\n case 'class':\r\n \t\t highlighterLogger.debug(`findElementBySmartSelector: strategy='${smartSelector.strategy}', selector=${smartSelector?.selector}, index=${smartSelector?.index}`);\r\n if (typeof smartSelector?.index === 'number') {\r\n return querySelectorAllByText(smartSelector.selector, root)[smartSelector.index];\r\n }\r\n else {\r\n return querySelectorAllByText(smartSelector.selector, root)[0];\r\n }\t\t \r\n case 'label':\r\n \t\t\tif (smartSelector.strategy === 'label for') {\r\n \t\t\t\thighlighterLogger.debug(`findElementBySmartSelector: strategy='label for', label_selector=${smartSelector.label_selector}`);\r\n \t\t\t\tconst label_for = querySelectorAllByText(smartSelector.label_selector, root)[0];\r\n \t\t\t\tcombinedSelector = smartSelector?.selector ? `#${label_for.getAttribute('for')} > ${smartSelector.selector}` : `#${label_for.getAttribute('for')}`;\r\n \t\t\t\treturn robustQuerySelector(combinedSelector, root);\r\n \t\t\t}\r\n \t\t\telse if (smartSelector.strategy === 'label by') {\r\n \t\t\t\thighlighterLogger.debug(`findElementBySmartSelector: strategy='label by', label_selector=${smartSelector.label_selector}`);\r\n \t\t\t\tconst label_by = querySelectorAllByText(smartSelector.label_selector, root)[0];\r\n \t\t\t\tcombinedSelector = smartSelector?.selector ? `[aria-labelledby=\"${label_by.getAttribute('id')}\"] > ${smartSelector.selector}` : `[aria-labelledby=\"${label_by.getAttribute('id')}\"]`;\r\n \t\t\t\treturn robustQuerySelector(combinedSelector, root);\r\n \t\t\t}\r\n \t\t\telse {\r\n \t\t\t\thighlighterLogger.error(`findElementBySmartSelector: unsupported label strategy: ${smartSelector.strategy}`);\r\n \t\t\t\treturn null;\r\n \t\t\t} \r\n default:\r\n highlighterLogger.error(`findElementBySmartSelector: Unsupported smart selector strategy: ${smartSelector.strategy}`);\r\n return null;\r\n }\r\n }\r\n\r\n function getElementInfo(element, index) { \t\t\t\r\n \tconst iframe = getContainingIFrame(element); \r\n \tconst iframe_selector = iframe ? generateCssPath(iframe) : \"\";\t\r\n \tconst smart_iframe_selector = iframe ? generateSmartSelector(iframe) : null;\r\n \t\r\n \t// Return element info with pre-calculated values\r\n \treturn {\r\n \t\tindex: index ?? -1,\r\n \t\ttag: element.tagName.toLowerCase(),\r\n \t\ttype: element.type || '',\r\n \t\ttext: getElementText(element) || getElementPlaceholder(element),\r\n \t\thtml: cleanHTML(element.outerHTML),\r\n \t\txpath: generateXPath(element),\r\n \t\tcss_selector: generateCssPath(element),\r\n \t\tbounding_box: element.getBoundingClientRect(),\t\t\r\n \t\tiframe_selector: iframe_selector,\t\t\r\n \t\tsmart_selector: generateSmartSelector(element),\r\n \t\tsmart_iframe_selector: smart_iframe_selector,\r\n \t\telement: element,\r\n \t\tdepth: getElementDepth(element)\r\n \t};\r\n }\n\n /**\r\n * Checks if a point is inside a bounding box\r\n * \r\n * @param point The point to check\r\n * @param box The bounding box\r\n * @returns boolean indicating if the point is inside the box\r\n */\r\n function isPointInsideBox(point, box) {\r\n return point.x >= box.x &&\r\n point.x <= box.x + box.width &&\r\n point.y >= box.y &&\r\n point.y <= box.y + box.height;\r\n }\r\n\r\n /**\r\n * Calculates the overlap area between two bounding boxes\r\n * \r\n * @param box1 First bounding box\r\n * @param box2 Second bounding box\r\n * @returns The overlap area\r\n */\r\n function calculateOverlap(box1, box2) {\r\n const xOverlap = Math.max(0,\r\n Math.min(box1.x + box1.width, box2.x + box2.width) -\r\n Math.max(box1.x, box2.x)\r\n );\r\n const yOverlap = Math.max(0,\r\n Math.min(box1.y + box1.height, box2.y + box2.height) -\r\n Math.max(box1.y, box2.y)\r\n );\r\n return xOverlap * yOverlap;\r\n }\r\n\r\n /**\r\n * Finds an exact match between candidate elements and the actual interaction element\r\n * \r\n * @param candidate_elements Array of candidate element infos\r\n * @param actualInteractionElementInfo The actual interaction element info\r\n * @returns The matching candidate element info, or null if no match is found\r\n */\r\n function findExactMatch(candidate_elements, actualInteractionElementInfo) {\r\n if (!actualInteractionElementInfo.element) {\r\n return null;\r\n }\r\n\r\n const exactMatch = candidate_elements.find(elementInfo => \r\n elementInfo.element && elementInfo.element === actualInteractionElementInfo.element\r\n );\r\n \r\n if (exactMatch) {\r\n highlighterLogger.debug('✅ Found exact element match:', {\r\n matchedElement: exactMatch.element?.tagName,\r\n matchedElementClass: exactMatch.element?.className,\r\n index: exactMatch.index\r\n });\r\n return exactMatch;\r\n }\r\n \r\n return null;\r\n }\r\n\r\n /**\r\n * Finds a match by traversing up the parent elements\r\n * \r\n * @param candidate_elements Array of candidate element infos\r\n * @param actualInteractionElementInfo The actual interaction element info\r\n * @returns The matching candidate element info, or null if no match is found\r\n */\r\n function findParentMatch(candidate_elements, actualInteractionElementInfo) {\r\n if (!actualInteractionElementInfo.element) {\r\n return null;\r\n }\r\n\r\n let element = actualInteractionElementInfo.element;\r\n while (element.parentElement) {\r\n element = element.parentElement;\r\n const parentMatch = candidate_elements.find(candidate => \r\n candidate.element && candidate.element === element\r\n );\r\n \r\n if (parentMatch) {\r\n highlighterLogger.debug('✅ Found parent element match:', {\r\n matchedElement: parentMatch.element?.tagName,\r\n matchedElementClass: parentMatch.element?.className,\r\n index: parentMatch.index,\r\n depth: element.tagName\r\n });\r\n return parentMatch;\r\n }\r\n \r\n // Stop if we hit another candidate element\r\n if (candidate_elements.some(candidate => \r\n candidate.element && candidate.element === element\r\n )) {\r\n highlighterLogger.debug('⚠️ Stopped parent search - hit another candidate element:', element.tagName);\r\n break;\r\n }\r\n }\r\n \r\n return null;\r\n }\r\n\r\n /**\r\n * Finds a match based on spatial relationships between elements\r\n * \r\n * @param candidate_elements Array of candidate element infos\r\n * @param actualInteractionElementInfo The actual interaction element info\r\n * @returns The matching candidate element info, or null if no match is found\r\n */\r\n function findSpatialMatch(candidate_elements, actualInteractionElementInfo) {\r\n if (!actualInteractionElementInfo.element || !actualInteractionElementInfo.bounding_box) {\r\n return null;\r\n }\r\n\r\n const actualBox = actualInteractionElementInfo.bounding_box;\r\n let bestMatch = null;\r\n let bestScore = 0;\r\n\r\n for (const candidateInfo of candidate_elements) {\r\n if (!candidateInfo.bounding_box) continue;\r\n \r\n const candidateBox = candidateInfo.bounding_box;\r\n let score = 0;\r\n\r\n // Check if actual element is contained within candidate\r\n if (isPointInsideBox({ x: actualBox.x, y: actualBox.y }, candidateBox) &&\r\n isPointInsideBox({ x: actualBox.x + actualBox.width, y: actualBox.y + actualBox.height }, candidateBox)) {\r\n score += 100; // High score for containment\r\n }\r\n\r\n // Calculate overlap area as a factor\r\n const overlap = calculateOverlap(actualBox, candidateBox);\r\n score += overlap;\r\n\r\n // Consider proximity if no containment\r\n if (score === 0) {\r\n const distance = Math.sqrt(\r\n Math.pow((actualBox.x + actualBox.width/2) - (candidateBox.x + candidateBox.width/2), 2) +\r\n Math.pow((actualBox.y + actualBox.height/2) - (candidateBox.y + candidateBox.height/2), 2)\r\n );\r\n // Convert distance to a score (closer = higher score)\r\n score = 1000 / (distance + 1);\r\n }\r\n\r\n if (score > bestScore) {\r\n bestScore = score;\r\n bestMatch = candidateInfo;\r\n highlighterLogger.debug('📏 New best spatial match:', {\r\n element: candidateInfo.element?.tagName,\r\n class: candidateInfo.element?.className,\r\n index: candidateInfo.index,\r\n score: score\r\n });\r\n }\r\n }\r\n\r\n if (bestMatch) {\r\n highlighterLogger.debug('✅ Final spatial match selected:', {\r\n element: bestMatch.element?.tagName,\r\n class: bestMatch.element?.className,\r\n index: bestMatch.index,\r\n finalScore: bestScore\r\n });\r\n return bestMatch;\r\n }\r\n\r\n return null;\r\n }\r\n\r\n /**\r\n * Finds a matching candidate element for an actual interaction element\r\n * \r\n * @param candidate_elements Array of candidate element infos\r\n * @param actualInteractionElementInfo The actual interaction element info\r\n * @returns The matching candidate element info, or null if no match is found\r\n */\r\n function findMatchingCandidateElementInfo(candidate_elements, actualInteractionElementInfo) {\r\n if (!actualInteractionElementInfo.element || !actualInteractionElementInfo.bounding_box) {\r\n highlighterLogger.error('❌ Missing required properties in actualInteractionElementInfo');\r\n return null;\r\n }\r\n\r\n highlighterLogger.debug('🔍 Starting element matching for:', {\r\n clickedElement: actualInteractionElementInfo.element.tagName,\r\n clickedElementClass: actualInteractionElementInfo.element.className,\r\n totalCandidates: candidate_elements.length\r\n });\r\n\r\n // First try exact element match\r\n const exactMatch = findExactMatch(candidate_elements, actualInteractionElementInfo);\r\n if (exactMatch) {\r\n return exactMatch;\r\n }\r\n highlighterLogger.debug('❌ No exact element match found, trying parent matching...');\r\n\r\n // Try finding closest clickable parent\r\n const parentMatch = findParentMatch(candidate_elements, actualInteractionElementInfo);\r\n if (parentMatch) {\r\n return parentMatch;\r\n }\r\n highlighterLogger.debug('❌ No parent match found, falling back to spatial matching...');\r\n\r\n // If no exact or parent match, look for spatial relationships\r\n const spatialMatch = findSpatialMatch(candidate_elements, actualInteractionElementInfo);\r\n if (spatialMatch) {\r\n return spatialMatch;\r\n }\r\n\r\n highlighterLogger.error('❌ No matching element found for actual interaction element:', actualInteractionElementInfo);\r\n return null;\r\n }\n\n const highlight = {\r\n execute: async function(elementTypes, handleScroll=false) {\r\n const elements = await findElements(elementTypes);\r\n highlightElements(elements, false,handleScroll);\r\n return elements;\r\n },\r\n\r\n unexecute: function(handleScroll=false) {\r\n unhighlightElements(handleScroll);\r\n },\r\n\r\n generateJSON: async function() {\r\n const json = {};\r\n\r\n // Capture viewport dimensions\r\n const viewportData = {\r\n width: window.innerWidth,\r\n height: window.innerHeight,\r\n documentWidth: document.documentElement.clientWidth,\r\n documentHeight: document.documentElement.clientHeight,\r\n timestamp: new Date().toISOString()\r\n };\r\n\r\n // Add viewport data to the JSON output\r\n json.viewport = viewportData;\r\n\r\n\r\n await Promise.all(Object.values(ElementTag).map(async elementType => {\r\n const elements = await findElements(elementType);\r\n json[elementType] = elements;\r\n }));\r\n\r\n // Serialize the JSON object\r\n const jsonString = JSON.stringify(json, null, 4); // Pretty print with 4 spaces\r\n\r\n highlighterLogger.log(`JSON: ${jsonString}`);\r\n return jsonString;\r\n },\r\n\r\n getElementInfo\r\n };\r\n\r\n\r\n function unhighlightElements(handleScroll=false) {\r\n const documents = getAllFrames();\r\n documents.forEach(doc => {\r\n const overlay = doc.getElementById('highlight-overlay');\r\n if (overlay) {\r\n if (handleScroll) {\r\n // Remove event listeners\r\n doc.removeEventListener('scroll', overlay.scrollHandler, true);\r\n doc.removeEventListener('resize', overlay.resizeHandler);\r\n }\r\n overlay.remove();\r\n }\r\n });\r\n }\r\n\r\n\r\n async function findElements(elementTypes, verbose=true) {\r\n const typesArray = Array.isArray(elementTypes) ? elementTypes : [elementTypes];\r\n highlighterLogger.debug('Starting element search for types:', typesArray);\r\n\r\n const elements = [];\r\n typesArray.forEach(elementType => {\r\n if (elementType === ElementTag.FILLABLE) {\r\n elements.push(...findFillables());\r\n }\r\n if (elementType === ElementTag.SELECTABLE) {\r\n elements.push(...findDropdowns());\r\n }\r\n if (elementType === ElementTag.CLICKABLE) {\r\n elements.push(...findClickables());\r\n elements.push(...findToggles());\r\n elements.push(...findCheckables());\r\n }\r\n if (elementType === ElementTag.NON_INTERACTIVE_ELEMENT) {\r\n elements.push(...findNonInteractiveElements());\r\n }\r\n });\r\n\r\n // console.log('Before uniquify:', elements.length);\r\n const elementsWithInfo = elements.map((element, index) => \r\n getElementInfo(element, index)\r\n );\r\n\r\n \r\n \r\n const uniqueElements = uniquifyElements(elementsWithInfo);\r\n highlighterLogger.debug(`Found ${uniqueElements.length} elements:`);\r\n \r\n // More comprehensive visibility check\r\n const visibleElements = uniqueElements.filter(elementInfo => {\r\n const el = elementInfo.element;\r\n const style = getComputedStyle(el);\r\n \r\n // Check various style properties that affect visibility\r\n if (style.display === 'none' || \r\n style.visibility === 'hidden') {\r\n return false;\r\n }\r\n \r\n // Check if element has non-zero dimensions\r\n const rect = el.getBoundingClientRect();\r\n if (rect.width === 0 || rect.height === 0) {\r\n return false;\r\n }\r\n \r\n // Check if element is within viewport\r\n if (rect.bottom < 0 || \r\n rect.top > window.innerHeight || \r\n rect.right < 0 || \r\n rect.left > window.innerWidth) {\r\n // Element is outside viewport, but still might be valid \r\n // if user scrolls to it, so we'll include it\r\n return true;\r\n }\r\n \r\n return true;\r\n });\r\n \r\n highlighterLogger.debug(`Out of which ${visibleElements.length} elements are visible:`);\r\n if (verbose) {\r\n visibleElements.forEach(info => {\r\n highlighterLogger.debug(`Element ${info.index}:`, info);\r\n });\r\n }\r\n \r\n return visibleElements;\r\n }\r\n\r\n // elements is an array of objects with index, xpath\r\n function highlightElements(elements, enableSmartSelectors=false, handleScroll=false) {\r\n // console.log('[highlightElements] called with', elements.length, 'elements');\r\n // Create overlay if it doesn't exist and store it in a dictionary\r\n const documents = getAllFrames(); \r\n let overlays = {};\r\n documents.forEach(doc => {\r\n let overlay = doc.getElementById('highlight-overlay');\r\n if (!overlay) {\r\n overlay = doc.createElement('div');\r\n overlay.id = 'highlight-overlay';\r\n overlay.style.cssText = `\r\n position: fixed;\r\n top: 0;\r\n left: 0;\r\n width: 100%;\r\n height: 100%;\r\n pointer-events: none;\r\n z-index: 2147483647;\r\n `;\r\n doc.body.appendChild(overlay);\r\n // console.log('[highlightElements] Created overlay in document:', doc);\r\n }\r\n overlays[doc.documentURI] = overlay;\r\n });\r\n \r\n\r\n const updateHighlights = (doc = null) => {\r\n if (doc) {\r\n overlays[doc.documentURI].innerHTML = '';\r\n } else {\r\n Object.values(overlays).forEach(overlay => { overlay.innerHTML = ''; });\r\n } \r\n elements.forEach((elementInfo, idx) => {\r\n //console.log(`[highlightElements] Processing element ${idx}:`, elementInfo.tag, elementInfo.css_selector, elementInfo.bounding_box);\r\n let element = elementInfo.element;\r\n if (!element) {\r\n element = findElementByInfo(elementInfo, enableSmartSelectors);\r\n if (!element) {\r\n highlighterLogger.warn('[highlightElements] Could not find element for:', elementInfo);\r\n return;\r\n }\r\n }\r\n //if highlights requested for a specific doc, skip unrelated elements\r\n if (doc && element.ownerDocument !== doc) {\r\n highlighterLogger.debug(\"[highlightElements] Skipped element since it doesn't belong to document\", doc);\r\n return;\r\n }\r\n const rect = element.getBoundingClientRect();\r\n if (rect.width === 0 || rect.height === 0) {\r\n highlighterLogger.warn('[highlightElements] Element has zero dimensions:', elementInfo);\r\n return;\r\n }\r\n // Create border highlight (red rectangle)\r\n // use ownerDocument to support iframes/frames\r\n const highlight = element.ownerDocument.createElement('div');\r\n highlight.style.cssText = `\r\n position: fixed;\r\n left: ${rect.x}px;\r\n top: ${rect.y}px;\r\n width: ${rect.width}px;\r\n height: ${rect.height}px;\r\n border: 1px solid rgb(255, 0, 0);\r\n transition: all 0.2s ease-in-out;\r\n `;\r\n // Create index label container - now positioned to the right and slightly up\r\n const labelContainer = element.ownerDocument.createElement('div');\r\n labelContainer.style.cssText = `\r\n position: absolute;\r\n right: -10px; /* Offset to the right */\r\n top: -10px; /* Offset upwards */\r\n padding: 4px;\r\n background-color: rgba(255, 255, 0, 0.6);\r\n display: flex;\r\n align-items: center;\r\n justify-content: center;\r\n `;\r\n const text = element.ownerDocument.createElement('span');\r\n text.style.cssText = `\r\n color: rgb(0, 0, 0, 0.8);\r\n font-family: 'Courier New', Courier, monospace;\r\n font-size: 12px;\r\n font-weight: bold;\r\n line-height: 1;\r\n `;\r\n text.textContent = elementInfo.index.toString();\r\n labelContainer.appendChild(text);\r\n highlight.appendChild(labelContainer); \r\n overlays[element.ownerDocument.documentURI].appendChild(highlight);\r\n \r\n });\r\n };\r\n\r\n // Initial highlight\r\n updateHighlights();\r\n\r\n if (handleScroll) {\r\n documents.forEach(doc => {\r\n // Update highlights on scroll and resize\r\n highlighterLogger.debug('registering scroll and resize handlers for document: ', doc);\r\n const scrollHandler = () => {\r\n requestAnimationFrame(() => updateHighlights(doc));\r\n };\r\n const resizeHandler = () => {\r\n updateHighlights(doc);\r\n };\r\n doc.addEventListener('scroll', scrollHandler, true);\r\n doc.addEventListener('resize', resizeHandler);\r\n // Store event handlers for cleanup\r\n overlays[doc.documentURI].scrollHandler = scrollHandler;\r\n overlays[doc.documentURI].resizeHandler = resizeHandler;\r\n }); \r\n }\r\n }\r\n\r\n // function unexecute() {\r\n // unhighlightElements();\r\n // }\r\n\r\n // Make it available globally for both Extension and Playwright\r\n if (typeof window !== 'undefined') {\r\n function stripElementRefs(elementInfo) {\r\n if (!elementInfo) return null;\r\n const { element, ...rest } = elementInfo;\r\n return rest;\r\n }\r\n\r\n window.ProboLabs = window.ProboLabs || {};\r\n\r\n // --- Caching State ---\r\n window.ProboLabs.candidates = [];\r\n window.ProboLabs.actual = null;\r\n window.ProboLabs.matchingCandidate = null;\r\n\r\n // --- Methods ---\r\n /**\r\n * Find and cache candidate elements of a given type (e.g., 'CLICKABLE').\r\n * NOTE: This function is async and must be awaited from Playwright/Node.\r\n */\r\n window.ProboLabs.findAndCacheCandidateElements = async function(elementType) {\r\n //console.log('[ProboLabs] findAndCacheCandidateElements called with:', elementType);\r\n const found = await findElements(elementType);\r\n window.ProboLabs.candidates = found;\r\n // console.log('[ProboLabs] candidates set to:', found, 'type:', typeof found, 'isArray:', Array.isArray(found));\r\n return found.length;\r\n };\r\n\r\n window.ProboLabs.findAndCacheActualElement = function(iframeSelector, elementSelector, isHover=false, useSmartSelectors=false) {\r\n // console.log('[ProboLabs] findAndCacheActualElement called with:', cssSelector, iframeSelector);\r\n highlighterLogger.debug(`[ProboLabs] findAndCacheActualElement called with: iframeSelector=${iframeSelector}, elementSelector=${elementSelector}, useSmartSelectors=${useSmartSelectors}`);\r\n let el = findElementBySelectors(iframeSelector, elementSelector, useSmartSelectors);\r\n if(isHover) {\r\n const visibleElement = findClosestVisibleElement(el);\r\n if (visibleElement) {\r\n el = visibleElement;\r\n }\r\n }\r\n if (!el) {\r\n window.ProboLabs.actual = null;\r\n // console.log('[ProboLabs] actual set to null');\r\n return false;\r\n }\r\n window.ProboLabs.actual = getElementInfo(el, -1);\r\n // console.log('[ProboLabs] actual set to:', window.ProboLabs.actual);\r\n return true;\r\n };\r\n\r\n window.ProboLabs.findAndCacheMatchingCandidate = function() {\r\n // console.log('[ProboLabs] findAndCacheMatchingCandidate called');\r\n if (!window.ProboLabs.candidates.length || !window.ProboLabs.actual) {\r\n window.ProboLabs.matchingCandidate = null;\r\n // console.log('[ProboLabs] matchingCandidate set to null');\r\n return false;\r\n }\r\n window.ProboLabs.matchingCandidate = findMatchingCandidateElementInfo(window.ProboLabs.candidates, window.ProboLabs.actual);\r\n // console.log('[ProboLabs] matchingCandidate set to:', window.ProboLabs.matchingCandidate);\r\n return !!window.ProboLabs.matchingCandidate;\r\n };\r\n\r\n window.ProboLabs.highlightCachedElements = function(which) {\r\n let elements = [];\r\n if (which === 'candidates') elements = window.ProboLabs.candidates;\r\n if (which === 'actual' && window.ProboLabs.actual) elements = [window.ProboLabs.actual];\r\n if (which === 'matching' && window.ProboLabs.matchingCandidate) elements = [window.ProboLabs.matchingCandidate];\r\n highlighterLogger.debug(`[ProboLabs] highlightCachedElements ${which} with ${elements.length} elements`);\r\n highlightElements(elements);\r\n };\r\n\r\n window.ProboLabs.unhighlight = function() {\r\n // console.log('[ProboLabs] unhighlight called');\r\n unhighlightElements();\r\n };\r\n\r\n window.ProboLabs.reset = function() {\r\n highlighterLogger.debug('[ProboLabs] reset called');\r\n window.ProboLabs.candidates = [];\r\n window.ProboLabs.actual = null;\r\n window.ProboLabs.matchingCandidate = null;\r\n unhighlightElements();\r\n };\r\n\r\n window.ProboLabs.getCandidates = function() {\r\n // console.log('[ProboLabs] getCandidates called. candidates:', window.ProboLabs.candidates, 'type:', typeof window.ProboLabs.candidates, 'isArray:', Array.isArray(window.ProboLabs.candidates));\r\n const arr = Array.isArray(window.ProboLabs.candidates) ? window.ProboLabs.candidates : [];\r\n return arr.map(stripElementRefs);\r\n };\r\n window.ProboLabs.getActual = function() {\r\n return stripElementRefs(window.ProboLabs.actual);\r\n };\r\n window.ProboLabs.getMatchingCandidate = function() {\r\n return stripElementRefs(window.ProboLabs.matchingCandidate);\r\n };\r\n\r\n window.ProboLabs.setLoggerDebugLevel = function(debugLevel) {\r\n highlighterLogger.setLogLevel(debugLevel);\r\n };\r\n\r\n // Retain existing API for backward compatibility\r\n window.ProboLabs.ElementTag = ElementTag;\r\n window.ProboLabs.highlightElements = highlightElements;\r\n window.ProboLabs.unhighlightElements = unhighlightElements;\r\n window.ProboLabs.findElements = findElements;\r\n window.ProboLabs.getElementInfo = getElementInfo;\r\n window.ProboLabs.highlight = window.ProboLabs.highlight;\r\n window.ProboLabs.unhighlight = window.ProboLabs.unhighlight;\r\n\r\n // --- Utility Functions ---\r\n function findClosestVisibleElement(element) {\r\n let current = element;\r\n while (current) {\r\n const style = window.getComputedStyle(current);\r\n if (\r\n style &&\r\n style.display !== 'none' &&\r\n style.visibility !== 'hidden' &&\r\n current.offsetWidth > 0 &&\r\n current.offsetHeight > 0\r\n ) {\r\n return current;\r\n }\r\n if (!current.parentElement || current === document.body) break;\r\n current = current.parentElement;\r\n }\r\n return null;\r\n }\r\n }\n\n exports.deserializeNodeFromJSON = deserializeNodeFromJSON;\n exports.detectScrollableContainers = detectScrollableContainers;\n exports.findElementByInfo = findElementByInfo;\n exports.findElementBySelectors = findElementBySelectors;\n exports.findElements = findElements;\n exports.generateCssPath = generateCssPath;\n exports.generateSmartSelector = generateSmartSelector;\n exports.getAriaLabelledByText = getAriaLabelledByText;\n exports.getContainingIFrame = getContainingIFrame;\n exports.getElementInfo = getElementInfo;\n exports.highlight = highlight;\n exports.highlightElements = highlightElements;\n exports.isScrollableContainer = isScrollableContainer;\n exports.serializeNodeToJSON = serializeNodeToJSON;\n exports.unhighlightElements = unhighlightElements;\n\n}));\n//# sourceMappingURL=probolabs.umd.js.map\n";
|
|
2
2
|
/**
|
|
3
3
|
* Element tag constants for different types of interactive elements
|
|
4
4
|
*/
|
|
@@ -55,6 +55,7 @@ var PlaywrightAction;
|
|
|
55
55
|
PlaywrightAction["UPLOAD_FILES"] = "UPLOAD_FILES";
|
|
56
56
|
PlaywrightAction["WAIT_FOR"] = "WAIT_FOR";
|
|
57
57
|
PlaywrightAction["WAIT_FOR_OTP"] = "WAIT_FOR_OTP";
|
|
58
|
+
PlaywrightAction["GEN_TOTP"] = "GEN_TOTP";
|
|
58
59
|
})(PlaywrightAction || (PlaywrightAction = {}));
|
|
59
60
|
|
|
60
61
|
/**
|
|
@@ -79,12 +80,20 @@ function resolveElementTag(action) {
|
|
|
79
80
|
case PlaywrightAction.WAIT_FOR:
|
|
80
81
|
return [ElementTag.CLICKABLE, ElementTag.FILLABLE, ElementTag.NON_INTERACTIVE_ELEMENT];
|
|
81
82
|
case PlaywrightAction.WAIT_FOR_OTP:
|
|
83
|
+
case PlaywrightAction.GEN_TOTP:
|
|
82
84
|
return [ElementTag.FILLABLE];
|
|
83
85
|
default:
|
|
84
86
|
console.error(`Unknown action: ${action}`);
|
|
85
87
|
throw new Error(`Unknown action: ${action}`);
|
|
86
88
|
}
|
|
87
89
|
}
|
|
90
|
+
// clickable element detection result
|
|
91
|
+
var IsClickable;
|
|
92
|
+
(function (IsClickable) {
|
|
93
|
+
IsClickable["YES"] = "YES";
|
|
94
|
+
IsClickable["NO"] = "NO";
|
|
95
|
+
IsClickable["MAYBE"] = "MAYBE";
|
|
96
|
+
})(IsClickable || (IsClickable = {}));
|
|
88
97
|
|
|
89
98
|
// WebSocketsMessageType enum for WebSocket and event message types shared across the app
|
|
90
99
|
var WebSocketsMessageType;
|
|
@@ -131,122 +140,30 @@ class ProboLogger {
|
|
|
131
140
|
this.level = level;
|
|
132
141
|
}
|
|
133
142
|
setLogLevel(level) {
|
|
143
|
+
console.log(`[${this.prefix}] Setting log level to: ${level} (was: ${this.level})`);
|
|
134
144
|
this.level = level;
|
|
135
145
|
}
|
|
136
146
|
shouldLog(level) {
|
|
137
147
|
return logLevelOrder[level] >= logLevelOrder[this.level];
|
|
138
148
|
}
|
|
139
|
-
preamble() {
|
|
149
|
+
preamble(level) {
|
|
140
150
|
const now = new Date();
|
|
141
151
|
const hours = String(now.getHours()).padStart(2, '0');
|
|
142
152
|
const minutes = String(now.getMinutes()).padStart(2, '0');
|
|
143
153
|
const seconds = String(now.getSeconds()).padStart(2, '0');
|
|
144
154
|
const milliseconds = String(now.getMilliseconds()).padStart(3, '0');
|
|
145
|
-
return `[${hours}:${minutes}:${seconds}.${milliseconds}] [${this.prefix}]`;
|
|
155
|
+
return `[${hours}:${minutes}:${seconds}.${milliseconds}] [${this.prefix}] [${level}]`;
|
|
146
156
|
}
|
|
147
157
|
debug(...args) { if (this.shouldLog(ProboLogLevel.DEBUG))
|
|
148
|
-
console.debug(this.preamble(), ...args); }
|
|
158
|
+
console.debug(this.preamble(ProboLogLevel.DEBUG), ...args); }
|
|
149
159
|
info(...args) { if (this.shouldLog(ProboLogLevel.INFO))
|
|
150
|
-
console.info(this.preamble(), ...args); }
|
|
160
|
+
console.info(this.preamble(ProboLogLevel.INFO), ...args); }
|
|
151
161
|
log(...args) { if (this.shouldLog(ProboLogLevel.LOG))
|
|
152
|
-
console.log(this.preamble(), ...args); }
|
|
162
|
+
console.log(this.preamble(ProboLogLevel.LOG), ...args); }
|
|
153
163
|
warn(...args) { if (this.shouldLog(ProboLogLevel.WARN))
|
|
154
|
-
console.warn(this.preamble(), ...args); }
|
|
164
|
+
console.warn(this.preamble(ProboLogLevel.WARN), ...args); }
|
|
155
165
|
error(...args) { if (this.shouldLog(ProboLogLevel.ERROR))
|
|
156
|
-
console.error(this.preamble(), ...args); }
|
|
157
|
-
}
|
|
158
|
-
/**
|
|
159
|
-
* Interfaces for element information (unchanged)
|
|
160
|
-
*/
|
|
161
|
-
/* export interface ElementInfo {
|
|
162
|
-
index: string;
|
|
163
|
-
tag: string;
|
|
164
|
-
type: string;
|
|
165
|
-
text: string;
|
|
166
|
-
html: string;
|
|
167
|
-
xpath: string;
|
|
168
|
-
css_selector: string;
|
|
169
|
-
bounding_box: {
|
|
170
|
-
x: number;
|
|
171
|
-
y: number;
|
|
172
|
-
width: number;
|
|
173
|
-
height: number;
|
|
174
|
-
top: number;
|
|
175
|
-
right: number;
|
|
176
|
-
bottom: number;
|
|
177
|
-
left: number;
|
|
178
|
-
};
|
|
179
|
-
iframe_selector: string;
|
|
180
|
-
element: any;
|
|
181
|
-
depth?: number;
|
|
182
|
-
getSelector(): string;
|
|
183
|
-
getDepth(): number;
|
|
184
|
-
}
|
|
185
|
-
*/
|
|
186
|
-
/* export interface CleanElementInfo {
|
|
187
|
-
index: string;
|
|
188
|
-
tag: string;
|
|
189
|
-
type: string;
|
|
190
|
-
text: string;
|
|
191
|
-
html: string;
|
|
192
|
-
xpath: string;
|
|
193
|
-
css_selector: string;
|
|
194
|
-
iframe_selector: string;
|
|
195
|
-
bounding_box: {
|
|
196
|
-
x: number;
|
|
197
|
-
y: number;
|
|
198
|
-
width: number;
|
|
199
|
-
height: number;
|
|
200
|
-
top: number;
|
|
201
|
-
right: number;
|
|
202
|
-
bottom: number;
|
|
203
|
-
left: number;
|
|
204
|
-
};
|
|
205
|
-
depth: number;
|
|
206
|
-
} */
|
|
207
|
-
// Element cleaner logging
|
|
208
|
-
const elementLogger = new ProboLogger('element-cleaner');
|
|
209
|
-
/**
|
|
210
|
-
* Cleans and returns a minimal element info structure.
|
|
211
|
-
*/
|
|
212
|
-
function cleanupElementInfo(elementInfo) {
|
|
213
|
-
var _a;
|
|
214
|
-
elementLogger.debug(`Cleaning up element info for ${elementInfo.tag} at index ${elementInfo.index}`);
|
|
215
|
-
const depth = (_a = elementInfo.depth) !== null && _a !== void 0 ? _a : elementInfo.getDepth();
|
|
216
|
-
const cleanEl = {
|
|
217
|
-
index: elementInfo.index,
|
|
218
|
-
tag: elementInfo.tag,
|
|
219
|
-
type: elementInfo.type,
|
|
220
|
-
text: elementInfo.text,
|
|
221
|
-
html: elementInfo.html,
|
|
222
|
-
xpath: elementInfo.xpath,
|
|
223
|
-
css_selector: elementInfo.css_selector,
|
|
224
|
-
iframe_selector: elementInfo.iframe_selector,
|
|
225
|
-
bounding_box: elementInfo.bounding_box,
|
|
226
|
-
depth
|
|
227
|
-
};
|
|
228
|
-
elementLogger.debug(`Cleaned element: ${JSON.stringify(cleanEl)}`);
|
|
229
|
-
return cleanEl;
|
|
230
|
-
}
|
|
231
|
-
/**
|
|
232
|
-
* Cleans highlighted elements in an instruction payload.
|
|
233
|
-
*/
|
|
234
|
-
function cleanupInstructionElements(instruction) {
|
|
235
|
-
var _a;
|
|
236
|
-
if (!((_a = instruction === null || instruction === void 0 ? void 0 : instruction.result) === null || _a === void 0 ? void 0 : _a.highlighted_elements)) {
|
|
237
|
-
elementLogger.debug('No highlighted elements to clean');
|
|
238
|
-
return instruction;
|
|
239
|
-
}
|
|
240
|
-
elementLogger.debug(`Cleaning ${instruction.result.highlighted_elements.length} highlighted elements`);
|
|
241
|
-
const cleaned = {
|
|
242
|
-
...instruction,
|
|
243
|
-
result: {
|
|
244
|
-
...instruction.result,
|
|
245
|
-
highlighted_elements: instruction.result.highlighted_elements.map((el) => cleanupElementInfo(el))
|
|
246
|
-
}
|
|
247
|
-
};
|
|
248
|
-
elementLogger.debug('Instruction cleaning completed');
|
|
249
|
-
return cleaned;
|
|
166
|
+
console.error(this.preamble(ProboLogLevel.ERROR), ...args); }
|
|
250
167
|
}
|
|
251
168
|
function matchRegex(str, regex) {
|
|
252
169
|
//parse the regex string
|
|
@@ -286,6 +203,49 @@ const setupBrowserConsoleLogs = (page, enableConsoleLogs, logger = null) => {
|
|
|
286
203
|
};
|
|
287
204
|
page.on('console', listener);
|
|
288
205
|
};
|
|
206
|
+
/**
|
|
207
|
+
* Safely interpolates template literals in a string using the provided context.
|
|
208
|
+
* Similar to JavaScript template literals, but executed safely at runtime.
|
|
209
|
+
* Recursively interpolates until no more template literals remain.
|
|
210
|
+
*
|
|
211
|
+
* This function only interpolates the argument string itself, not the entire context.
|
|
212
|
+
* When a template literal resolves to another string containing template literals,
|
|
213
|
+
* it recursively interpolates that result.
|
|
214
|
+
*
|
|
215
|
+
* @param str The string containing template literal syntax (e.g., "Hello ${name}")
|
|
216
|
+
* @param context The context object containing variables for interpolation
|
|
217
|
+
* @param maxDepth Maximum recursion depth to prevent infinite loops (default: 10)
|
|
218
|
+
* @returns The interpolated string, or the original string if no interpolation is needed
|
|
219
|
+
*/
|
|
220
|
+
function interpolateTemplate(str, context, maxDepth = 10) {
|
|
221
|
+
if (typeof str !== 'string' || !str.includes('${')) {
|
|
222
|
+
return str;
|
|
223
|
+
}
|
|
224
|
+
if (maxDepth <= 0) {
|
|
225
|
+
console.warn('⚠️ Maximum interpolation depth reached, returning partially interpolated string');
|
|
226
|
+
return str;
|
|
227
|
+
}
|
|
228
|
+
try {
|
|
229
|
+
// Escape backticks in the template to prevent template literal injection
|
|
230
|
+
const escapedTemplate = str.replace(/\\/g, '\\\\').replace(/`/g, '\\`');
|
|
231
|
+
// Create a safe template execution function
|
|
232
|
+
const compiled = new Function('ctx', `
|
|
233
|
+
const {${Object.keys(context).join(',')}} = ctx;
|
|
234
|
+
return \`${escapedTemplate}\`;
|
|
235
|
+
`);
|
|
236
|
+
const rendered = compiled(context);
|
|
237
|
+
// If the result still contains template literals, recursively interpolate
|
|
238
|
+
if (rendered.includes('${') && rendered !== str) {
|
|
239
|
+
return interpolateTemplate(rendered, context, maxDepth - 1);
|
|
240
|
+
}
|
|
241
|
+
return rendered;
|
|
242
|
+
}
|
|
243
|
+
catch (e) {
|
|
244
|
+
const error = e instanceof Error ? e : new Error(String(e));
|
|
245
|
+
console.error('⚠️ Template evaluation failed:', error);
|
|
246
|
+
throw error;
|
|
247
|
+
}
|
|
248
|
+
}
|
|
289
249
|
|
|
290
250
|
function getDefaultExportFromCjs (x) {
|
|
291
251
|
return x && x.__esModule && Object.prototype.hasOwnProperty.call(x, 'default') ? x['default'] : x;
|
|
@@ -883,23 +843,26 @@ class ApiClient {
|
|
|
883
843
|
return;
|
|
884
844
|
});
|
|
885
845
|
}
|
|
886
|
-
async resolveNextInstruction(stepId, instruction, aiModel) {
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
return data.instruction;
|
|
846
|
+
/* async resolveNextInstruction(stepId: string, instruction: Instruction | null, aiModel?: string) {
|
|
847
|
+
apiLogger.debug(`resolving next instruction: ${instruction}`);
|
|
848
|
+
return this.requestWithRetry('resolveNextInstruction', async () => {
|
|
849
|
+
apiLogger.debug(`API client: Resolving next instruction for step ${stepId}`);
|
|
850
|
+
|
|
851
|
+
const cleanInstruction = cleanupInstructionElements(instruction);
|
|
852
|
+
|
|
853
|
+
const response = await fetch(`${this.apiUrl}/step-runners/${stepId}/run/`, {
|
|
854
|
+
method: 'POST',
|
|
855
|
+
headers: this.getHeaders(),
|
|
856
|
+
body: JSON.stringify({
|
|
857
|
+
executed_instruction: cleanInstruction,
|
|
858
|
+
ai_model: aiModel
|
|
859
|
+
}),
|
|
901
860
|
});
|
|
902
|
-
|
|
861
|
+
|
|
862
|
+
const data = await this.handleResponse(response);
|
|
863
|
+
return data.instruction;
|
|
864
|
+
});
|
|
865
|
+
} */
|
|
903
866
|
async uploadScreenshot(screenshot_bytes) {
|
|
904
867
|
return this.requestWithRetry('uploadScreenshot', async () => {
|
|
905
868
|
const response = await fetch(`${this.apiUrl}/upload-screenshots/`, {
|
|
@@ -968,7 +931,7 @@ class ApiClient {
|
|
|
968
931
|
// Backend will create new step or update existing one based on interaction_id
|
|
969
932
|
apiLogger.debug(`converting interaction #${interaction.interactionId} to step`);
|
|
970
933
|
return this.requestWithRetry('interactionToStep', async () => {
|
|
971
|
-
var _a, _b;
|
|
934
|
+
var _a, _b, _c, _d;
|
|
972
935
|
const response = await fetch(`${this.apiUrl}/interaction-to-step/`, {
|
|
973
936
|
method: 'POST',
|
|
974
937
|
headers: this.getHeaders(),
|
|
@@ -980,6 +943,8 @@ class ApiClient {
|
|
|
980
943
|
argument: interaction.argument,
|
|
981
944
|
element_css_selector: ((_a = interaction.elementInfo) === null || _a === void 0 ? void 0 : _a.css_selector) || '',
|
|
982
945
|
iframe_selector: ((_b = interaction.elementInfo) === null || _b === void 0 ? void 0 : _b.iframe_selector) || '',
|
|
946
|
+
smart_selector: ((_c = interaction.elementInfo) === null || _c === void 0 ? void 0 : _c.smart_selector) || null,
|
|
947
|
+
smart_iframe_selector: ((_d = interaction.elementInfo) === null || _d === void 0 ? void 0 : _d.smart_iframe_selector) || null,
|
|
983
948
|
prompt: interaction.nativeDescription,
|
|
984
949
|
vanilla_prompt: interaction.nativeDescription,
|
|
985
950
|
is_vanilla_prompt_robust: interaction.isNativeDescriptionElaborate || false,
|
|
@@ -1079,6 +1044,9 @@ var AIModel;
|
|
|
1079
1044
|
AIModel["GPT4"] = "gpt4";
|
|
1080
1045
|
AIModel["GPT4_MINI"] = "gpt4-mini";
|
|
1081
1046
|
AIModel["CLAUDE_3_5"] = "claude-3.5";
|
|
1047
|
+
AIModel["CLAUDE_SONNET_4_5"] = "claude-sonnet-4.5";
|
|
1048
|
+
AIModel["CLAUDE_HAIKU_4_5"] = "claude-haiku-4.5";
|
|
1049
|
+
AIModel["CLAUDE_OPUS_4_1"] = "claude-opus-4.1";
|
|
1082
1050
|
AIModel["GROK_2"] = "grok-2";
|
|
1083
1051
|
AIModel["LLAMA_4_SCOUT"] = "llama-4-scout";
|
|
1084
1052
|
AIModel["DEEPSEEK_V3"] = "deepseek-v3";
|
|
@@ -1104,7 +1072,7 @@ const DEFAULT_PLAYWRIGHT_TIMEOUT_CONFIG = {
|
|
|
1104
1072
|
};
|
|
1105
1073
|
|
|
1106
1074
|
// Default logger instance
|
|
1107
|
-
const proboLogger = new ProboLogger('
|
|
1075
|
+
const proboLogger = new ProboLogger('proboLib');
|
|
1108
1076
|
|
|
1109
1077
|
/**
|
|
1110
1078
|
* Wait for DOM mutations to settle using MutationObserver logic
|
|
@@ -1183,8 +1151,10 @@ async function findClosestVisibleElement(locator) {
|
|
|
1183
1151
|
}
|
|
1184
1152
|
|
|
1185
1153
|
class Highlighter {
|
|
1186
|
-
constructor(enableConsoleLogs = true) {
|
|
1154
|
+
constructor(enableSmartSelectors = false, enableConsoleLogs = true, debugLevel = ProboLogLevel.INFO) {
|
|
1155
|
+
this.enableSmartSelectors = enableSmartSelectors;
|
|
1187
1156
|
this.enableConsoleLogs = enableConsoleLogs;
|
|
1157
|
+
this.debugLevel = debugLevel;
|
|
1188
1158
|
}
|
|
1189
1159
|
async ensureHighlighterScript(page, maxRetries = 3) {
|
|
1190
1160
|
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
@@ -1194,11 +1164,12 @@ class Highlighter {
|
|
|
1194
1164
|
proboLogger.debug('Injecting highlighter script...');
|
|
1195
1165
|
await page.evaluate(highlighterCode);
|
|
1196
1166
|
// Verify the script was injected correctly
|
|
1197
|
-
const verified = await page.evaluate(`
|
|
1198
|
-
//console.log('ProboLabs global:', window.ProboLabs);
|
|
1199
|
-
typeof window.ProboLabs?.highlight?.execute === 'function'
|
|
1167
|
+
const verified = await page.evaluate(`
|
|
1168
|
+
//console.log('ProboLabs global:', window.ProboLabs);
|
|
1169
|
+
typeof window.ProboLabs?.highlight?.execute === 'function'
|
|
1200
1170
|
`);
|
|
1201
1171
|
proboLogger.debug('Script injection verified:', verified);
|
|
1172
|
+
await page.evaluate(`window.ProboLabs.setLoggerDebugLevel('${this.debugLevel}');`);
|
|
1202
1173
|
}
|
|
1203
1174
|
return; // Success - exit the function
|
|
1204
1175
|
}
|
|
@@ -1240,10 +1211,10 @@ class Highlighter {
|
|
|
1240
1211
|
(_b = (_a = window === null || window === void 0 ? void 0 : window.ProboLabs) === null || _a === void 0 ? void 0 : _a.highlight) === null || _b === void 0 ? void 0 : _b.unexecute();
|
|
1241
1212
|
});
|
|
1242
1213
|
}
|
|
1243
|
-
async highlightElement(page, element_css_selector, iframe_selector, element_index) {
|
|
1214
|
+
async highlightElement(page, element_css_selector, iframe_selector, smart_selector, smart_iframe_selector, element_index) {
|
|
1244
1215
|
await this.ensureHighlighterScript(page);
|
|
1245
|
-
proboLogger.debug('Highlighting element with:', { element_css_selector, iframe_selector, element_index });
|
|
1246
|
-
await page.evaluate(({ css_selector, iframe_selector, index }) => {
|
|
1216
|
+
proboLogger.debug('Highlighting element with:', { element_css_selector, iframe_selector, smart_selector, smart_iframe_selector, enableSmartSelectors: this.enableSmartSelectors, element_index });
|
|
1217
|
+
await page.evaluate(({ css_selector, iframe_selector, smart_selector, smart_iframe_selector, index }) => {
|
|
1247
1218
|
const proboLabs = window.ProboLabs;
|
|
1248
1219
|
if (!proboLabs) {
|
|
1249
1220
|
proboLogger.warn('ProboLabs not initialized');
|
|
@@ -1253,13 +1224,17 @@ class Highlighter {
|
|
|
1253
1224
|
const elementInfo = {
|
|
1254
1225
|
css_selector: css_selector,
|
|
1255
1226
|
iframe_selector: iframe_selector,
|
|
1227
|
+
smart_selector: smart_selector,
|
|
1228
|
+
smart_iframe_selector: smart_iframe_selector,
|
|
1256
1229
|
index: index
|
|
1257
1230
|
};
|
|
1258
1231
|
// Call highlightElements directly
|
|
1259
|
-
proboLabs.highlightElements([elementInfo]);
|
|
1232
|
+
proboLabs.highlightElements([elementInfo], this.enableSmartSelectors);
|
|
1260
1233
|
}, {
|
|
1261
1234
|
css_selector: element_css_selector,
|
|
1262
1235
|
iframe_selector: iframe_selector,
|
|
1236
|
+
smart_selector: smart_selector,
|
|
1237
|
+
smart_iframe_selector: smart_iframe_selector,
|
|
1263
1238
|
index: element_index
|
|
1264
1239
|
});
|
|
1265
1240
|
}
|
|
@@ -1280,12 +1255,12 @@ class Highlighter {
|
|
|
1280
1255
|
* Find and cache the actual interaction element by CSS and iframe selector.
|
|
1281
1256
|
* Returns true if found, false otherwise.
|
|
1282
1257
|
*/
|
|
1283
|
-
async findAndCacheActualElement(page,
|
|
1258
|
+
async findAndCacheActualElement(page, iframeSelector, elementSelector, isHover = false) {
|
|
1284
1259
|
await this.ensureHighlighterScript(page);
|
|
1285
1260
|
const result = await page.evaluate((params) => {
|
|
1286
1261
|
var _a, _b;
|
|
1287
|
-
return (_b = (_a = window.ProboLabs) === null || _a === void 0 ? void 0 : _a.findAndCacheActualElement) === null || _b === void 0 ? void 0 : _b.call(_a, params.
|
|
1288
|
-
}, {
|
|
1262
|
+
return (_b = (_a = window.ProboLabs) === null || _a === void 0 ? void 0 : _a.findAndCacheActualElement) === null || _b === void 0 ? void 0 : _b.call(_a, params.iframeSelector, params.elementSelector, params.isHover, params.useSmartSelectors);
|
|
1263
|
+
}, { iframeSelector: iframeSelector, elementSelector: elementSelector, isHover: isHover, useSmartSelectors: this.enableSmartSelectors });
|
|
1289
1264
|
return result !== null && result !== void 0 ? result : false;
|
|
1290
1265
|
}
|
|
1291
1266
|
/**
|
|
@@ -1806,7 +1781,7 @@ class NavTracker {
|
|
|
1806
1781
|
this.inflightGraceMs = (_g = options.inflightGraceMs) !== null && _g !== void 0 ? _g : 4000;
|
|
1807
1782
|
this.waitForStabilityVerbose = (_h = options.waitForStabilityVerbose) !== null && _h !== void 0 ? _h : false;
|
|
1808
1783
|
proboLogger.debug(`NavTracker constructor set values: quietTimeout=${this.waitForStabilityQuietTimeout}, initialDelay=${this.waitForStabilityInitialDelay}, globalTimeout=${this.waitForStabilityGlobalTimeout}, verbose=${this.waitForStabilityVerbose}`);
|
|
1809
|
-
this.instanceId = Math.random().toString(36).
|
|
1784
|
+
this.instanceId = Math.random().toString(36).substring(2, 11);
|
|
1810
1785
|
// Initialize timestamps
|
|
1811
1786
|
const now = Date.now();
|
|
1812
1787
|
this.lastHardNavAt = now;
|
|
@@ -3928,21 +3903,23 @@ const gen = (suffix, blockLen, outputLen)=>createHasher(()=>new Keccak(blockLen,
|
|
|
3928
3903
|
}
|
|
3929
3904
|
|
|
3930
3905
|
class ProboPlaywright {
|
|
3931
|
-
constructor(timeoutConfig = {}, page = null) {
|
|
3906
|
+
constructor({ enableSmartSelectors = false, timeoutConfig = {}, debugLevel = ProboLogLevel.INFO }, page = null) {
|
|
3932
3907
|
this.page = null;
|
|
3933
|
-
|
|
3908
|
+
this.params = {};
|
|
3909
|
+
this.enableSmartSelectors = enableSmartSelectors;
|
|
3934
3910
|
this.timeoutConfig = {
|
|
3935
3911
|
...DEFAULT_PLAYWRIGHT_TIMEOUT_CONFIG,
|
|
3936
3912
|
...timeoutConfig
|
|
3937
3913
|
};
|
|
3938
3914
|
this.setPage(page);
|
|
3915
|
+
proboLogger.setLogLevel(debugLevel);
|
|
3939
3916
|
}
|
|
3940
3917
|
/**
|
|
3941
|
-
|
|
3942
|
-
|
|
3943
|
-
|
|
3944
|
-
|
|
3945
|
-
|
|
3918
|
+
* Sets the Playwright page instance for this ProboPlaywright instance.
|
|
3919
|
+
* Also applies the configured default navigation and action timeouts to the page.
|
|
3920
|
+
*
|
|
3921
|
+
* @param page - The Playwright Page instance to use, or null to unset.
|
|
3922
|
+
*/
|
|
3946
3923
|
setPage(page) {
|
|
3947
3924
|
this.page = page;
|
|
3948
3925
|
if (this.page) {
|
|
@@ -3951,19 +3928,52 @@ class ProboPlaywright {
|
|
|
3951
3928
|
}
|
|
3952
3929
|
}
|
|
3953
3930
|
/**
|
|
3954
|
-
*
|
|
3955
|
-
*
|
|
3956
|
-
*
|
|
3957
|
-
|
|
3958
|
-
|
|
3959
|
-
|
|
3931
|
+
* Sets the parameters object for template literal interpolation
|
|
3932
|
+
* Stores a reference to the params object so mutations are automatically reflected
|
|
3933
|
+
* @param params The parameters object containing values to use for interpolation
|
|
3934
|
+
*/
|
|
3935
|
+
setParams(params) {
|
|
3936
|
+
this.params = params;
|
|
3937
|
+
}
|
|
3938
|
+
/**
|
|
3939
|
+
* Interpolates a string using the current params and optional additional context
|
|
3940
|
+
* @param str The string to interpolate (may contain ${variable} syntax)
|
|
3941
|
+
* @param additionalContext Optional additional context to merge with params
|
|
3942
|
+
* @returns The interpolated string
|
|
3960
3943
|
*/
|
|
3944
|
+
interpolate(str, additionalContext = {}) {
|
|
3945
|
+
const context = { ...this.params, ...additionalContext };
|
|
3946
|
+
return interpolateTemplate(str, context);
|
|
3947
|
+
}
|
|
3948
|
+
/**
|
|
3949
|
+
* Executes a single step in the test scenario with the specified action on the target element.
|
|
3950
|
+
* Handles iframe navigation, element highlighting, and various Playwright actions like click, fill, validate, etc.
|
|
3951
|
+
*
|
|
3952
|
+
* @param params - Configuration object containing element selectors, action type, arguments, and display options
|
|
3953
|
+
* @returns Promise that resolves to a string for extract actions, boolean for assert actions, or void for other actions
|
|
3954
|
+
* @throws Error if element is not found or validation fails
|
|
3955
|
+
*/
|
|
3961
3956
|
async runStep(params) {
|
|
3962
|
-
|
|
3957
|
+
let { action, argument = '', iframeSelector = '', elementSelector = '', smartSelector = null, smartIFrameSelector = null, annotation = '', } = params;
|
|
3963
3958
|
// 0. Check that page is set
|
|
3964
3959
|
if (!this.page) {
|
|
3965
|
-
throw new Error('
|
|
3960
|
+
throw new Error('Page is not set');
|
|
3966
3961
|
}
|
|
3962
|
+
// Interpolate argument if it's a string with template literals
|
|
3963
|
+
if (typeof argument === 'string' && argument.includes('${')) {
|
|
3964
|
+
argument = this.interpolate(argument);
|
|
3965
|
+
}
|
|
3966
|
+
else if (Array.isArray(argument)) {
|
|
3967
|
+
// Handle array arguments (e.g., for UPLOAD_FILES or SELECT_DROPDOWN)
|
|
3968
|
+
argument = argument.map((arg) => {
|
|
3969
|
+
if (typeof arg === 'string' && arg.includes('${')) {
|
|
3970
|
+
return this.interpolate(arg);
|
|
3971
|
+
}
|
|
3972
|
+
return arg;
|
|
3973
|
+
});
|
|
3974
|
+
}
|
|
3975
|
+
//trace message to help with debug
|
|
3976
|
+
proboLogger.info(`runStep(${JSON.stringify({ ...params, argument })})`);
|
|
3967
3977
|
// 1. Check if we need to visit a url
|
|
3968
3978
|
if (action === PlaywrightAction.VISIT_URL || action === PlaywrightAction.VISIT_BASE_URL) {
|
|
3969
3979
|
try {
|
|
@@ -3994,10 +4004,11 @@ class ProboPlaywright {
|
|
|
3994
4004
|
const startTime = Date.now();
|
|
3995
4005
|
let locator;
|
|
3996
4006
|
if (iframeSelector && iframeSelector.length > 0) {
|
|
3997
|
-
|
|
4007
|
+
const frameLocator = await this.getLocator(iframeSelector, smartIFrameSelector, true);
|
|
4008
|
+
locator = await this.getLocator(elementSelector, smartSelector, false, frameLocator);
|
|
3998
4009
|
}
|
|
3999
4010
|
else {
|
|
4000
|
-
locator = this.
|
|
4011
|
+
locator = await this.getLocator(elementSelector, smartSelector);
|
|
4001
4012
|
}
|
|
4002
4013
|
// Fail fast: immediately validate that the element exists for non-wait actions
|
|
4003
4014
|
const locator_timeout = (action === PlaywrightAction.WAIT_FOR) ? params.timeout || 10000 : this.timeoutConfig.playwrightLocatorTimeout;
|
|
@@ -4005,7 +4016,25 @@ class ProboPlaywright {
|
|
|
4005
4016
|
await locator.waitFor({ state: 'attached', timeout: locator_timeout });
|
|
4006
4017
|
}
|
|
4007
4018
|
catch (e) {
|
|
4008
|
-
|
|
4019
|
+
if (this.enableSmartSelectors) { //fallback to CSS selector
|
|
4020
|
+
try {
|
|
4021
|
+
proboLogger.warn(`Element not found with smart selector: ${JSON.stringify(smartSelector)} ${smartIFrameSelector ? `with iframe smart selector: ${JSON.stringify(smartIFrameSelector)}` : ''}. Falling back to CSS selector`);
|
|
4022
|
+
if (iframeSelector && iframeSelector.length > 0) {
|
|
4023
|
+
const frameLocator = await this.getLocatorOrFrame(iframeSelector, true);
|
|
4024
|
+
locator = await this.getLocatorOrFrame(elementSelector, false, frameLocator);
|
|
4025
|
+
}
|
|
4026
|
+
else {
|
|
4027
|
+
locator = await this.getLocatorOrFrame(elementSelector, false);
|
|
4028
|
+
}
|
|
4029
|
+
await locator.waitFor({ state: 'attached', timeout: 200 }); //only small timeout is needed as we've already waited for the locator_timeout
|
|
4030
|
+
}
|
|
4031
|
+
catch (e) {
|
|
4032
|
+
throw new Error(`Element not found with CSS selector: ${elementSelector} ${iframeSelector ? `in iframe: ${iframeSelector}` : ''} after ${locator_timeout}ms`);
|
|
4033
|
+
}
|
|
4034
|
+
}
|
|
4035
|
+
else {
|
|
4036
|
+
throw new Error(`Element not found with CSS selector: ${elementSelector} ${iframeSelector ? `in iframe: ${iframeSelector}` : ''} after ${locator_timeout}ms`);
|
|
4037
|
+
}
|
|
4009
4038
|
}
|
|
4010
4039
|
if (action === PlaywrightAction.HOVER) {
|
|
4011
4040
|
const visibleLocator = await findClosestVisibleElement(locator);
|
|
@@ -4024,13 +4053,13 @@ class ProboPlaywright {
|
|
|
4024
4053
|
case PlaywrightAction.CLICK:
|
|
4025
4054
|
case PlaywrightAction.CHECK_CHECKBOX:
|
|
4026
4055
|
case PlaywrightAction.SELECT_RADIO:
|
|
4027
|
-
await this.
|
|
4056
|
+
await this.robustMouseAction(locator, 'click');
|
|
4028
4057
|
break;
|
|
4029
4058
|
case PlaywrightAction.FILL_IN:
|
|
4030
4059
|
await this.robustFill(locator, argument);
|
|
4031
4060
|
break;
|
|
4032
4061
|
case PlaywrightAction.SELECT_DROPDOWN:
|
|
4033
|
-
await locator.selectOption(argument);
|
|
4062
|
+
await locator.selectOption(argument, { timeout: this.timeoutConfig.playwrightActionTimeout });
|
|
4034
4063
|
break;
|
|
4035
4064
|
case PlaywrightAction.SET_SLIDER:
|
|
4036
4065
|
await this.setSliderValue(locator, argument);
|
|
@@ -4039,11 +4068,35 @@ class ProboPlaywright {
|
|
|
4039
4068
|
// till we figure out how to get the inbox name we will wait for ANY OTP in all inboxes
|
|
4040
4069
|
const otp = await OTP.waitForOTP({ checkRecentMessagesSinceMs: 120000 });
|
|
4041
4070
|
if (otp) {
|
|
4042
|
-
|
|
4043
|
-
await locator.fill(otp);
|
|
4071
|
+
proboLogger.log(`✅ OTP found: ${otp}`);
|
|
4072
|
+
await locator.fill(otp, { timeout: this.timeoutConfig.playwrightActionTimeout });
|
|
4073
|
+
}
|
|
4074
|
+
else {
|
|
4075
|
+
proboLogger.log(`❌ OTP not found`);
|
|
4076
|
+
}
|
|
4077
|
+
break;
|
|
4078
|
+
case PlaywrightAction.GEN_TOTP:
|
|
4079
|
+
// Use secret from argument and auxiliary config (digits/algorithm) from totpConfig
|
|
4080
|
+
// If totpConfig is not provided, use defaults (digits: 6, algorithm: 'SHA1')
|
|
4081
|
+
const totpAux = params.totpConfig;
|
|
4082
|
+
const secretArg = params.argument;
|
|
4083
|
+
if (secretArg) {
|
|
4084
|
+
try {
|
|
4085
|
+
// Use provided config or defaults
|
|
4086
|
+
const digits = (totpAux === null || totpAux === void 0 ? void 0 : totpAux.digits) || 6;
|
|
4087
|
+
const algorithm = (totpAux === null || totpAux === void 0 ? void 0 : totpAux.algorithm) || 'SHA1';
|
|
4088
|
+
const totpCode = this.generateOTP(secretArg, digits, algorithm);
|
|
4089
|
+
proboLogger.log(`✅ TOTP generated (digits: ${digits}, algorithm: ${algorithm})`);
|
|
4090
|
+
await locator.fill(totpCode, { timeout: this.timeoutConfig.playwrightActionTimeout });
|
|
4091
|
+
}
|
|
4092
|
+
catch (error) {
|
|
4093
|
+
proboLogger.error(`❌ TOTP generation failed: ${error}`);
|
|
4094
|
+
throw new Error(`TOTP generation failed: ${error}`);
|
|
4095
|
+
}
|
|
4044
4096
|
}
|
|
4045
4097
|
else {
|
|
4046
|
-
|
|
4098
|
+
proboLogger.log(`❌ Missing TOTP argument`);
|
|
4099
|
+
throw new Error(`Missing TOTP argument`);
|
|
4047
4100
|
}
|
|
4048
4101
|
break;
|
|
4049
4102
|
case PlaywrightAction.ASSERT_CONTAINS_VALUE:
|
|
@@ -4051,38 +4104,39 @@ class ProboPlaywright {
|
|
|
4051
4104
|
if (!matchRegex(containerText, argument)) {
|
|
4052
4105
|
throw new Error(`Validation failed. Expected text "${containerText}" to match "${argument}".`);
|
|
4053
4106
|
}
|
|
4054
|
-
|
|
4107
|
+
return true; // Return true for successful assertion
|
|
4055
4108
|
case PlaywrightAction.ASSERT_EXACT_VALUE:
|
|
4056
4109
|
const actualText = await this.getTextValue(locator);
|
|
4057
4110
|
if (actualText !== argument) {
|
|
4058
4111
|
throw new Error(`Validation failed. Expected text "${argument}", but got "${actualText}".`);
|
|
4059
4112
|
}
|
|
4060
|
-
|
|
4113
|
+
return true; // Return true for successful assertion
|
|
4061
4114
|
case PlaywrightAction.HOVER:
|
|
4062
|
-
//console.log('HOVER', locator);
|
|
4063
4115
|
if (locator) {
|
|
4064
|
-
|
|
4065
|
-
|
|
4116
|
+
await this.robustMouseAction(locator, 'hover');
|
|
4117
|
+
}
|
|
4118
|
+
else {
|
|
4119
|
+
throw new Error('not executing HOVER because no visible ancestor found');
|
|
4066
4120
|
}
|
|
4067
4121
|
break;
|
|
4068
4122
|
case PlaywrightAction.SCROLL_TO_ELEMENT:
|
|
4069
4123
|
// Restore exact scroll positions from recording
|
|
4070
4124
|
const scrollData = JSON.parse(argument);
|
|
4071
4125
|
try {
|
|
4072
|
-
|
|
4126
|
+
proboLogger.log('🔄 Restoring scroll position for container:', locator, 'scrollTop:', scrollData.scrollTop, 'scrollLeft:', scrollData.scrollLeft);
|
|
4073
4127
|
await locator.evaluate((el, scrollData) => {
|
|
4074
4128
|
// el.scrollTop = scrollData.scrollTop;
|
|
4075
4129
|
// el.scrollLeft = scrollData.scrollLeft;
|
|
4076
4130
|
el.scrollTo({ left: scrollData.scrollLeft, top: scrollData.scrollTop, behavior: 'smooth' });
|
|
4077
|
-
}, { scrollTop: scrollData.scrollTop, scrollLeft: scrollData.scrollLeft }, { timeout:
|
|
4131
|
+
}, { scrollTop: scrollData.scrollTop, scrollLeft: scrollData.scrollLeft }, { timeout: this.timeoutConfig.playwrightActionTimeout });
|
|
4078
4132
|
}
|
|
4079
4133
|
catch (e) {
|
|
4080
|
-
|
|
4134
|
+
proboLogger.error('🔄 Failed to restore scroll position for container:', locator, 'scrollTop:', scrollData.scrollTop, 'scrollLeft:', scrollData.scrollLeft, 'error:', e);
|
|
4081
4135
|
}
|
|
4082
4136
|
await this.page.waitForTimeout(500);
|
|
4083
4137
|
break;
|
|
4084
4138
|
case PlaywrightAction.UPLOAD_FILES:
|
|
4085
|
-
await locator.setInputFiles(argument);
|
|
4139
|
+
await locator.setInputFiles(argument, { timeout: this.timeoutConfig.playwrightActionTimeout });
|
|
4086
4140
|
break;
|
|
4087
4141
|
case PlaywrightAction.EXTRACT_VALUE:
|
|
4088
4142
|
let extractedText = await this.getTextValue(locator);
|
|
@@ -4103,7 +4157,7 @@ class ProboPlaywright {
|
|
|
4103
4157
|
// Check if the text matches (using the same logic as ASSERT_CONTAINS_VALUE)
|
|
4104
4158
|
if (matchRegex(currentText, expectedText)) {
|
|
4105
4159
|
textMatches = true;
|
|
4106
|
-
|
|
4160
|
+
proboLogger.log(`✅ Wait for text completed successfully. Found: "${currentText}"`);
|
|
4107
4161
|
}
|
|
4108
4162
|
else {
|
|
4109
4163
|
// Text doesn't match yet, wait for the polling interval
|
|
@@ -4132,6 +4186,13 @@ class ProboPlaywright {
|
|
|
4132
4186
|
throw new Error(`Unhandled action: ${action}`);
|
|
4133
4187
|
}
|
|
4134
4188
|
}
|
|
4189
|
+
/**
|
|
4190
|
+
* Public method to generate TOTP code using the provided secret, digits, and algorithm
|
|
4191
|
+
* @param secret - The TOTP secret (base32 encoded)
|
|
4192
|
+
* @param digits - Number of digits in the TOTP code (default: 6)
|
|
4193
|
+
* @param algorithm - Hash algorithm to use (default: 'SHA1')
|
|
4194
|
+
* @returns The generated TOTP code
|
|
4195
|
+
*/
|
|
4135
4196
|
generateOTP(secret, digits = 6, algorithm = 'SHA1') {
|
|
4136
4197
|
const otp = new TOTP({
|
|
4137
4198
|
algorithm,
|
|
@@ -4140,44 +4201,122 @@ class ProboPlaywright {
|
|
|
4140
4201
|
});
|
|
4141
4202
|
return otp.generate();
|
|
4142
4203
|
}
|
|
4204
|
+
async getLocator(elementSelector, smartSelector, getIFrameLocator = false, frameLocator = null) {
|
|
4205
|
+
if (!this.page) {
|
|
4206
|
+
throw new Error('Page is not set');
|
|
4207
|
+
}
|
|
4208
|
+
if (this.enableSmartSelectors && smartSelector && Object.keys(smartSelector).length > 0) {
|
|
4209
|
+
const { strategy, selector, index, label_selector } = smartSelector;
|
|
4210
|
+
proboLogger.debug(`getLocator: smartSelector=${JSON.stringify(smartSelector)}`);
|
|
4211
|
+
let labelLocator;
|
|
4212
|
+
let combinedSelector;
|
|
4213
|
+
let id;
|
|
4214
|
+
const mainStrategy = smartSelector.strategy.split(' ')[0]; //extract the main strategy from the full strategy string
|
|
4215
|
+
switch (mainStrategy) {
|
|
4216
|
+
case 'role':
|
|
4217
|
+
case 'placeholder':
|
|
4218
|
+
case 'name':
|
|
4219
|
+
case 'data-testid':
|
|
4220
|
+
case 'id':
|
|
4221
|
+
case 'css':
|
|
4222
|
+
case 'class':
|
|
4223
|
+
proboLogger.debug(`getLocator: strategy=${mainStrategy}, index=${index}`);
|
|
4224
|
+
if (typeof index === 'number') {
|
|
4225
|
+
const matchedLocators = await this.getLocatorOrFrame(selector, getIFrameLocator, frameLocator);
|
|
4226
|
+
if (!getIFrameLocator) {
|
|
4227
|
+
proboLogger.debug(`getLocator: matched ${await matchedLocators.count()} elements`);
|
|
4228
|
+
for (const [index, element] of (await matchedLocators.all()).entries()) {
|
|
4229
|
+
// Get outerHTML
|
|
4230
|
+
const html = await element.evaluate(el => el.outerHTML);
|
|
4231
|
+
proboLogger.debug(`[${index}] ${html}`);
|
|
4232
|
+
}
|
|
4233
|
+
}
|
|
4234
|
+
return matchedLocators.nth(index);
|
|
4235
|
+
}
|
|
4236
|
+
else {
|
|
4237
|
+
return this.getLocatorOrFrame(selector, getIFrameLocator, frameLocator);
|
|
4238
|
+
}
|
|
4239
|
+
case 'label':
|
|
4240
|
+
if (smartSelector.strategy === 'label for') {
|
|
4241
|
+
proboLogger.debug(`getLocator: strategy=${smartSelector.strategy}, label_selector=${label_selector}`);
|
|
4242
|
+
labelLocator = this.getLocatorOrFrame(label_selector, false, frameLocator);
|
|
4243
|
+
id = await labelLocator.getAttribute('for');
|
|
4244
|
+
proboLogger.debug(`getLocator: labelLocator.getAttribute('for')=${id}`);
|
|
4245
|
+
combinedSelector = selector ? `#${id} > ${selector}` : `#${id}`;
|
|
4246
|
+
proboLogger.debug(`getLocator: combinedSelector=${combinedSelector}`);
|
|
4247
|
+
return this.getLocatorOrFrame(combinedSelector, getIFrameLocator, frameLocator);
|
|
4248
|
+
}
|
|
4249
|
+
else if (smartSelector.strategy === 'label by') {
|
|
4250
|
+
proboLogger.debug(`getLocator: strategy=${smartSelector.strategy}, label_selector=${label_selector}`);
|
|
4251
|
+
labelLocator = this.getLocatorOrFrame(label_selector, false, frameLocator);
|
|
4252
|
+
id = await labelLocator.getAttribute('id');
|
|
4253
|
+
proboLogger.debug(`getLocator: labelLocator.getAttribute('id')=${id}`);
|
|
4254
|
+
combinedSelector = selector ? `[aria-labelledby="${id}"] > ${selector}` : `[aria-labelledby="${id}"]`;
|
|
4255
|
+
proboLogger.debug(`getLocator: combinedSelector=${combinedSelector}`);
|
|
4256
|
+
return this.getLocatorOrFrame(combinedSelector, getIFrameLocator, frameLocator);
|
|
4257
|
+
}
|
|
4258
|
+
else {
|
|
4259
|
+
throw new Error(`Unsupported label strategy: ${smartSelector.strategy}`);
|
|
4260
|
+
}
|
|
4261
|
+
default:
|
|
4262
|
+
throw new Error(`Unsupported smart selector strategy: ${strategy}`);
|
|
4263
|
+
}
|
|
4264
|
+
}
|
|
4265
|
+
// fallback to standard locator
|
|
4266
|
+
return this.getLocatorOrFrame(elementSelector, getIFrameLocator, frameLocator);
|
|
4267
|
+
}
|
|
4268
|
+
getLocatorOrFrame(selector, getIFrameLocator, frameLocator = null) {
|
|
4269
|
+
if (!this.page) {
|
|
4270
|
+
throw new Error('Page is not set');
|
|
4271
|
+
}
|
|
4272
|
+
if (getIFrameLocator) {
|
|
4273
|
+
proboLogger.debug(`getLocatorOrFrame: getting frame locator for selector=${selector}`);
|
|
4274
|
+
if (frameLocator) {
|
|
4275
|
+
throw new Error(`getLocatorOrFrame: frameLocator must be null when getIFrameLocator is true`);
|
|
4276
|
+
}
|
|
4277
|
+
return this.page.frameLocator(selector);
|
|
4278
|
+
}
|
|
4279
|
+
proboLogger.debug(`getLocatorOrFrame: ${frameLocator ? `frameLocator.locator("${selector}")` : `page.locator("${selector}")`}`);
|
|
4280
|
+
return frameLocator ? frameLocator.locator(selector) : this.page.locator(selector);
|
|
4281
|
+
}
|
|
4143
4282
|
/**
|
|
4144
|
-
|
|
4145
|
-
|
|
4146
|
-
|
|
4147
|
-
|
|
4148
|
-
|
|
4149
|
-
|
|
4283
|
+
* Creates a visual highlight overlay on the target element with optional annotation text.
|
|
4284
|
+
* The highlight appears as a red border around the element and can include descriptive text.
|
|
4285
|
+
*
|
|
4286
|
+
* @param locator - The Playwright locator for the element to highlight
|
|
4287
|
+
* @param annotation - Optional text annotation to display above/below the highlighted element
|
|
4288
|
+
*/
|
|
4150
4289
|
async highlight(locator, annotation = null) {
|
|
4151
4290
|
try {
|
|
4152
4291
|
await locator.evaluate((el) => {
|
|
4153
4292
|
const overlay = el.ownerDocument.createElement('div');
|
|
4154
4293
|
overlay.id = 'highlight-overlay';
|
|
4155
|
-
overlay.style.cssText = `
|
|
4156
|
-
position: fixed;
|
|
4157
|
-
top: 0;
|
|
4158
|
-
left: 0;
|
|
4159
|
-
width: 100%;
|
|
4160
|
-
height: 100%;
|
|
4161
|
-
pointer-events: none;
|
|
4162
|
-
z-index: 2147483647;
|
|
4294
|
+
overlay.style.cssText = `
|
|
4295
|
+
position: fixed;
|
|
4296
|
+
top: 0;
|
|
4297
|
+
left: 0;
|
|
4298
|
+
width: 100%;
|
|
4299
|
+
height: 100%;
|
|
4300
|
+
pointer-events: none;
|
|
4301
|
+
z-index: 2147483647;
|
|
4163
4302
|
`;
|
|
4164
4303
|
el.ownerDocument.body.appendChild(overlay);
|
|
4165
4304
|
const bbox = el.getBoundingClientRect();
|
|
4166
4305
|
const highlight = el.ownerDocument.createElement('div');
|
|
4167
|
-
highlight.style.cssText = `
|
|
4168
|
-
position: fixed;
|
|
4169
|
-
left: ${bbox.x}px;
|
|
4170
|
-
top: ${bbox.y}px;
|
|
4171
|
-
width: ${bbox.width}px;
|
|
4172
|
-
height: ${bbox.height}px;
|
|
4173
|
-
border: 2px solid rgb(255, 0, 0);
|
|
4174
|
-
transition: all 0.2s ease-in-out;
|
|
4306
|
+
highlight.style.cssText = `
|
|
4307
|
+
position: fixed;
|
|
4308
|
+
left: ${bbox.x}px;
|
|
4309
|
+
top: ${bbox.y}px;
|
|
4310
|
+
width: ${bbox.width}px;
|
|
4311
|
+
height: ${bbox.height}px;
|
|
4312
|
+
border: 2px solid rgb(255, 0, 0);
|
|
4313
|
+
transition: all 0.2s ease-in-out;
|
|
4175
4314
|
`;
|
|
4176
4315
|
overlay.appendChild(highlight);
|
|
4177
4316
|
}, { timeout: 500 });
|
|
4178
4317
|
}
|
|
4179
4318
|
catch (e) {
|
|
4180
|
-
|
|
4319
|
+
proboLogger.log('highlight: failed to run locator.evaluate()', e);
|
|
4181
4320
|
}
|
|
4182
4321
|
if (annotation) {
|
|
4183
4322
|
await locator.evaluate((el, annotation) => {
|
|
@@ -4185,19 +4324,19 @@ class ProboPlaywright {
|
|
|
4185
4324
|
if (overlay) {
|
|
4186
4325
|
const bbox = el.getBoundingClientRect();
|
|
4187
4326
|
const annotationEl = el.ownerDocument.createElement('div');
|
|
4188
|
-
annotationEl.style.cssText = `
|
|
4189
|
-
position: fixed;
|
|
4190
|
-
left: ${bbox.x}px;
|
|
4191
|
-
top: ${bbox.y - 25}px;
|
|
4192
|
-
padding: 2px 6px;
|
|
4193
|
-
background-color: rgba(255, 255, 0, 0.6);
|
|
4194
|
-
color: black;
|
|
4195
|
-
font-size: 16px;
|
|
4196
|
-
font-family: 'Courier New', Courier, monospace;
|
|
4197
|
-
font-weight: bold;
|
|
4198
|
-
border-radius: 3px;
|
|
4199
|
-
pointer-events: none;
|
|
4200
|
-
z-index: 2147483647;
|
|
4327
|
+
annotationEl.style.cssText = `
|
|
4328
|
+
position: fixed;
|
|
4329
|
+
left: ${bbox.x}px;
|
|
4330
|
+
top: ${bbox.y - 25}px;
|
|
4331
|
+
padding: 2px 6px;
|
|
4332
|
+
background-color: rgba(255, 255, 0, 0.6);
|
|
4333
|
+
color: black;
|
|
4334
|
+
font-size: 16px;
|
|
4335
|
+
font-family: 'Courier New', Courier, monospace;
|
|
4336
|
+
font-weight: bold;
|
|
4337
|
+
border-radius: 3px;
|
|
4338
|
+
pointer-events: none;
|
|
4339
|
+
z-index: 2147483647;
|
|
4201
4340
|
`;
|
|
4202
4341
|
annotationEl.textContent = annotation;
|
|
4203
4342
|
// If element is too close to top of window, position annotation below
|
|
@@ -4211,11 +4350,11 @@ class ProboPlaywright {
|
|
|
4211
4350
|
}
|
|
4212
4351
|
;
|
|
4213
4352
|
/**
|
|
4214
|
-
|
|
4215
|
-
|
|
4216
|
-
|
|
4217
|
-
|
|
4218
|
-
|
|
4353
|
+
* Removes the highlight overlay from the target element.
|
|
4354
|
+
* Cleans up the visual highlighting created by the highlight method.
|
|
4355
|
+
*
|
|
4356
|
+
* @param locator - The Playwright locator for the element to unhighlight
|
|
4357
|
+
*/
|
|
4219
4358
|
async unhighlight(locator) {
|
|
4220
4359
|
try {
|
|
4221
4360
|
await locator.evaluate((el) => {
|
|
@@ -4226,48 +4365,48 @@ class ProboPlaywright {
|
|
|
4226
4365
|
}, { timeout: 500 });
|
|
4227
4366
|
}
|
|
4228
4367
|
catch (e) {
|
|
4229
|
-
|
|
4368
|
+
proboLogger.log('unhighlight: failed to run locator.evaluate()', e);
|
|
4230
4369
|
}
|
|
4231
4370
|
}
|
|
4232
4371
|
;
|
|
4233
4372
|
/**
|
|
4234
|
-
|
|
4235
|
-
|
|
4236
|
-
|
|
4237
|
-
|
|
4238
|
-
|
|
4239
|
-
|
|
4373
|
+
* Attempts to fill a form field with the specified value using multiple fallback strategies.
|
|
4374
|
+
* First tries the standard fill method, then falls back to click + type if needed.
|
|
4375
|
+
*
|
|
4376
|
+
* @param locator - The Playwright locator for the input element
|
|
4377
|
+
* @param value - The text value to fill into the input field
|
|
4378
|
+
*/
|
|
4240
4379
|
async robustFill(locator, value) {
|
|
4241
4380
|
if (!this.page) {
|
|
4242
|
-
throw new Error('
|
|
4381
|
+
throw new Error('Page is not set');
|
|
4243
4382
|
}
|
|
4244
4383
|
try {
|
|
4245
|
-
await locator.fill(value);
|
|
4384
|
+
await locator.fill(value, { timeout: this.timeoutConfig.playwrightActionTimeout });
|
|
4246
4385
|
return;
|
|
4247
4386
|
}
|
|
4248
4387
|
catch (err) {
|
|
4249
|
-
|
|
4388
|
+
proboLogger.warn('robustFill: failed to run locator.fill(), trying fallback', err);
|
|
4250
4389
|
}
|
|
4251
4390
|
// fallback: click and type
|
|
4252
4391
|
try {
|
|
4253
|
-
await locator.focus();
|
|
4392
|
+
await locator.focus({ timeout: this.timeoutConfig.playwrightActionTimeout });
|
|
4254
4393
|
await this.page.keyboard.type(value);
|
|
4255
4394
|
return;
|
|
4256
4395
|
}
|
|
4257
4396
|
catch (err) {
|
|
4258
|
-
|
|
4397
|
+
proboLogger.warn('robustFill: failed to run locator.focus() and page.keyboard.type()', err);
|
|
4259
4398
|
}
|
|
4260
4399
|
}
|
|
4261
4400
|
;
|
|
4262
4401
|
async robustTypeKeys(value) {
|
|
4263
4402
|
if (!this.page) {
|
|
4264
|
-
throw new Error('
|
|
4403
|
+
throw new Error('Page is not set');
|
|
4265
4404
|
}
|
|
4266
4405
|
/* try {
|
|
4267
|
-
|
|
4268
|
-
|
|
4406
|
+
await locator.press(value);
|
|
4407
|
+
return;
|
|
4269
4408
|
} catch (err) {
|
|
4270
|
-
|
|
4409
|
+
proboLogger.warn('robustTypeKeys: failed to run locator.type()', err);
|
|
4271
4410
|
} */
|
|
4272
4411
|
// fallback: click and type
|
|
4273
4412
|
try {
|
|
@@ -4276,69 +4415,98 @@ class ProboPlaywright {
|
|
|
4276
4415
|
return;
|
|
4277
4416
|
}
|
|
4278
4417
|
catch (err) {
|
|
4279
|
-
|
|
4418
|
+
proboLogger.warn('robustTypeKeys: failed to run page.keyboard.type()', err);
|
|
4280
4419
|
}
|
|
4281
4420
|
}
|
|
4282
4421
|
/**
|
|
4283
|
-
|
|
4284
|
-
|
|
4285
|
-
|
|
4286
|
-
|
|
4287
|
-
|
|
4288
|
-
|
|
4289
|
-
|
|
4422
|
+
* Performs a robust mouse action (click or hover) using multiple fallback strategies.
|
|
4423
|
+
* Attempts standard click first, then mouse click at center coordinates, and finally native DOM events.
|
|
4424
|
+
*
|
|
4425
|
+
* @param locator - The Playwright locator for the element to click
|
|
4426
|
+
* @param action - The mouse action to perform ('click' or 'hover')
|
|
4427
|
+
* @throws Error if all mouse action methods fail
|
|
4428
|
+
*/
|
|
4429
|
+
async robustMouseAction(locator, action) {
|
|
4290
4430
|
if (!this.page) {
|
|
4291
|
-
throw new Error('
|
|
4431
|
+
throw new Error('Page is not set');
|
|
4292
4432
|
}
|
|
4293
4433
|
// start with a standard click
|
|
4294
4434
|
try {
|
|
4295
|
-
|
|
4435
|
+
if (action === 'click') {
|
|
4436
|
+
await locator.click({ noWaitAfter: false, timeout: this.timeoutConfig.playwrightActionTimeout });
|
|
4437
|
+
}
|
|
4438
|
+
else if (action === 'hover') {
|
|
4439
|
+
await locator.hover({ timeout: this.timeoutConfig.playwrightActionTimeout });
|
|
4440
|
+
}
|
|
4441
|
+
else {
|
|
4442
|
+
throw new Error(`Unsupported mouse action: ${action}`);
|
|
4443
|
+
}
|
|
4296
4444
|
return;
|
|
4297
4445
|
}
|
|
4298
4446
|
catch (err) {
|
|
4299
|
-
|
|
4447
|
+
proboLogger.warn(`robustMouseAction: failed to run locator.${action}(), trying mouse.${action}()`);
|
|
4300
4448
|
}
|
|
4301
4449
|
// try clicking using mouse at the center of the element
|
|
4302
4450
|
try {
|
|
4303
4451
|
const bbox = await locator.boundingBox({ timeout: this.timeoutConfig.playwrightLocatorTimeout });
|
|
4304
4452
|
if (bbox) {
|
|
4305
|
-
|
|
4453
|
+
if (action === 'click') {
|
|
4454
|
+
await this.page.mouse.click(bbox.x + bbox.width / 2, bbox.y + bbox.height / 2);
|
|
4455
|
+
}
|
|
4456
|
+
else if (action === 'hover') {
|
|
4457
|
+
await this.page.mouse.move(bbox.x + bbox.width / 2, bbox.y + bbox.height / 2);
|
|
4458
|
+
}
|
|
4306
4459
|
return;
|
|
4307
4460
|
}
|
|
4308
4461
|
else {
|
|
4309
|
-
|
|
4462
|
+
proboLogger.warn('robustMouseAction: bounding box not found');
|
|
4310
4463
|
}
|
|
4311
4464
|
}
|
|
4312
4465
|
catch (err2) {
|
|
4313
|
-
|
|
4466
|
+
proboLogger.warn(`robustMouseAction: failed to run page.mouse.${action}(), trying dispatching an event`);
|
|
4314
4467
|
}
|
|
4315
4468
|
// fallback: dispatch native mouse events manually
|
|
4316
4469
|
try {
|
|
4317
|
-
|
|
4318
|
-
|
|
4319
|
-
|
|
4320
|
-
|
|
4321
|
-
|
|
4322
|
-
|
|
4470
|
+
if (action === 'click') {
|
|
4471
|
+
await locator.evaluate((el) => {
|
|
4472
|
+
['mousedown', 'mouseup', 'click'].forEach(type => {
|
|
4473
|
+
const event = new MouseEvent(type, {
|
|
4474
|
+
bubbles: true,
|
|
4475
|
+
cancelable: true,
|
|
4476
|
+
view: window
|
|
4477
|
+
});
|
|
4478
|
+
el.dispatchEvent(event);
|
|
4323
4479
|
});
|
|
4324
|
-
|
|
4325
|
-
|
|
4326
|
-
|
|
4480
|
+
}, { timeout: this.timeoutConfig.playwrightActionTimeout });
|
|
4481
|
+
}
|
|
4482
|
+
else if (action === 'hover') {
|
|
4483
|
+
await locator.evaluate((el) => {
|
|
4484
|
+
['mouseenter', 'mouseover', 'focusin', 'focus'].forEach(type => {
|
|
4485
|
+
const event = new MouseEvent(type, {
|
|
4486
|
+
bubbles: true,
|
|
4487
|
+
cancelable: true,
|
|
4488
|
+
view: window
|
|
4489
|
+
});
|
|
4490
|
+
el.dispatchEvent(event);
|
|
4491
|
+
});
|
|
4492
|
+
}, { timeout: this.timeoutConfig.playwrightActionTimeout });
|
|
4493
|
+
}
|
|
4327
4494
|
}
|
|
4328
4495
|
catch (err3) {
|
|
4329
|
-
|
|
4330
|
-
|
|
4496
|
+
proboLogger.error(`robustMouseAction: all ${action} methods failed:`);
|
|
4497
|
+
// not re-throwing playwright errors as they are long and not useful for the user
|
|
4498
|
+
throw new Error(`robustMouseAction: all ${action} methods failed`);
|
|
4331
4499
|
}
|
|
4332
4500
|
}
|
|
4333
4501
|
;
|
|
4334
4502
|
/**
|
|
4335
|
-
|
|
4336
|
-
|
|
4337
|
-
|
|
4338
|
-
|
|
4339
|
-
|
|
4340
|
-
|
|
4341
|
-
|
|
4503
|
+
* Extracts text content from an element using multiple strategies.
|
|
4504
|
+
* Tries textContent first, then inputValue, and finally looks for nested input elements.
|
|
4505
|
+
* Returns normalized and trimmed text for consistent comparison.
|
|
4506
|
+
*
|
|
4507
|
+
* @param locator - The Playwright locator for the element to extract text from
|
|
4508
|
+
* @returns Normalized text content with consistent whitespace handling
|
|
4509
|
+
*/
|
|
4342
4510
|
async getTextValue(locator) {
|
|
4343
4511
|
let textValue = await locator.textContent();
|
|
4344
4512
|
if (!textValue) {
|
|
@@ -4346,7 +4514,7 @@ class ProboPlaywright {
|
|
|
4346
4514
|
textValue = await locator.inputValue();
|
|
4347
4515
|
}
|
|
4348
4516
|
catch (err) {
|
|
4349
|
-
|
|
4517
|
+
// proboLogger.warn('getTextValue: failed to run locator.inputValue()', err);
|
|
4350
4518
|
}
|
|
4351
4519
|
}
|
|
4352
4520
|
if (!textValue) {
|
|
@@ -4354,7 +4522,7 @@ class ProboPlaywright {
|
|
|
4354
4522
|
textValue = await locator.locator('input').inputValue();
|
|
4355
4523
|
}
|
|
4356
4524
|
catch (err) {
|
|
4357
|
-
|
|
4525
|
+
// proboLogger.warn('getTextValue: failed to run locator.locator("input").inputValue()', err);
|
|
4358
4526
|
}
|
|
4359
4527
|
}
|
|
4360
4528
|
if (!textValue) {
|
|
@@ -4365,16 +4533,31 @@ class ProboPlaywright {
|
|
|
4365
4533
|
}
|
|
4366
4534
|
;
|
|
4367
4535
|
async setSliderValue(locator, value) {
|
|
4368
|
-
|
|
4369
|
-
|
|
4370
|
-
|
|
4371
|
-
|
|
4372
|
-
|
|
4536
|
+
try {
|
|
4537
|
+
await locator.fill(value, { timeout: this.timeoutConfig.playwrightActionTimeout });
|
|
4538
|
+
return;
|
|
4539
|
+
}
|
|
4540
|
+
catch (err) {
|
|
4541
|
+
proboLogger.warn('setSliderValue: failed to run locator.fill(), trying fallback', err);
|
|
4542
|
+
}
|
|
4543
|
+
// fallback
|
|
4544
|
+
try {
|
|
4545
|
+
await locator.focus();
|
|
4546
|
+
await locator.evaluate((el, value) => {
|
|
4547
|
+
el.value = value;
|
|
4548
|
+
el.dispatchEvent(new Event('input', { bubbles: true }));
|
|
4549
|
+
el.dispatchEvent(new Event('change', { bubbles: true }));
|
|
4550
|
+
}, value, { timeout: this.timeoutConfig.playwrightActionTimeout });
|
|
4551
|
+
}
|
|
4552
|
+
catch (err) {
|
|
4553
|
+
proboLogger.error('setSliderValue: failed to run locator.evaluate()', err);
|
|
4554
|
+
}
|
|
4373
4555
|
}
|
|
4374
4556
|
} /* class ProboPlaywright */
|
|
4375
4557
|
|
|
4376
4558
|
class Probo {
|
|
4377
|
-
constructor({ scenarioName, token = '', apiUrl = '', enableConsoleLogs = false, logToConsole = true, logToFile = false, debugLevel = ProboLogLevel.INFO, aiModel = AIModel.AZURE_GPT4_MINI, timeoutConfig = {} }) {
|
|
4559
|
+
constructor({ scenarioName, token = '', apiUrl = '', enableConsoleLogs = false, enableSmartSelectors = false, logToConsole = true, logToFile = false, debugLevel = ProboLogLevel.INFO, aiModel = AIModel.AZURE_GPT4_MINI, timeoutConfig = {} }) {
|
|
4560
|
+
this.params = {};
|
|
4378
4561
|
// Configure logger transports and level
|
|
4379
4562
|
// configureLogger({ logToConsole, logToFile, level: debugLevel });
|
|
4380
4563
|
proboLogger.setLogLevel(debugLevel);
|
|
@@ -4388,27 +4571,51 @@ class Probo {
|
|
|
4388
4571
|
proboLogger.error("API endpoint wasn't provided. Pass 'apiUrl' or set PROBO_API_ENDPOINT");
|
|
4389
4572
|
throw new Error('Probo API endpoint not provided');
|
|
4390
4573
|
}
|
|
4391
|
-
this.highlighter = new Highlighter(enableConsoleLogs);
|
|
4574
|
+
this.highlighter = new Highlighter(enableSmartSelectors, enableConsoleLogs);
|
|
4392
4575
|
this.apiClient = new ApiClient(apiEndPoint, apiKey);
|
|
4576
|
+
this.debugLevel = debugLevel;
|
|
4393
4577
|
this.enableConsoleLogs = enableConsoleLogs;
|
|
4394
4578
|
this.scenarioName = scenarioName;
|
|
4395
4579
|
this.aiModel = aiModel;
|
|
4580
|
+
this.enableSmartSelectors = enableSmartSelectors;
|
|
4396
4581
|
this.timeoutConfig = { ...DEFAULT_PLAYWRIGHT_TIMEOUT_CONFIG, ...timeoutConfig };
|
|
4397
4582
|
// set the log level for the api client
|
|
4398
|
-
apiLogger.setLogLevel(debugLevel);
|
|
4583
|
+
apiLogger.setLogLevel(this.debugLevel);
|
|
4399
4584
|
proboLogger.info(`Initializing: scenario=${scenarioName}, apiUrl=${apiEndPoint}, ` +
|
|
4400
4585
|
`enableConsoleLogs=${enableConsoleLogs}, debugLevel=${debugLevel}, aiModel=${aiModel}`);
|
|
4401
4586
|
}
|
|
4402
|
-
|
|
4587
|
+
/**
|
|
4588
|
+
* Sets the parameters object for template literal interpolation
|
|
4589
|
+
* Stores a reference to the params object so mutations are automatically reflected
|
|
4590
|
+
* @param params The parameters object containing values to use for interpolation
|
|
4591
|
+
*/
|
|
4592
|
+
setParams(params) {
|
|
4593
|
+
this.params = params;
|
|
4594
|
+
}
|
|
4595
|
+
/**
|
|
4596
|
+
* Interpolates a string using the current params and optional additional context
|
|
4597
|
+
* @param str The string to interpolate (may contain ${variable} syntax)
|
|
4598
|
+
* @param additionalContext Optional additional context to merge with params
|
|
4599
|
+
* @returns The interpolated string
|
|
4600
|
+
*/
|
|
4601
|
+
interpolate(str, additionalContext = {}) {
|
|
4602
|
+
const context = { ...this.params, ...additionalContext };
|
|
4603
|
+
return interpolateTemplate(str, context);
|
|
4604
|
+
}
|
|
4605
|
+
async askAI(page, question, options, assertAnswer = '') {
|
|
4403
4606
|
var _a, _b;
|
|
4404
|
-
const response = await this.askAIHelper(page, question, options);
|
|
4607
|
+
const response = await this.askAIHelper(page, question, options, assertAnswer);
|
|
4405
4608
|
if ((_a = response === null || response === void 0 ? void 0 : response.result) === null || _a === void 0 ? void 0 : _a.error) {
|
|
4406
4609
|
throw new Error(response.result.error);
|
|
4407
4610
|
}
|
|
4408
4611
|
return (_b = response === null || response === void 0 ? void 0 : response.result) === null || _b === void 0 ? void 0 : _b.answer;
|
|
4409
4612
|
}
|
|
4410
4613
|
async runStep(page, stepPrompt, argument = null, options = { aiModel: this.aiModel, timeoutConfig: this.timeoutConfig }) {
|
|
4614
|
+
// Interpolate argument if it's a string with template literals
|
|
4615
|
+
argument = this.interpolate(argument);
|
|
4411
4616
|
const runStepStartTime = Date.now();
|
|
4617
|
+
//trace message to help with debug
|
|
4618
|
+
proboLogger.info(`runStep(${stepPrompt}, ${argument})`);
|
|
4412
4619
|
try {
|
|
4413
4620
|
// Determine which AI model to use for this step
|
|
4414
4621
|
const aiModelToUse = options.aiModel !== undefined ? options.aiModel : this.aiModel;
|
|
@@ -4478,7 +4685,7 @@ class Probo {
|
|
|
4478
4685
|
proboLogger.debug(`⏱️ findBestCandidateElement took ${findBestCandidateTime}ms`);
|
|
4479
4686
|
proboLogger.debug(`AI selected candidate element at index: ${index}`);
|
|
4480
4687
|
// STEP 8: Find the actual element object from the candidates
|
|
4481
|
-
const actualElement = candidate_elements.find(element =>
|
|
4688
|
+
const actualElement = candidate_elements.find(element => element.index === index);
|
|
4482
4689
|
if (!actualElement) {
|
|
4483
4690
|
throw new Error(`No candidate element found with index ${index}. Available indices: ${candidate_elements.map(e => e.index).join(', ')}`);
|
|
4484
4691
|
}
|
|
@@ -4517,11 +4724,17 @@ class Probo {
|
|
|
4517
4724
|
proboLogger.log(`Found existing step with ID: ${result.id} going to perform action: ${result.action} with value: ${actionArgument}`);
|
|
4518
4725
|
try {
|
|
4519
4726
|
// Create ProboPlaywright instance with the page
|
|
4520
|
-
const proboPlaywright = new ProboPlaywright(
|
|
4727
|
+
const proboPlaywright = new ProboPlaywright({
|
|
4728
|
+
enableSmartSelectors: this.enableSmartSelectors,
|
|
4729
|
+
debugLevel: this.debugLevel,
|
|
4730
|
+
timeoutConfig: this.timeoutConfig
|
|
4731
|
+
}, page);
|
|
4521
4732
|
// Call runStep with the cached action
|
|
4522
4733
|
const runStepResult = await proboPlaywright.runStep({
|
|
4523
4734
|
iframeSelector: result.iframe_selector,
|
|
4524
4735
|
elementSelector: result.element_css_selector,
|
|
4736
|
+
smartSelector: result.smart_selector,
|
|
4737
|
+
smartIFrameSelector: result.smart_iframe_selector,
|
|
4525
4738
|
action: result.action,
|
|
4526
4739
|
argument: actionArgument,
|
|
4527
4740
|
});
|
|
@@ -4565,8 +4778,8 @@ class Probo {
|
|
|
4565
4778
|
async unhighlightElements(page) {
|
|
4566
4779
|
return this.highlighter.unhighlightElements(page);
|
|
4567
4780
|
}
|
|
4568
|
-
async highlightElement(page, element_css_selector, iframe_selector, element_index) {
|
|
4569
|
-
return this.highlighter.highlightElement(page, element_css_selector, iframe_selector, element_index);
|
|
4781
|
+
async highlightElement(page, element_css_selector, iframe_selector, smart_selector, smart_iframe_selector, element_index) {
|
|
4782
|
+
return this.highlighter.highlightElement(page, element_css_selector, iframe_selector, smart_selector, smart_iframe_selector, element_index);
|
|
4570
4783
|
}
|
|
4571
4784
|
async waitForMutationsToSettle(page, timeout, initTimeout) {
|
|
4572
4785
|
const mutationTimeout = timeout !== null && timeout !== void 0 ? timeout : DEFAULT_PLAYWRIGHT_TIMEOUT_CONFIG.mutationsTimeout;
|
|
@@ -4576,7 +4789,7 @@ class Probo {
|
|
|
4576
4789
|
async screenshot(page) {
|
|
4577
4790
|
proboLogger.debug(`taking screenshot of current page: ${page.url()}`);
|
|
4578
4791
|
// await page.evaluate(() => document.fonts?.ready.catch(() => {}));
|
|
4579
|
-
const screenshot_bytes = await page.screenshot({ fullPage: true, animations: 'disabled' });
|
|
4792
|
+
const screenshot_bytes = await page.screenshot({ fullPage: true, animations: 'disabled', timeout: 10000 });
|
|
4580
4793
|
// make an api call to upload the screenshot to cloud
|
|
4581
4794
|
proboLogger.debug('uploading image data to cloud');
|
|
4582
4795
|
const screenshot_url = await this.apiClient.uploadScreenshot(screenshot_bytes);
|
|
@@ -4587,17 +4800,25 @@ class Probo {
|
|
|
4587
4800
|
const element_css_selector = actualElement.css_selector;
|
|
4588
4801
|
const iframe_selector = actualElement.iframe_selector;
|
|
4589
4802
|
const element_index = actualElement.index;
|
|
4590
|
-
|
|
4803
|
+
const smart_selector = actualElement.smart_selector;
|
|
4804
|
+
const smart_iframe_selector = actualElement.smart_iframe_selector;
|
|
4805
|
+
proboLogger.debug('Handling perform action:', { action, element_css_selector, iframe_selector, smart_selector, smart_iframe_selector, element_index });
|
|
4591
4806
|
if (action !== PlaywrightAction.VISIT_URL) {
|
|
4592
4807
|
await this.unhighlightElements(page);
|
|
4593
4808
|
proboLogger.debug('Unhighlighted elements');
|
|
4594
|
-
await this.highlightElement(page, element_css_selector, iframe_selector, element_index);
|
|
4809
|
+
await this.highlightElement(page, element_css_selector, iframe_selector, smart_selector, smart_iframe_selector, element_index);
|
|
4595
4810
|
proboLogger.debug('Highlighted element');
|
|
4596
4811
|
}
|
|
4597
4812
|
const pre_action_screenshot_url = await this.screenshot(page);
|
|
4598
|
-
const returnValue = await new ProboPlaywright(
|
|
4813
|
+
const returnValue = await new ProboPlaywright({
|
|
4814
|
+
enableSmartSelectors: this.enableSmartSelectors,
|
|
4815
|
+
debugLevel: this.debugLevel,
|
|
4816
|
+
timeoutConfig: this.timeoutConfig
|
|
4817
|
+
}, page).runStep({
|
|
4599
4818
|
iframeSelector: iframe_selector,
|
|
4600
4819
|
elementSelector: element_css_selector,
|
|
4820
|
+
smartSelector: smart_selector,
|
|
4821
|
+
smartIFrameSelector: smart_iframe_selector,
|
|
4601
4822
|
action: action,
|
|
4602
4823
|
argument: value,
|
|
4603
4824
|
highlightTimeout: 0
|
|
@@ -4611,7 +4832,8 @@ class Probo {
|
|
|
4611
4832
|
});
|
|
4612
4833
|
return returnValue;
|
|
4613
4834
|
}
|
|
4614
|
-
async askAIHelper(page, question, options) {
|
|
4835
|
+
async askAIHelper(page, question, options, assertAnswer = '') {
|
|
4836
|
+
var _a, _b;
|
|
4615
4837
|
// Set default value for createStep to true if not provided
|
|
4616
4838
|
const createStep = (options === null || options === void 0 ? void 0 : options.createStep) !== undefined ? options.createStep : true;
|
|
4617
4839
|
const stepId = options === null || options === void 0 ? void 0 : options.stepId;
|
|
@@ -4679,6 +4901,14 @@ class Probo {
|
|
|
4679
4901
|
proboLogger.debug(`✅ [askAI] Step created successfully with ID: ${stepId}`);
|
|
4680
4902
|
}
|
|
4681
4903
|
// Return the answer from the result, or the reasoning if no answer
|
|
4904
|
+
if (assertAnswer) {
|
|
4905
|
+
const actualAnswer = (_b = (_a = serverResponse === null || serverResponse === void 0 ? void 0 : serverResponse.result) === null || _a === void 0 ? void 0 : _a.answer) === null || _b === void 0 ? void 0 : _b.toString();
|
|
4906
|
+
const interpolatedAssertAnswer = this.interpolate(assertAnswer);
|
|
4907
|
+
if (actualAnswer !== interpolatedAssertAnswer) {
|
|
4908
|
+
proboLogger.error(`❌ [askAI] Expected answer to be ${interpolatedAssertAnswer}, but got ${actualAnswer}`);
|
|
4909
|
+
throw new Error(`Expected answer to be ${interpolatedAssertAnswer}, but got ${actualAnswer}`);
|
|
4910
|
+
}
|
|
4911
|
+
}
|
|
4682
4912
|
return serverResponse;
|
|
4683
4913
|
}
|
|
4684
4914
|
catch (error) {
|
|
@@ -4688,5 +4918,5 @@ class Probo {
|
|
|
4688
4918
|
}
|
|
4689
4919
|
}
|
|
4690
4920
|
|
|
4691
|
-
export { Highlighter, NavTracker, OTP, PlaywrightAction, Probo, ProboLogLevel, ProboPlaywright, findClosestVisibleElement };
|
|
4921
|
+
export { AIModel, Highlighter, NavTracker, OTP, PlaywrightAction, Probo, ProboLogLevel, ProboPlaywright, findClosestVisibleElement };
|
|
4692
4922
|
//# sourceMappingURL=index.js.map
|