@probolabs/playwright 1.0.12 → 1.0.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (10) hide show
  1. package/README.md +227 -227
  2. package/dist/fixtures.js.map +1 -1
  3. package/dist/index.d.ts +23 -14
  4. package/dist/index.js +1097 -1057
  5. package/dist/index.js.map +1 -1
  6. package/dist/types/index.d.ts.map +1 -1
  7. package/dist/types/otp.d.ts.map +1 -1
  8. package/dist/types/replay-utils.d.ts.map +1 -1
  9. package/loaded_extensions/README.md +23 -23
  10. package/package.json +2 -2
package/dist/index.js CHANGED
@@ -1,4 +1,14 @@
1
- const highlighterCode = "(function (global, factory) {\n typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports) :\n typeof define === 'function' && define.amd ? define(['exports'], factory) :\n (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global.ProboLabs = {}));\n})(this, (function (exports) { 'use strict';\n\n const ElementTag = {\r\n CLICKABLE: \"CLICKABLE\", // button, link, toggle switch, checkbox, radio, dropdowns, clickable divs\r\n FILLABLE: \"FILLABLE\", // input, textarea content_editable, date picker??\r\n SELECTABLE: \"SELECTABLE\", // select\r\n NON_INTERACTIVE_ELEMENT: 'NON_INTERACTIVE_ELEMENT',\r\n };\r\n\r\n class ElementInfo {\r\n constructor(element, index, {tag, type, text, html, xpath, css_selector, bounding_box, iframe_selector, short_css_selector, short_iframe_selector}) {\r\n this.index = index.toString();\r\n this.tag = tag;\r\n this.type = type;\r\n this.text = text;\r\n this.html = html;\r\n this.xpath = xpath;\r\n this.css_selector = css_selector;\r\n this.bounding_box = bounding_box;\r\n this.iframe_selector = iframe_selector;\r\n this.element = element;\r\n this.depth = -1;\r\n this.short_css_selector = short_css_selector;\r\n this.short_iframe_selector = short_iframe_selector;\r\n }\r\n\r\n getSelector() {\r\n return this.xpath ? this.xpath : this.css_selector;\r\n }\r\n\r\n getDepth() {\r\n if (this.depth >= 0) {\r\n return this.depth;\r\n }\r\n \r\n this.depth = 0;\r\n let currentElement = this.element;\r\n \r\n while (currentElement.nodeType === Node.ELEMENT_NODE) { \r\n this.depth++;\r\n currentElement = getParentNode(currentElement);\r\n }\r\n \r\n return this.depth;\r\n }\r\n }\r\n\r\n function getParentNode(element) {\r\n if (!element || element.nodeType !== Node.ELEMENT_NODE) return null;\r\n \r\n let parent = null;\r\n // SF is using slots and shadow DOM heavily\r\n // However, there might be slots in the light DOM which shouldn't be traversed\r\n if (element.assignedSlot && element.getRootNode() instanceof ShadowRoot)\r\n parent = element.assignedSlot;\r\n else \r\n parent = element.parentNode;\r\n \r\n // Check if we're at a shadow root\r\n if (parent && parent.nodeType !== Node.ELEMENT_NODE && parent.getRootNode() instanceof ShadowRoot) \r\n parent = parent.getRootNode().host; \r\n\r\n return parent;\r\n }\n\n // License: MIT\n // Author: Anton Medvedev <anton@medv.io>\n // Source: https://github.com/antonmedv/finder\n const acceptedAttrNames = new Set(['role', 'name', 'aria-label', 'rel', 'href']);\n /** Check if attribute name and value are word-like. */\n function attr(name, value) {\n let nameIsOk = acceptedAttrNames.has(name);\n nameIsOk ||= name.startsWith('data-') && wordLike(name);\n let valueIsOk = wordLike(value) && value.length < 100;\n valueIsOk ||= value.startsWith('#') && wordLike(value.slice(1));\n return nameIsOk && valueIsOk;\n }\n /** Check if id name is word-like. */\n function idName(name) {\n return wordLike(name);\n }\n /** Check if class name is word-like. */\n function className(name) {\n return wordLike(name);\n }\n /** Check if tag name is word-like. */\n function tagName(name) {\n return true;\n }\n /** Finds unique CSS selectors for the given element. */\n function finder(input, options) {\n if (input.nodeType !== Node.ELEMENT_NODE) {\n throw new Error(`Can't generate CSS selector for non-element node type.`);\n }\n if (input.tagName.toLowerCase() === 'html') {\n return 'html';\n }\n const defaults = {\n root: document.body,\n idName: idName,\n className: className,\n tagName: tagName,\n attr: attr,\n timeoutMs: 1000,\n seedMinLength: 3,\n optimizedMinLength: 2,\n maxNumberOfPathChecks: Infinity,\n };\n const startTime = new Date();\n const config = { ...defaults, ...options };\n const rootDocument = findRootDocument(config.root, defaults);\n let foundPath;\n let count = 0;\n for (const candidate of search(input, config, rootDocument)) {\n const elapsedTimeMs = new Date().getTime() - startTime.getTime();\n if (elapsedTimeMs > config.timeoutMs ||\n count >= config.maxNumberOfPathChecks) {\n const fPath = fallback(input, rootDocument);\n if (!fPath) {\n throw new Error(`Timeout: Can't find a unique selector after ${config.timeoutMs}ms`);\n }\n return selector(fPath);\n }\n count++;\n if (unique(candidate, rootDocument)) {\n foundPath = candidate;\n break;\n }\n }\n if (!foundPath) {\n throw new Error(`Selector was not found.`);\n }\n const optimized = [\n ...optimize(foundPath, input, config, rootDocument, startTime),\n ];\n optimized.sort(byPenalty);\n if (optimized.length > 0) {\n return selector(optimized[0]);\n }\n return selector(foundPath);\n }\n function* search(input, config, rootDocument) {\n const stack = [];\n let paths = [];\n let current = input;\n let i = 0;\n while (current && current !== rootDocument) {\n const level = tie(current, config);\n for (const node of level) {\n node.level = i;\n }\n stack.push(level);\n current = current.parentElement;\n i++;\n paths.push(...combinations(stack));\n if (i >= config.seedMinLength) {\n paths.sort(byPenalty);\n for (const candidate of paths) {\n yield candidate;\n }\n paths = [];\n }\n }\n paths.sort(byPenalty);\n for (const candidate of paths) {\n yield candidate;\n }\n }\n function wordLike(name) {\n if (/^[a-z\\-]{3,}$/i.test(name)) {\n const words = name.split(/-|[A-Z]/);\n for (const word of words) {\n if (word.length <= 2) {\n return false;\n }\n if (/[^aeiou]{4,}/i.test(word)) {\n return false;\n }\n }\n return true;\n }\n return false;\n }\n function tie(element, config) {\n const level = [];\n const elementId = element.getAttribute('id');\n if (elementId && config.idName(elementId)) {\n level.push({\n name: '#' + CSS.escape(elementId),\n penalty: 0,\n });\n }\n for (let i = 0; i < element.classList.length; i++) {\n const name = element.classList[i];\n if (config.className(name)) {\n level.push({\n name: '.' + CSS.escape(name),\n penalty: 1,\n });\n }\n }\n for (let i = 0; i < element.attributes.length; i++) {\n const attr = element.attributes[i];\n if (config.attr(attr.name, attr.value)) {\n level.push({\n name: `[${CSS.escape(attr.name)}=\"${CSS.escape(attr.value)}\"]`,\n penalty: 2,\n });\n }\n }\n const tagName = element.tagName.toLowerCase();\n if (config.tagName(tagName)) {\n level.push({\n name: tagName,\n penalty: 5,\n });\n const index = indexOf(element, tagName);\n if (index !== undefined) {\n level.push({\n name: nthOfType(tagName, index),\n penalty: 10,\n });\n }\n }\n const nth = indexOf(element);\n if (nth !== undefined) {\n level.push({\n name: nthChild(tagName, nth),\n penalty: 50,\n });\n }\n return level;\n }\n function selector(path) {\n let node = path[0];\n let query = node.name;\n for (let i = 1; i < path.length; i++) {\n const level = path[i].level || 0;\n if (node.level === level - 1) {\n query = `${path[i].name} > ${query}`;\n }\n else {\n query = `${path[i].name} ${query}`;\n }\n node = path[i];\n }\n return query;\n }\n function penalty(path) {\n return path.map((node) => node.penalty).reduce((acc, i) => acc + i, 0);\n }\n function byPenalty(a, b) {\n return penalty(a) - penalty(b);\n }\n function indexOf(input, tagName) {\n const parent = input.parentNode;\n if (!parent) {\n return undefined;\n }\n let child = parent.firstChild;\n if (!child) {\n return undefined;\n }\n let i = 0;\n while (child) {\n if (child.nodeType === Node.ELEMENT_NODE &&\n (tagName === undefined ||\n child.tagName.toLowerCase() === tagName)) {\n i++;\n }\n if (child === input) {\n break;\n }\n child = child.nextSibling;\n }\n return i;\n }\n function fallback(input, rootDocument) {\n let i = 0;\n let current = input;\n const path = [];\n while (current && current !== rootDocument) {\n const tagName = current.tagName.toLowerCase();\n const index = indexOf(current, tagName);\n if (index === undefined) {\n return;\n }\n path.push({\n name: nthOfType(tagName, index),\n penalty: NaN,\n level: i,\n });\n current = current.parentElement;\n i++;\n }\n if (unique(path, rootDocument)) {\n return path;\n }\n }\n function nthChild(tagName, index) {\n if (tagName === 'html') {\n return 'html';\n }\n return `${tagName}:nth-child(${index})`;\n }\n function nthOfType(tagName, index) {\n if (tagName === 'html') {\n return 'html';\n }\n return `${tagName}:nth-of-type(${index})`;\n }\n function* combinations(stack, path = []) {\n if (stack.length > 0) {\n for (let node of stack[0]) {\n yield* combinations(stack.slice(1, stack.length), path.concat(node));\n }\n }\n else {\n yield path;\n }\n }\n function findRootDocument(rootNode, defaults) {\n if (rootNode.nodeType === Node.DOCUMENT_NODE) {\n return rootNode;\n }\n if (rootNode === defaults.root) {\n return rootNode.ownerDocument;\n }\n return rootNode;\n }\n function unique(path, rootDocument) {\n const css = selector(path);\n switch (rootDocument.querySelectorAll(css).length) {\n case 0:\n throw new Error(`Can't select any node with this selector: ${css}`);\n case 1:\n return true;\n default:\n return false;\n }\n }\n function* optimize(path, input, config, rootDocument, startTime) {\n if (path.length > 2 && path.length > config.optimizedMinLength) {\n for (let i = 1; i < path.length - 1; i++) {\n const elapsedTimeMs = new Date().getTime() - startTime.getTime();\n if (elapsedTimeMs > config.timeoutMs) {\n return;\n }\n const newPath = [...path];\n newPath.splice(i, 1);\n if (unique(newPath, rootDocument) &&\n rootDocument.querySelector(selector(newPath)) === input) {\n yield newPath;\n yield* optimize(newPath, input, config, rootDocument, startTime);\n }\n }\n }\n }\n\n // import { realpath } from \"fs\";\r\n\r\n function getAllDocumentElementsIncludingShadow(selectors, root = document) {\r\n const elements = Array.from(root.querySelectorAll(selectors));\r\n\r\n root.querySelectorAll('*').forEach(el => {\r\n if (el.shadowRoot) {\r\n elements.push(...getAllDocumentElementsIncludingShadow(selectors, el.shadowRoot));\r\n }\r\n });\r\n return elements;\r\n }\r\n\r\n function getAllFrames(root = document) {\r\n const result = [root];\r\n const frames = getAllDocumentElementsIncludingShadow('frame, iframe', root); \r\n frames.forEach(frame => {\r\n try {\r\n const frameDocument = frame.contentDocument || frame.contentWindow.document;\r\n if (frameDocument) {\r\n result.push(frameDocument);\r\n }\r\n } catch (e) {\r\n // Skip cross-origin frames\r\n console.warn('Could not access frame content:', e.message);\r\n }\r\n });\r\n\r\n return result;\r\n }\r\n\r\n function getAllElementsIncludingShadow(selectors, root = document) {\r\n const elements = [];\r\n\r\n getAllFrames(root).forEach(doc => {\r\n elements.push(...getAllDocumentElementsIncludingShadow(selectors, doc));\r\n });\r\n\r\n return elements;\r\n }\r\n\r\n /**\r\n * Deeply searches through DOM trees including Shadow DOM and frames/iframes\r\n * @param {string} selector - CSS selector to search for\r\n * @param {Document|Element} [root=document] - Starting point for the search\r\n * @param {Object} [options] - Search options\r\n * @param {boolean} [options.searchShadow=true] - Whether to search Shadow DOM\r\n * @param {boolean} [options.searchFrames=true] - Whether to search frames/iframes\r\n * @returns {Element[]} Array of found elements\r\n \r\n function getAllElementsIncludingShadow(selector, root = document, options = {}) {\r\n const {\r\n searchShadow = true,\r\n searchFrames = true\r\n } = options;\r\n\r\n const results = new Set();\r\n \r\n // Helper to check if an element is valid and not yet found\r\n const addIfValid = (element) => {\r\n if (element && !results.has(element)) {\r\n results.add(element);\r\n }\r\n };\r\n\r\n // Helper to process a single document or element\r\n function processNode(node) {\r\n // Search regular DOM\r\n node.querySelectorAll(selector).forEach(addIfValid);\r\n\r\n if (searchShadow) {\r\n // Search all shadow roots\r\n const treeWalker = document.createTreeWalker(\r\n node,\r\n NodeFilter.SHOW_ELEMENT,\r\n {\r\n acceptNode: (element) => {\r\n return element.shadowRoot ? \r\n NodeFilter.FILTER_ACCEPT : \r\n NodeFilter.FILTER_SKIP;\r\n }\r\n }\r\n );\r\n\r\n while (treeWalker.nextNode()) {\r\n const element = treeWalker.currentNode;\r\n if (element.shadowRoot) {\r\n // Search within shadow root\r\n element.shadowRoot.querySelectorAll(selector).forEach(addIfValid);\r\n // Recursively process the shadow root for nested shadow DOMs\r\n processNode(element.shadowRoot);\r\n }\r\n }\r\n }\r\n\r\n if (searchFrames) {\r\n // Search frames and iframes\r\n const frames = node.querySelectorAll('frame, iframe');\r\n frames.forEach(frame => {\r\n try {\r\n const frameDocument = frame.contentDocument;\r\n if (frameDocument) {\r\n processNode(frameDocument);\r\n }\r\n } catch (e) {\r\n // Skip cross-origin frames\r\n console.warn('Could not access frame content:', e.message);\r\n }\r\n });\r\n }\r\n }\r\n\r\n // Start processing from the root\r\n processNode(root);\r\n\r\n return Array.from(results);\r\n }\r\n */\r\n // <div x=1 y=2 role='combobox'> </div>\r\n function findDropdowns() {\r\n const dropdowns = [];\r\n \r\n // Native select elements\r\n dropdowns.push(...getAllElementsIncludingShadow('select'));\r\n \r\n // Elements with dropdown roles that don't have <input>..</input>\r\n const roleElements = getAllElementsIncludingShadow('[role=\"combobox\"], [role=\"listbox\"], [role=\"dropdown\"], [role=\"option\"], [role=\"menu\"], [role=\"menuitem\"]').filter(el => {\r\n return el.tagName.toLowerCase() !== 'input' || ![\"button\", \"checkbox\", \"radio\"].includes(el.getAttribute(\"type\"));\r\n });\r\n dropdowns.push(...roleElements);\r\n \r\n // Common dropdown class patterns\r\n const dropdownPattern = /.*(dropdown|select|combobox|menu).*/i;\r\n const elements = getAllElementsIncludingShadow('*');\r\n const dropdownClasses = Array.from(elements).filter(el => {\r\n const hasDropdownClass = dropdownPattern.test(el.className);\r\n const validTag = ['li', 'ul', 'span', 'div', 'p', 'a', 'button'].includes(el.tagName.toLowerCase());\r\n const style = window.getComputedStyle(el); \r\n const result = hasDropdownClass && validTag && (style.cursor === 'pointer' || el.tagName.toLowerCase() === 'a' || el.tagName.toLowerCase() === 'button');\r\n return result;\r\n });\r\n \r\n dropdowns.push(...dropdownClasses);\r\n \r\n // Elements with aria-haspopup attribute\r\n dropdowns.push(...getAllElementsIncludingShadow('[aria-haspopup=\"true\"], [aria-haspopup=\"listbox\"], [aria-haspopup=\"menu\"]'));\r\n\r\n // Improve navigation element detection\r\n // Semantic nav elements with list items\r\n dropdowns.push(...getAllElementsIncludingShadow('nav ul li, nav ol li'));\r\n \r\n // Navigation elements in common design patterns\r\n dropdowns.push(...getAllElementsIncludingShadow('header a, .header a, .nav a, .navigation a, .menu a, .sidebar a, aside a'));\r\n \r\n // Elements in primary navigation areas with common attributes\r\n dropdowns.push(...getAllElementsIncludingShadow('[role=\"navigation\"] a, [aria-label*=\"navigation\"] a, [aria-label*=\"menu\"] a'));\r\n\r\n return dropdowns;\r\n }\r\n\r\n function findClickables() {\r\n const clickables = [];\r\n \r\n const checkboxPattern = /checkbox/i;\r\n // Collect all clickable elements first\r\n const nativeLinks = [...getAllElementsIncludingShadow('a')];\r\n const nativeButtons = [...getAllElementsIncludingShadow('button')];\r\n const inputButtons = [...getAllElementsIncludingShadow('input[type=\"button\"], input[type=\"submit\"], input[type=\"reset\"]')];\r\n const roleButtons = [...getAllElementsIncludingShadow('[role=\"button\"]')];\r\n // const tabbable = [...getAllElementsIncludingShadow('[tabindex=\"0\"]')];\r\n const clickHandlers = [...getAllElementsIncludingShadow('[onclick]')];\r\n const dropdowns = findDropdowns();\r\n const nativeCheckboxes = [...getAllElementsIncludingShadow('input[type=\"checkbox\"]')]; \r\n const fauxCheckboxes = getAllElementsIncludingShadow('*').filter(el => {\r\n if (checkboxPattern.test(el.className)) {\r\n const realCheckboxes = getAllElementsIncludingShadow('input[type=\"checkbox\"]', el);\r\n if (realCheckboxes.length === 1) {\r\n const boundingRect = realCheckboxes[0].getBoundingClientRect();\r\n return boundingRect.width <= 1 && boundingRect.height <= 1 \r\n }\r\n }\r\n return false;\r\n });\r\n const nativeRadios = [...getAllElementsIncludingShadow('input[type=\"radio\"]')];\r\n const toggles = findToggles();\r\n const pointerElements = findElementsWithPointer();\r\n // Add all elements at once\r\n clickables.push(\r\n ...nativeLinks,\r\n ...nativeButtons,\r\n ...inputButtons,\r\n ...roleButtons,\r\n // ...tabbable,\r\n ...clickHandlers,\r\n ...dropdowns,\r\n ...nativeCheckboxes,\r\n ...fauxCheckboxes,\r\n ...nativeRadios,\r\n ...toggles,\r\n ...pointerElements\r\n );\r\n\r\n // Only uniquify once at the end\r\n return clickables; // Let findElements handle the uniquification\r\n }\r\n\r\n function findToggles() {\r\n const toggles = [];\r\n const checkboxes = getAllElementsIncludingShadow('input[type=\"checkbox\"]');\r\n const togglePattern = /switch|toggle|slider/i;\r\n\r\n checkboxes.forEach(checkbox => {\r\n let isToggle = false;\r\n\r\n // Check the checkbox itself\r\n if (togglePattern.test(checkbox.className) || togglePattern.test(checkbox.getAttribute('role') || '')) {\r\n isToggle = true;\r\n }\r\n\r\n // Check parent elements (up to 3 levels)\r\n if (!isToggle) {\r\n let element = checkbox;\r\n for (let i = 0; i < 3; i++) {\r\n const parent = element.parentElement;\r\n if (!parent) break;\r\n\r\n const className = parent.className || '';\r\n const role = parent.getAttribute('role') || '';\r\n\r\n if (togglePattern.test(className) || togglePattern.test(role)) {\r\n isToggle = true;\r\n break;\r\n }\r\n element = parent;\r\n }\r\n }\r\n\r\n // Check next sibling\r\n if (!isToggle) {\r\n const nextSibling = checkbox.nextElementSibling;\r\n if (nextSibling) {\r\n const className = nextSibling.className || '';\r\n const role = nextSibling.getAttribute('role') || '';\r\n if (togglePattern.test(className) || togglePattern.test(role)) {\r\n isToggle = true;\r\n }\r\n }\r\n }\r\n\r\n if (isToggle) {\r\n toggles.push(checkbox);\r\n }\r\n });\r\n\r\n return toggles;\r\n }\r\n\r\n function findNonInteractiveElements() {\r\n // Get all elements in the document\r\n const all = Array.from(getAllElementsIncludingShadow('*'));\r\n \r\n // Filter elements based on Python implementation rules\r\n return all.filter(element => {\r\n if (!element.firstElementChild) {\r\n const tag = element.tagName.toLowerCase(); \r\n if (!['select', 'button', 'a'].includes(tag)) {\r\n const validTags = ['p', 'span', 'div', 'input', 'textarea','td','th'].includes(tag) || /^h\\d$/.test(tag) || /text/.test(tag);\r\n const boundingRect = element.getBoundingClientRect();\r\n return validTags && boundingRect.height > 1 && boundingRect.width > 1;\r\n }\r\n }\r\n return false;\r\n });\r\n }\r\n\r\n\r\n\r\n // export function findNonInteractiveElements() {\r\n // const all = [];\r\n // try {\r\n // const elements = getAllElementsIncludingShadow('*');\r\n // all.push(...elements);\r\n // } catch (e) {\r\n // console.warn('Error getting elements:', e);\r\n // }\r\n \r\n // console.debug('Total elements found:', all.length);\r\n \r\n // return all.filter(element => {\r\n // try {\r\n // const tag = element.tagName.toLowerCase(); \r\n\r\n // // Special handling for input elements\r\n // if (tag === 'input' || tag === 'textarea') {\r\n // const boundingRect = element.getBoundingClientRect();\r\n // const value = element.value || '';\r\n // const placeholder = element.placeholder || '';\r\n // return boundingRect.height > 1 && \r\n // boundingRect.width > 1 && \r\n // (value.trim() !== '' || placeholder.trim() !== '');\r\n // }\r\n\r\n \r\n // // Check if it's a valid tag for text content\r\n // const validTags = ['p', 'span', 'div', 'label', 'th', 'td', 'li', 'button', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'a', 'select'].includes(tag) || \r\n // /^h\\d$/.test(tag) || \r\n // /text/.test(tag);\r\n\r\n // const boundingRect = element.getBoundingClientRect();\r\n\r\n // // Get direct text content, excluding child element text\r\n // let directText = '';\r\n // for (const node of element.childNodes) {\r\n // // Only include text nodes (nodeType 3)\r\n // if (node.nodeType === 3) {\r\n // directText += node.textContent || '';\r\n // }\r\n // }\r\n \r\n // // If no direct text and it's a table cell or heading, check label content\r\n // if (!directText.trim() && (tag === 'th' || tag === 'td' || tag === 'h1')) {\r\n // const labels = element.getElementsByTagName('label');\r\n // for (const label of labels) {\r\n // directText += label.textContent || '';\r\n // }\r\n // }\r\n\r\n // // If still no text and it's a heading, get all text content\r\n // if (!directText.trim() && tag === 'h1') {\r\n // directText = element.textContent || '';\r\n // }\r\n\r\n // directText = directText.trim();\r\n\r\n // // Debug logging\r\n // if (directText) {\r\n // console.debugg('Text element found:', {\r\n // tag,\r\n // text: directText,\r\n // dimensions: boundingRect,\r\n // element\r\n // });\r\n // }\r\n\r\n // return validTags && \r\n // boundingRect.height > 1 && \r\n // boundingRect.width > 1 && \r\n // directText !== '';\r\n \r\n // } catch (e) {\r\n // console.warn('Error processing element:', e);\r\n // return false;\r\n // }\r\n // });\r\n // }\r\n\r\n\r\n\r\n\r\n\r\n function findElementsWithPointer() {\r\n const elements = [];\r\n const allElements = getAllElementsIncludingShadow('*');\r\n \r\n console.log('Checking elements with pointer style...');\r\n \r\n allElements.forEach(element => {\r\n // Skip SVG elements for now\r\n if (element instanceof SVGElement || element.tagName.toLowerCase() === 'svg') {\r\n return;\r\n }\r\n \r\n const style = window.getComputedStyle(element);\r\n if (style.cursor === 'pointer') {\r\n elements.push(element);\r\n }\r\n });\r\n \r\n console.log(`Found ${elements.length} elements with pointer cursor`);\r\n return elements;\r\n }\r\n\r\n function findCheckables() {\r\n const elements = [];\r\n\r\n elements.push(...getAllElementsIncludingShadow('input[type=\"checkbox\"]'));\r\n elements.push(...getAllElementsIncludingShadow('input[type=\"radio\"]'));\r\n const all_elements = getAllElementsIncludingShadow('label');\r\n const radioClasses = Array.from(all_elements).filter(el => {\r\n return /.*radio.*/i.test(el.className); \r\n });\r\n elements.push(...radioClasses);\r\n return elements;\r\n }\r\n\r\n function findFillables() {\r\n const elements = [];\r\n\r\n const inputs = [...getAllElementsIncludingShadow('input:not([type=\"radio\"]):not([type=\"checkbox\"])')];\r\n console.log('Found inputs:', inputs.length, inputs);\r\n elements.push(...inputs);\r\n \r\n const textareas = [...getAllElementsIncludingShadow('textarea')];\r\n console.log('Found textareas:', textareas.length);\r\n elements.push(...textareas);\r\n \r\n const editables = [...getAllElementsIncludingShadow('[contenteditable=\"true\"]')];\r\n console.log('Found editables:', editables.length);\r\n elements.push(...editables);\r\n\r\n return elements;\r\n }\n\n // Helper function to check if element is a form control\r\n function isFormControl(elementInfo) {\r\n return /^(input|select|textarea|button|label)$/i.test(elementInfo.tag);\r\n }\r\n\r\n const isDropdownItem = (elementInfo) => {\r\n const dropdownPatterns = [\r\n /dropdown[-_]?item/i, // matches: dropdown-item, dropdownitem, dropdown_item\r\n /menu[-_]?item/i, // matches: menu-item, menuitem, menu_item\r\n /dropdown[-_]?link/i, // matches: dropdown-link, dropdownlink, dropdown_link\r\n /list[-_]?item/i, // matches: list-item, listitem, list_item\r\n /select[-_]?item/i, // matches: select-item, selectitem, select_item \r\n ];\r\n\r\n const rolePatterns = [\r\n /menu[-_]?item/i, // matches: menuitem, menu-item\r\n /option/i, // matches: option\r\n /list[-_]?item/i, // matches: listitem, list-item\r\n /tree[-_]?item/i // matches: treeitem, tree-item\r\n ];\r\n\r\n const hasMatchingClass = elementInfo.element.className && \r\n dropdownPatterns.some(pattern => \r\n pattern.test(elementInfo.element.className)\r\n );\r\n\r\n const hasMatchingRole = elementInfo.element.getAttribute('role') && \r\n rolePatterns.some(pattern => \r\n pattern.test(elementInfo.element.getAttribute('role'))\r\n );\r\n\r\n return hasMatchingClass || hasMatchingRole;\r\n };\r\n\r\n /**\r\n * Finds the first element matching a CSS selector, traversing Shadow DOM if necessary\r\n * @param {string} selector - CSS selector to search for\r\n * @param {Element} [root=document] - Root element to start searching from\r\n * @returns {Element|null} - The first matching element or null if not found\r\n */\r\n function querySelectorShadow(selector, root = document) {\r\n // First try to find in light DOM\r\n let element = root.querySelector(selector);\r\n if (element) return element;\r\n \r\n // Get all elements with shadow root\r\n const shadowElements = Array.from(root.querySelectorAll('*'))\r\n .filter(el => el.shadowRoot);\r\n \r\n // Search through each shadow root until we find a match\r\n for (const el of shadowElements) {\r\n element = querySelectorShadow(selector, el.shadowRoot);\r\n if (element) return element;\r\n }\r\n \r\n return null;\r\n }\r\n\r\n const getElementByXPathOrCssSelector = (element_info) => {\r\n console.log('getElementByXPathOrCssSelector:', element_info);\r\n\r\n findElement(document, element_info.iframe_selector, element_info.css_selector);\r\n };\r\n\r\n const findElement = (root, iframeSelector, cssSelector) => {\r\n let element;\r\n \r\n if (iframeSelector) { \r\n const frames = getAllDocumentElementsIncludingShadow('iframe', root);\r\n \r\n // Iterate over all frames and compare their CSS selectors\r\n for (const frame of frames) {\r\n const selector = generateCssPath(frame);\r\n if (selector === iframeSelector) {\r\n const frameDocument = frame.contentDocument || frame.contentWindow.document;\r\n element = querySelectorShadow(cssSelector, frameDocument);\r\n console.log('found element ', element);\r\n break;\r\n } \r\n } }\r\n else\r\n element = querySelectorShadow(cssSelector, root);\r\n \r\n if (!element) {\r\n console.warn('Failed to find element with CSS selector:', cssSelector);\r\n }\r\n\r\n return element;\r\n };\r\n\r\n\r\n function isDecendent(parent, child) {\r\n let element = child;\r\n while (element && element !== parent && element.nodeType === Node.ELEMENT_NODE) { \r\n element = getParentNode(element); \r\n }\r\n return element === parent;\r\n }\r\n\r\n function generateXPath(element) {\r\n return '/'+extractElementPath(element).map(item => `${item.tagName}${item.onlyChild ? '' : `[${item.index}]`}`).join('/');\r\n }\r\n\r\n function generateCssPath(element) {\r\n return extractElementPath(element).map(item => `${item.tagName}:nth-of-type(${item.index})`).join(' > ');\r\n }\r\n\r\n function extractElementPath(element) {\r\n if (!element) {\r\n console.error('ERROR: No element provided to generatePath');\r\n return [];\r\n }\r\n const path = [];\r\n // traversing up the DOM tree\r\n while (element && element.nodeType === Node.ELEMENT_NODE) { \r\n let tagName = element.nodeName.toLowerCase();\r\n \r\n let sibling = element;\r\n let index = 1;\r\n \r\n while (sibling = sibling.previousElementSibling) {\r\n if (sibling.nodeName.toLowerCase() === tagName) index++;\r\n }\r\n sibling = element;\r\n \r\n let onlyChild = (index === 1);\r\n while (onlyChild && (sibling = sibling.nextElementSibling)) {\r\n if (sibling.nodeName.toLowerCase() === tagName) onlyChild = false;\r\n }\r\n \r\n // add a tuple with tagName, index (nth), and onlyChild \r\n path.unshift({\r\n tagName: tagName,\r\n index: index,\r\n onlyChild: onlyChild \r\n }); \r\n\r\n element = getParentNode(element);\r\n }\r\n \r\n return path;\r\n }\r\n\r\n function cleanHTML(rawHTML) {\r\n const parser = new DOMParser();\r\n const doc = parser.parseFromString(rawHTML, \"text/html\");\r\n\r\n function cleanElement(element) {\r\n const allowedAttributes = new Set([\r\n \"role\",\r\n \"type\",\r\n \"class\",\r\n \"href\",\r\n \"alt\",\r\n \"title\",\r\n \"readonly\",\r\n \"checked\",\r\n \"enabled\",\r\n \"disabled\",\r\n ]);\r\n\r\n [...element.attributes].forEach(attr => {\r\n const name = attr.name.toLowerCase();\r\n const value = attr.value;\r\n\r\n const isTestAttribute = /^(testid|test-id|data-test-id)$/.test(name);\r\n const isDataAttribute = name.startsWith(\"data-\") && value;\r\n const isBooleanAttribute = [\"readonly\", \"checked\", \"enabled\", \"disabled\"].includes(name);\r\n\r\n if (!allowedAttributes.has(name) && !isDataAttribute && !isTestAttribute && !isBooleanAttribute) {\r\n element.removeAttribute(name);\r\n }\r\n });\r\n\r\n // Handle SVG content - more aggressive replacement\r\n if (element.tagName.toLowerCase() === \"svg\") {\r\n // Remove all attributes except class and role\r\n [...element.attributes].forEach(attr => {\r\n const name = attr.name.toLowerCase();\r\n if (name !== \"class\" && name !== \"role\") {\r\n element.removeAttribute(name);\r\n }\r\n });\r\n element.innerHTML = \"CONTENT REMOVED\";\r\n } else {\r\n // Recursively clean child elements\r\n Array.from(element.children).forEach(cleanElement);\r\n }\r\n\r\n // Only remove empty elements that aren't semantic or icon elements\r\n const keepEmptyElements = ['i', 'span', 'svg', 'button', 'input'];\r\n if (!keepEmptyElements.includes(element.tagName.toLowerCase()) && \r\n !element.children.length && \r\n !element.textContent.trim()) {\r\n element.remove();\r\n }\r\n }\r\n\r\n // Process all elements in the document body\r\n Array.from(doc.body.children).forEach(cleanElement);\r\n return doc.body.innerHTML;\r\n }\r\n\r\n function getContainingIframe(element) {\r\n // If not in an iframe, return null\r\n if (element.ownerDocument.defaultView === window.top) {\r\n return null;\r\n }\r\n \r\n // Try to find the iframe in the parent document that contains our element\r\n try {\r\n const parentDocument = element.ownerDocument.defaultView.parent.document;\r\n const iframes = parentDocument.querySelectorAll('iframe');\r\n \r\n for (const iframe of iframes) {\r\n if (iframe.contentWindow === element.ownerDocument.defaultView) {\r\n return iframe;\r\n }\r\n }\r\n } catch (e) {\r\n // Cross-origin restriction\r\n return \"Cross-origin iframe - cannot access details\";\r\n }\r\n \r\n return null;\r\n }\r\n\r\n function getElementInfo(element, index) {\r\n // Get text content with spaces between elements\r\n /* function getTextContent(element) {\r\n const walker = document.createTreeWalker(\r\n element,\r\n NodeFilter.SHOW_TEXT,\r\n null,\r\n false\r\n );\r\n\r\n let text = '';\r\n let node;\r\n\r\n while (node = walker.nextNode()) {\r\n const trimmedText = node.textContent.trim();\r\n if (trimmedText) {\r\n // Add space if there's already text\r\n if (text) {\r\n text += ' ';\r\n }\r\n text += trimmedText;\r\n }\r\n }\r\n\r\n return text;\r\n } */\r\n\r\n const xpath = generateXPath(element);\r\n const css_selector = generateCssPath(element);\r\n //disabled since it's blocking event handling in recorder\r\n const short_css_selector = ''; //getRobustSelector(element);\r\n\r\n const iframe = getContainingIframe(element); \r\n const iframe_selector = iframe ? generateCssPath(iframe) : \"\";\r\n //disabled since it's blocking event handling in recorder\r\n const short_iframe_selector = ''; //iframe ? getRobustSelector(iframe) : \"\";\r\n\r\n // Return element info with pre-calculated values\r\n return new ElementInfo(element, index, {\r\n tag: element.tagName.toLowerCase(),\r\n type: element.type || '',\r\n text: element.innerText || element.placeholder || '', //getTextContent(element),\r\n html: cleanHTML(element.outerHTML),\r\n xpath: xpath,\r\n css_selector: css_selector,\r\n bounding_box: element.getBoundingClientRect(),\r\n iframe_selector: iframe_selector,\r\n short_css_selector: short_css_selector,\r\n short_iframe_selector: short_iframe_selector\r\n });\r\n }\r\n\r\n function getAriaLabelledByText(elementInfo, includeHidden=true) {\r\n if (!elementInfo.element.hasAttribute('aria-labelledby')) return '';\r\n\r\n const ids = elementInfo.element.getAttribute('aria-labelledby').split(/\\s+/);\r\n let labelText = '';\r\n\r\n //locate root (document or iFrame document if element is contained in an iframe)\r\n let root = document;\r\n if (elementInfo.iframe_selector) { \r\n const frames = getAllDocumentElementsIncludingShadow('iframe', document);\r\n \r\n // Iterate over all frames and compare their CSS selectors\r\n for (const frame of frames) {\r\n const selector = generateCssPath(frame);\r\n if (selector === elementInfo.iframe_selector) {\r\n root = frame.contentDocument || frame.contentWindow.document; \r\n break;\r\n }\r\n } \r\n }\r\n\r\n ids.forEach(id => {\r\n const el = querySelectorShadow(`#${CSS.escape(id)}`, root);\r\n if (el) {\r\n if (includeHidden || el.offsetParent !== null || getComputedStyle(el).display !== 'none') {\r\n labelText += el.textContent.trim() + ' ';\r\n }\r\n }\r\n });\r\n\r\n return labelText.trim();\r\n }\r\n\r\n\r\n\r\n const filterZeroDimensions = (elementInfo) => {\r\n const rect = elementInfo.bounding_box;\r\n //single pixel elements are typically faux controls and should be filtered too\r\n const hasSize = rect.width > 1 && rect.height > 1;\r\n const style = window.getComputedStyle(elementInfo.element);\r\n const isVisible = style.display !== 'none' && style.visibility !== 'hidden';\r\n \r\n if (!hasSize || !isVisible) {\r\n \r\n return false;\r\n }\r\n return true;\r\n };\r\n\r\n\r\n\r\n function uniquifyElements(elements) {\r\n const seen = new Set();\r\n\r\n console.log(`Starting uniquification with ${elements.length} elements`);\r\n\r\n // Filter out testing infrastructure elements first\r\n const filteredInfrastructure = elements.filter(element_info => {\r\n // Skip the highlight-overlay element completely - it's part of the testing infrastructure\r\n if (element_info.element.id === 'highlight-overlay' || \r\n (element_info.css_selector && element_info.css_selector.includes('#highlight-overlay'))) {\r\n console.log('Filtered out testing infrastructure element:', element_info.css_selector);\r\n return false;\r\n }\r\n \r\n // Filter out UI framework container/manager elements\r\n const el = element_info.element;\r\n // UI framework container checks - generic detection for any framework\r\n if ((el.getAttribute('data-rendered-by') || \r\n el.getAttribute('data-reactroot') || \r\n el.getAttribute('ng-version') || \r\n el.getAttribute('data-component-id') ||\r\n el.getAttribute('data-root') ||\r\n el.getAttribute('data-framework')) && \r\n (el.className && \r\n typeof el.className === 'string' && \r\n (el.className.includes('Container') || \r\n el.className.includes('container') || \r\n el.className.includes('Manager') || \r\n el.className.includes('manager')))) {\r\n console.log('Filtered out UI framework container element:', element_info.css_selector);\r\n return false;\r\n }\r\n \r\n // Direct filter for framework container elements that shouldn't be interactive\r\n // Consolidating multiple container detection patterns into one efficient check\r\n const isFullViewport = element_info.bounding_box && \r\n element_info.bounding_box.x <= 5 && \r\n element_info.bounding_box.y <= 5 && \r\n element_info.bounding_box.width >= (window.innerWidth * 0.95) && \r\n element_info.bounding_box.height >= (window.innerHeight * 0.95);\r\n \r\n // Empty content check\r\n const isEmpty = !el.innerText || el.innerText.trim() === '';\r\n \r\n // Check if it's a framework container element\r\n if (element_info.element.tagName === 'DIV' && \r\n isFullViewport && \r\n isEmpty && \r\n (\r\n // Pattern matching for root containers\r\n (element_info.xpath && \r\n (element_info.xpath.match(/^\\/html\\[\\d+\\]\\/body\\[\\d+\\]\\/div\\[\\d+\\]\\/div\\[\\d+\\]$/) || \r\n element_info.xpath.match(/^\\/\\/\\*\\[@id='[^']+'\\]\\/div\\[\\d+\\]$/))) ||\r\n \r\n // Simple DOM structure\r\n (element_info.css_selector.split(' > ').length <= 4 && element_info.depth <= 5) ||\r\n \r\n // Empty or container-like classes\r\n (!el.className || el.className === '' || \r\n (typeof el.className === 'string' && \r\n (el.className.includes('overlay') || \r\n el.className.includes('container') || \r\n el.className.includes('wrapper'))))\r\n )) {\r\n console.log('Filtered out framework container element:', element_info.css_selector);\r\n return false;\r\n }\r\n \r\n return true;\r\n });\r\n\r\n // First filter out elements with zero dimensions\r\n const nonZeroElements = filteredInfrastructure.filter(filterZeroDimensions);\r\n // sort by CSS selector depth so parents are processed first\r\n nonZeroElements.sort((a, b) => a.getDepth() - b.getDepth());\r\n console.log(`After dimension filtering: ${nonZeroElements.length} elements remain (${elements.length - nonZeroElements.length} removed)`);\r\n \r\n const filteredByParent = nonZeroElements.filter(element_info => {\r\n\r\n const parent = findClosestParent(seen, element_info);\r\n const keep = parent == null || shouldKeepNestedElement(element_info, parent);\r\n // console.log(\"node \", element_info.index, \": keep=\", keep, \" parent=\", parent);\r\n // if (!keep && !element_info.xpath) {\r\n // console.log(\"Filtered out element \", element_info,\" because it's a nested element of \", parent);\r\n // }\r\n if (keep)\r\n seen.add(element_info.css_selector);\r\n\r\n return keep;\r\n });\r\n\r\n console.log(`After parent/child filtering: ${filteredByParent.length} elements remain (${nonZeroElements.length - filteredByParent.length} removed)`);\r\n\r\n // Final overlap filtering\r\n const filteredResults = filteredByParent.filter(element => {\r\n\r\n // Look for any element that came BEFORE this one in the array\r\n const hasEarlierOverlap = filteredByParent.some(other => {\r\n // Only check elements that came before (lower index)\r\n if (filteredByParent.indexOf(other) >= filteredByParent.indexOf(element)) {\r\n return false;\r\n }\r\n \r\n const isOverlapping = areElementsOverlapping(element, other); \r\n return isOverlapping;\r\n }); \r\n\r\n // Keep element if it has no earlier overlapping elements\r\n return !hasEarlierOverlap;\r\n });\r\n \r\n \r\n \r\n // Check for overlay removal\r\n console.log(`After filtering: ${filteredResults.length} (${filteredByParent.length - filteredResults.length} removed by overlap)`);\r\n \r\n const nonOverlaidElements = filteredResults.filter(element => {\r\n return !isOverlaid(element);\r\n });\r\n\r\n console.log(`Final elements after overlay removal: ${nonOverlaidElements.length} (${filteredResults.length - nonOverlaidElements.length} removed)`);\r\n \r\n return nonOverlaidElements;\r\n\r\n }\r\n\r\n\r\n\r\n const areElementsOverlapping = (element1, element2) => {\r\n if (element1.css_selector === element2.css_selector) {\r\n return true;\r\n }\r\n \r\n const box1 = element1.bounding_box;\r\n const box2 = element2.bounding_box;\r\n \r\n return box1.x === box2.x &&\r\n box1.y === box2.y &&\r\n box1.width === box2.width &&\r\n box1.height === box2.height;\r\n // element1.text === element2.text &&\r\n // element2.tag === 'a';\r\n };\r\n\r\n function findClosestParent(seen, element_info) { \r\n \r\n // Split the xpath into segments\r\n const segments = element_info.css_selector.split(' > ');\r\n \r\n // Try increasingly shorter paths until we find one in the seen set\r\n for (let i = segments.length - 1; i > 0; i--) {\r\n const parentPath = segments.slice(0, i).join(' > ');\r\n if (seen.has(parentPath)) {\r\n return parentPath;\r\n }\r\n }\r\n\r\n return null;\r\n }\r\n\r\n function shouldKeepNestedElement(elementInfo, parentPath) {\r\n let result = false;\r\n const parentSegments = parentPath.split(' > ');\r\n\r\n const isParentLink = /^a(:nth-of-type\\(\\d+\\))?$/.test(parentSegments[parentSegments.length - 1]);\r\n if (isParentLink) {\r\n return false; \r\n }\r\n // If this is a checkbox/radio input\r\n if (elementInfo.tag === 'input' && \r\n (elementInfo.type === 'checkbox' || elementInfo.type === 'radio')) {\r\n \r\n // Check if parent is a label by looking at the parent xpath's last segment\r\n \r\n const isParentLabel = /^label(:nth-of-type\\(\\d+\\))?$/.test(parentSegments[parentSegments.length - 1]);\r\n \r\n // If parent is a label, don't keep the input (we'll keep the label instead)\r\n if (isParentLabel) {\r\n return false;\r\n }\r\n }\r\n \r\n // Keep all other form controls and dropdown items\r\n if (isFormControl(elementInfo) || isDropdownItem(elementInfo)) {\r\n result = true;\r\n }\r\n\r\n if(isTableCell(elementInfo)) {\r\n result = true;\r\n }\r\n \r\n \r\n // console.log(`shouldKeepNestedElement: ${elementInfo.tag} ${elementInfo.text} ${elementInfo.xpath} -> ${parentXPath} -> ${result}`);\r\n return result;\r\n }\r\n\r\n\r\n function isTableCell(elementInfo) {\r\n const element = elementInfo.element;\r\n if(!element || !(element instanceof HTMLElement)) {\r\n return false;\r\n }\r\n const validTags = new Set(['td', 'th']);\r\n const validRoles = new Set(['cell', 'gridcell', 'columnheader', 'rowheader']);\r\n \r\n const tag = element.tagName.toLowerCase();\r\n const role = element.getAttribute('role')?.toLowerCase();\r\n\r\n if (validTags.has(tag) || (role && validRoles.has(role))) {\r\n return true;\r\n }\r\n return false;\r\n \r\n }\r\n\r\n function isOverlaid(elementInfo) {\r\n const element = elementInfo.element;\r\n const boundingRect = elementInfo.bounding_box;\r\n \r\n\r\n \r\n \r\n // Create a diagnostic logging function that only logs when needed\r\n const diagnosticLog = (...args) => {\r\n { // set to true for debugging\r\n console.log('[OVERLAY-DEBUG]', ...args);\r\n }\r\n };\r\n\r\n // Special handling for tooltips\r\n if (elementInfo.element.className && typeof elementInfo.element.className === 'string' && \r\n elementInfo.element.className.includes('tooltip')) {\r\n diagnosticLog('Element is a tooltip, not considering it overlaid');\r\n return false;\r\n }\r\n \r\n \r\n \r\n // Get element at the center point to check if it's covered by a popup/modal\r\n const middleX = boundingRect.x + boundingRect.width/2;\r\n const middleY = boundingRect.y + boundingRect.height/2;\r\n const elementAtMiddle = element.ownerDocument.elementFromPoint(middleX, middleY);\r\n \r\n if (elementAtMiddle && \r\n elementAtMiddle !== element && \r\n !isDecendent(element, elementAtMiddle) && \r\n !isDecendent(elementAtMiddle, element)) {\r\n\r\n \r\n return true;\r\n }\r\n \r\n \r\n return false;\r\n \r\n }\r\n\r\n\r\n\r\n /**\r\n * Get the “best” short, unique, and robust CSS selector for an element.\r\n * \r\n * @param {Element} element\r\n * @returns {string} A selector guaranteed to find exactly that element in its context\r\n */\r\n function getRobustSelector(element) {\r\n // 1. Figure out the real “root” (iframe doc, shadow root, or main doc)\r\n const root = (() => {\r\n const rootNode = element.getRootNode();\r\n if (rootNode instanceof ShadowRoot) {\r\n return rootNode;\r\n }\r\n return element.ownerDocument;\r\n })();\r\n\r\n // 2. Options to bias toward stable attrs and away from auto-generated classes\r\n const options = {\r\n root,\r\n // only use data-*, id or aria-label by default\r\n attr(name, value) {\r\n if (name === 'id' || name.startsWith('data-') || name === 'aria-label') {\r\n return true;\r\n }\r\n return false;\r\n },\r\n // skip framework junk\r\n filter(name, value) {\r\n if (name.startsWith('ng-') || name.startsWith('_ngcontent') || /^p-/.test(name)) {\r\n return false;\r\n }\r\n return true;\r\n },\r\n // let finder try really short seeds\r\n seedMinLength: 1,\r\n optimizedMinLength: 1,\r\n };\r\n\r\n let selector;\r\n try {\r\n selector = finder(element, options);\r\n // 3. Verify it really works in the context\r\n const found = root.querySelectorAll(selector);\r\n if (found.length !== 1 || found[0] !== element) {\r\n throw new Error('not unique or not found');\r\n }\r\n return selector;\r\n } catch (err) {\r\n // 4. Fallback: full path (you already have this utility)\r\n console.warn('[getRobustSelector] finder failed, falling back to full path:', err);\r\n return generateCssPath(element); // you’d import or define this elsewhere\r\n }\r\n }\r\n\r\n /**\r\n * Checks if an element is scrollable (has scrollable content)\r\n * \r\n * @param element - The element to check\r\n * @returns boolean indicating if the element is scrollable\r\n */\r\n function isScrollableContainer(element) {\r\n if (!element) return false;\r\n \r\n const style = window.getComputedStyle(element);\r\n \r\n // Reliable way to detect if an element has scrollbars or is scrollable\r\n const hasScrollHeight = element.scrollHeight > element.clientHeight;\r\n const hasScrollWidth = element.scrollWidth > element.clientWidth;\r\n \r\n // Check actual style properties\r\n const hasOverflowY = style.overflowY === 'auto' || \r\n style.overflowY === 'scroll' || \r\n style.overflowY === 'overlay';\r\n const hasOverflowX = style.overflowX === 'auto' || \r\n style.overflowX === 'scroll' || \r\n style.overflowX === 'overlay';\r\n \r\n // Check common class names and attributes for scrollable containers across frameworks\r\n const hasScrollClasses = element.classList.contains('scroll') || \r\n element.classList.contains('scrollable') ||\r\n element.classList.contains('overflow') ||\r\n element.classList.contains('overflow-auto') ||\r\n element.classList.contains('overflow-scroll') ||\r\n element.getAttribute('data-scrollable') === 'true';\r\n \r\n // Check for height/max-height constraints that often indicate scrolling content\r\n const hasHeightConstraint = style.maxHeight && \r\n style.maxHeight !== 'none' && \r\n style.maxHeight !== 'auto';\r\n \r\n // An element is scrollable if it has:\r\n // 1. Actual scrollbars in use (most reliable check) OR\r\n // 2. Overflow styles allowing scrolling AND content that would require scrolling\r\n return (hasScrollHeight && hasOverflowY) || \r\n (hasScrollWidth && hasOverflowX) ||\r\n (hasScrollClasses && (hasScrollHeight || hasScrollWidth)) ||\r\n (hasHeightConstraint && hasScrollHeight);\r\n }\r\n\r\n /**\r\n * Detects scrollable containers that are ancestors of the target element\r\n * \r\n * This function traverses up the DOM tree from the target element and identifies\r\n * all scrollable containers (elements that have scrollable content).\r\n * \r\n * @param target - The target element to start the search from\r\n * @returns Array of objects with selector and scroll properties\r\n */\r\n function detectScrollableContainers(target) {\r\n const scrollableContainers = [];\r\n \r\n if (!target) {\r\n return scrollableContainers;\r\n }\r\n \r\n console.log('🔍 [detectScrollableContainers] Starting detection for target:', target.tagName, target.id, target.className);\r\n \r\n // Detect if target is inside an iframe\r\n const iframe = getContainingIframe(target);\r\n const iframe_selector = iframe ? generateCssPath(iframe) : \"\";\r\n \r\n console.log('🔍 [detectScrollableContainers] Iframe context:', iframe ? 'inside iframe' : 'main document', 'selector:', iframe_selector);\r\n \r\n // Start from the target element and traverse up the DOM tree\r\n let currentElement = target;\r\n let depth = 0;\r\n const MAX_DEPTH = 10; // Limit traversal depth to avoid infinite loops\r\n \r\n while (currentElement && currentElement.nodeType === Node.ELEMENT_NODE && depth < MAX_DEPTH) { \r\n // Check if the current element is scrollable\r\n if (isScrollableContainer(currentElement)) {\r\n console.log('🔍 [detectScrollableContainers] Found scrollable container at depth', depth, ':', currentElement.tagName, currentElement.id, currentElement.className);\r\n \r\n const container = {\r\n containerEl: currentElement,\r\n selector: generateCssPath(currentElement),\r\n iframe_selector: iframe_selector,\r\n scrollTop: currentElement.scrollTop,\r\n scrollLeft: currentElement.scrollLeft,\r\n scrollHeight: currentElement.scrollHeight,\r\n scrollWidth: currentElement.scrollWidth,\r\n clientHeight: currentElement.clientHeight,\r\n clientWidth: currentElement.clientWidth\r\n };\r\n \r\n scrollableContainers.push(container);\r\n }\r\n \r\n // Move to parent element\r\n currentElement = getParentNode(currentElement);\r\n \r\n depth++;\r\n }\r\n \r\n console.log('🔍 [detectScrollableContainers] Detection complete. Found', scrollableContainers.length, 'scrollable containers');\r\n return scrollableContainers;\r\n }\n\n class DOMSerializer {\r\n constructor(options = {}) {\r\n this.options = {\r\n includeStyles: true,\r\n includeScripts: false, // Security consideration\r\n includeFrames: true,\r\n includeShadowDOM: true,\r\n maxDepth: 50,\r\n ...options\r\n };\r\n this.serializedFrames = new Map();\r\n this.shadowRoots = new Map();\r\n }\r\n \r\n /**\r\n * Serialize a complete document or element\r\n */\r\n serialize(rootElement = document) {\r\n try {\r\n const serialized = {\r\n type: 'document',\r\n doctype: this.serializeDoctype(rootElement),\r\n documentElement: this.serializeElement(rootElement.documentElement || rootElement),\r\n frames: [],\r\n timestamp: Date.now(),\r\n url: rootElement.URL || window.location?.href,\r\n metadata: {\r\n title: rootElement.title,\r\n charset: rootElement.characterSet,\r\n contentType: rootElement.contentType\r\n }\r\n };\r\n \r\n // Serialize frames and iframes if enabled\r\n if (this.options.includeFrames) {\r\n serialized.frames = this.serializeFrames(rootElement);\r\n }\r\n \r\n return serialized;\r\n } catch (error) {\r\n console.error('Serialization error:', error);\r\n throw new Error(`DOM serialization failed: ${error.message}`);\r\n }\r\n }\r\n \r\n /**\r\n * Serialize document type declaration\r\n */\r\n serializeDoctype(doc) {\r\n if (!doc.doctype) return null;\r\n \r\n return {\r\n name: doc.doctype.name,\r\n publicId: doc.doctype.publicId,\r\n systemId: doc.doctype.systemId\r\n };\r\n }\r\n \r\n /**\r\n * Serialize an individual element and its children\r\n */\r\n serializeElement(element, depth = 0) {\r\n if (depth > this.options.maxDepth) {\r\n return { type: 'text', content: '<!-- Max depth exceeded -->' };\r\n }\r\n \r\n const nodeType = element.nodeType;\r\n \r\n switch (nodeType) {\r\n case Node.ELEMENT_NODE:\r\n return this.serializeElementNode(element, depth);\r\n case Node.TEXT_NODE:\r\n return this.serializeTextNode(element);\r\n case Node.COMMENT_NODE:\r\n return this.serializeCommentNode(element);\r\n case Node.DOCUMENT_FRAGMENT_NODE:\r\n return this.serializeDocumentFragment(element, depth);\r\n default:\r\n return null;\r\n }\r\n }\r\n \r\n /**\r\n * Serialize element node with attributes and children\r\n */\r\n serializeElementNode(element, depth) {\r\n const tagName = element.tagName.toLowerCase();\r\n \r\n // Skip script tags for security unless explicitly enabled\r\n if (tagName === 'script' && !this.options.includeScripts) {\r\n return { type: 'comment', content: '<!-- Script tag removed for security -->' };\r\n }\r\n \r\n const serialized = {\r\n type: 'element',\r\n tagName: tagName,\r\n attributes: this.serializeAttributes(element),\r\n children: [],\r\n shadowRoot: null\r\n };\r\n \r\n // Handle Shadow DOM\r\n if (this.options.includeShadowDOM && element.shadowRoot) {\r\n serialized.shadowRoot = this.serializeShadowRoot(element.shadowRoot, depth + 1);\r\n }\r\n \r\n // Handle special elements\r\n if (tagName === 'iframe' || tagName === 'frame') {\r\n serialized.frameData = this.serializeFrameElement(element);\r\n }\r\n \r\n // Serialize children\r\n for (const child of element.childNodes) {\r\n const serializedChild = this.serializeElement(child, depth + 1);\r\n if (serializedChild) {\r\n serialized.children.push(serializedChild);\r\n }\r\n }\r\n \r\n // Include computed styles if enabled\r\n if (this.options.includeStyles && element.nodeType === Node.ELEMENT_NODE) {\r\n serialized.computedStyle = this.serializeComputedStyle(element);\r\n }\r\n \r\n return serialized;\r\n }\r\n \r\n /**\r\n * Serialize element attributes\r\n */\r\n serializeAttributes(element) {\r\n const attributes = {};\r\n \r\n if (element.attributes) {\r\n for (const attr of element.attributes) {\r\n attributes[attr.name] = attr.value;\r\n }\r\n }\r\n \r\n return attributes;\r\n }\r\n \r\n /**\r\n * Serialize computed styles\r\n */\r\n serializeComputedStyle(element) {\r\n try {\r\n const computedStyle = window.getComputedStyle(element);\r\n const styles = {};\r\n \r\n // Only serialize non-default values to reduce size\r\n const importantStyles = [\r\n 'display', 'position', 'width', 'height', 'margin', 'padding',\r\n 'border', 'background', 'color', 'font-family', 'font-size',\r\n 'text-align', 'visibility', 'z-index', 'transform'\r\n ];\r\n \r\n for (const prop of importantStyles) {\r\n const value = computedStyle.getPropertyValue(prop);\r\n if (value && value !== 'initial' && value !== 'normal') {\r\n styles[prop] = value;\r\n }\r\n }\r\n \r\n return styles;\r\n } catch (error) {\r\n return {};\r\n }\r\n }\r\n \r\n /**\r\n * Serialize text node\r\n */\r\n serializeTextNode(node) {\r\n return {\r\n type: 'text',\r\n content: node.textContent\r\n };\r\n }\r\n \r\n /**\r\n * Serialize comment node\r\n */\r\n serializeCommentNode(node) {\r\n return {\r\n type: 'comment',\r\n content: node.textContent\r\n };\r\n }\r\n \r\n /**\r\n * Serialize document fragment\r\n */\r\n serializeDocumentFragment(fragment, depth) {\r\n const serialized = {\r\n type: 'fragment',\r\n children: []\r\n };\r\n \r\n for (const child of fragment.childNodes) {\r\n const serializedChild = this.serializeElement(child, depth + 1);\r\n if (serializedChild) {\r\n serialized.children.push(serializedChild);\r\n }\r\n }\r\n \r\n return serialized;\r\n }\r\n \r\n /**\r\n * Serialize Shadow DOM\r\n */\r\n serializeShadowRoot(shadowRoot, depth) {\r\n const serialized = {\r\n type: 'shadowRoot',\r\n mode: shadowRoot.mode,\r\n children: []\r\n };\r\n \r\n for (const child of shadowRoot.childNodes) {\r\n const serializedChild = this.serializeElement(child, depth + 1);\r\n if (serializedChild) {\r\n serialized.children.push(serializedChild);\r\n }\r\n }\r\n \r\n return serialized;\r\n }\r\n \r\n /**\r\n * Serialize frame/iframe elements\r\n */\r\n serializeFrameElement(frameElement) {\r\n const frameData = {\r\n src: frameElement.src,\r\n name: frameElement.name,\r\n id: frameElement.id,\r\n sandbox: frameElement.sandbox?.toString() || '',\r\n allowfullscreen: frameElement.allowFullscreen\r\n };\r\n \r\n // Try to access frame content (may fail due to CORS)\r\n try {\r\n const frameDoc = frameElement.contentDocument;\r\n if (frameDoc && this.options.includeFrames) {\r\n frameData.content = this.serialize(frameDoc);\r\n }\r\n } catch (error) {\r\n frameData.accessError = 'Cross-origin frame content not accessible';\r\n }\r\n \r\n return frameData;\r\n }\r\n \r\n /**\r\n * Serialize all frames in document\r\n */\r\n serializeFrames(doc) {\r\n const frames = [];\r\n const frameElements = doc.querySelectorAll('iframe, frame');\r\n \r\n for (const frameElement of frameElements) {\r\n try {\r\n const frameDoc = frameElement.contentDocument;\r\n if (frameDoc) {\r\n frames.push({\r\n element: this.serializeElement(frameElement),\r\n content: this.serialize(frameDoc)\r\n });\r\n }\r\n } catch (error) {\r\n frames.push({\r\n element: this.serializeElement(frameElement),\r\n error: 'Frame content not accessible'\r\n });\r\n }\r\n }\r\n \r\n return frames;\r\n }\r\n \r\n /**\r\n * Deserialize serialized DOM data back to DOM nodes\r\n */\r\n deserialize(serializedData, targetDocument = document) {\r\n try {\r\n if (serializedData.type === 'document') {\r\n return this.deserializeDocument(serializedData, targetDocument);\r\n } else {\r\n return this.deserializeElement(serializedData, targetDocument);\r\n }\r\n } catch (error) {\r\n console.error('Deserialization error:', error);\r\n throw new Error(`DOM deserialization failed: ${error.message}`);\r\n }\r\n }\r\n \r\n /**\r\n * Deserialize complete document\r\n */\r\n deserializeDocument(serializedDoc, targetDoc) {\r\n // Create new document if needed\r\n const doc = targetDoc || document.implementation.createHTMLDocument();\r\n \r\n // Set doctype if present\r\n if (serializedDoc.doctype) {\r\n const doctype = document.implementation.createDocumentType(\r\n serializedDoc.doctype.name,\r\n serializedDoc.doctype.publicId,\r\n serializedDoc.doctype.systemId\r\n );\r\n doc.replaceChild(doctype, doc.doctype);\r\n }\r\n \r\n // Deserialize document element\r\n if (serializedDoc.documentElement) {\r\n const newDocElement = this.deserializeElement(serializedDoc.documentElement, doc);\r\n doc.replaceChild(newDocElement, doc.documentElement);\r\n }\r\n \r\n // Handle metadata\r\n if (serializedDoc.metadata) {\r\n doc.title = serializedDoc.metadata.title || '';\r\n }\r\n \r\n return doc;\r\n }\r\n \r\n /**\r\n * Deserialize individual element\r\n */\r\n deserializeElement(serializedNode, doc) {\r\n switch (serializedNode.type) {\r\n case 'element':\r\n return this.deserializeElementNode(serializedNode, doc);\r\n case 'text':\r\n return doc.createTextNode(serializedNode.content);\r\n case 'comment':\r\n return doc.createComment(serializedNode.content);\r\n case 'fragment':\r\n return this.deserializeDocumentFragment(serializedNode, doc);\r\n case 'shadowRoot':\r\n // Shadow roots are handled during element creation\r\n return null;\r\n default:\r\n return null;\r\n }\r\n }\r\n \r\n /**\r\n * Deserialize element node\r\n */\r\n deserializeElementNode(serializedElement, doc) {\r\n const element = doc.createElement(serializedElement.tagName);\r\n \r\n // Set attributes\r\n if (serializedElement.attributes) {\r\n for (const [name, value] of Object.entries(serializedElement.attributes)) {\r\n try {\r\n element.setAttribute(name, value);\r\n } catch (error) {\r\n console.warn(`Failed to set attribute ${name}:`, error);\r\n }\r\n }\r\n }\r\n \r\n // Apply computed styles if available\r\n if (serializedElement.computedStyle && this.options.includeStyles) {\r\n for (const [prop, value] of Object.entries(serializedElement.computedStyle)) {\r\n try {\r\n element.style.setProperty(prop, value);\r\n } catch (error) {\r\n console.warn(`Failed to set style ${prop}:`, error);\r\n }\r\n }\r\n }\r\n \r\n // Create shadow root if present\r\n if (serializedElement.shadowRoot && element.attachShadow) {\r\n try {\r\n const shadowRoot = element.attachShadow({ \r\n mode: serializedElement.shadowRoot.mode || 'open' \r\n });\r\n \r\n // Deserialize shadow root children\r\n for (const child of serializedElement.shadowRoot.children) {\r\n const childElement = this.deserializeElement(child, doc);\r\n if (childElement) {\r\n shadowRoot.appendChild(childElement);\r\n }\r\n }\r\n } catch (error) {\r\n console.warn('Failed to create shadow root:', error);\r\n }\r\n }\r\n \r\n // Deserialize children\r\n if (serializedElement.children) {\r\n for (const child of serializedElement.children) {\r\n const childElement = this.deserializeElement(child, doc);\r\n if (childElement) {\r\n element.appendChild(childElement);\r\n }\r\n }\r\n }\r\n \r\n // Handle frame content\r\n if (serializedElement.frameData && serializedElement.frameData.content) {\r\n // Frame content deserialization would happen after the frame loads\r\n element.addEventListener('load', () => {\r\n try {\r\n const frameDoc = element.contentDocument;\r\n if (frameDoc) {\r\n this.deserializeDocument(serializedElement.frameData.content, frameDoc);\r\n }\r\n } catch (error) {\r\n console.warn('Failed to deserialize frame content:', error);\r\n }\r\n });\r\n }\r\n \r\n return element;\r\n }\r\n \r\n /**\r\n * Deserialize document fragment\r\n */\r\n deserializeDocumentFragment(serializedFragment, doc) {\r\n const fragment = doc.createDocumentFragment();\r\n \r\n if (serializedFragment.children) {\r\n for (const child of serializedFragment.children) {\r\n const childElement = this.deserializeElement(child, doc);\r\n if (childElement) {\r\n fragment.appendChild(childElement);\r\n }\r\n }\r\n }\r\n \r\n return fragment;\r\n }\r\n }\r\n \r\n // Usage example and utility functions\r\n class DOMUtils {\r\n /**\r\n * Create serializer with common presets\r\n */\r\n static createSerializer(preset = 'default') {\r\n const presets = {\r\n default: {\r\n includeStyles: true,\r\n includeScripts: false,\r\n includeFrames: true,\r\n includeShadowDOM: true\r\n },\r\n minimal: {\r\n includeStyles: false,\r\n includeScripts: false,\r\n includeFrames: false,\r\n includeShadowDOM: false\r\n },\r\n complete: {\r\n includeStyles: true,\r\n includeScripts: true,\r\n includeFrames: true,\r\n includeShadowDOM: true\r\n },\r\n secure: {\r\n includeStyles: true,\r\n includeScripts: false,\r\n includeFrames: false,\r\n includeShadowDOM: true\r\n }\r\n };\r\n \r\n return new DOMSerializer(presets[preset] || presets.default);\r\n }\r\n \r\n /**\r\n * Serialize DOM to JSON string\r\n */\r\n static serializeToJSON(element, options) {\r\n const serializer = new DOMSerializer(options);\r\n const serialized = serializer.serialize(element);\r\n return JSON.stringify(serialized, null, 2);\r\n }\r\n \r\n /**\r\n * Deserialize from JSON string\r\n */\r\n static deserializeFromJSON(jsonString, targetDocument) {\r\n const serialized = JSON.parse(jsonString);\r\n const serializer = new DOMSerializer();\r\n return serializer.deserialize(serialized, targetDocument);\r\n }\r\n \r\n /**\r\n * Clone DOM with full fidelity including Shadow DOM\r\n */\r\n static deepClone(element, options) {\r\n const serializer = new DOMSerializer(options);\r\n const serialized = serializer.serialize(element);\r\n return serializer.deserialize(serialized, element.ownerDocument);\r\n }\r\n \r\n /**\r\n * Compare two DOM structures\r\n */\r\n static compare(element1, element2, options) {\r\n const serializer = new DOMSerializer(options);\r\n const serialized1 = serializer.serialize(element1);\r\n const serialized2 = serializer.serialize(element2);\r\n \r\n return JSON.stringify(serialized1) === JSON.stringify(serialized2);\r\n }\r\n }\r\n \r\n /*\r\n // Export for use\r\n if (typeof module !== 'undefined' && module.exports) {\r\n module.exports = { DOMSerializer, DOMUtils };\r\n } else if (typeof window !== 'undefined') {\r\n window.DOMSerializer = DOMSerializer;\r\n window.DOMUtils = DOMUtils;\r\n }\r\n */\r\n\r\n /* Usage Examples:\r\n \r\n // Basic serialization\r\n const serializer = new DOMSerializer();\r\n const serialized = serializer.serialize(document);\r\n console.log(JSON.stringify(serialized, null, 2));\r\n \r\n // Deserialize back to DOM\r\n const clonedDoc = serializer.deserialize(serialized);\r\n \r\n // Using presets\r\n const minimalSerializer = DOMUtils.createSerializer('minimal');\r\n const secureSerializer = DOMUtils.createSerializer('secure');\r\n \r\n // Serialize specific element with Shadow DOM\r\n const customElement = document.querySelector('my-custom-element');\r\n const serializedElement = serializer.serialize(customElement);\r\n \r\n // JSON utilities\r\n const jsonString = DOMUtils.serializeToJSON(document.body);\r\n const restored = DOMUtils.deserializeFromJSON(jsonString);\r\n \r\n // Deep clone with Shadow DOM support\r\n const clone = DOMUtils.deepClone(document.body, { includeShadowDOM: true });\r\n \r\n */\r\n\r\n function serializeNodeToJSON(nodeElement) {\r\n return DOMUtils.serializeToJSON(nodeElement, {includeStyles: false});\r\n }\r\n\r\n function deserializeNodeFromJSON(jsonString) {\r\n return DOMUtils.deserializeFromJSON(jsonString);\r\n }\n\n /**\r\n * Checks if a point is inside a bounding box\r\n * \r\n * @param point The point to check\r\n * @param box The bounding box\r\n * @returns boolean indicating if the point is inside the box\r\n */\r\n function isPointInsideBox(point, box) {\r\n return point.x >= box.x &&\r\n point.x <= box.x + box.width &&\r\n point.y >= box.y &&\r\n point.y <= box.y + box.height;\r\n }\r\n\r\n /**\r\n * Calculates the overlap area between two bounding boxes\r\n * \r\n * @param box1 First bounding box\r\n * @param box2 Second bounding box\r\n * @returns The overlap area\r\n */\r\n function calculateOverlap(box1, box2) {\r\n const xOverlap = Math.max(0,\r\n Math.min(box1.x + box1.width, box2.x + box2.width) -\r\n Math.max(box1.x, box2.x)\r\n );\r\n const yOverlap = Math.max(0,\r\n Math.min(box1.y + box1.height, box2.y + box2.height) -\r\n Math.max(box1.y, box2.y)\r\n );\r\n return xOverlap * yOverlap;\r\n }\r\n\r\n /**\r\n * Finds an exact match between candidate elements and the actual interaction element\r\n * \r\n * @param candidate_elements Array of candidate element infos\r\n * @param actualInteractionElementInfo The actual interaction element info\r\n * @returns The matching candidate element info, or null if no match is found\r\n */\r\n function findExactMatch(candidate_elements, actualInteractionElementInfo) {\r\n if (!actualInteractionElementInfo.element) {\r\n return null;\r\n }\r\n\r\n const exactMatch = candidate_elements.find(elementInfo => \r\n elementInfo.element && elementInfo.element === actualInteractionElementInfo.element\r\n );\r\n \r\n if (exactMatch) {\r\n console.log('✅ Found exact element match:', {\r\n matchedElement: exactMatch.element?.tagName,\r\n matchedElementClass: exactMatch.element?.className,\r\n index: exactMatch.index\r\n });\r\n return exactMatch;\r\n }\r\n \r\n return null;\r\n }\r\n\r\n /**\r\n * Finds a match by traversing up the parent elements\r\n * \r\n * @param candidate_elements Array of candidate element infos\r\n * @param actualInteractionElementInfo The actual interaction element info\r\n * @returns The matching candidate element info, or null if no match is found\r\n */\r\n function findParentMatch(candidate_elements, actualInteractionElementInfo) {\r\n if (!actualInteractionElementInfo.element) {\r\n return null;\r\n }\r\n\r\n let element = actualInteractionElementInfo.element;\r\n while (element.parentElement) {\r\n element = element.parentElement;\r\n const parentMatch = candidate_elements.find(candidate => \r\n candidate.element && candidate.element === element\r\n );\r\n \r\n if (parentMatch) {\r\n console.log('✅ Found parent element match:', {\r\n matchedElement: parentMatch.element?.tagName,\r\n matchedElementClass: parentMatch.element?.className,\r\n index: parentMatch.index,\r\n depth: element.tagName\r\n });\r\n return parentMatch;\r\n }\r\n \r\n // Stop if we hit another candidate element\r\n if (candidate_elements.some(candidate => \r\n candidate.element && candidate.element === element\r\n )) {\r\n console.log('⚠️ Stopped parent search - hit another candidate element:', element.tagName);\r\n break;\r\n }\r\n }\r\n \r\n return null;\r\n }\r\n\r\n /**\r\n * Finds a match based on spatial relationships between elements\r\n * \r\n * @param candidate_elements Array of candidate element infos\r\n * @param actualInteractionElementInfo The actual interaction element info\r\n * @returns The matching candidate element info, or null if no match is found\r\n */\r\n function findSpatialMatch(candidate_elements, actualInteractionElementInfo) {\r\n if (!actualInteractionElementInfo.element || !actualInteractionElementInfo.bounding_box) {\r\n return null;\r\n }\r\n\r\n const actualBox = actualInteractionElementInfo.bounding_box;\r\n let bestMatch = null;\r\n let bestScore = 0;\r\n\r\n for (const candidateInfo of candidate_elements) {\r\n if (!candidateInfo.bounding_box) continue;\r\n \r\n const candidateBox = candidateInfo.bounding_box;\r\n let score = 0;\r\n\r\n // Check if actual element is contained within candidate\r\n if (isPointInsideBox({ x: actualBox.x, y: actualBox.y }, candidateBox) &&\r\n isPointInsideBox({ x: actualBox.x + actualBox.width, y: actualBox.y + actualBox.height }, candidateBox)) {\r\n score += 100; // High score for containment\r\n }\r\n\r\n // Calculate overlap area as a factor\r\n const overlap = calculateOverlap(actualBox, candidateBox);\r\n score += overlap;\r\n\r\n // Consider proximity if no containment\r\n if (score === 0) {\r\n const distance = Math.sqrt(\r\n Math.pow((actualBox.x + actualBox.width/2) - (candidateBox.x + candidateBox.width/2), 2) +\r\n Math.pow((actualBox.y + actualBox.height/2) - (candidateBox.y + candidateBox.height/2), 2)\r\n );\r\n // Convert distance to a score (closer = higher score)\r\n score = 1000 / (distance + 1);\r\n }\r\n\r\n if (score > bestScore) {\r\n bestScore = score;\r\n bestMatch = candidateInfo;\r\n console.log('📏 New best spatial match:', {\r\n element: candidateInfo.element?.tagName,\r\n class: candidateInfo.element?.className,\r\n index: candidateInfo.index,\r\n score: score\r\n });\r\n }\r\n }\r\n\r\n if (bestMatch) {\r\n console.log('✅ Final spatial match selected:', {\r\n element: bestMatch.element?.tagName,\r\n class: bestMatch.element?.className,\r\n index: bestMatch.index,\r\n finalScore: bestScore\r\n });\r\n return bestMatch;\r\n }\r\n\r\n return null;\r\n }\r\n\r\n /**\r\n * Finds a matching candidate element for an actual interaction element\r\n * \r\n * @param candidate_elements Array of candidate element infos\r\n * @param actualInteractionElementInfo The actual interaction element info\r\n * @returns The matching candidate element info, or null if no match is found\r\n */\r\n function findMatchingCandidateElementInfo(candidate_elements, actualInteractionElementInfo) {\r\n if (!actualInteractionElementInfo.element || !actualInteractionElementInfo.bounding_box) {\r\n console.error('❌ Missing required properties in actualInteractionElementInfo');\r\n return null;\r\n }\r\n\r\n console.log('🔍 Starting element matching for:', {\r\n clickedElement: actualInteractionElementInfo.element.tagName,\r\n clickedElementClass: actualInteractionElementInfo.element.className,\r\n totalCandidates: candidate_elements.length\r\n });\r\n\r\n // First try exact element match\r\n const exactMatch = findExactMatch(candidate_elements, actualInteractionElementInfo);\r\n if (exactMatch) {\r\n return exactMatch;\r\n }\r\n console.log('❌ No exact element match found, trying parent matching...');\r\n\r\n // Try finding closest clickable parent\r\n const parentMatch = findParentMatch(candidate_elements, actualInteractionElementInfo);\r\n if (parentMatch) {\r\n return parentMatch;\r\n }\r\n console.log('❌ No parent match found, falling back to spatial matching...');\r\n\r\n // If no exact or parent match, look for spatial relationships\r\n const spatialMatch = findSpatialMatch(candidate_elements, actualInteractionElementInfo);\r\n if (spatialMatch) {\r\n return spatialMatch;\r\n }\r\n\r\n console.error('❌ No matching element found for actual interaction element:', actualInteractionElementInfo);\r\n return null;\r\n }\n\n const highlight = {\r\n execute: async function(elementTypes, handleScroll=false) {\r\n const elements = await findElements(elementTypes);\r\n highlightElements(elements, handleScroll);\r\n return elements;\r\n },\r\n\r\n unexecute: function(handleScroll=false) {\r\n unhighlightElements(handleScroll);\r\n },\r\n\r\n generateJSON: async function() {\r\n const json = {};\r\n\r\n // Capture viewport dimensions\r\n const viewportData = {\r\n width: window.innerWidth,\r\n height: window.innerHeight,\r\n documentWidth: document.documentElement.clientWidth,\r\n documentHeight: document.documentElement.clientHeight,\r\n timestamp: new Date().toISOString()\r\n };\r\n\r\n // Add viewport data to the JSON output\r\n json.viewport = viewportData;\r\n\r\n\r\n await Promise.all(Object.values(ElementTag).map(async elementType => {\r\n const elements = await findElements(elementType);\r\n json[elementType] = elements;\r\n }));\r\n\r\n // Serialize the JSON object\r\n const jsonString = JSON.stringify(json, null, 4); // Pretty print with 4 spaces\r\n\r\n console.log(`JSON: ${jsonString}`);\r\n return jsonString;\r\n },\r\n\r\n getElementInfo\r\n };\r\n\r\n\r\n function unhighlightElements(handleScroll=false) {\r\n const documents = getAllFrames();\r\n documents.forEach(doc => {\r\n const overlay = doc.getElementById('highlight-overlay');\r\n if (overlay) {\r\n if (handleScroll) {\r\n // Remove event listeners\r\n doc.removeEventListener('scroll', overlay.scrollHandler, true);\r\n doc.removeEventListener('resize', overlay.resizeHandler);\r\n }\r\n overlay.remove();\r\n }\r\n });\r\n }\r\n\r\n\r\n\r\n\r\n async function findElements(elementTypes, verbose=true) {\r\n const typesArray = Array.isArray(elementTypes) ? elementTypes : [elementTypes];\r\n console.log('Starting element search for types:', typesArray);\r\n\r\n const elements = [];\r\n typesArray.forEach(elementType => {\r\n if (elementType === ElementTag.FILLABLE) {\r\n elements.push(...findFillables());\r\n }\r\n if (elementType === ElementTag.SELECTABLE) {\r\n elements.push(...findDropdowns());\r\n }\r\n if (elementType === ElementTag.CLICKABLE) {\r\n elements.push(...findClickables());\r\n elements.push(...findToggles());\r\n elements.push(...findCheckables());\r\n }\r\n if (elementType === ElementTag.NON_INTERACTIVE_ELEMENT) {\r\n elements.push(...findNonInteractiveElements());\r\n }\r\n });\r\n\r\n // console.log('Before uniquify:', elements.length);\r\n const elementsWithInfo = elements.map((element, index) => \r\n getElementInfo(element, index)\r\n );\r\n\r\n \r\n \r\n const uniqueElements = uniquifyElements(elementsWithInfo);\r\n console.log(`Found ${uniqueElements.length} elements:`);\r\n \r\n // More comprehensive visibility check\r\n const visibleElements = uniqueElements.filter(elementInfo => {\r\n const el = elementInfo.element;\r\n const style = getComputedStyle(el);\r\n \r\n // Check various style properties that affect visibility\r\n if (style.display === 'none' || \r\n style.visibility === 'hidden') {\r\n return false;\r\n }\r\n \r\n // Check if element has non-zero dimensions\r\n const rect = el.getBoundingClientRect();\r\n if (rect.width === 0 || rect.height === 0) {\r\n return false;\r\n }\r\n \r\n // Check if element is within viewport\r\n if (rect.bottom < 0 || \r\n rect.top > window.innerHeight || \r\n rect.right < 0 || \r\n rect.left > window.innerWidth) {\r\n // Element is outside viewport, but still might be valid \r\n // if user scrolls to it, so we'll include it\r\n return true;\r\n }\r\n \r\n return true;\r\n });\r\n \r\n console.log(`Out of which ${visibleElements.length} elements are visible:`);\r\n if (verbose) {\r\n visibleElements.forEach(info => {\r\n console.log(`Element ${info.index}:`, info);\r\n });\r\n }\r\n \r\n return visibleElements;\r\n }\r\n\r\n // elements is an array of objects with index, xpath\r\n function highlightElements(elements, handleScroll=false) {\r\n // console.log('[highlightElements] called with', elements.length, 'elements');\r\n // Create overlay if it doesn't exist and store it in a dictionary\r\n const documents = getAllFrames(); \r\n let overlays = {};\r\n documents.forEach(doc => {\r\n let overlay = doc.getElementById('highlight-overlay');\r\n if (!overlay) {\r\n overlay = doc.createElement('div');\r\n overlay.id = 'highlight-overlay';\r\n overlay.style.cssText = `\r\n position: fixed;\r\n top: 0;\r\n left: 0;\r\n width: 100%;\r\n height: 100%;\r\n pointer-events: none;\r\n z-index: 2147483647;\r\n `;\r\n doc.body.appendChild(overlay);\r\n // console.log('[highlightElements] Created overlay in document:', doc);\r\n }\r\n overlays[doc.documentURI] = overlay;\r\n });\r\n \r\n\r\n const updateHighlights = (doc = null) => {\r\n if (doc) {\r\n overlays[doc.documentURI].innerHTML = '';\r\n } else {\r\n Object.values(overlays).forEach(overlay => { overlay.innerHTML = ''; });\r\n } \r\n elements.forEach((elementInfo, idx) => {\r\n //console.log(`[highlightElements] Processing element ${idx}:`, elementInfo.tag, elementInfo.css_selector, elementInfo.bounding_box);\r\n let element = elementInfo.element; //getElementByXPathOrCssSelector(elementInfo);\r\n if (!element) {\r\n element = getElementByXPathOrCssSelector(elementInfo);\r\n if (!element) {\r\n console.warn('[highlightElements] Could not find element for:', elementInfo);\r\n return;\r\n }\r\n }\r\n //if highlights requested for a specific doc, skip unrelated elements\r\n if (doc && element.ownerDocument !== doc) {\r\n console.log(\"[highlightElements] Skipped element since it doesn't belong to document\", doc);\r\n return;\r\n }\r\n const rect = element.getBoundingClientRect();\r\n if (rect.width === 0 || rect.height === 0) {\r\n console.warn('[highlightElements] Element has zero dimensions:', elementInfo);\r\n return;\r\n }\r\n // Create border highlight (red rectangle)\r\n // use ownerDocument to support iframes/frames\r\n const highlight = element.ownerDocument.createElement('div');\r\n highlight.style.cssText = `\r\n position: fixed;\r\n left: ${rect.x}px;\r\n top: ${rect.y}px;\r\n width: ${rect.width}px;\r\n height: ${rect.height}px;\r\n border: 1px solid rgb(255, 0, 0);\r\n transition: all 0.2s ease-in-out;\r\n `;\r\n // Create index label container - now positioned to the right and slightly up\r\n const labelContainer = element.ownerDocument.createElement('div');\r\n labelContainer.style.cssText = `\r\n position: absolute;\r\n right: -10px; /* Offset to the right */\r\n top: -10px; /* Offset upwards */\r\n padding: 4px;\r\n background-color: rgba(255, 255, 0, 0.6);\r\n display: flex;\r\n align-items: center;\r\n justify-content: center;\r\n `;\r\n const text = element.ownerDocument.createElement('span');\r\n text.style.cssText = `\r\n color: rgb(0, 0, 0, 0.8);\r\n font-family: 'Courier New', Courier, monospace;\r\n font-size: 12px;\r\n font-weight: bold;\r\n line-height: 1;\r\n `;\r\n text.textContent = elementInfo.index;\r\n labelContainer.appendChild(text);\r\n highlight.appendChild(labelContainer); \r\n overlays[element.ownerDocument.documentURI].appendChild(highlight);\r\n \r\n });\r\n };\r\n\r\n // Initial highlight\r\n updateHighlights();\r\n\r\n if (handleScroll) {\r\n documents.forEach(doc => {\r\n // Update highlights on scroll and resize\r\n console.log('registering scroll and resize handlers for document: ', doc);\r\n const scrollHandler = () => {\r\n requestAnimationFrame(() => updateHighlights(doc));\r\n };\r\n const resizeHandler = () => {\r\n updateHighlights(doc);\r\n };\r\n doc.addEventListener('scroll', scrollHandler, true);\r\n doc.addEventListener('resize', resizeHandler);\r\n // Store event handlers for cleanup\r\n overlays[doc.documentURI].scrollHandler = scrollHandler;\r\n overlays[doc.documentURI].resizeHandler = resizeHandler;\r\n }); \r\n }\r\n }\r\n\r\n // function unexecute() {\r\n // unhighlightElements();\r\n // }\r\n\r\n // Make it available globally for both Extension and Playwright\r\n if (typeof window !== 'undefined') {\r\n function stripElementRefs(elementInfo) {\r\n if (!elementInfo) return null;\r\n const { element, ...rest } = elementInfo;\r\n return rest;\r\n }\r\n\r\n window.ProboLabs = window.ProboLabs || {};\r\n\r\n // --- Caching State ---\r\n window.ProboLabs.candidates = [];\r\n window.ProboLabs.actual = null;\r\n window.ProboLabs.matchingCandidate = null;\r\n\r\n // --- Methods ---\r\n /**\r\n * Find and cache candidate elements of a given type (e.g., 'CLICKABLE').\r\n * NOTE: This function is async and must be awaited from Playwright/Node.\r\n */\r\n window.ProboLabs.findAndCacheCandidateElements = async function(elementType) {\r\n //console.log('[ProboLabs] findAndCacheCandidateElements called with:', elementType);\r\n const found = await findElements(elementType);\r\n window.ProboLabs.candidates = found;\r\n // console.log('[ProboLabs] candidates set to:', found, 'type:', typeof found, 'isArray:', Array.isArray(found));\r\n return found.length;\r\n };\r\n\r\n window.ProboLabs.findAndCacheActualElement = function(cssSelector, iframeSelector, isHover=false) {\r\n // console.log('[ProboLabs] findAndCacheActualElement called with:', cssSelector, iframeSelector);\r\n let el = findElement(document, iframeSelector, cssSelector);\r\n if(isHover) {\r\n const visibleElement = findClosestVisibleElement(el);\r\n if (visibleElement) {\r\n el = visibleElement;\r\n }\r\n }\r\n if (!el) {\r\n window.ProboLabs.actual = null;\r\n // console.log('[ProboLabs] actual set to null');\r\n return false;\r\n }\r\n window.ProboLabs.actual = getElementInfo(el, -1);\r\n // console.log('[ProboLabs] actual set to:', window.ProboLabs.actual);\r\n return true;\r\n };\r\n\r\n window.ProboLabs.findAndCacheMatchingCandidate = function() {\r\n // console.log('[ProboLabs] findAndCacheMatchingCandidate called');\r\n if (!window.ProboLabs.candidates.length || !window.ProboLabs.actual) {\r\n window.ProboLabs.matchingCandidate = null;\r\n // console.log('[ProboLabs] matchingCandidate set to null');\r\n return false;\r\n }\r\n window.ProboLabs.matchingCandidate = findMatchingCandidateElementInfo(window.ProboLabs.candidates, window.ProboLabs.actual);\r\n // console.log('[ProboLabs] matchingCandidate set to:', window.ProboLabs.matchingCandidate);\r\n return !!window.ProboLabs.matchingCandidate;\r\n };\r\n\r\n window.ProboLabs.highlightCachedElements = function(which) {\r\n let elements = [];\r\n if (which === 'candidates') elements = window.ProboLabs.candidates;\r\n if (which === 'actual' && window.ProboLabs.actual) elements = [window.ProboLabs.actual];\r\n if (which === 'matching' && window.ProboLabs.matchingCandidate) elements = [window.ProboLabs.matchingCandidate];\r\n console.log(`[ProboLabs] highlightCachedElements ${which} with ${elements.length} elements`);\r\n highlightElements(elements);\r\n };\r\n\r\n window.ProboLabs.unhighlight = function() {\r\n // console.log('[ProboLabs] unhighlight called');\r\n unhighlightElements();\r\n };\r\n\r\n window.ProboLabs.reset = function() {\r\n console.log('[ProboLabs] reset called');\r\n window.ProboLabs.candidates = [];\r\n window.ProboLabs.actual = null;\r\n window.ProboLabs.matchingCandidate = null;\r\n unhighlightElements();\r\n };\r\n\r\n window.ProboLabs.getCandidates = function() {\r\n // console.log('[ProboLabs] getCandidates called. candidates:', window.ProboLabs.candidates, 'type:', typeof window.ProboLabs.candidates, 'isArray:', Array.isArray(window.ProboLabs.candidates));\r\n const arr = Array.isArray(window.ProboLabs.candidates) ? window.ProboLabs.candidates : [];\r\n return arr.map(stripElementRefs);\r\n };\r\n window.ProboLabs.getActual = function() {\r\n return stripElementRefs(window.ProboLabs.actual);\r\n };\r\n window.ProboLabs.getMatchingCandidate = function() {\r\n return stripElementRefs(window.ProboLabs.matchingCandidate);\r\n };\r\n\r\n // Retain existing API for backward compatibility\r\n window.ProboLabs.ElementTag = ElementTag;\r\n window.ProboLabs.highlightElements = highlightElements;\r\n window.ProboLabs.unhighlightElements = unhighlightElements;\r\n window.ProboLabs.findElements = findElements;\r\n window.ProboLabs.getElementInfo = getElementInfo;\r\n window.ProboLabs.highlight = window.ProboLabs.highlight;\r\n window.ProboLabs.unhighlight = window.ProboLabs.unhighlight;\r\n\r\n // --- Utility Functions ---\r\n function findClosestVisibleElement(element) {\r\n let current = element;\r\n while (current) {\r\n const style = window.getComputedStyle(current);\r\n if (\r\n style &&\r\n style.display !== 'none' &&\r\n style.visibility !== 'hidden' &&\r\n current.offsetWidth > 0 &&\r\n current.offsetHeight > 0\r\n ) {\r\n return current;\r\n }\r\n if (!current.parentElement || current === document.body) break;\r\n current = current.parentElement;\r\n }\r\n return null;\r\n }\r\n }\n\n exports.ElementInfo = ElementInfo;\n exports.ElementTag = ElementTag;\n exports.deserializeNodeFromJSON = deserializeNodeFromJSON;\n exports.detectScrollableContainers = detectScrollableContainers;\n exports.findElement = findElement;\n exports.findElements = findElements;\n exports.generateCssPath = generateCssPath;\n exports.getAriaLabelledByText = getAriaLabelledByText;\n exports.getContainingIframe = getContainingIframe;\n exports.getElementInfo = getElementInfo;\n exports.getParentNode = getParentNode;\n exports.getRobustSelector = getRobustSelector;\n exports.highlight = highlight;\n exports.highlightElements = highlightElements;\n exports.isScrollableContainer = isScrollableContainer;\n exports.serializeNodeToJSON = serializeNodeToJSON;\n exports.unhighlightElements = unhighlightElements;\n\n}));\n//# sourceMappingURL=probolabs.umd.js.map\n";
1
+ const highlighterCode = "(function (global, factory) {\n typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports) :\n typeof define === 'function' && define.amd ? define(['exports'], factory) :\n (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global.ProboLabs = {}));\n})(this, (function (exports) { 'use strict';\n\n const ElementTag = {\n CLICKABLE: \"CLICKABLE\", // button, link, toggle switch, checkbox, radio, dropdowns, clickable divs\n FILLABLE: \"FILLABLE\", // input, textarea content_editable, date picker??\n SELECTABLE: \"SELECTABLE\", // select\n NON_INTERACTIVE_ELEMENT: 'NON_INTERACTIVE_ELEMENT',\n };\n\n class ElementInfo {\n constructor(element, index, {tag, type, text, html, xpath, css_selector, bounding_box, iframe_selector, short_css_selector, short_iframe_selector}) {\n this.index = index.toString();\n this.tag = tag;\n this.type = type;\n this.text = text;\n this.html = html;\n this.xpath = xpath;\n this.css_selector = css_selector;\n this.bounding_box = bounding_box;\n this.iframe_selector = iframe_selector;\n this.element = element;\n this.depth = -1;\n this.short_css_selector = short_css_selector;\n this.short_iframe_selector = short_iframe_selector;\n }\n\n getSelector() {\n return this.xpath ? this.xpath : this.css_selector;\n }\n\n getDepth() {\n if (this.depth >= 0) {\n return this.depth;\n }\n \n this.depth = 0;\n let currentElement = this.element;\n \n while (currentElement.nodeType === Node.ELEMENT_NODE) { \n this.depth++;\n currentElement = getParentNode(currentElement);\n }\n \n return this.depth;\n }\n }\n\n function getParentNode(element) {\n if (!element || element.nodeType !== Node.ELEMENT_NODE) return null;\n \n let parent = null;\n // SF is using slots and shadow DOM heavily\n // However, there might be slots in the light DOM which shouldn't be traversed\n if (element.assignedSlot && element.getRootNode() instanceof ShadowRoot)\n parent = element.assignedSlot;\n else \n parent = element.parentNode;\n \n // Check if we're at a shadow root\n if (parent && parent.nodeType !== Node.ELEMENT_NODE && parent.getRootNode() instanceof ShadowRoot) \n parent = parent.getRootNode().host; \n\n return parent;\n }\n\n // License: MIT\n // Author: Anton Medvedev <anton@medv.io>\n // Source: https://github.com/antonmedv/finder\n const acceptedAttrNames = new Set(['role', 'name', 'aria-label', 'rel', 'href']);\n /** Check if attribute name and value are word-like. */\n function attr(name, value) {\n let nameIsOk = acceptedAttrNames.has(name);\n nameIsOk ||= name.startsWith('data-') && wordLike(name);\n let valueIsOk = wordLike(value) && value.length < 100;\n valueIsOk ||= value.startsWith('#') && wordLike(value.slice(1));\n return nameIsOk && valueIsOk;\n }\n /** Check if id name is word-like. */\n function idName(name) {\n return wordLike(name);\n }\n /** Check if class name is word-like. */\n function className(name) {\n return wordLike(name);\n }\n /** Check if tag name is word-like. */\n function tagName(name) {\n return true;\n }\n /** Finds unique CSS selectors for the given element. */\n function finder(input, options) {\n if (input.nodeType !== Node.ELEMENT_NODE) {\n throw new Error(`Can't generate CSS selector for non-element node type.`);\n }\n if (input.tagName.toLowerCase() === 'html') {\n return 'html';\n }\n const defaults = {\n root: document.body,\n idName: idName,\n className: className,\n tagName: tagName,\n attr: attr,\n timeoutMs: 1000,\n seedMinLength: 3,\n optimizedMinLength: 2,\n maxNumberOfPathChecks: Infinity,\n };\n const startTime = new Date();\n const config = { ...defaults, ...options };\n const rootDocument = findRootDocument(config.root, defaults);\n let foundPath;\n let count = 0;\n for (const candidate of search(input, config, rootDocument)) {\n const elapsedTimeMs = new Date().getTime() - startTime.getTime();\n if (elapsedTimeMs > config.timeoutMs ||\n count >= config.maxNumberOfPathChecks) {\n const fPath = fallback(input, rootDocument);\n if (!fPath) {\n throw new Error(`Timeout: Can't find a unique selector after ${config.timeoutMs}ms`);\n }\n return selector(fPath);\n }\n count++;\n if (unique(candidate, rootDocument)) {\n foundPath = candidate;\n break;\n }\n }\n if (!foundPath) {\n throw new Error(`Selector was not found.`);\n }\n const optimized = [\n ...optimize(foundPath, input, config, rootDocument, startTime),\n ];\n optimized.sort(byPenalty);\n if (optimized.length > 0) {\n return selector(optimized[0]);\n }\n return selector(foundPath);\n }\n function* search(input, config, rootDocument) {\n const stack = [];\n let paths = [];\n let current = input;\n let i = 0;\n while (current && current !== rootDocument) {\n const level = tie(current, config);\n for (const node of level) {\n node.level = i;\n }\n stack.push(level);\n current = current.parentElement;\n i++;\n paths.push(...combinations(stack));\n if (i >= config.seedMinLength) {\n paths.sort(byPenalty);\n for (const candidate of paths) {\n yield candidate;\n }\n paths = [];\n }\n }\n paths.sort(byPenalty);\n for (const candidate of paths) {\n yield candidate;\n }\n }\n function wordLike(name) {\n if (/^[a-z\\-]{3,}$/i.test(name)) {\n const words = name.split(/-|[A-Z]/);\n for (const word of words) {\n if (word.length <= 2) {\n return false;\n }\n if (/[^aeiou]{4,}/i.test(word)) {\n return false;\n }\n }\n return true;\n }\n return false;\n }\n function tie(element, config) {\n const level = [];\n const elementId = element.getAttribute('id');\n if (elementId && config.idName(elementId)) {\n level.push({\n name: '#' + CSS.escape(elementId),\n penalty: 0,\n });\n }\n for (let i = 0; i < element.classList.length; i++) {\n const name = element.classList[i];\n if (config.className(name)) {\n level.push({\n name: '.' + CSS.escape(name),\n penalty: 1,\n });\n }\n }\n for (let i = 0; i < element.attributes.length; i++) {\n const attr = element.attributes[i];\n if (config.attr(attr.name, attr.value)) {\n level.push({\n name: `[${CSS.escape(attr.name)}=\"${CSS.escape(attr.value)}\"]`,\n penalty: 2,\n });\n }\n }\n const tagName = element.tagName.toLowerCase();\n if (config.tagName(tagName)) {\n level.push({\n name: tagName,\n penalty: 5,\n });\n const index = indexOf(element, tagName);\n if (index !== undefined) {\n level.push({\n name: nthOfType(tagName, index),\n penalty: 10,\n });\n }\n }\n const nth = indexOf(element);\n if (nth !== undefined) {\n level.push({\n name: nthChild(tagName, nth),\n penalty: 50,\n });\n }\n return level;\n }\n function selector(path) {\n let node = path[0];\n let query = node.name;\n for (let i = 1; i < path.length; i++) {\n const level = path[i].level || 0;\n if (node.level === level - 1) {\n query = `${path[i].name} > ${query}`;\n }\n else {\n query = `${path[i].name} ${query}`;\n }\n node = path[i];\n }\n return query;\n }\n function penalty(path) {\n return path.map((node) => node.penalty).reduce((acc, i) => acc + i, 0);\n }\n function byPenalty(a, b) {\n return penalty(a) - penalty(b);\n }\n function indexOf(input, tagName) {\n const parent = input.parentNode;\n if (!parent) {\n return undefined;\n }\n let child = parent.firstChild;\n if (!child) {\n return undefined;\n }\n let i = 0;\n while (child) {\n if (child.nodeType === Node.ELEMENT_NODE &&\n (tagName === undefined ||\n child.tagName.toLowerCase() === tagName)) {\n i++;\n }\n if (child === input) {\n break;\n }\n child = child.nextSibling;\n }\n return i;\n }\n function fallback(input, rootDocument) {\n let i = 0;\n let current = input;\n const path = [];\n while (current && current !== rootDocument) {\n const tagName = current.tagName.toLowerCase();\n const index = indexOf(current, tagName);\n if (index === undefined) {\n return;\n }\n path.push({\n name: nthOfType(tagName, index),\n penalty: NaN,\n level: i,\n });\n current = current.parentElement;\n i++;\n }\n if (unique(path, rootDocument)) {\n return path;\n }\n }\n function nthChild(tagName, index) {\n if (tagName === 'html') {\n return 'html';\n }\n return `${tagName}:nth-child(${index})`;\n }\n function nthOfType(tagName, index) {\n if (tagName === 'html') {\n return 'html';\n }\n return `${tagName}:nth-of-type(${index})`;\n }\n function* combinations(stack, path = []) {\n if (stack.length > 0) {\n for (let node of stack[0]) {\n yield* combinations(stack.slice(1, stack.length), path.concat(node));\n }\n }\n else {\n yield path;\n }\n }\n function findRootDocument(rootNode, defaults) {\n if (rootNode.nodeType === Node.DOCUMENT_NODE) {\n return rootNode;\n }\n if (rootNode === defaults.root) {\n return rootNode.ownerDocument;\n }\n return rootNode;\n }\n function unique(path, rootDocument) {\n const css = selector(path);\n switch (rootDocument.querySelectorAll(css).length) {\n case 0:\n throw new Error(`Can't select any node with this selector: ${css}`);\n case 1:\n return true;\n default:\n return false;\n }\n }\n function* optimize(path, input, config, rootDocument, startTime) {\n if (path.length > 2 && path.length > config.optimizedMinLength) {\n for (let i = 1; i < path.length - 1; i++) {\n const elapsedTimeMs = new Date().getTime() - startTime.getTime();\n if (elapsedTimeMs > config.timeoutMs) {\n return;\n }\n const newPath = [...path];\n newPath.splice(i, 1);\n if (unique(newPath, rootDocument) &&\n rootDocument.querySelector(selector(newPath)) === input) {\n yield newPath;\n yield* optimize(newPath, input, config, rootDocument, startTime);\n }\n }\n }\n }\n\n // import { realpath } from \"fs\";\n\n function getAllDocumentElementsIncludingShadow(selectors, root = document) {\n const elements = Array.from(root.querySelectorAll(selectors));\n\n root.querySelectorAll('*').forEach(el => {\n if (el.shadowRoot) {\n elements.push(...getAllDocumentElementsIncludingShadow(selectors, el.shadowRoot));\n }\n });\n return elements;\n }\n\n function getAllFrames(root = document) {\n const result = [root];\n const frames = getAllDocumentElementsIncludingShadow('frame, iframe', root); \n frames.forEach(frame => {\n try {\n const frameDocument = frame.contentDocument || frame.contentWindow.document;\n if (frameDocument) {\n result.push(frameDocument);\n }\n } catch (e) {\n // Skip cross-origin frames\n console.warn('Could not access frame content:', e.message);\n }\n });\n\n return result;\n }\n\n function getAllElementsIncludingShadow(selectors, root = document) {\n const elements = [];\n\n getAllFrames(root).forEach(doc => {\n elements.push(...getAllDocumentElementsIncludingShadow(selectors, doc));\n });\n\n return elements;\n }\n\n /**\n * Deeply searches through DOM trees including Shadow DOM and frames/iframes\n * @param {string} selector - CSS selector to search for\n * @param {Document|Element} [root=document] - Starting point for the search\n * @param {Object} [options] - Search options\n * @param {boolean} [options.searchShadow=true] - Whether to search Shadow DOM\n * @param {boolean} [options.searchFrames=true] - Whether to search frames/iframes\n * @returns {Element[]} Array of found elements\n \n function getAllElementsIncludingShadow(selector, root = document, options = {}) {\n const {\n searchShadow = true,\n searchFrames = true\n } = options;\n\n const results = new Set();\n \n // Helper to check if an element is valid and not yet found\n const addIfValid = (element) => {\n if (element && !results.has(element)) {\n results.add(element);\n }\n };\n\n // Helper to process a single document or element\n function processNode(node) {\n // Search regular DOM\n node.querySelectorAll(selector).forEach(addIfValid);\n\n if (searchShadow) {\n // Search all shadow roots\n const treeWalker = document.createTreeWalker(\n node,\n NodeFilter.SHOW_ELEMENT,\n {\n acceptNode: (element) => {\n return element.shadowRoot ? \n NodeFilter.FILTER_ACCEPT : \n NodeFilter.FILTER_SKIP;\n }\n }\n );\n\n while (treeWalker.nextNode()) {\n const element = treeWalker.currentNode;\n if (element.shadowRoot) {\n // Search within shadow root\n element.shadowRoot.querySelectorAll(selector).forEach(addIfValid);\n // Recursively process the shadow root for nested shadow DOMs\n processNode(element.shadowRoot);\n }\n }\n }\n\n if (searchFrames) {\n // Search frames and iframes\n const frames = node.querySelectorAll('frame, iframe');\n frames.forEach(frame => {\n try {\n const frameDocument = frame.contentDocument;\n if (frameDocument) {\n processNode(frameDocument);\n }\n } catch (e) {\n // Skip cross-origin frames\n console.warn('Could not access frame content:', e.message);\n }\n });\n }\n }\n\n // Start processing from the root\n processNode(root);\n\n return Array.from(results);\n }\n */\n // <div x=1 y=2 role='combobox'> </div>\n function findDropdowns() {\n const dropdowns = [];\n \n // Native select elements\n dropdowns.push(...getAllElementsIncludingShadow('select'));\n \n // Elements with dropdown roles that don't have <input>..</input>\n const roleElements = getAllElementsIncludingShadow('[role=\"combobox\"], [role=\"listbox\"], [role=\"dropdown\"], [role=\"option\"], [role=\"menu\"], [role=\"menuitem\"]').filter(el => {\n return el.tagName.toLowerCase() !== 'input' || ![\"button\", \"checkbox\", \"radio\"].includes(el.getAttribute(\"type\"));\n });\n dropdowns.push(...roleElements);\n \n // Common dropdown class patterns\n const dropdownPattern = /.*(dropdown|select|combobox|menu).*/i;\n const elements = getAllElementsIncludingShadow('*');\n const dropdownClasses = Array.from(elements).filter(el => {\n const hasDropdownClass = dropdownPattern.test(el.className);\n const validTag = ['li', 'ul', 'span', 'div', 'p', 'a', 'button'].includes(el.tagName.toLowerCase());\n const style = window.getComputedStyle(el); \n const result = hasDropdownClass && validTag && (style.cursor === 'pointer' || el.tagName.toLowerCase() === 'a' || el.tagName.toLowerCase() === 'button');\n return result;\n });\n \n dropdowns.push(...dropdownClasses);\n \n // Elements with aria-haspopup attribute\n dropdowns.push(...getAllElementsIncludingShadow('[aria-haspopup=\"true\"], [aria-haspopup=\"listbox\"], [aria-haspopup=\"menu\"]'));\n\n // Improve navigation element detection\n // Semantic nav elements with list items\n dropdowns.push(...getAllElementsIncludingShadow('nav ul li, nav ol li'));\n \n // Navigation elements in common design patterns\n dropdowns.push(...getAllElementsIncludingShadow('header a, .header a, .nav a, .navigation a, .menu a, .sidebar a, aside a'));\n \n // Elements in primary navigation areas with common attributes\n dropdowns.push(...getAllElementsIncludingShadow('[role=\"navigation\"] a, [aria-label*=\"navigation\"] a, [aria-label*=\"menu\"] a'));\n\n return dropdowns;\n }\n\n function findClickables() {\n const clickables = [];\n \n const checkboxPattern = /checkbox/i;\n // Collect all clickable elements first\n const nativeLinks = [...getAllElementsIncludingShadow('a')];\n const nativeButtons = [...getAllElementsIncludingShadow('button')];\n const inputButtons = [...getAllElementsIncludingShadow('input[type=\"button\"], input[type=\"submit\"], input[type=\"reset\"]')];\n const roleButtons = [...getAllElementsIncludingShadow('[role=\"button\"]')];\n // const tabbable = [...getAllElementsIncludingShadow('[tabindex=\"0\"]')];\n const clickHandlers = [...getAllElementsIncludingShadow('[onclick]')];\n const dropdowns = findDropdowns();\n const nativeCheckboxes = [...getAllElementsIncludingShadow('input[type=\"checkbox\"]')]; \n const fauxCheckboxes = getAllElementsIncludingShadow('*').filter(el => {\n if (checkboxPattern.test(el.className)) {\n const realCheckboxes = getAllElementsIncludingShadow('input[type=\"checkbox\"]', el);\n if (realCheckboxes.length === 1) {\n const boundingRect = realCheckboxes[0].getBoundingClientRect();\n return boundingRect.width <= 1 && boundingRect.height <= 1 \n }\n }\n return false;\n });\n const nativeRadios = [...getAllElementsIncludingShadow('input[type=\"radio\"]')];\n const toggles = findToggles();\n const pointerElements = findElementsWithPointer();\n // Add all elements at once\n clickables.push(\n ...nativeLinks,\n ...nativeButtons,\n ...inputButtons,\n ...roleButtons,\n // ...tabbable,\n ...clickHandlers,\n ...dropdowns,\n ...nativeCheckboxes,\n ...fauxCheckboxes,\n ...nativeRadios,\n ...toggles,\n ...pointerElements\n );\n\n // Only uniquify once at the end\n return clickables; // Let findElements handle the uniquification\n }\n\n function findToggles() {\n const toggles = [];\n const checkboxes = getAllElementsIncludingShadow('input[type=\"checkbox\"]');\n const togglePattern = /switch|toggle|slider/i;\n\n checkboxes.forEach(checkbox => {\n let isToggle = false;\n\n // Check the checkbox itself\n if (togglePattern.test(checkbox.className) || togglePattern.test(checkbox.getAttribute('role') || '')) {\n isToggle = true;\n }\n\n // Check parent elements (up to 3 levels)\n if (!isToggle) {\n let element = checkbox;\n for (let i = 0; i < 3; i++) {\n const parent = element.parentElement;\n if (!parent) break;\n\n const className = parent.className || '';\n const role = parent.getAttribute('role') || '';\n\n if (togglePattern.test(className) || togglePattern.test(role)) {\n isToggle = true;\n break;\n }\n element = parent;\n }\n }\n\n // Check next sibling\n if (!isToggle) {\n const nextSibling = checkbox.nextElementSibling;\n if (nextSibling) {\n const className = nextSibling.className || '';\n const role = nextSibling.getAttribute('role') || '';\n if (togglePattern.test(className) || togglePattern.test(role)) {\n isToggle = true;\n }\n }\n }\n\n if (isToggle) {\n toggles.push(checkbox);\n }\n });\n\n return toggles;\n }\n\n function findNonInteractiveElements() {\n // Get all elements in the document\n const all = Array.from(getAllElementsIncludingShadow('*'));\n \n // Filter elements based on Python implementation rules\n return all.filter(element => {\n if (!element.firstElementChild) {\n const tag = element.tagName.toLowerCase(); \n if (!['select', 'button', 'a'].includes(tag)) {\n const validTags = ['p', 'span', 'div', 'input', 'textarea','td','th'].includes(tag) || /^h\\d$/.test(tag) || /text/.test(tag);\n const boundingRect = element.getBoundingClientRect();\n return validTags && boundingRect.height > 1 && boundingRect.width > 1;\n }\n }\n return false;\n });\n }\n\n\n\n // export function findNonInteractiveElements() {\n // const all = [];\n // try {\n // const elements = getAllElementsIncludingShadow('*');\n // all.push(...elements);\n // } catch (e) {\n // console.warn('Error getting elements:', e);\n // }\n \n // console.debug('Total elements found:', all.length);\n \n // return all.filter(element => {\n // try {\n // const tag = element.tagName.toLowerCase(); \n\n // // Special handling for input elements\n // if (tag === 'input' || tag === 'textarea') {\n // const boundingRect = element.getBoundingClientRect();\n // const value = element.value || '';\n // const placeholder = element.placeholder || '';\n // return boundingRect.height > 1 && \n // boundingRect.width > 1 && \n // (value.trim() !== '' || placeholder.trim() !== '');\n // }\n\n \n // // Check if it's a valid tag for text content\n // const validTags = ['p', 'span', 'div', 'label', 'th', 'td', 'li', 'button', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'a', 'select'].includes(tag) || \n // /^h\\d$/.test(tag) || \n // /text/.test(tag);\n\n // const boundingRect = element.getBoundingClientRect();\n\n // // Get direct text content, excluding child element text\n // let directText = '';\n // for (const node of element.childNodes) {\n // // Only include text nodes (nodeType 3)\n // if (node.nodeType === 3) {\n // directText += node.textContent || '';\n // }\n // }\n \n // // If no direct text and it's a table cell or heading, check label content\n // if (!directText.trim() && (tag === 'th' || tag === 'td' || tag === 'h1')) {\n // const labels = element.getElementsByTagName('label');\n // for (const label of labels) {\n // directText += label.textContent || '';\n // }\n // }\n\n // // If still no text and it's a heading, get all text content\n // if (!directText.trim() && tag === 'h1') {\n // directText = element.textContent || '';\n // }\n\n // directText = directText.trim();\n\n // // Debug logging\n // if (directText) {\n // console.debugg('Text element found:', {\n // tag,\n // text: directText,\n // dimensions: boundingRect,\n // element\n // });\n // }\n\n // return validTags && \n // boundingRect.height > 1 && \n // boundingRect.width > 1 && \n // directText !== '';\n \n // } catch (e) {\n // console.warn('Error processing element:', e);\n // return false;\n // }\n // });\n // }\n\n\n\n\n\n function findElementsWithPointer() {\n const elements = [];\n const allElements = getAllElementsIncludingShadow('*');\n \n console.log('Checking elements with pointer style...');\n \n allElements.forEach(element => {\n // Skip SVG elements for now\n if (element instanceof SVGElement || element.tagName.toLowerCase() === 'svg') {\n return;\n }\n \n const style = window.getComputedStyle(element);\n if (style.cursor === 'pointer') {\n elements.push(element);\n }\n });\n \n console.log(`Found ${elements.length} elements with pointer cursor`);\n return elements;\n }\n\n function findCheckables() {\n const elements = [];\n\n elements.push(...getAllElementsIncludingShadow('input[type=\"checkbox\"]'));\n elements.push(...getAllElementsIncludingShadow('input[type=\"radio\"]'));\n const all_elements = getAllElementsIncludingShadow('label');\n const radioClasses = Array.from(all_elements).filter(el => {\n return /.*radio.*/i.test(el.className); \n });\n elements.push(...radioClasses);\n return elements;\n }\n\n function findFillables() {\n const elements = [];\n\n const inputs = [...getAllElementsIncludingShadow('input:not([type=\"radio\"]):not([type=\"checkbox\"])')];\n console.log('Found inputs:', inputs.length, inputs);\n elements.push(...inputs);\n \n const textareas = [...getAllElementsIncludingShadow('textarea')];\n console.log('Found textareas:', textareas.length);\n elements.push(...textareas);\n \n const editables = [...getAllElementsIncludingShadow('[contenteditable=\"true\"]')];\n console.log('Found editables:', editables.length);\n elements.push(...editables);\n\n return elements;\n }\n\n // Helper function to check if element is a form control\n function isFormControl(elementInfo) {\n return /^(input|select|textarea|button|label)$/i.test(elementInfo.tag);\n }\n\n const isDropdownItem = (elementInfo) => {\n const dropdownPatterns = [\n /dropdown[-_]?item/i, // matches: dropdown-item, dropdownitem, dropdown_item\n /menu[-_]?item/i, // matches: menu-item, menuitem, menu_item\n /dropdown[-_]?link/i, // matches: dropdown-link, dropdownlink, dropdown_link\n /list[-_]?item/i, // matches: list-item, listitem, list_item\n /select[-_]?item/i, // matches: select-item, selectitem, select_item \n ];\n\n const rolePatterns = [\n /menu[-_]?item/i, // matches: menuitem, menu-item\n /option/i, // matches: option\n /list[-_]?item/i, // matches: listitem, list-item\n /tree[-_]?item/i // matches: treeitem, tree-item\n ];\n\n const hasMatchingClass = elementInfo.element.className && \n dropdownPatterns.some(pattern => \n pattern.test(elementInfo.element.className)\n );\n\n const hasMatchingRole = elementInfo.element.getAttribute('role') && \n rolePatterns.some(pattern => \n pattern.test(elementInfo.element.getAttribute('role'))\n );\n\n return hasMatchingClass || hasMatchingRole;\n };\n\n /**\n * Finds the first element matching a CSS selector, traversing Shadow DOM if necessary\n * @param {string} selector - CSS selector to search for\n * @param {Element} [root=document] - Root element to start searching from\n * @returns {Element|null} - The first matching element or null if not found\n */\n function querySelectorShadow(selector, root = document) {\n // First try to find in light DOM\n let element = root.querySelector(selector);\n if (element) return element;\n \n // Get all elements with shadow root\n const shadowElements = Array.from(root.querySelectorAll('*'))\n .filter(el => el.shadowRoot);\n \n // Search through each shadow root until we find a match\n for (const el of shadowElements) {\n element = querySelectorShadow(selector, el.shadowRoot);\n if (element) return element;\n }\n \n return null;\n }\n\n const getElementByXPathOrCssSelector = (element_info) => {\n console.log('getElementByXPathOrCssSelector:', element_info);\n\n findElement(document, element_info.iframe_selector, element_info.css_selector);\n };\n\n const findElement = (root, iframeSelector, cssSelector) => {\n let element;\n \n if (iframeSelector) { \n const frames = getAllDocumentElementsIncludingShadow('iframe', root);\n \n // Iterate over all frames and compare their CSS selectors\n for (const frame of frames) {\n const selector = generateCssPath(frame);\n if (selector === iframeSelector) {\n const frameDocument = frame.contentDocument || frame.contentWindow.document;\n element = querySelectorShadow(cssSelector, frameDocument);\n console.log('found element ', element);\n break;\n } \n } }\n else\n element = querySelectorShadow(cssSelector, root);\n \n if (!element) {\n console.warn('Failed to find element with CSS selector:', cssSelector);\n }\n\n return element;\n };\n\n\n function isDecendent(parent, child) {\n let element = child;\n while (element && element !== parent && element.nodeType === Node.ELEMENT_NODE) { \n element = getParentNode(element); \n }\n return element === parent;\n }\n\n function generateXPath(element) {\n return '/'+extractElementPath(element).map(item => `${item.tagName}${item.onlyChild ? '' : `[${item.index}]`}`).join('/');\n }\n\n function generateCssPath(element) {\n return extractElementPath(element).map(item => `${item.tagName}:nth-of-type(${item.index})`).join(' > ');\n }\n\n function extractElementPath(element) {\n if (!element) {\n console.error('ERROR: No element provided to generatePath');\n return [];\n }\n const path = [];\n // traversing up the DOM tree\n while (element && element.nodeType === Node.ELEMENT_NODE) { \n let tagName = element.nodeName.toLowerCase();\n \n let sibling = element;\n let index = 1;\n \n while (sibling = sibling.previousElementSibling) {\n if (sibling.nodeName.toLowerCase() === tagName) index++;\n }\n sibling = element;\n \n let onlyChild = (index === 1);\n while (onlyChild && (sibling = sibling.nextElementSibling)) {\n if (sibling.nodeName.toLowerCase() === tagName) onlyChild = false;\n }\n \n // add a tuple with tagName, index (nth), and onlyChild \n path.unshift({\n tagName: tagName,\n index: index,\n onlyChild: onlyChild \n }); \n\n element = getParentNode(element);\n }\n \n return path;\n }\n\n function cleanHTML(rawHTML) {\n const parser = new DOMParser();\n const doc = parser.parseFromString(rawHTML, \"text/html\");\n\n function cleanElement(element) {\n const allowedAttributes = new Set([\n \"role\",\n \"type\",\n \"class\",\n \"href\",\n \"alt\",\n \"title\",\n \"readonly\",\n \"checked\",\n \"enabled\",\n \"disabled\",\n ]);\n\n [...element.attributes].forEach(attr => {\n const name = attr.name.toLowerCase();\n const value = attr.value;\n\n const isTestAttribute = /^(testid|test-id|data-test-id)$/.test(name);\n const isDataAttribute = name.startsWith(\"data-\") && value;\n const isBooleanAttribute = [\"readonly\", \"checked\", \"enabled\", \"disabled\"].includes(name);\n\n if (!allowedAttributes.has(name) && !isDataAttribute && !isTestAttribute && !isBooleanAttribute) {\n element.removeAttribute(name);\n }\n });\n\n // Handle SVG content - more aggressive replacement\n if (element.tagName.toLowerCase() === \"svg\") {\n // Remove all attributes except class and role\n [...element.attributes].forEach(attr => {\n const name = attr.name.toLowerCase();\n if (name !== \"class\" && name !== \"role\") {\n element.removeAttribute(name);\n }\n });\n element.innerHTML = \"CONTENT REMOVED\";\n } else {\n // Recursively clean child elements\n Array.from(element.children).forEach(cleanElement);\n }\n\n // Only remove empty elements that aren't semantic or icon elements\n const keepEmptyElements = ['i', 'span', 'svg', 'button', 'input'];\n if (!keepEmptyElements.includes(element.tagName.toLowerCase()) && \n !element.children.length && \n !element.textContent.trim()) {\n element.remove();\n }\n }\n\n // Process all elements in the document body\n Array.from(doc.body.children).forEach(cleanElement);\n return doc.body.innerHTML;\n }\n\n function getContainingIframe(element) {\n // If not in an iframe, return null\n if (element.ownerDocument.defaultView === window.top) {\n return null;\n }\n \n // Try to find the iframe in the parent document that contains our element\n try {\n const parentDocument = element.ownerDocument.defaultView.parent.document;\n const iframes = parentDocument.querySelectorAll('iframe');\n \n for (const iframe of iframes) {\n if (iframe.contentWindow === element.ownerDocument.defaultView) {\n return iframe;\n }\n }\n } catch (e) {\n // Cross-origin restriction\n return \"Cross-origin iframe - cannot access details\";\n }\n \n return null;\n }\n\n function getElementInfo(element, index) {\n // Get text content with spaces between elements\n /* function getTextContent(element) {\n const walker = document.createTreeWalker(\n element,\n NodeFilter.SHOW_TEXT,\n null,\n false\n );\n\n let text = '';\n let node;\n\n while (node = walker.nextNode()) {\n const trimmedText = node.textContent.trim();\n if (trimmedText) {\n // Add space if there's already text\n if (text) {\n text += ' ';\n }\n text += trimmedText;\n }\n }\n\n return text;\n } */\n \n const xpath = generateXPath(element);\n const css_selector = generateCssPath(element);\n //disabled since it's blocking event handling in recorder\n const short_css_selector = ''; //getRobustSelector(element);\n\n const iframe = getContainingIframe(element); \n const iframe_selector = iframe ? generateCssPath(iframe) : \"\";\n //disabled since it's blocking event handling in recorder\n const short_iframe_selector = ''; //iframe ? getRobustSelector(iframe) : \"\";\n\n // Return element info with pre-calculated values\n return new ElementInfo(element, index, {\n tag: element.tagName.toLowerCase(),\n type: element.type || '',\n text: element.innerText || element.placeholder || '', //getTextContent(element),\n html: cleanHTML(element.outerHTML),\n xpath: xpath,\n css_selector: css_selector,\n bounding_box: element.getBoundingClientRect(),\n iframe_selector: iframe_selector,\n short_css_selector: short_css_selector,\n short_iframe_selector: short_iframe_selector\n });\n }\n\n function getAriaLabelledByText(elementInfo, includeHidden=true) {\n if (!elementInfo.element.hasAttribute('aria-labelledby')) return '';\n\n const ids = elementInfo.element.getAttribute('aria-labelledby').split(/\\s+/);\n let labelText = '';\n\n //locate root (document or iFrame document if element is contained in an iframe)\n let root = document;\n if (elementInfo.iframe_selector) { \n const frames = getAllDocumentElementsIncludingShadow('iframe', document);\n \n // Iterate over all frames and compare their CSS selectors\n for (const frame of frames) {\n const selector = generateCssPath(frame);\n if (selector === elementInfo.iframe_selector) {\n root = frame.contentDocument || frame.contentWindow.document; \n break;\n }\n } \n }\n\n ids.forEach(id => {\n const el = querySelectorShadow(`#${CSS.escape(id)}`, root);\n if (el) {\n if (includeHidden || el.offsetParent !== null || getComputedStyle(el).display !== 'none') {\n labelText += el.textContent.trim() + ' ';\n }\n }\n });\n\n return labelText.trim();\n }\n\n\n\n const filterZeroDimensions = (elementInfo) => {\n const rect = elementInfo.bounding_box;\n //single pixel elements are typically faux controls and should be filtered too\n const hasSize = rect.width > 1 && rect.height > 1;\n const style = window.getComputedStyle(elementInfo.element);\n const isVisible = style.display !== 'none' && style.visibility !== 'hidden';\n \n if (!hasSize || !isVisible) {\n \n return false;\n }\n return true;\n };\n\n\n\n function uniquifyElements(elements) {\n const seen = new Set();\n\n console.log(`Starting uniquification with ${elements.length} elements`);\n\n // Filter out testing infrastructure elements first\n const filteredInfrastructure = elements.filter(element_info => {\n // Skip the highlight-overlay element completely - it's part of the testing infrastructure\n if (element_info.element.id === 'highlight-overlay' || \n (element_info.css_selector && element_info.css_selector.includes('#highlight-overlay'))) {\n console.log('Filtered out testing infrastructure element:', element_info.css_selector);\n return false;\n }\n \n // Filter out UI framework container/manager elements\n const el = element_info.element;\n // UI framework container checks - generic detection for any framework\n if ((el.getAttribute('data-rendered-by') || \n el.getAttribute('data-reactroot') || \n el.getAttribute('ng-version') || \n el.getAttribute('data-component-id') ||\n el.getAttribute('data-root') ||\n el.getAttribute('data-framework')) && \n (el.className && \n typeof el.className === 'string' && \n (el.className.includes('Container') || \n el.className.includes('container') || \n el.className.includes('Manager') || \n el.className.includes('manager')))) {\n console.log('Filtered out UI framework container element:', element_info.css_selector);\n return false;\n }\n \n // Direct filter for framework container elements that shouldn't be interactive\n // Consolidating multiple container detection patterns into one efficient check\n const isFullViewport = element_info.bounding_box && \n element_info.bounding_box.x <= 5 && \n element_info.bounding_box.y <= 5 && \n element_info.bounding_box.width >= (window.innerWidth * 0.95) && \n element_info.bounding_box.height >= (window.innerHeight * 0.95);\n \n // Empty content check\n const isEmpty = !el.innerText || el.innerText.trim() === '';\n \n // Check if it's a framework container element\n if (element_info.element.tagName === 'DIV' && \n isFullViewport && \n isEmpty && \n (\n // Pattern matching for root containers\n (element_info.xpath && \n (element_info.xpath.match(/^\\/html\\[\\d+\\]\\/body\\[\\d+\\]\\/div\\[\\d+\\]\\/div\\[\\d+\\]$/) || \n element_info.xpath.match(/^\\/\\/\\*\\[@id='[^']+'\\]\\/div\\[\\d+\\]$/))) ||\n \n // Simple DOM structure\n (element_info.css_selector.split(' > ').length <= 4 && element_info.depth <= 5) ||\n \n // Empty or container-like classes\n (!el.className || el.className === '' || \n (typeof el.className === 'string' && \n (el.className.includes('overlay') || \n el.className.includes('container') || \n el.className.includes('wrapper'))))\n )) {\n console.log('Filtered out framework container element:', element_info.css_selector);\n return false;\n }\n \n return true;\n });\n\n // First filter out elements with zero dimensions\n const nonZeroElements = filteredInfrastructure.filter(filterZeroDimensions);\n // sort by CSS selector depth so parents are processed first\n nonZeroElements.sort((a, b) => a.getDepth() - b.getDepth());\n console.log(`After dimension filtering: ${nonZeroElements.length} elements remain (${elements.length - nonZeroElements.length} removed)`);\n \n const filteredByParent = nonZeroElements.filter(element_info => {\n\n const parent = findClosestParent(seen, element_info);\n const keep = parent == null || shouldKeepNestedElement(element_info, parent);\n // console.log(\"node \", element_info.index, \": keep=\", keep, \" parent=\", parent);\n // if (!keep && !element_info.xpath) {\n // console.log(\"Filtered out element \", element_info,\" because it's a nested element of \", parent);\n // }\n if (keep)\n seen.add(element_info.css_selector);\n\n return keep;\n });\n\n console.log(`After parent/child filtering: ${filteredByParent.length} elements remain (${nonZeroElements.length - filteredByParent.length} removed)`);\n\n // Final overlap filtering\n const filteredResults = filteredByParent.filter(element => {\n\n // Look for any element that came BEFORE this one in the array\n const hasEarlierOverlap = filteredByParent.some(other => {\n // Only check elements that came before (lower index)\n if (filteredByParent.indexOf(other) >= filteredByParent.indexOf(element)) {\n return false;\n }\n \n const isOverlapping = areElementsOverlapping(element, other); \n return isOverlapping;\n }); \n\n // Keep element if it has no earlier overlapping elements\n return !hasEarlierOverlap;\n });\n \n \n \n // Check for overlay removal\n console.log(`After filtering: ${filteredResults.length} (${filteredByParent.length - filteredResults.length} removed by overlap)`);\n \n const nonOverlaidElements = filteredResults.filter(element => {\n return !isOverlaid(element);\n });\n\n console.log(`Final elements after overlay removal: ${nonOverlaidElements.length} (${filteredResults.length - nonOverlaidElements.length} removed)`);\n \n return nonOverlaidElements;\n\n }\n\n\n\n const areElementsOverlapping = (element1, element2) => {\n if (element1.css_selector === element2.css_selector) {\n return true;\n }\n \n const box1 = element1.bounding_box;\n const box2 = element2.bounding_box;\n \n return box1.x === box2.x &&\n box1.y === box2.y &&\n box1.width === box2.width &&\n box1.height === box2.height;\n // element1.text === element2.text &&\n // element2.tag === 'a';\n };\n\n function findClosestParent(seen, element_info) { \n \n // Split the xpath into segments\n const segments = element_info.css_selector.split(' > ');\n \n // Try increasingly shorter paths until we find one in the seen set\n for (let i = segments.length - 1; i > 0; i--) {\n const parentPath = segments.slice(0, i).join(' > ');\n if (seen.has(parentPath)) {\n return parentPath;\n }\n }\n\n return null;\n }\n\n function shouldKeepNestedElement(elementInfo, parentPath) {\n let result = false;\n const parentSegments = parentPath.split(' > ');\n\n const isParentLink = /^a(:nth-of-type\\(\\d+\\))?$/.test(parentSegments[parentSegments.length - 1]);\n if (isParentLink) {\n return false; \n }\n // If this is a checkbox/radio input\n if (elementInfo.tag === 'input' && \n (elementInfo.type === 'checkbox' || elementInfo.type === 'radio')) {\n \n // Check if parent is a label by looking at the parent xpath's last segment\n \n const isParentLabel = /^label(:nth-of-type\\(\\d+\\))?$/.test(parentSegments[parentSegments.length - 1]);\n \n // If parent is a label, don't keep the input (we'll keep the label instead)\n if (isParentLabel) {\n return false;\n }\n }\n \n // Keep all other form controls and dropdown items\n if (isFormControl(elementInfo) || isDropdownItem(elementInfo)) {\n result = true;\n }\n\n if(isTableCell(elementInfo)) {\n result = true;\n }\n \n \n // console.log(`shouldKeepNestedElement: ${elementInfo.tag} ${elementInfo.text} ${elementInfo.xpath} -> ${parentXPath} -> ${result}`);\n return result;\n }\n\n\n function isTableCell(elementInfo) {\n const element = elementInfo.element;\n if(!element || !(element instanceof HTMLElement)) {\n return false;\n }\n const validTags = new Set(['td', 'th']);\n const validRoles = new Set(['cell', 'gridcell', 'columnheader', 'rowheader']);\n \n const tag = element.tagName.toLowerCase();\n const role = element.getAttribute('role')?.toLowerCase();\n\n if (validTags.has(tag) || (role && validRoles.has(role))) {\n return true;\n }\n return false;\n \n }\n\n function isOverlaid(elementInfo) {\n const element = elementInfo.element;\n const boundingRect = elementInfo.bounding_box;\n \n\n \n \n // Create a diagnostic logging function that only logs when needed\n const diagnosticLog = (...args) => {\n { // set to true for debugging\n console.log('[OVERLAY-DEBUG]', ...args);\n }\n };\n\n // Special handling for tooltips\n if (elementInfo.element.className && typeof elementInfo.element.className === 'string' && \n elementInfo.element.className.includes('tooltip')) {\n diagnosticLog('Element is a tooltip, not considering it overlaid');\n return false;\n }\n \n \n \n // Get element at the center point to check if it's covered by a popup/modal\n const middleX = boundingRect.x + boundingRect.width/2;\n const middleY = boundingRect.y + boundingRect.height/2;\n const elementAtMiddle = element.ownerDocument.elementFromPoint(middleX, middleY);\n \n if (elementAtMiddle && \n elementAtMiddle !== element && \n !isDecendent(element, elementAtMiddle) && \n !isDecendent(elementAtMiddle, element)) {\n\n \n return true;\n }\n \n \n return false;\n \n }\n\n\n\n /**\n * Get the “best” short, unique, and robust CSS selector for an element.\n * \n * @param {Element} element\n * @returns {string} A selector guaranteed to find exactly that element in its context\n */\n function getRobustSelector(element) {\n // 1. Figure out the real “root” (iframe doc, shadow root, or main doc)\n const root = (() => {\n const rootNode = element.getRootNode();\n if (rootNode instanceof ShadowRoot) {\n return rootNode;\n }\n return element.ownerDocument;\n })();\n\n // 2. Options to bias toward stable attrs and away from auto-generated classes\n const options = {\n root,\n // only use data-*, id or aria-label by default\n attr(name, value) {\n if (name === 'id' || name.startsWith('data-') || name === 'aria-label') {\n return true;\n }\n return false;\n },\n // skip framework junk\n filter(name, value) {\n if (name.startsWith('ng-') || name.startsWith('_ngcontent') || /^p-/.test(name)) {\n return false;\n }\n return true;\n },\n // let finder try really short seeds\n seedMinLength: 1,\n optimizedMinLength: 1,\n };\n\n let selector;\n try {\n selector = finder(element, options);\n // 3. Verify it really works in the context\n const found = root.querySelectorAll(selector);\n if (found.length !== 1 || found[0] !== element) {\n throw new Error('not unique or not found');\n }\n return selector;\n } catch (err) {\n // 4. Fallback: full path (you already have this utility)\n console.warn('[getRobustSelector] finder failed, falling back to full path:', err);\n return generateCssPath(element); // you’d import or define this elsewhere\n }\n }\n\n /**\n * Checks if an element is scrollable (has scrollable content)\n * \n * @param element - The element to check\n * @returns boolean indicating if the element is scrollable\n */\n function isScrollableContainer(element) {\n if (!element) return false;\n \n const style = window.getComputedStyle(element);\n \n // Reliable way to detect if an element has scrollbars or is scrollable\n const hasScrollHeight = element.scrollHeight > element.clientHeight;\n const hasScrollWidth = element.scrollWidth > element.clientWidth;\n \n // Check actual style properties\n const hasOverflowY = style.overflowY === 'auto' || \n style.overflowY === 'scroll' || \n style.overflowY === 'overlay';\n const hasOverflowX = style.overflowX === 'auto' || \n style.overflowX === 'scroll' || \n style.overflowX === 'overlay';\n \n // Check common class names and attributes for scrollable containers across frameworks\n const hasScrollClasses = element.classList.contains('scroll') || \n element.classList.contains('scrollable') ||\n element.classList.contains('overflow') ||\n element.classList.contains('overflow-auto') ||\n element.classList.contains('overflow-scroll') ||\n element.getAttribute('data-scrollable') === 'true';\n \n // Check for height/max-height constraints that often indicate scrolling content\n const hasHeightConstraint = style.maxHeight && \n style.maxHeight !== 'none' && \n style.maxHeight !== 'auto';\n \n // An element is scrollable if it has:\n // 1. Actual scrollbars in use (most reliable check) OR\n // 2. Overflow styles allowing scrolling AND content that would require scrolling\n return (hasScrollHeight && hasOverflowY) || \n (hasScrollWidth && hasOverflowX) ||\n (hasScrollClasses && (hasScrollHeight || hasScrollWidth)) ||\n (hasHeightConstraint && hasScrollHeight);\n }\n\n /**\n * Detects scrollable containers that are ancestors of the target element\n * \n * This function traverses up the DOM tree from the target element and identifies\n * all scrollable containers (elements that have scrollable content).\n * \n * @param target - The target element to start the search from\n * @returns Array of objects with selector and scroll properties\n */\n function detectScrollableContainers(target) {\n const scrollableContainers = [];\n \n if (!target) {\n return scrollableContainers;\n }\n \n console.log('🔍 [detectScrollableContainers] Starting detection for target:', target.tagName, target.id, target.className);\n \n // Detect if target is inside an iframe\n const iframe = getContainingIframe(target);\n const iframe_selector = iframe ? generateCssPath(iframe) : \"\";\n \n console.log('🔍 [detectScrollableContainers] Iframe context:', iframe ? 'inside iframe' : 'main document', 'selector:', iframe_selector);\n \n // Start from the target element and traverse up the DOM tree\n let currentElement = target;\n let depth = 0;\n const MAX_DEPTH = 10; // Limit traversal depth to avoid infinite loops\n \n while (currentElement && currentElement.nodeType === Node.ELEMENT_NODE && depth < MAX_DEPTH) { \n // Check if the current element is scrollable\n if (isScrollableContainer(currentElement)) {\n console.log('🔍 [detectScrollableContainers] Found scrollable container at depth', depth, ':', currentElement.tagName, currentElement.id, currentElement.className);\n \n const container = {\n containerEl: currentElement,\n selector: generateCssPath(currentElement),\n iframe_selector: iframe_selector,\n scrollTop: currentElement.scrollTop,\n scrollLeft: currentElement.scrollLeft,\n scrollHeight: currentElement.scrollHeight,\n scrollWidth: currentElement.scrollWidth,\n clientHeight: currentElement.clientHeight,\n clientWidth: currentElement.clientWidth\n };\n \n scrollableContainers.push(container);\n }\n \n // Move to parent element\n currentElement = getParentNode(currentElement);\n \n depth++;\n }\n \n console.log('🔍 [detectScrollableContainers] Detection complete. Found', scrollableContainers.length, 'scrollable containers');\n return scrollableContainers;\n }\n\n class DOMSerializer {\n constructor(options = {}) {\n this.options = {\n includeStyles: true,\n includeScripts: false, // Security consideration\n includeFrames: true,\n includeShadowDOM: true,\n maxDepth: 50,\n ...options\n };\n this.serializedFrames = new Map();\n this.shadowRoots = new Map();\n }\n \n /**\n * Serialize a complete document or element\n */\n serialize(rootElement = document) {\n try {\n const serialized = {\n type: 'document',\n doctype: this.serializeDoctype(rootElement),\n documentElement: this.serializeElement(rootElement.documentElement || rootElement),\n frames: [],\n timestamp: Date.now(),\n url: rootElement.URL || window.location?.href,\n metadata: {\n title: rootElement.title,\n charset: rootElement.characterSet,\n contentType: rootElement.contentType\n }\n };\n \n // Serialize frames and iframes if enabled\n if (this.options.includeFrames) {\n serialized.frames = this.serializeFrames(rootElement);\n }\n \n return serialized;\n } catch (error) {\n console.error('Serialization error:', error);\n throw new Error(`DOM serialization failed: ${error.message}`);\n }\n }\n \n /**\n * Serialize document type declaration\n */\n serializeDoctype(doc) {\n if (!doc.doctype) return null;\n \n return {\n name: doc.doctype.name,\n publicId: doc.doctype.publicId,\n systemId: doc.doctype.systemId\n };\n }\n \n /**\n * Serialize an individual element and its children\n */\n serializeElement(element, depth = 0) {\n if (depth > this.options.maxDepth) {\n return { type: 'text', content: '<!-- Max depth exceeded -->' };\n }\n \n const nodeType = element.nodeType;\n \n switch (nodeType) {\n case Node.ELEMENT_NODE:\n return this.serializeElementNode(element, depth);\n case Node.TEXT_NODE:\n return this.serializeTextNode(element);\n case Node.COMMENT_NODE:\n return this.serializeCommentNode(element);\n case Node.DOCUMENT_FRAGMENT_NODE:\n return this.serializeDocumentFragment(element, depth);\n default:\n return null;\n }\n }\n \n /**\n * Serialize element node with attributes and children\n */\n serializeElementNode(element, depth) {\n const tagName = element.tagName.toLowerCase();\n \n // Skip script tags for security unless explicitly enabled\n if (tagName === 'script' && !this.options.includeScripts) {\n return { type: 'comment', content: '<!-- Script tag removed for security -->' };\n }\n \n const serialized = {\n type: 'element',\n tagName: tagName,\n attributes: this.serializeAttributes(element),\n children: [],\n shadowRoot: null\n };\n \n // Handle Shadow DOM\n if (this.options.includeShadowDOM && element.shadowRoot) {\n serialized.shadowRoot = this.serializeShadowRoot(element.shadowRoot, depth + 1);\n }\n \n // Handle special elements\n if (tagName === 'iframe' || tagName === 'frame') {\n serialized.frameData = this.serializeFrameElement(element);\n }\n \n // Serialize children\n for (const child of element.childNodes) {\n const serializedChild = this.serializeElement(child, depth + 1);\n if (serializedChild) {\n serialized.children.push(serializedChild);\n }\n }\n \n // Include computed styles if enabled\n if (this.options.includeStyles && element.nodeType === Node.ELEMENT_NODE) {\n serialized.computedStyle = this.serializeComputedStyle(element);\n }\n \n return serialized;\n }\n \n /**\n * Serialize element attributes\n */\n serializeAttributes(element) {\n const attributes = {};\n \n if (element.attributes) {\n for (const attr of element.attributes) {\n attributes[attr.name] = attr.value;\n }\n }\n \n return attributes;\n }\n \n /**\n * Serialize computed styles\n */\n serializeComputedStyle(element) {\n try {\n const computedStyle = window.getComputedStyle(element);\n const styles = {};\n \n // Only serialize non-default values to reduce size\n const importantStyles = [\n 'display', 'position', 'width', 'height', 'margin', 'padding',\n 'border', 'background', 'color', 'font-family', 'font-size',\n 'text-align', 'visibility', 'z-index', 'transform'\n ];\n \n for (const prop of importantStyles) {\n const value = computedStyle.getPropertyValue(prop);\n if (value && value !== 'initial' && value !== 'normal') {\n styles[prop] = value;\n }\n }\n \n return styles;\n } catch (error) {\n return {};\n }\n }\n \n /**\n * Serialize text node\n */\n serializeTextNode(node) {\n return {\n type: 'text',\n content: node.textContent\n };\n }\n \n /**\n * Serialize comment node\n */\n serializeCommentNode(node) {\n return {\n type: 'comment',\n content: node.textContent\n };\n }\n \n /**\n * Serialize document fragment\n */\n serializeDocumentFragment(fragment, depth) {\n const serialized = {\n type: 'fragment',\n children: []\n };\n \n for (const child of fragment.childNodes) {\n const serializedChild = this.serializeElement(child, depth + 1);\n if (serializedChild) {\n serialized.children.push(serializedChild);\n }\n }\n \n return serialized;\n }\n \n /**\n * Serialize Shadow DOM\n */\n serializeShadowRoot(shadowRoot, depth) {\n const serialized = {\n type: 'shadowRoot',\n mode: shadowRoot.mode,\n children: []\n };\n \n for (const child of shadowRoot.childNodes) {\n const serializedChild = this.serializeElement(child, depth + 1);\n if (serializedChild) {\n serialized.children.push(serializedChild);\n }\n }\n \n return serialized;\n }\n \n /**\n * Serialize frame/iframe elements\n */\n serializeFrameElement(frameElement) {\n const frameData = {\n src: frameElement.src,\n name: frameElement.name,\n id: frameElement.id,\n sandbox: frameElement.sandbox?.toString() || '',\n allowfullscreen: frameElement.allowFullscreen\n };\n \n // Try to access frame content (may fail due to CORS)\n try {\n const frameDoc = frameElement.contentDocument;\n if (frameDoc && this.options.includeFrames) {\n frameData.content = this.serialize(frameDoc);\n }\n } catch (error) {\n frameData.accessError = 'Cross-origin frame content not accessible';\n }\n \n return frameData;\n }\n \n /**\n * Serialize all frames in document\n */\n serializeFrames(doc) {\n const frames = [];\n const frameElements = doc.querySelectorAll('iframe, frame');\n \n for (const frameElement of frameElements) {\n try {\n const frameDoc = frameElement.contentDocument;\n if (frameDoc) {\n frames.push({\n element: this.serializeElement(frameElement),\n content: this.serialize(frameDoc)\n });\n }\n } catch (error) {\n frames.push({\n element: this.serializeElement(frameElement),\n error: 'Frame content not accessible'\n });\n }\n }\n \n return frames;\n }\n \n /**\n * Deserialize serialized DOM data back to DOM nodes\n */\n deserialize(serializedData, targetDocument = document) {\n try {\n if (serializedData.type === 'document') {\n return this.deserializeDocument(serializedData, targetDocument);\n } else {\n return this.deserializeElement(serializedData, targetDocument);\n }\n } catch (error) {\n console.error('Deserialization error:', error);\n throw new Error(`DOM deserialization failed: ${error.message}`);\n }\n }\n \n /**\n * Deserialize complete document\n */\n deserializeDocument(serializedDoc, targetDoc) {\n // Create new document if needed\n const doc = targetDoc || document.implementation.createHTMLDocument();\n \n // Set doctype if present\n if (serializedDoc.doctype) {\n const doctype = document.implementation.createDocumentType(\n serializedDoc.doctype.name,\n serializedDoc.doctype.publicId,\n serializedDoc.doctype.systemId\n );\n doc.replaceChild(doctype, doc.doctype);\n }\n \n // Deserialize document element\n if (serializedDoc.documentElement) {\n const newDocElement = this.deserializeElement(serializedDoc.documentElement, doc);\n doc.replaceChild(newDocElement, doc.documentElement);\n }\n \n // Handle metadata\n if (serializedDoc.metadata) {\n doc.title = serializedDoc.metadata.title || '';\n }\n \n return doc;\n }\n \n /**\n * Deserialize individual element\n */\n deserializeElement(serializedNode, doc) {\n switch (serializedNode.type) {\n case 'element':\n return this.deserializeElementNode(serializedNode, doc);\n case 'text':\n return doc.createTextNode(serializedNode.content);\n case 'comment':\n return doc.createComment(serializedNode.content);\n case 'fragment':\n return this.deserializeDocumentFragment(serializedNode, doc);\n case 'shadowRoot':\n // Shadow roots are handled during element creation\n return null;\n default:\n return null;\n }\n }\n \n /**\n * Deserialize element node\n */\n deserializeElementNode(serializedElement, doc) {\n const element = doc.createElement(serializedElement.tagName);\n \n // Set attributes\n if (serializedElement.attributes) {\n for (const [name, value] of Object.entries(serializedElement.attributes)) {\n try {\n element.setAttribute(name, value);\n } catch (error) {\n console.warn(`Failed to set attribute ${name}:`, error);\n }\n }\n }\n \n // Apply computed styles if available\n if (serializedElement.computedStyle && this.options.includeStyles) {\n for (const [prop, value] of Object.entries(serializedElement.computedStyle)) {\n try {\n element.style.setProperty(prop, value);\n } catch (error) {\n console.warn(`Failed to set style ${prop}:`, error);\n }\n }\n }\n \n // Create shadow root if present\n if (serializedElement.shadowRoot && element.attachShadow) {\n try {\n const shadowRoot = element.attachShadow({ \n mode: serializedElement.shadowRoot.mode || 'open' \n });\n \n // Deserialize shadow root children\n for (const child of serializedElement.shadowRoot.children) {\n const childElement = this.deserializeElement(child, doc);\n if (childElement) {\n shadowRoot.appendChild(childElement);\n }\n }\n } catch (error) {\n console.warn('Failed to create shadow root:', error);\n }\n }\n \n // Deserialize children\n if (serializedElement.children) {\n for (const child of serializedElement.children) {\n const childElement = this.deserializeElement(child, doc);\n if (childElement) {\n element.appendChild(childElement);\n }\n }\n }\n \n // Handle frame content\n if (serializedElement.frameData && serializedElement.frameData.content) {\n // Frame content deserialization would happen after the frame loads\n element.addEventListener('load', () => {\n try {\n const frameDoc = element.contentDocument;\n if (frameDoc) {\n this.deserializeDocument(serializedElement.frameData.content, frameDoc);\n }\n } catch (error) {\n console.warn('Failed to deserialize frame content:', error);\n }\n });\n }\n \n return element;\n }\n \n /**\n * Deserialize document fragment\n */\n deserializeDocumentFragment(serializedFragment, doc) {\n const fragment = doc.createDocumentFragment();\n \n if (serializedFragment.children) {\n for (const child of serializedFragment.children) {\n const childElement = this.deserializeElement(child, doc);\n if (childElement) {\n fragment.appendChild(childElement);\n }\n }\n }\n \n return fragment;\n }\n }\n \n // Usage example and utility functions\n class DOMUtils {\n /**\n * Create serializer with common presets\n */\n static createSerializer(preset = 'default') {\n const presets = {\n default: {\n includeStyles: true,\n includeScripts: false,\n includeFrames: true,\n includeShadowDOM: true\n },\n minimal: {\n includeStyles: false,\n includeScripts: false,\n includeFrames: false,\n includeShadowDOM: false\n },\n complete: {\n includeStyles: true,\n includeScripts: true,\n includeFrames: true,\n includeShadowDOM: true\n },\n secure: {\n includeStyles: true,\n includeScripts: false,\n includeFrames: false,\n includeShadowDOM: true\n }\n };\n \n return new DOMSerializer(presets[preset] || presets.default);\n }\n \n /**\n * Serialize DOM to JSON string\n */\n static serializeToJSON(element, options) {\n const serializer = new DOMSerializer(options);\n const serialized = serializer.serialize(element);\n return JSON.stringify(serialized, null, 2);\n }\n \n /**\n * Deserialize from JSON string\n */\n static deserializeFromJSON(jsonString, targetDocument) {\n const serialized = JSON.parse(jsonString);\n const serializer = new DOMSerializer();\n return serializer.deserialize(serialized, targetDocument);\n }\n \n /**\n * Clone DOM with full fidelity including Shadow DOM\n */\n static deepClone(element, options) {\n const serializer = new DOMSerializer(options);\n const serialized = serializer.serialize(element);\n return serializer.deserialize(serialized, element.ownerDocument);\n }\n \n /**\n * Compare two DOM structures\n */\n static compare(element1, element2, options) {\n const serializer = new DOMSerializer(options);\n const serialized1 = serializer.serialize(element1);\n const serialized2 = serializer.serialize(element2);\n \n return JSON.stringify(serialized1) === JSON.stringify(serialized2);\n }\n }\n \n /*\n // Export for use\n if (typeof module !== 'undefined' && module.exports) {\n module.exports = { DOMSerializer, DOMUtils };\n } else if (typeof window !== 'undefined') {\n window.DOMSerializer = DOMSerializer;\n window.DOMUtils = DOMUtils;\n }\n */\n\n /* Usage Examples:\n \n // Basic serialization\n const serializer = new DOMSerializer();\n const serialized = serializer.serialize(document);\n console.log(JSON.stringify(serialized, null, 2));\n \n // Deserialize back to DOM\n const clonedDoc = serializer.deserialize(serialized);\n \n // Using presets\n const minimalSerializer = DOMUtils.createSerializer('minimal');\n const secureSerializer = DOMUtils.createSerializer('secure');\n \n // Serialize specific element with Shadow DOM\n const customElement = document.querySelector('my-custom-element');\n const serializedElement = serializer.serialize(customElement);\n \n // JSON utilities\n const jsonString = DOMUtils.serializeToJSON(document.body);\n const restored = DOMUtils.deserializeFromJSON(jsonString);\n \n // Deep clone with Shadow DOM support\n const clone = DOMUtils.deepClone(document.body, { includeShadowDOM: true });\n \n */\n\n function serializeNodeToJSON(nodeElement) {\n return DOMUtils.serializeToJSON(nodeElement, {includeStyles: false});\n }\n\n function deserializeNodeFromJSON(jsonString) {\n return DOMUtils.deserializeFromJSON(jsonString);\n }\n\n /**\n * Checks if a point is inside a bounding box\n * \n * @param point The point to check\n * @param box The bounding box\n * @returns boolean indicating if the point is inside the box\n */\n function isPointInsideBox(point, box) {\n return point.x >= box.x &&\n point.x <= box.x + box.width &&\n point.y >= box.y &&\n point.y <= box.y + box.height;\n }\n\n /**\n * Calculates the overlap area between two bounding boxes\n * \n * @param box1 First bounding box\n * @param box2 Second bounding box\n * @returns The overlap area\n */\n function calculateOverlap(box1, box2) {\n const xOverlap = Math.max(0,\n Math.min(box1.x + box1.width, box2.x + box2.width) -\n Math.max(box1.x, box2.x)\n );\n const yOverlap = Math.max(0,\n Math.min(box1.y + box1.height, box2.y + box2.height) -\n Math.max(box1.y, box2.y)\n );\n return xOverlap * yOverlap;\n }\n\n /**\n * Finds an exact match between candidate elements and the actual interaction element\n * \n * @param candidate_elements Array of candidate element infos\n * @param actualInteractionElementInfo The actual interaction element info\n * @returns The matching candidate element info, or null if no match is found\n */\n function findExactMatch(candidate_elements, actualInteractionElementInfo) {\n if (!actualInteractionElementInfo.element) {\n return null;\n }\n\n const exactMatch = candidate_elements.find(elementInfo => \n elementInfo.element && elementInfo.element === actualInteractionElementInfo.element\n );\n \n if (exactMatch) {\n console.log('✅ Found exact element match:', {\n matchedElement: exactMatch.element?.tagName,\n matchedElementClass: exactMatch.element?.className,\n index: exactMatch.index\n });\n return exactMatch;\n }\n \n return null;\n }\n\n /**\n * Finds a match by traversing up the parent elements\n * \n * @param candidate_elements Array of candidate element infos\n * @param actualInteractionElementInfo The actual interaction element info\n * @returns The matching candidate element info, or null if no match is found\n */\n function findParentMatch(candidate_elements, actualInteractionElementInfo) {\n if (!actualInteractionElementInfo.element) {\n return null;\n }\n\n let element = actualInteractionElementInfo.element;\n while (element.parentElement) {\n element = element.parentElement;\n const parentMatch = candidate_elements.find(candidate => \n candidate.element && candidate.element === element\n );\n \n if (parentMatch) {\n console.log('✅ Found parent element match:', {\n matchedElement: parentMatch.element?.tagName,\n matchedElementClass: parentMatch.element?.className,\n index: parentMatch.index,\n depth: element.tagName\n });\n return parentMatch;\n }\n \n // Stop if we hit another candidate element\n if (candidate_elements.some(candidate => \n candidate.element && candidate.element === element\n )) {\n console.log('⚠️ Stopped parent search - hit another candidate element:', element.tagName);\n break;\n }\n }\n \n return null;\n }\n\n /**\n * Finds a match based on spatial relationships between elements\n * \n * @param candidate_elements Array of candidate element infos\n * @param actualInteractionElementInfo The actual interaction element info\n * @returns The matching candidate element info, or null if no match is found\n */\n function findSpatialMatch(candidate_elements, actualInteractionElementInfo) {\n if (!actualInteractionElementInfo.element || !actualInteractionElementInfo.bounding_box) {\n return null;\n }\n\n const actualBox = actualInteractionElementInfo.bounding_box;\n let bestMatch = null;\n let bestScore = 0;\n\n for (const candidateInfo of candidate_elements) {\n if (!candidateInfo.bounding_box) continue;\n \n const candidateBox = candidateInfo.bounding_box;\n let score = 0;\n\n // Check if actual element is contained within candidate\n if (isPointInsideBox({ x: actualBox.x, y: actualBox.y }, candidateBox) &&\n isPointInsideBox({ x: actualBox.x + actualBox.width, y: actualBox.y + actualBox.height }, candidateBox)) {\n score += 100; // High score for containment\n }\n\n // Calculate overlap area as a factor\n const overlap = calculateOverlap(actualBox, candidateBox);\n score += overlap;\n\n // Consider proximity if no containment\n if (score === 0) {\n const distance = Math.sqrt(\n Math.pow((actualBox.x + actualBox.width/2) - (candidateBox.x + candidateBox.width/2), 2) +\n Math.pow((actualBox.y + actualBox.height/2) - (candidateBox.y + candidateBox.height/2), 2)\n );\n // Convert distance to a score (closer = higher score)\n score = 1000 / (distance + 1);\n }\n\n if (score > bestScore) {\n bestScore = score;\n bestMatch = candidateInfo;\n console.log('📏 New best spatial match:', {\n element: candidateInfo.element?.tagName,\n class: candidateInfo.element?.className,\n index: candidateInfo.index,\n score: score\n });\n }\n }\n\n if (bestMatch) {\n console.log('✅ Final spatial match selected:', {\n element: bestMatch.element?.tagName,\n class: bestMatch.element?.className,\n index: bestMatch.index,\n finalScore: bestScore\n });\n return bestMatch;\n }\n\n return null;\n }\n\n /**\n * Finds a matching candidate element for an actual interaction element\n * \n * @param candidate_elements Array of candidate element infos\n * @param actualInteractionElementInfo The actual interaction element info\n * @returns The matching candidate element info, or null if no match is found\n */\n function findMatchingCandidateElementInfo(candidate_elements, actualInteractionElementInfo) {\n if (!actualInteractionElementInfo.element || !actualInteractionElementInfo.bounding_box) {\n console.error('❌ Missing required properties in actualInteractionElementInfo');\n return null;\n }\n\n console.log('🔍 Starting element matching for:', {\n clickedElement: actualInteractionElementInfo.element.tagName,\n clickedElementClass: actualInteractionElementInfo.element.className,\n totalCandidates: candidate_elements.length\n });\n\n // First try exact element match\n const exactMatch = findExactMatch(candidate_elements, actualInteractionElementInfo);\n if (exactMatch) {\n return exactMatch;\n }\n console.log('❌ No exact element match found, trying parent matching...');\n\n // Try finding closest clickable parent\n const parentMatch = findParentMatch(candidate_elements, actualInteractionElementInfo);\n if (parentMatch) {\n return parentMatch;\n }\n console.log('❌ No parent match found, falling back to spatial matching...');\n\n // If no exact or parent match, look for spatial relationships\n const spatialMatch = findSpatialMatch(candidate_elements, actualInteractionElementInfo);\n if (spatialMatch) {\n return spatialMatch;\n }\n\n console.error('❌ No matching element found for actual interaction element:', actualInteractionElementInfo);\n return null;\n }\n\n const highlight = {\n execute: async function(elementTypes, handleScroll=false) {\n const elements = await findElements(elementTypes);\n highlightElements(elements, handleScroll);\n return elements;\n },\n\n unexecute: function(handleScroll=false) {\n unhighlightElements(handleScroll);\n },\n\n generateJSON: async function() {\n const json = {};\n\n // Capture viewport dimensions\n const viewportData = {\n width: window.innerWidth,\n height: window.innerHeight,\n documentWidth: document.documentElement.clientWidth,\n documentHeight: document.documentElement.clientHeight,\n timestamp: new Date().toISOString()\n };\n\n // Add viewport data to the JSON output\n json.viewport = viewportData;\n\n\n await Promise.all(Object.values(ElementTag).map(async elementType => {\n const elements = await findElements(elementType);\n json[elementType] = elements;\n }));\n\n // Serialize the JSON object\n const jsonString = JSON.stringify(json, null, 4); // Pretty print with 4 spaces\n\n console.log(`JSON: ${jsonString}`);\n return jsonString;\n },\n\n getElementInfo\n };\n\n\n function unhighlightElements(handleScroll=false) {\n const documents = getAllFrames();\n documents.forEach(doc => {\n const overlay = doc.getElementById('highlight-overlay');\n if (overlay) {\n if (handleScroll) {\n // Remove event listeners\n doc.removeEventListener('scroll', overlay.scrollHandler, true);\n doc.removeEventListener('resize', overlay.resizeHandler);\n }\n overlay.remove();\n }\n });\n }\n\n\n\n\n async function findElements(elementTypes, verbose=true) {\n const typesArray = Array.isArray(elementTypes) ? elementTypes : [elementTypes];\n console.log('Starting element search for types:', typesArray);\n\n const elements = [];\n typesArray.forEach(elementType => {\n if (elementType === ElementTag.FILLABLE) {\n elements.push(...findFillables());\n }\n if (elementType === ElementTag.SELECTABLE) {\n elements.push(...findDropdowns());\n }\n if (elementType === ElementTag.CLICKABLE) {\n elements.push(...findClickables());\n elements.push(...findToggles());\n elements.push(...findCheckables());\n }\n if (elementType === ElementTag.NON_INTERACTIVE_ELEMENT) {\n elements.push(...findNonInteractiveElements());\n }\n });\n\n // console.log('Before uniquify:', elements.length);\n const elementsWithInfo = elements.map((element, index) => \n getElementInfo(element, index)\n );\n\n \n \n const uniqueElements = uniquifyElements(elementsWithInfo);\n console.log(`Found ${uniqueElements.length} elements:`);\n \n // More comprehensive visibility check\n const visibleElements = uniqueElements.filter(elementInfo => {\n const el = elementInfo.element;\n const style = getComputedStyle(el);\n \n // Check various style properties that affect visibility\n if (style.display === 'none' || \n style.visibility === 'hidden') {\n return false;\n }\n \n // Check if element has non-zero dimensions\n const rect = el.getBoundingClientRect();\n if (rect.width === 0 || rect.height === 0) {\n return false;\n }\n \n // Check if element is within viewport\n if (rect.bottom < 0 || \n rect.top > window.innerHeight || \n rect.right < 0 || \n rect.left > window.innerWidth) {\n // Element is outside viewport, but still might be valid \n // if user scrolls to it, so we'll include it\n return true;\n }\n \n return true;\n });\n \n console.log(`Out of which ${visibleElements.length} elements are visible:`);\n if (verbose) {\n visibleElements.forEach(info => {\n console.log(`Element ${info.index}:`, info);\n });\n }\n \n return visibleElements;\n }\n\n // elements is an array of objects with index, xpath\n function highlightElements(elements, handleScroll=false) {\n // console.log('[highlightElements] called with', elements.length, 'elements');\n // Create overlay if it doesn't exist and store it in a dictionary\n const documents = getAllFrames(); \n let overlays = {};\n documents.forEach(doc => {\n let overlay = doc.getElementById('highlight-overlay');\n if (!overlay) {\n overlay = doc.createElement('div');\n overlay.id = 'highlight-overlay';\n overlay.style.cssText = `\n position: fixed;\n top: 0;\n left: 0;\n width: 100%;\n height: 100%;\n pointer-events: none;\n z-index: 2147483647;\n `;\n doc.body.appendChild(overlay);\n // console.log('[highlightElements] Created overlay in document:', doc);\n }\n overlays[doc.documentURI] = overlay;\n });\n \n\n const updateHighlights = (doc = null) => {\n if (doc) {\n overlays[doc.documentURI].innerHTML = '';\n } else {\n Object.values(overlays).forEach(overlay => { overlay.innerHTML = ''; });\n } \n elements.forEach((elementInfo, idx) => {\n //console.log(`[highlightElements] Processing element ${idx}:`, elementInfo.tag, elementInfo.css_selector, elementInfo.bounding_box);\n let element = elementInfo.element; //getElementByXPathOrCssSelector(elementInfo);\n if (!element) {\n element = getElementByXPathOrCssSelector(elementInfo);\n if (!element) {\n console.warn('[highlightElements] Could not find element for:', elementInfo);\n return;\n }\n }\n //if highlights requested for a specific doc, skip unrelated elements\n if (doc && element.ownerDocument !== doc) {\n console.log(\"[highlightElements] Skipped element since it doesn't belong to document\", doc);\n return;\n }\n const rect = element.getBoundingClientRect();\n if (rect.width === 0 || rect.height === 0) {\n console.warn('[highlightElements] Element has zero dimensions:', elementInfo);\n return;\n }\n // Create border highlight (red rectangle)\n // use ownerDocument to support iframes/frames\n const highlight = element.ownerDocument.createElement('div');\n highlight.style.cssText = `\n position: fixed;\n left: ${rect.x}px;\n top: ${rect.y}px;\n width: ${rect.width}px;\n height: ${rect.height}px;\n border: 1px solid rgb(255, 0, 0);\n transition: all 0.2s ease-in-out;\n `;\n // Create index label container - now positioned to the right and slightly up\n const labelContainer = element.ownerDocument.createElement('div');\n labelContainer.style.cssText = `\n position: absolute;\n right: -10px; /* Offset to the right */\n top: -10px; /* Offset upwards */\n padding: 4px;\n background-color: rgba(255, 255, 0, 0.6);\n display: flex;\n align-items: center;\n justify-content: center;\n `;\n const text = element.ownerDocument.createElement('span');\n text.style.cssText = `\n color: rgb(0, 0, 0, 0.8);\n font-family: 'Courier New', Courier, monospace;\n font-size: 12px;\n font-weight: bold;\n line-height: 1;\n `;\n text.textContent = elementInfo.index;\n labelContainer.appendChild(text);\n highlight.appendChild(labelContainer); \n overlays[element.ownerDocument.documentURI].appendChild(highlight);\n \n });\n };\n\n // Initial highlight\n updateHighlights();\n\n if (handleScroll) {\n documents.forEach(doc => {\n // Update highlights on scroll and resize\n console.log('registering scroll and resize handlers for document: ', doc);\n const scrollHandler = () => {\n requestAnimationFrame(() => updateHighlights(doc));\n };\n const resizeHandler = () => {\n updateHighlights(doc);\n };\n doc.addEventListener('scroll', scrollHandler, true);\n doc.addEventListener('resize', resizeHandler);\n // Store event handlers for cleanup\n overlays[doc.documentURI].scrollHandler = scrollHandler;\n overlays[doc.documentURI].resizeHandler = resizeHandler;\n }); \n }\n }\n\n // function unexecute() {\n // unhighlightElements();\n // }\n\n // Make it available globally for both Extension and Playwright\n if (typeof window !== 'undefined') {\n function stripElementRefs(elementInfo) {\n if (!elementInfo) return null;\n const { element, ...rest } = elementInfo;\n return rest;\n }\n\n window.ProboLabs = window.ProboLabs || {};\n\n // --- Caching State ---\n window.ProboLabs.candidates = [];\n window.ProboLabs.actual = null;\n window.ProboLabs.matchingCandidate = null;\n\n // --- Methods ---\n /**\n * Find and cache candidate elements of a given type (e.g., 'CLICKABLE').\n * NOTE: This function is async and must be awaited from Playwright/Node.\n */\n window.ProboLabs.findAndCacheCandidateElements = async function(elementType) {\n //console.log('[ProboLabs] findAndCacheCandidateElements called with:', elementType);\n const found = await findElements(elementType);\n window.ProboLabs.candidates = found;\n // console.log('[ProboLabs] candidates set to:', found, 'type:', typeof found, 'isArray:', Array.isArray(found));\n return found.length;\n };\n\n window.ProboLabs.findAndCacheActualElement = function(cssSelector, iframeSelector, isHover=false) {\n // console.log('[ProboLabs] findAndCacheActualElement called with:', cssSelector, iframeSelector);\n let el = findElement(document, iframeSelector, cssSelector);\n if(isHover) {\n const visibleElement = findClosestVisibleElement(el);\n if (visibleElement) {\n el = visibleElement;\n }\n }\n if (!el) {\n window.ProboLabs.actual = null;\n // console.log('[ProboLabs] actual set to null');\n return false;\n }\n window.ProboLabs.actual = getElementInfo(el, -1);\n // console.log('[ProboLabs] actual set to:', window.ProboLabs.actual);\n return true;\n };\n\n window.ProboLabs.findAndCacheMatchingCandidate = function() {\n // console.log('[ProboLabs] findAndCacheMatchingCandidate called');\n if (!window.ProboLabs.candidates.length || !window.ProboLabs.actual) {\n window.ProboLabs.matchingCandidate = null;\n // console.log('[ProboLabs] matchingCandidate set to null');\n return false;\n }\n window.ProboLabs.matchingCandidate = findMatchingCandidateElementInfo(window.ProboLabs.candidates, window.ProboLabs.actual);\n // console.log('[ProboLabs] matchingCandidate set to:', window.ProboLabs.matchingCandidate);\n return !!window.ProboLabs.matchingCandidate;\n };\n\n window.ProboLabs.highlightCachedElements = function(which) {\n let elements = [];\n if (which === 'candidates') elements = window.ProboLabs.candidates;\n if (which === 'actual' && window.ProboLabs.actual) elements = [window.ProboLabs.actual];\n if (which === 'matching' && window.ProboLabs.matchingCandidate) elements = [window.ProboLabs.matchingCandidate];\n console.log(`[ProboLabs] highlightCachedElements ${which} with ${elements.length} elements`);\n highlightElements(elements);\n };\n\n window.ProboLabs.unhighlight = function() {\n // console.log('[ProboLabs] unhighlight called');\n unhighlightElements();\n };\n\n window.ProboLabs.reset = function() {\n console.log('[ProboLabs] reset called');\n window.ProboLabs.candidates = [];\n window.ProboLabs.actual = null;\n window.ProboLabs.matchingCandidate = null;\n unhighlightElements();\n };\n\n window.ProboLabs.getCandidates = function() {\n // console.log('[ProboLabs] getCandidates called. candidates:', window.ProboLabs.candidates, 'type:', typeof window.ProboLabs.candidates, 'isArray:', Array.isArray(window.ProboLabs.candidates));\n const arr = Array.isArray(window.ProboLabs.candidates) ? window.ProboLabs.candidates : [];\n return arr.map(stripElementRefs);\n };\n window.ProboLabs.getActual = function() {\n return stripElementRefs(window.ProboLabs.actual);\n };\n window.ProboLabs.getMatchingCandidate = function() {\n return stripElementRefs(window.ProboLabs.matchingCandidate);\n };\n\n // Retain existing API for backward compatibility\n window.ProboLabs.ElementTag = ElementTag;\n window.ProboLabs.highlightElements = highlightElements;\n window.ProboLabs.unhighlightElements = unhighlightElements;\n window.ProboLabs.findElements = findElements;\n window.ProboLabs.getElementInfo = getElementInfo;\n window.ProboLabs.highlight = window.ProboLabs.highlight;\n window.ProboLabs.unhighlight = window.ProboLabs.unhighlight;\n\n // --- Utility Functions ---\n function findClosestVisibleElement(element) {\n let current = element;\n while (current) {\n const style = window.getComputedStyle(current);\n if (\n style &&\n style.display !== 'none' &&\n style.visibility !== 'hidden' &&\n current.offsetWidth > 0 &&\n current.offsetHeight > 0\n ) {\n return current;\n }\n if (!current.parentElement || current === document.body) break;\n current = current.parentElement;\n }\n return null;\n }\n }\n\n exports.ElementInfo = ElementInfo;\n exports.ElementTag = ElementTag;\n exports.deserializeNodeFromJSON = deserializeNodeFromJSON;\n exports.detectScrollableContainers = detectScrollableContainers;\n exports.findElement = findElement;\n exports.findElements = findElements;\n exports.generateCssPath = generateCssPath;\n exports.getAriaLabelledByText = getAriaLabelledByText;\n exports.getContainingIframe = getContainingIframe;\n exports.getElementInfo = getElementInfo;\n exports.getParentNode = getParentNode;\n exports.getRobustSelector = getRobustSelector;\n exports.highlight = highlight;\n exports.highlightElements = highlightElements;\n exports.isScrollableContainer = isScrollableContainer;\n exports.serializeNodeToJSON = serializeNodeToJSON;\n exports.unhighlightElements = unhighlightElements;\n\n}));\n//# sourceMappingURL=probolabs.umd.js.map\n";
2
+ /**
3
+ * Element tag constants for different types of interactive elements
4
+ */
5
+ const ElementTag = {
6
+ CLICKABLE: "CLICKABLE",
7
+ FILLABLE: "FILLABLE",
8
+ SELECTABLE: "SELECTABLE",
9
+ NON_INTERACTIVE_ELEMENT: 'NON_INTERACTIVE_ELEMENT',
10
+ };
11
+
2
12
  var ApplyAIStatus;
3
13
  (function (ApplyAIStatus) {
4
14
  ApplyAIStatus["PREPARE_START"] = "PREPARE_START";
@@ -47,6 +57,35 @@ var PlaywrightAction;
47
57
  PlaywrightAction["WAIT_FOR_OTP"] = "WAIT_FOR_OTP";
48
58
  })(PlaywrightAction || (PlaywrightAction = {}));
49
59
 
60
+ /**
61
+ * Resolves an action type to the corresponding ElementTag
62
+ *
63
+ * @param action The action type (CLICK, FILL_IN, SELECT_DROPDOWN)
64
+ * @returns The corresponding ElementTag
65
+ */
66
+ function resolveElementTag(action) {
67
+ switch (action) {
68
+ case PlaywrightAction.CLICK:
69
+ return [ElementTag.CLICKABLE, ElementTag.FILLABLE];
70
+ case PlaywrightAction.FILL_IN:
71
+ return [ElementTag.FILLABLE];
72
+ case PlaywrightAction.SELECT_DROPDOWN:
73
+ return [ElementTag.SELECTABLE];
74
+ case PlaywrightAction.HOVER:
75
+ return [ElementTag.CLICKABLE, ElementTag.FILLABLE, ElementTag.NON_INTERACTIVE_ELEMENT];
76
+ case PlaywrightAction.ASSERT_EXACT_VALUE:
77
+ case PlaywrightAction.ASSERT_CONTAINS_VALUE:
78
+ case PlaywrightAction.EXTRACT_VALUE:
79
+ case PlaywrightAction.WAIT_FOR:
80
+ return [ElementTag.CLICKABLE, ElementTag.FILLABLE, ElementTag.NON_INTERACTIVE_ELEMENT];
81
+ case PlaywrightAction.WAIT_FOR_OTP:
82
+ return [ElementTag.FILLABLE];
83
+ default:
84
+ console.error(`Unknown action: ${action}`);
85
+ throw new Error(`Unknown action: ${action}`);
86
+ }
87
+ }
88
+
50
89
  // WebSocketsMessageType enum for WebSocket and event message types shared across the app
51
90
  var WebSocketsMessageType;
52
91
  (function (WebSocketsMessageType) {
@@ -557,10 +596,11 @@ const errorMessages = new Set([
557
596
  'Failed to fetch', // Chrome
558
597
  'NetworkError when attempting to fetch resource.', // Firefox
559
598
  'The Internet connection appears to be offline.', // Safari 16
560
- 'Load failed', // Safari 17+
561
599
  'Network request failed', // `cross-fetch`
562
600
  'fetch failed', // Undici (Node.js)
563
601
  'terminated', // Undici (Node.js)
602
+ ' A network error occurred.', // Bun (WebKit)
603
+ 'Network connection lost', // Cloudflare Workers (fetch)
564
604
  ]);
565
605
 
566
606
  function isNetworkError(error) {
@@ -573,13 +613,22 @@ function isNetworkError(error) {
573
613
  return false;
574
614
  }
575
615
 
576
- // We do an extra check for Safari 17+ as it has a very generic error message.
577
- // Network errors in Safari have no stack.
578
- if (error.message === 'Load failed') {
579
- return error.stack === undefined;
616
+ const {message, stack} = error;
617
+
618
+ // Safari 17+ has generic message but no stack for network errors
619
+ if (message === 'Load failed') {
620
+ return stack === undefined
621
+ // Sentry adds its own stack trace to the fetch error, so also check for that
622
+ || '__sentry_captured__' in error;
623
+ }
624
+
625
+ // Deno network errors start with specific text
626
+ if (message.startsWith('error sending request for url')) {
627
+ return true;
580
628
  }
581
629
 
582
- return errorMessages.has(error.message);
630
+ // Standard network error messages
631
+ return errorMessages.has(message);
583
632
  }
584
633
 
585
634
  class AbortError extends Error {
@@ -608,7 +657,7 @@ const decorateErrorWithCounts = (error, attemptNumber, options) => {
608
657
  return error;
609
658
  };
610
659
 
611
- async function pRetry$2(input, options) {
660
+ async function pRetry(input, options) {
612
661
  return new Promise((resolve, reject) => {
613
662
  options = {...options};
614
663
  options.onFailedAttempt ??= () => {};
@@ -693,6 +742,68 @@ class ApiClient {
693
742
  this.maxRetries = maxRetries;
694
743
  this.initialBackoff = initialBackoff;
695
744
  }
745
+ /**
746
+ * Determines if an error should be retried.
747
+ * Only retries on timeout and network errors, not on client/server errors.
748
+ */
749
+ isRetryableError(error) {
750
+ var _a, _b, _c, _d, _e;
751
+ // Network/connection errors should be retried
752
+ if ((_a = error.message) === null || _a === void 0 ? void 0 : _a.includes('fetch failed'))
753
+ return true;
754
+ if ((_b = error.message) === null || _b === void 0 ? void 0 : _b.includes('network'))
755
+ return true;
756
+ if ((_c = error.message) === null || _c === void 0 ? void 0 : _c.includes('ETIMEDOUT'))
757
+ return true;
758
+ if ((_d = error.message) === null || _d === void 0 ? void 0 : _d.includes('ECONNRESET'))
759
+ return true;
760
+ if ((_e = error.message) === null || _e === void 0 ? void 0 : _e.includes('ECONNREFUSED'))
761
+ return true;
762
+ if (error.code === 'ETIMEDOUT')
763
+ return true;
764
+ if (error.code === 'ECONNRESET')
765
+ return true;
766
+ if (error.code === 'ECONNREFUSED')
767
+ return true;
768
+ // If it's an ApiError, check the status code
769
+ if (error instanceof ApiError) {
770
+ // Retry on timeout-related status codes
771
+ if (error.status === 408)
772
+ return true; // Request Timeout
773
+ if (error.status === 502)
774
+ return true; // Bad Gateway (temporary)
775
+ if (error.status === 503)
776
+ return true; // Service Unavailable (temporary)
777
+ if (error.status === 504)
778
+ return true; // Gateway Timeout
779
+ if (error.status === 0)
780
+ return true; // Network error
781
+ // Don't retry on client errors (4xx) or server errors (5xx)
782
+ // These indicate problems that won't be fixed by retrying
783
+ return false;
784
+ }
785
+ // For unknown errors, don't retry to avoid masking issues
786
+ return false;
787
+ }
788
+ /**
789
+ * Generic helper to wrap API requests with retry logic and consistent error handling.
790
+ */
791
+ async requestWithRetry(operationName, operation) {
792
+ return pRetry(operation, {
793
+ retries: this.maxRetries,
794
+ minTimeout: this.initialBackoff,
795
+ shouldRetry: (error) => {
796
+ const shouldRetry = this.isRetryableError(error);
797
+ if (!shouldRetry) {
798
+ apiLogger.error(`${operationName} failed with non-retryable error: ${error.message || error}`);
799
+ }
800
+ return shouldRetry;
801
+ },
802
+ onFailedAttempt: error => {
803
+ apiLogger.warn(`${operationName} failed (retryable), attempt ${error.attemptNumber} of ${error.retriesLeft + error.attemptNumber}. Error: ${error.message}`);
804
+ }
805
+ });
806
+ }
696
807
  async handleResponse(response) {
697
808
  var _a;
698
809
  try {
@@ -735,7 +846,7 @@ class ApiClient {
735
846
  }
736
847
  async createStep(options) {
737
848
  apiLogger.debug('creating step ', options.stepPrompt);
738
- return pRetry$2(async () => {
849
+ return this.requestWithRetry('createStep', async () => {
739
850
  const response = await fetch(`${this.apiUrl}/step-runners/`, {
740
851
  method: 'POST',
741
852
  headers: this.getHeaders(),
@@ -743,44 +854,38 @@ class ApiClient {
743
854
  step_id: options.stepIdFromServer,
744
855
  scenario_name: options.scenarioName,
745
856
  step_prompt: options.stepPrompt,
857
+ argument: options.argument,
746
858
  initial_screenshot: options.initial_screenshot_url,
747
859
  initial_html_content: options.initial_html_content,
748
860
  use_cache: options.use_cache,
749
- url: options.url
861
+ url: options.url,
862
+ action: options.action,
863
+ vanilla_prompt: options.vanilla_prompt,
864
+ is_vanilla_prompt_robust: options.is_vanilla_prompt_robust,
865
+ target_element_name: options.target_element_name,
866
+ position: options.position
750
867
  }),
751
868
  });
752
869
  const data = await this.handleResponse(response);
753
870
  return data.step.id;
754
- }, {
755
- retries: this.maxRetries,
756
- minTimeout: this.initialBackoff,
757
- onFailedAttempt: error => {
758
- apiLogger.warn(`API call failed, attempt ${error.attemptNumber} of ${error.retriesLeft + error.attemptNumber}...`);
759
- }
760
871
  });
761
872
  }
762
873
  async patchStep(stepId, fields) {
763
- // Use PATCH /steps/:id/ endpoint
764
- apiLogger.debug(`patching step #${stepId}`);
765
- return pRetry$2(async () => {
874
+ // Use PATCH /steps/:id/ endpoint with partial=true
875
+ apiLogger.debug(`patching step #${stepId} with fields:`, Object.keys(fields));
876
+ return this.requestWithRetry('patchStep', async () => {
766
877
  const response = await fetch(`${this.apiUrl}/steps/${stepId}/`, {
767
878
  method: 'PATCH',
768
879
  headers: this.getHeaders(),
769
880
  body: JSON.stringify(fields)
770
881
  });
771
- const data = await this.handleResponse(response);
882
+ await this.handleResponse(response);
772
883
  return;
773
- }, {
774
- retries: this.maxRetries,
775
- minTimeout: this.initialBackoff,
776
- onFailedAttempt: error => {
777
- apiLogger.warn(`API call failed, attempt ${error.attemptNumber} of ${error.retriesLeft + error.attemptNumber}...`);
778
- }
779
884
  });
780
885
  }
781
886
  async resolveNextInstruction(stepId, instruction, aiModel) {
782
887
  apiLogger.debug(`resolving next instruction: ${instruction}`);
783
- return pRetry$2(async () => {
888
+ return this.requestWithRetry('resolveNextInstruction', async () => {
784
889
  apiLogger.debug(`API client: Resolving next instruction for step ${stepId}`);
785
890
  const cleanInstruction = cleanupInstructionElements(instruction);
786
891
  const response = await fetch(`${this.apiUrl}/step-runners/${stepId}/run/`, {
@@ -793,16 +898,10 @@ class ApiClient {
793
898
  });
794
899
  const data = await this.handleResponse(response);
795
900
  return data.instruction;
796
- }, {
797
- retries: this.maxRetries,
798
- minTimeout: this.initialBackoff,
799
- onFailedAttempt: error => {
800
- apiLogger.warn(`API call failed, attempt ${error.attemptNumber} of ${error.retriesLeft + error.attemptNumber}...`);
801
- }
802
901
  });
803
902
  }
804
903
  async uploadScreenshot(screenshot_bytes) {
805
- return pRetry$2(async () => {
904
+ return this.requestWithRetry('uploadScreenshot', async () => {
806
905
  const response = await fetch(`${this.apiUrl}/upload-screenshots/`, {
807
906
  method: 'POST',
808
907
  headers: {
@@ -812,17 +911,22 @@ class ApiClient {
812
911
  });
813
912
  const data = await this.handleResponse(response);
814
913
  return data.screenshot_url;
815
- }, {
816
- retries: this.maxRetries,
817
- minTimeout: this.initialBackoff,
818
- onFailedAttempt: error => {
819
- apiLogger.warn(`API call failed, attempt ${error.attemptNumber} of ${error.retriesLeft + error.attemptNumber}...`);
820
- }
914
+ });
915
+ }
916
+ async findStepById(stepId) {
917
+ apiLogger.debug(`Finding step by id: ${stepId}`);
918
+ return this.requestWithRetry('findStepById', async () => {
919
+ const response = await fetch(`${this.apiUrl}/steps/${stepId}/`, {
920
+ method: 'GET',
921
+ headers: this.getHeaders(),
922
+ });
923
+ const data = await this.handleResponse(response);
924
+ return data.step;
821
925
  });
822
926
  }
823
927
  async findStepByPrompt(prompt, scenarioName, url = '') {
824
928
  apiLogger.debug(`Finding step by prompt: ${prompt} and scenario: ${scenarioName}`);
825
- return pRetry$2(async () => {
929
+ return this.requestWithRetry('findStepByPrompt', async () => {
826
930
  const response = await fetch(`${this.apiUrl}/step-runners/find-step-by-prompt/`, {
827
931
  method: 'POST',
828
932
  headers: this.getHeaders(),
@@ -847,16 +951,10 @@ class ApiClient {
847
951
  // For any other error, rethrow
848
952
  throw error;
849
953
  }
850
- }, {
851
- retries: this.maxRetries,
852
- minTimeout: this.initialBackoff,
853
- onFailedAttempt: error => {
854
- apiLogger.warn(`API call failed, attempt ${error.attemptNumber} of ${error.retriesLeft + error.attemptNumber}...`);
855
- }
856
954
  });
857
955
  }
858
956
  async resetStep(stepId) {
859
- return pRetry$2(async () => {
957
+ return this.requestWithRetry('resetStep', async () => {
860
958
  const response = await fetch(`${this.apiUrl}/steps/${stepId}/reset/`, {
861
959
  method: 'POST',
862
960
  headers: this.getHeaders(),
@@ -867,8 +965,9 @@ class ApiClient {
867
965
  }
868
966
  async interactionToStep(scenarioName, interaction, position = -1) {
869
967
  // Use POST /interaction-to-step/ endpoint
968
+ // Backend will create new step or update existing one based on interaction_id
870
969
  apiLogger.debug(`converting interaction #${interaction.interactionId} to step`);
871
- return pRetry$2(async () => {
970
+ return this.requestWithRetry('interactionToStep', async () => {
872
971
  var _a, _b;
873
972
  const response = await fetch(`${this.apiUrl}/interaction-to-step/`, {
874
973
  method: 'POST',
@@ -891,18 +990,12 @@ class ApiClient {
891
990
  });
892
991
  const data = await this.handleResponse(response);
893
992
  return [data.result.step_id, data.result.matched_step, data.scenario_id];
894
- }, {
895
- retries: this.maxRetries,
896
- minTimeout: this.initialBackoff,
897
- onFailedAttempt: error => {
898
- apiLogger.warn(`API call failed, attempt ${error.attemptNumber} of ${error.retriesLeft + error.attemptNumber}...`);
899
- }
900
993
  });
901
994
  }
902
995
  async actionToPrompt(action2promptInput, aiModel) {
903
996
  // Use POST /action-to-prompt/ endpoint
904
997
  apiLogger.debug(`running action2prompt for step #${action2promptInput.step_id}`);
905
- return pRetry$2(async () => {
998
+ return this.requestWithRetry('actionToPrompt', async () => {
906
999
  const response = await fetch(`${this.apiUrl}/steps/${action2promptInput.step_id}/action_to_prompt/`, {
907
1000
  method: 'POST',
908
1001
  headers: this.getHeaders(),
@@ -910,30 +1003,23 @@ class ApiClient {
910
1003
  });
911
1004
  const data = await this.handleResponse(response);
912
1005
  return data;
913
- }, {
914
- retries: this.maxRetries,
915
- minTimeout: this.initialBackoff,
916
- onFailedAttempt: error => {
917
- apiLogger.warn(`API call failed, attempt ${error.attemptNumber} of ${error.retriesLeft + error.attemptNumber}...`);
918
- }
919
1006
  });
920
1007
  }
921
1008
  async summarizeScenario(scenarioId, aiModel) {
922
- const response = await fetch(`${this.apiUrl}/api/scenarios/${scenarioId}/summary`, {
923
- method: 'POST',
924
- headers: this.getHeaders(),
925
- body: JSON.stringify({ 'model': aiModel })
1009
+ return this.requestWithRetry('summarizeScenario', async () => {
1010
+ const response = await fetch(`${this.apiUrl}/api/scenarios/${scenarioId}/summary`, {
1011
+ method: 'POST',
1012
+ headers: this.getHeaders(),
1013
+ body: JSON.stringify({ 'model': aiModel })
1014
+ });
1015
+ const data = await this.handleResponse(response);
1016
+ return data;
926
1017
  });
927
- const data = await response.json();
928
- if (!response.ok) {
929
- throw new Error(data.error || 'Failed to summarize scenario');
930
- }
931
- return data;
932
1018
  }
933
1019
  async askAI(question, scenarioName, screenshot, aiModel) {
934
1020
  apiLogger.debug(`Asking AI question: "${question}", scenarioName: ${scenarioName}`);
935
1021
  apiLogger.debug(`headers: ${JSON.stringify(this.getHeaders())}`);
936
- return pRetry$2(async () => {
1022
+ return this.requestWithRetry('askAI', async () => {
937
1023
  const response = await fetch(`${this.apiUrl}/api/ask-ai/`, {
938
1024
  method: 'POST',
939
1025
  headers: this.getHeaders(),
@@ -946,12 +1032,37 @@ class ApiClient {
946
1032
  });
947
1033
  const data = await this.handleResponse(response);
948
1034
  return data;
949
- }, {
950
- retries: this.maxRetries,
951
- minTimeout: this.initialBackoff,
952
- onFailedAttempt: error => {
953
- apiLogger.warn(`API call failed, attempt ${error.attemptNumber} of ${error.retriesLeft + error.attemptNumber}...`);
954
- }
1035
+ });
1036
+ }
1037
+ async screenshotReasoning(stepId, prompt, aiModel) {
1038
+ apiLogger.debug(`Performing screenshot reasoning for step: ${stepId}`);
1039
+ return this.requestWithRetry('screenshotReasoning', async () => {
1040
+ const response = await fetch(`${this.apiUrl}/steps/${stepId}/screenshot_reasoning/`, {
1041
+ method: 'POST',
1042
+ headers: this.getHeaders(),
1043
+ body: JSON.stringify({
1044
+ prompt: prompt,
1045
+ model: aiModel
1046
+ }),
1047
+ });
1048
+ const data = await this.handleResponse(response);
1049
+ return data.action;
1050
+ });
1051
+ }
1052
+ async findBestCandidateElement(stepId, candidatesScreenshotUrl, candidateElements, aiModel) {
1053
+ apiLogger.debug(`Finding best candidate element for step: ${stepId}`);
1054
+ return this.requestWithRetry('findBestCandidateElement', async () => {
1055
+ const response = await fetch(`${this.apiUrl}/steps/${stepId}/find_best_candidate_element/`, {
1056
+ method: 'POST',
1057
+ headers: this.getHeaders(),
1058
+ body: JSON.stringify({
1059
+ screenshot_url: candidatesScreenshotUrl,
1060
+ candidate_elements: candidateElements,
1061
+ model: aiModel
1062
+ }),
1063
+ });
1064
+ const data = await this.handleResponse(response);
1065
+ return data.index;
955
1066
  });
956
1067
  }
957
1068
  } /* ApiClient */
@@ -992,29 +1103,6 @@ const DEFAULT_PLAYWRIGHT_TIMEOUT_CONFIG = {
992
1103
  waitForStabilityVerbose: false,
993
1104
  };
994
1105
 
995
- ({
996
- // API Configuration
997
- apiKey: '',
998
- apiEndPoint: 'https://api.probolabs.ai',
999
- baseUrl: undefined,
1000
- // Scenario Configuration
1001
- scenarioName: 'new recording',
1002
- scenarioId: undefined,
1003
- aiModel: 'azure-gpt4-mini',
1004
- activeParamSet: 0,
1005
- // Browser Configuration
1006
- resetBrowserBeforeReplay: true,
1007
- // Script Configuration
1008
- scriptTimeout: 30000,
1009
- // UI Configuration
1010
- hover_enabled: false,
1011
- // Logging Configuration
1012
- enableConsoleLogs: false,
1013
- debugLevel: 'INFO',
1014
- // Timeout Configuration (spread from PlaywrightTimeoutConfig)
1015
- ...DEFAULT_PLAYWRIGHT_TIMEOUT_CONFIG,
1016
- });
1017
-
1018
1106
  // Default logger instance
1019
1107
  const proboLogger = new ProboLogger('probolib');
1020
1108
 
@@ -1106,9 +1194,9 @@ class Highlighter {
1106
1194
  proboLogger.debug('Injecting highlighter script...');
1107
1195
  await page.evaluate(highlighterCode);
1108
1196
  // Verify the script was injected correctly
1109
- const verified = await page.evaluate(`
1110
- //console.log('ProboLabs global:', window.ProboLabs);
1111
- typeof window.ProboLabs?.highlight?.execute === 'function'
1197
+ const verified = await page.evaluate(`
1198
+ //console.log('ProboLabs global:', window.ProboLabs);
1199
+ typeof window.ProboLabs?.highlight?.execute === 'function'
1112
1200
  `);
1113
1201
  proboLogger.debug('Script injection verified:', verified);
1114
1202
  }
@@ -1269,103 +1357,6 @@ class Highlighter {
1269
1357
  }
1270
1358
  }
1271
1359
 
1272
- var pRetry$1 = {exports: {}};
1273
-
1274
- var hasRequiredPRetry;
1275
-
1276
- function requirePRetry () {
1277
- if (hasRequiredPRetry) return pRetry$1.exports;
1278
- hasRequiredPRetry = 1;
1279
- const retry = requireRetry();
1280
-
1281
- const networkErrorMsgs = [
1282
- 'Failed to fetch', // Chrome
1283
- 'NetworkError when attempting to fetch resource.', // Firefox
1284
- 'The Internet connection appears to be offline.', // Safari
1285
- 'Network request failed' // `cross-fetch`
1286
- ];
1287
-
1288
- class AbortError extends Error {
1289
- constructor(message) {
1290
- super();
1291
-
1292
- if (message instanceof Error) {
1293
- this.originalError = message;
1294
- ({message} = message);
1295
- } else {
1296
- this.originalError = new Error(message);
1297
- this.originalError.stack = this.stack;
1298
- }
1299
-
1300
- this.name = 'AbortError';
1301
- this.message = message;
1302
- }
1303
- }
1304
-
1305
- const decorateErrorWithCounts = (error, attemptNumber, options) => {
1306
- // Minus 1 from attemptNumber because the first attempt does not count as a retry
1307
- const retriesLeft = options.retries - (attemptNumber - 1);
1308
-
1309
- error.attemptNumber = attemptNumber;
1310
- error.retriesLeft = retriesLeft;
1311
- return error;
1312
- };
1313
-
1314
- const isNetworkError = errorMessage => networkErrorMsgs.includes(errorMessage);
1315
-
1316
- const pRetry = (input, options) => new Promise((resolve, reject) => {
1317
- options = {
1318
- onFailedAttempt: () => {},
1319
- retries: 10,
1320
- ...options
1321
- };
1322
-
1323
- const operation = retry.operation(options);
1324
-
1325
- operation.attempt(async attemptNumber => {
1326
- try {
1327
- resolve(await input(attemptNumber));
1328
- } catch (error) {
1329
- if (!(error instanceof Error)) {
1330
- reject(new TypeError(`Non-error was thrown: "${error}". You should only throw errors.`));
1331
- return;
1332
- }
1333
-
1334
- if (error instanceof AbortError) {
1335
- operation.stop();
1336
- reject(error.originalError);
1337
- } else if (error instanceof TypeError && !isNetworkError(error.message)) {
1338
- operation.stop();
1339
- reject(error);
1340
- } else {
1341
- decorateErrorWithCounts(error, attemptNumber, options);
1342
-
1343
- try {
1344
- await options.onFailedAttempt(error);
1345
- } catch (error) {
1346
- reject(error);
1347
- return;
1348
- }
1349
-
1350
- if (!operation.retry(error)) {
1351
- reject(operation.mainError());
1352
- }
1353
- }
1354
- }
1355
- });
1356
- });
1357
-
1358
- pRetry$1.exports = pRetry;
1359
- // TODO: remove this in the next major version
1360
- pRetry$1.exports.default = pRetry;
1361
-
1362
- pRetry$1.exports.AbortError = AbortError;
1363
- return pRetry$1.exports;
1364
- }
1365
-
1366
- var pRetryExports = requirePRetry();
1367
- var pRetry = /*@__PURE__*/getDefaultExportFromCjs(pRetryExports);
1368
-
1369
1360
  const MAILINATOR_API_KEY = '5bfef31518e84dfbb861b36f59259695';
1370
1361
  const MAILINATOR_DOMAIN = 'probolabs.testinator.com';
1371
1362
  /**
@@ -1560,7 +1551,8 @@ class OTP {
1560
1551
  throw new Error(`Mailinator API error: ${response.status} ${response.statusText}`);
1561
1552
  }
1562
1553
  const data = await response.json();
1563
- return data.msgs || data.messages || data || [];
1554
+ const messages = data.msgs || data.messages || data || [];
1555
+ return messages;
1564
1556
  }
1565
1557
  catch (error) {
1566
1558
  console.error('Error fetching messages from all inboxes:', error);
@@ -1569,29 +1561,29 @@ class OTP {
1569
1561
  }
1570
1562
  /**
1571
1563
  * Waits for an OTP to arrive in the inbox and extracts it
1572
- * @param inbox - The inbox name to monitor (optional - if not provided, searches all inboxes)
1573
- * @param timeout - Maximum time to wait in milliseconds (default: 30000)
1574
- * @param checkInterval - How often to check in milliseconds (default: 1000)
1575
- * @param checkRecentMessagesSinceMs - When > 0, check messages from the last X milliseconds and return the most recent OTP (default: 0)
1564
+ * @param options - Configuration options for waiting for OTP
1565
+ * @param options.inbox - The inbox name to monitor (optional - if not provided, searches all inboxes)
1566
+ * @param options.timeout - Maximum time to wait in milliseconds (default: 30000)
1567
+ * @param options.checkInterval - How often to check in milliseconds (default: 1000)
1568
+ * @param options.checkRecentMessagesSinceMs - When > 0, check messages from the last X milliseconds and return the most recent OTP (default: 0)
1576
1569
  * @returns Promise<string | null> - The extracted OTP code or null if timeout/no OTP found
1577
1570
  */
1578
- static async waitForOTP(inbox, timeout = 30000, checkInterval = 1000, checkRecentMessagesSinceMs = 0) {
1571
+ static async waitForOTP(options = {}) {
1572
+ const { inbox, timeout = 30000, checkInterval = 1000, checkRecentMessagesSinceMs = 0 } = options;
1573
+ console.log(`[waitForOTP] inbox: ${inbox || 'all'}, timeout: ${timeout}ms, checkInterval: ${checkInterval}ms, checkRecentSince: ${checkRecentMessagesSinceMs}ms`);
1579
1574
  const startTime = Date.now();
1580
1575
  // If checkRecentMessagesSinceMs > 0, check for recent messages first
1581
1576
  if (checkRecentMessagesSinceMs > 0) {
1582
- console.log(`Checking for OTP in messages from the last ${checkRecentMessagesSinceMs}ms...`);
1583
1577
  const recentMessagesCutoff = Date.now() - checkRecentMessagesSinceMs;
1584
1578
  const recentMessages = inbox
1585
1579
  ? await OTP.fetchLastMessages(inbox)
1586
1580
  : await OTP.fetchAllInboxMessages();
1587
- // Filter messages from the specified time window
1588
1581
  const messagesFromWindow = recentMessages.filter(msg => msg.time >= recentMessagesCutoff);
1589
- // Sort by time (most recent first) and check for OTP
1590
1582
  const sortedMessages = messagesFromWindow.sort((a, b) => b.time - a.time);
1591
1583
  for (const message of sortedMessages) {
1592
1584
  let otp = OTP.extractOTPFromMessage(message);
1593
1585
  if (otp) {
1594
- console.log(`Found OTP in recent message: ${otp}`);
1586
+ console.log(`✅ [waitForOTP] Found OTP: ${otp}`);
1595
1587
  return otp;
1596
1588
  }
1597
1589
  // If no OTP found in summary, fetch full message details
@@ -1599,15 +1591,20 @@ class OTP {
1599
1591
  const fullMessage = await OTP.fetchMessage(message.id);
1600
1592
  otp = OTP.extractOTPFromMessage(fullMessage);
1601
1593
  if (otp) {
1602
- console.log(`Found OTP in recent full message: ${otp}`);
1594
+ console.log(`✅ [waitForOTP] Found OTP: ${otp}`);
1603
1595
  return otp;
1604
1596
  }
1605
1597
  }
1606
1598
  catch (error) {
1607
- console.warn(`Error fetching full message ${message.id}:`, error);
1599
+ // Silently continue to next message
1608
1600
  }
1609
1601
  }
1610
- console.log(`No OTP found in recent messages, starting to monitor for new messages...`);
1602
+ if (messagesFromWindow.length === 0) {
1603
+ console.log(`❌ [waitForOTP] No messages found in the last ${checkRecentMessagesSinceMs}ms`);
1604
+ }
1605
+ else {
1606
+ console.log(`❌ [waitForOTP] Checked ${messagesFromWindow.length} recent message(s) but none contained OTP`);
1607
+ }
1611
1608
  }
1612
1609
  // Get initial messages for monitoring new ones (from specific inbox or all inboxes)
1613
1610
  const initialMessages = inbox
@@ -1628,795 +1625,826 @@ class OTP {
1628
1625
  // First try to extract OTP from the message summary (faster)
1629
1626
  let otp = OTP.extractOTPFromMessage(newMessage);
1630
1627
  if (otp) {
1628
+ console.log(`✅ [waitForOTP] Found OTP: ${otp}`);
1631
1629
  return otp;
1632
1630
  }
1633
1631
  // If no OTP found in summary, fetch full message details
1634
1632
  const fullMessage = await OTP.fetchMessage(newMessage.id);
1635
1633
  otp = OTP.extractOTPFromMessage(fullMessage);
1636
1634
  if (otp) {
1635
+ console.log(`✅ [waitForOTP] Found OTP: ${otp}`);
1637
1636
  return otp;
1638
1637
  }
1639
1638
  }
1640
1639
  }
1641
1640
  catch (error) {
1642
- console.warn('Error checking for new messages:', error);
1641
+ // Silently continue polling
1643
1642
  }
1644
1643
  }
1644
+ console.log(`❌ [waitForOTP] Timeout reached (${timeout}ms) - no OTP found`);
1645
1645
  return null; // Timeout reached or no OTP found
1646
1646
  }
1647
1647
  }
1648
1648
 
1649
- class ProboPlaywright {
1650
- constructor(config = DEFAULT_PLAYWRIGHT_TIMEOUT_CONFIG, page = null) {
1651
- this.page = null;
1652
- // Merge provided config with defaults to ensure all properties are defined
1653
- this.config = {
1654
- ...DEFAULT_PLAYWRIGHT_TIMEOUT_CONFIG,
1655
- ...config
1656
- };
1657
- this.setPage(page);
1658
- }
1649
+ /**
1650
+ * Global navigation tracker that monitors page navigation events and network activity
1651
+ * using CDP (Chrome DevTools Protocol) for comprehensive network monitoring
1652
+ *
1653
+ * This is a singleton class - only one instance can exist at a time
1654
+ */
1655
+ class NavTracker {
1659
1656
  /**
1660
- * Sets the Playwright page instance for this ProboPlaywright instance.
1661
- * Also applies the configured default navigation and action timeouts to the page.
1662
- *
1663
- * @param page - The Playwright Page instance to use, or null to unset.
1657
+ * Private constructor - use getInstance() to get the singleton instance
1664
1658
  */
1665
- setPage(page) {
1659
+ constructor(page, options = {}) {
1660
+ var _a, _b, _c, _d, _f, _g, _h;
1661
+ this.navigationCount = 0;
1662
+ this.lastNavTime = null;
1663
+ this.isListening = false;
1664
+ this.client = null;
1665
+ this.inflight = new Set();
1666
+ this.lastHardNavAt = 0;
1667
+ this.lastSoftNavAt = 0;
1668
+ this.lastNetworkActivityAt = 0;
1669
+ this.totalRequestsTracked = 0;
1670
+ // Define relevant resource types and content types
1671
+ this.RELEVANT_RESOURCE_TYPES = [
1672
+ 'Document',
1673
+ 'Stylesheet',
1674
+ 'Image',
1675
+ 'Font',
1676
+ 'Script',
1677
+ 'XHR',
1678
+ 'Fetch'
1679
+ ];
1680
+ this.RELEVANT_CONTENT_TYPES = [
1681
+ 'text/html',
1682
+ 'text/css',
1683
+ 'application/javascript',
1684
+ 'image/',
1685
+ 'font/',
1686
+ 'application/json',
1687
+ ];
1688
+ // Additional patterns to filter out
1689
+ this.IGNORED_URL_PATTERNS = [
1690
+ // Analytics and tracking
1691
+ 'analytics',
1692
+ 'tracking',
1693
+ 'telemetry',
1694
+ 'beacon',
1695
+ 'metrics',
1696
+ // Ad-related
1697
+ 'doubleclick',
1698
+ 'adsystem',
1699
+ 'adserver',
1700
+ 'advertising',
1701
+ // Social media widgets
1702
+ 'facebook.com/plugins',
1703
+ 'platform.twitter',
1704
+ 'linkedin.com/embed',
1705
+ // Live chat and support
1706
+ 'livechat',
1707
+ 'zendesk',
1708
+ 'intercom',
1709
+ 'crisp.chat',
1710
+ 'hotjar',
1711
+ // Push notifications
1712
+ 'push-notifications',
1713
+ 'onesignal',
1714
+ 'pushwoosh',
1715
+ // Background sync/heartbeat
1716
+ 'heartbeat',
1717
+ 'ping',
1718
+ 'alive',
1719
+ // WebRTC and streaming
1720
+ 'webrtc',
1721
+ 'rtmp://',
1722
+ 'wss://',
1723
+ // Common CDNs for dynamic content
1724
+ 'cloudfront.net',
1725
+ 'fastly.net',
1726
+ ];
1666
1727
  this.page = page;
1667
- if (this.page) {
1668
- this.page.setDefaultNavigationTimeout(this.config.playwrightNavigationTimeout);
1669
- this.page.setDefaultTimeout(this.config.playwrightActionTimeout);
1670
- }
1671
- }
1672
- /**
1673
- * Executes a single step in the test scenario with the specified action on the target element.
1674
- * Handles iframe navigation, element highlighting, and various Playwright actions like click, fill, validate, etc.
1675
- *
1676
- * @param params - Configuration object containing element selectors, action type, arguments, and display options
1677
- * @returns Promise that resolves to a result object for extract actions, or void for other actions
1678
- * @throws Error if element is not found or validation fails
1679
- */
1680
- async runStep(params) {
1681
- const { action, argument = '', iframeSelector = '', elementSelector = '', annotation = '', } = params;
1682
- // 0. Check that page is set
1683
- if (!this.page) {
1684
- throw new Error('ProboPlaywright: Page is not set');
1728
+ this.waitForStabilityQuietTimeout = (_a = options.waitForStabilityQuietTimeout) !== null && _a !== void 0 ? _a : 2000;
1729
+ this.waitForStabilityInitialDelay = (_b = options.waitForStabilityInitialDelay) !== null && _b !== void 0 ? _b : 500;
1730
+ this.waitForStabilityGlobalTimeout = (_c = options.waitForStabilityGlobalTimeout) !== null && _c !== void 0 ? _c : 15000;
1731
+ this.pollMs = (_d = options.pollMs) !== null && _d !== void 0 ? _d : 100;
1732
+ this.maxInflight = (_f = options.maxInflight) !== null && _f !== void 0 ? _f : 0;
1733
+ this.inflightGraceMs = (_g = options.inflightGraceMs) !== null && _g !== void 0 ? _g : 4000;
1734
+ this.waitForStabilityVerbose = (_h = options.waitForStabilityVerbose) !== null && _h !== void 0 ? _h : false;
1735
+ proboLogger.debug(`NavTracker constructor set values: quietTimeout=${this.waitForStabilityQuietTimeout}, initialDelay=${this.waitForStabilityInitialDelay}, globalTimeout=${this.waitForStabilityGlobalTimeout}, verbose=${this.waitForStabilityVerbose}`);
1736
+ this.instanceId = Math.random().toString(36).substr(2, 9);
1737
+ // Initialize timestamps
1738
+ const now = Date.now();
1739
+ this.lastHardNavAt = now;
1740
+ this.lastSoftNavAt = now;
1741
+ this.lastNetworkActivityAt = now;
1742
+ // Note: start() is called asynchronously in getInstance() to ensure proper initialization
1743
+ }
1744
+ /**
1745
+ * Start listening for navigation and network events using CDP (private method)
1746
+ */
1747
+ async start() {
1748
+ if (this.isListening) {
1749
+ proboLogger.debug(`NavTracker[${this.instanceId}]: already listening, ignoring start()`);
1750
+ return;
1685
1751
  }
1686
- // 1. Check if we need to visit a url
1687
- if (action === PlaywrightAction.VISIT_URL || action === PlaywrightAction.VISIT_BASE_URL) {
1688
- try {
1689
- await this.page.goto(argument, { timeout: this.config.playwrightNavigationTimeout });
1690
- }
1691
- catch (e) {
1692
- throw new Error(`Failed to navigate to ${argument}`);
1752
+ try {
1753
+ // Set up CDP session
1754
+ this.client = await this.page.context().newCDPSession(this.page);
1755
+ await this.client.send('Page.enable');
1756
+ await this.client.send('Network.enable');
1757
+ await this.client.send('Runtime.enable');
1758
+ await this.client.send('Page.setLifecycleEventsEnabled', { enabled: true });
1759
+ // Set up navigation event handlers
1760
+ this.client.on('Page.frameNavigated', (e) => {
1761
+ var _a;
1762
+ if (!((_a = e.frame) === null || _a === void 0 ? void 0 : _a.parentId)) { // main frame has no parentId
1763
+ this.lastHardNavAt = Date.now();
1764
+ this.navigationCount++;
1765
+ this.lastNavTime = Date.now();
1766
+ if (this.waitForStabilityVerbose) {
1767
+ proboLogger.debug(`NavTracker[${this.instanceId}]: Hard navigation detected at ${this.lastHardNavAt}`);
1768
+ }
1769
+ }
1770
+ });
1771
+ this.client.on('Page.navigatedWithinDocument', (_e) => {
1772
+ this.lastSoftNavAt = Date.now();
1773
+ if (this.waitForStabilityVerbose) {
1774
+ proboLogger.debug(`NavTracker[${this.instanceId}]: Soft navigation detected at ${this.lastSoftNavAt}`);
1775
+ }
1776
+ });
1777
+ // Set up network event handlers
1778
+ this.client.on('Network.requestWillBeSent', (e) => {
1779
+ this.onNetworkRequest(e);
1780
+ });
1781
+ this.client.on('Network.loadingFinished', (e) => {
1782
+ this.onNetworkResponse(e, 'finished');
1783
+ });
1784
+ this.client.on('Network.loadingFailed', (e) => {
1785
+ this.onNetworkResponse(e, 'failed');
1786
+ });
1787
+ this.isListening = true;
1788
+ proboLogger.debug(`NavTracker[${this.instanceId}]: started CDP-based monitoring`);
1789
+ }
1790
+ catch (error) {
1791
+ proboLogger.error(`NavTracker[${this.instanceId}]: Failed to start CDP monitoring: ${error}`);
1792
+ // Fall back to basic navigation tracking
1793
+ this.page.on("framenavigated", (frame) => {
1794
+ if (frame === this.page.mainFrame()) {
1795
+ this.navigationCount++;
1796
+ this.lastNavTime = Date.now();
1797
+ this.lastHardNavAt = Date.now();
1798
+ }
1799
+ });
1800
+ this.isListening = true;
1801
+ }
1802
+ }
1803
+ /**
1804
+ * Stop listening for navigation and network events (private method)
1805
+ */
1806
+ stop() {
1807
+ if (!this.isListening) {
1808
+ proboLogger.debug(`NavTracker[${this.instanceId}]: not listening, ignoring stop()`);
1809
+ return;
1810
+ }
1811
+ try {
1812
+ if (this.client) {
1813
+ // Check if the page is still available before detaching
1814
+ if (this.page && !this.page.isClosed()) {
1815
+ this.client.detach();
1816
+ }
1817
+ else {
1818
+ proboLogger.debug(`NavTracker[${this.instanceId}]: Page is closed, skipping CDP detach`);
1819
+ }
1820
+ this.client = null;
1693
1821
  }
1822
+ }
1823
+ catch (error) {
1824
+ proboLogger.debug(`NavTracker[${this.instanceId}]: Error detaching CDP client: ${error}`);
1825
+ }
1826
+ this.isListening = false;
1827
+ proboLogger.debug(`NavTracker[${this.instanceId}]: stopped CDP monitoring`);
1828
+ }
1829
+ /**
1830
+ * Handle network request events
1831
+ */
1832
+ onNetworkRequest(e) {
1833
+ var _a, _b, _c;
1834
+ const requestType = e.type;
1835
+ const url = (_b = (_a = e.request) === null || _a === void 0 ? void 0 : _a.url) !== null && _b !== void 0 ? _b : '';
1836
+ // Filter by resource type
1837
+ if (!this.RELEVANT_RESOURCE_TYPES.includes(requestType)) {
1694
1838
  return;
1695
1839
  }
1696
- // 2. Get the locator (iframe or not)
1697
- const startTime = Date.now();
1698
- let locator;
1699
- if (iframeSelector && iframeSelector.length > 0) {
1700
- locator = this.page.frameLocator(iframeSelector).locator(elementSelector);
1840
+ // Filter out streaming, websocket, and other real-time requests
1841
+ if (['WebSocket', 'EventSource', 'Media', 'Manifest', 'Other'].includes(requestType)) {
1842
+ return;
1701
1843
  }
1702
- else {
1703
- locator = this.page.locator(elementSelector);
1844
+ // Filter out by URL patterns
1845
+ const urlLower = url.toLowerCase();
1846
+ if (this.IGNORED_URL_PATTERNS.some(pattern => urlLower.includes(pattern))) {
1847
+ return;
1704
1848
  }
1705
- // Fail fast: immediately validate that the element exists for non-wait actions
1706
- const locator_timeout = (action === PlaywrightAction.WAIT_FOR) ? params.timeout || 10000 : this.config.playwrightLocatorTimeout;
1707
- try {
1708
- await locator.waitFor({ state: 'attached', timeout: locator_timeout });
1849
+ // Filter out data URLs and blob URLs
1850
+ if (urlLower.startsWith('data:') || urlLower.startsWith('blob:')) {
1851
+ return;
1709
1852
  }
1710
- catch (e) {
1711
- throw new Error(`Element not found with selector: ${elementSelector}${iframeSelector ? ` in iframe: ${iframeSelector}` : ''} after ${locator_timeout}ms`);
1853
+ // Filter out requests with certain headers
1854
+ const headers = ((_c = e.request) === null || _c === void 0 ? void 0 : _c.headers) || {};
1855
+ if (headers['purpose'] === 'prefetch' || ['video', 'audio'].includes(headers['sec-fetch-dest'])) {
1856
+ return;
1712
1857
  }
1713
- if (action === PlaywrightAction.HOVER) {
1714
- const visibleLocator = await findClosestVisibleElement(locator);
1715
- if (visibleLocator) {
1716
- locator = visibleLocator;
1717
- }
1858
+ this.inflight.add(e.requestId);
1859
+ this.totalRequestsTracked++;
1860
+ this.lastNetworkActivityAt = Date.now();
1861
+ if (this.waitForStabilityVerbose) {
1862
+ proboLogger.debug(`NavTracker[${this.instanceId}]: Network request started: ${requestType} - ${url}`);
1718
1863
  }
1719
- // 3. Highlight, wait, unhighlight if highlightTimeout > 0
1720
- if (this.config.highlightTimeout > 0) {
1721
- await this.highlight(locator, annotation);
1722
- await this.page.waitForTimeout(this.config.highlightTimeout);
1723
- await this.unhighlight(locator);
1864
+ }
1865
+ /**
1866
+ * Handle network response events
1867
+ */
1868
+ onNetworkResponse(e, status) {
1869
+ const requestId = e.requestId;
1870
+ if (!this.inflight.has(requestId)) {
1871
+ return;
1724
1872
  }
1725
- // 4. Action logic
1726
- switch (action) {
1727
- case PlaywrightAction.CLICK:
1728
- case PlaywrightAction.CHECK_CHECKBOX:
1729
- case PlaywrightAction.SELECT_RADIO:
1730
- await this.robustClick(locator);
1731
- break;
1732
- case PlaywrightAction.FILL_IN:
1733
- await this.robustFill(locator, argument);
1734
- break;
1735
- case PlaywrightAction.SELECT_DROPDOWN:
1736
- await locator.selectOption(argument);
1737
- break;
1738
- case PlaywrightAction.SET_SLIDER:
1739
- await this.setSliderValue(locator, argument);
1740
- break;
1741
- case PlaywrightAction.WAIT_FOR_OTP:
1742
- // till we figure out how to get the inbox name we will wait for ANY OTP in all inboxes
1743
- const otp = await OTP.waitForOTP();
1744
- if (otp) {
1745
- console.log(`✅ OTP found: ${otp}`);
1746
- await locator.fill(otp);
1747
- }
1748
- else {
1749
- console.log(`❌ OTP not found`);
1750
- }
1751
- break;
1752
- case PlaywrightAction.ASSERT_CONTAINS_VALUE:
1753
- const containerText = await this.getTextValue(locator);
1754
- if (!matchRegex(containerText, argument)) {
1755
- throw new Error(`Validation failed. Expected text "${containerText}" to match "${argument}".`);
1756
- }
1757
- break;
1758
- case PlaywrightAction.ASSERT_EXACT_VALUE:
1759
- const actualText = await this.getTextValue(locator);
1760
- if (actualText !== argument) {
1761
- throw new Error(`Validation failed. Expected text "${argument}", but got "${actualText}".`);
1762
- }
1763
- break;
1764
- case PlaywrightAction.HOVER:
1765
- //console.log('HOVER', locator);
1766
- if (locator) {
1767
- //console.log('executing HOVER on closest visible ancestor');
1768
- await locator.hover();
1769
- }
1770
- break;
1771
- case PlaywrightAction.SCROLL_TO_ELEMENT:
1772
- // Restore exact scroll positions from recording
1773
- const scrollData = JSON.parse(argument);
1774
- try {
1775
- console.log('🔄 Restoring scroll position for container:', locator, 'scrollTop:', scrollData.scrollTop, 'scrollLeft:', scrollData.scrollLeft);
1776
- await locator.evaluate((el, scrollData) => {
1777
- // el.scrollTop = scrollData.scrollTop;
1778
- // el.scrollLeft = scrollData.scrollLeft;
1779
- el.scrollTo({ left: scrollData.scrollLeft, top: scrollData.scrollTop, behavior: 'smooth' });
1780
- }, { scrollTop: scrollData.scrollTop, scrollLeft: scrollData.scrollLeft }, { timeout: 2000 });
1873
+ this.inflight.delete(requestId);
1874
+ this.lastNetworkActivityAt = Date.now();
1875
+ if (this.waitForStabilityVerbose) {
1876
+ proboLogger.debug(`NavTracker[${this.instanceId}]: Network request ${status} (${this.inflight.size} remaining)`);
1877
+ }
1878
+ }
1879
+ /**
1880
+ * Check if navigation and network activity has stabilized
1881
+ */
1882
+ hasNavigationStabilized() {
1883
+ const now = Date.now();
1884
+ // Use the most recent activity timestamp
1885
+ const lastActivityAt = Math.max(this.lastHardNavAt, this.lastSoftNavAt, this.lastNetworkActivityAt);
1886
+ const quietSinceMs = now - lastActivityAt;
1887
+ const inflightOk = this.inflight.size <= this.maxInflight || (now - this.lastHardNavAt) > this.inflightGraceMs;
1888
+ const isStabilized = quietSinceMs >= this.waitForStabilityQuietTimeout && inflightOk;
1889
+ if (this.waitForStabilityVerbose) {
1890
+ proboLogger.debug(`NavTracker[${this.instanceId}]: hasNavigationStabilized() - quietSinceMs=${quietSinceMs}ms, waitForStabilityQuietTimeout=${this.waitForStabilityQuietTimeout}ms, inflight=${this.inflight.size}, inflightOk=${inflightOk}, stabilized=${isStabilized}`);
1891
+ }
1892
+ return isStabilized;
1893
+ }
1894
+ /**
1895
+ * Wait for navigation and network activity to stabilize
1896
+ * Uses CDP-based monitoring for comprehensive network activity tracking
1897
+ */
1898
+ async waitForNavigationToStabilize() {
1899
+ const now = Date.now();
1900
+ const lastActivityAt = Math.max(this.lastHardNavAt, this.lastSoftNavAt, this.lastNetworkActivityAt);
1901
+ const timeSinceLastActivity = now - lastActivityAt;
1902
+ proboLogger.debug(`NavTracker[${this.instanceId}]: waiting for navigation and network to stabilize (quietTimeout: ${this.waitForStabilityQuietTimeout}ms, initialDelay: ${this.waitForStabilityInitialDelay}ms, globalTimeout: ${this.waitForStabilityGlobalTimeout}ms, verbose: ${this.waitForStabilityVerbose}, lastActivity: ${timeSinceLastActivity}ms ago)`);
1903
+ // Ensure CDP monitoring is properly initialized
1904
+ if (!this.isListening) {
1905
+ proboLogger.warn(`NavTracker[${this.instanceId}]: CDP monitoring not initialized, initializing now...`);
1906
+ await this.start();
1907
+ }
1908
+ const startTime = Date.now();
1909
+ const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
1910
+ try {
1911
+ // Initial delay to catch any new network activity triggered by user actions
1912
+ if (this.waitForStabilityInitialDelay > 0) {
1913
+ if (this.waitForStabilityVerbose) {
1914
+ proboLogger.debug(`NavTracker[${this.instanceId}]: initial delay of ${this.waitForStabilityInitialDelay}ms to catch new network activity`);
1781
1915
  }
1782
- catch (e) {
1783
- console.error('🔄 Failed to restore scroll position for container:', locator, 'scrollTop:', scrollData.scrollTop, 'scrollLeft:', scrollData.scrollLeft, 'error:', e);
1916
+ await sleep(this.waitForStabilityInitialDelay);
1917
+ }
1918
+ // Wait a short time to catch any missed events
1919
+ await sleep(100);
1920
+ // Main stabilization loop
1921
+ while (true) {
1922
+ const now = Date.now();
1923
+ // Check for timeout
1924
+ if (now - startTime > this.waitForStabilityGlobalTimeout) {
1925
+ proboLogger.warn(`NavTracker[${this.instanceId}]: Timeout reached after ${this.waitForStabilityGlobalTimeout}ms with ${this.inflight.size} pending requests`);
1926
+ break;
1784
1927
  }
1785
- await this.page.waitForTimeout(500);
1786
- break;
1787
- case PlaywrightAction.UPLOAD_FILES:
1788
- await locator.setInputFiles(argument);
1789
- break;
1790
- case PlaywrightAction.EXTRACT_VALUE:
1791
- let extractedText = await this.getTextValue(locator);
1792
- return extractedText;
1793
- case PlaywrightAction.WAIT_FOR:
1794
- const expectedText = argument;
1795
- const pollingInterval = params.pollingInterval || 500; // Default 500ms
1796
- const timeout = params.timeout || 10000; // Default 10 seconds
1797
- let textMatches = false;
1798
- let currentText = '';
1799
- while (!textMatches && (Date.now() - startTime) < timeout) {
1800
- try {
1801
- // Check if element is visible first
1802
- const isVisible = await locator.isVisible();
1803
- if (isVisible) {
1804
- // Get the current text content only if element is visible
1805
- currentText = await this.getTextValue(locator);
1806
- // Check if the text matches (using the same logic as ASSERT_CONTAINS_VALUE)
1807
- if (matchRegex(currentText, expectedText)) {
1808
- textMatches = true;
1809
- console.log(`✅ Wait for text completed successfully. Found: "${currentText}"`);
1810
- }
1811
- else {
1812
- // Text doesn't match yet, wait for the polling interval
1813
- if ((Date.now() - startTime) < timeout) {
1814
- await this.page.waitForTimeout(pollingInterval);
1815
- }
1816
- }
1817
- }
1818
- else {
1819
- // Element is not visible, wait for the polling interval
1820
- if ((Date.now() - startTime) < timeout) {
1821
- await this.page.waitForTimeout(pollingInterval);
1822
- }
1823
- }
1824
- }
1825
- catch (e) {
1826
- throw new Error(`Wait for text failed while trying to extract text from selector: ${elementSelector}${iframeSelector ? ` in iframe: ${iframeSelector}` : ''}`);
1827
- }
1928
+ // Check if stabilized
1929
+ if (this.hasNavigationStabilized()) {
1930
+ const quietSinceMs = now - Math.max(this.lastHardNavAt, this.lastSoftNavAt, this.lastNetworkActivityAt);
1931
+ proboLogger.debug(`NavTracker[${this.instanceId}]: Page stabilized after ${quietSinceMs}ms of quiet time`);
1932
+ break;
1828
1933
  }
1829
- // Timeout reached without a match
1830
- if (!textMatches) {
1831
- throw new Error(`Wait for text failed. Expected "${expectedText}" to match "${currentText}" after ${timeout}ms of polling every ${pollingInterval}ms`);
1934
+ // Log progress every 2 seconds in verbose mode
1935
+ if (this.waitForStabilityVerbose && now % 2000 < this.pollMs) {
1936
+ const quietSinceMs = now - Math.max(this.lastHardNavAt, this.lastSoftNavAt, this.lastNetworkActivityAt);
1937
+ proboLogger.debug(`NavTracker[${this.instanceId}]: Status - quiet=${quietSinceMs}ms/${this.waitForStabilityQuietTimeout}ms, inflight=${this.inflight.size}/${this.maxInflight}`);
1832
1938
  }
1833
- break;
1834
- default:
1835
- throw new Error(`Unhandled action: ${action}`);
1939
+ await sleep(this.pollMs);
1940
+ }
1941
+ }
1942
+ catch (error) {
1943
+ proboLogger.error(`NavTracker[${this.instanceId}]: Error during stabilization: ${error}`);
1836
1944
  }
1837
1945
  }
1946
+ // ============================================================================
1947
+ // SINGLETON METHODS
1948
+ // ============================================================================
1838
1949
  /**
1839
- * Creates a visual highlight overlay on the target element with optional annotation text.
1840
- * The highlight appears as a red border around the element and can include descriptive text.
1841
- *
1842
- * @param locator - The Playwright locator for the element to highlight
1843
- * @param annotation - Optional text annotation to display above/below the highlighted element
1950
+ * Get the singleton instance of NavTracker
1951
+ * @param page The page to track (required for first creation)
1952
+ * @param options Optional configuration
1953
+ * @returns The singleton NavTracker instance
1844
1954
  */
1845
- async highlight(locator, annotation = null) {
1846
- try {
1847
- await locator.evaluate((el) => {
1848
- const overlay = el.ownerDocument.createElement('div');
1849
- overlay.id = 'highlight-overlay';
1850
- overlay.style.cssText = `
1851
- position: fixed;
1852
- top: 0;
1853
- left: 0;
1854
- width: 100%;
1855
- height: 100%;
1856
- pointer-events: none;
1857
- z-index: 2147483647;
1858
- `;
1859
- el.ownerDocument.body.appendChild(overlay);
1860
- const bbox = el.getBoundingClientRect();
1861
- const highlight = el.ownerDocument.createElement('div');
1862
- highlight.style.cssText = `
1863
- position: fixed;
1864
- left: ${bbox.x}px;
1865
- top: ${bbox.y}px;
1866
- width: ${bbox.width}px;
1867
- height: ${bbox.height}px;
1868
- border: 2px solid rgb(255, 0, 0);
1869
- transition: all 0.2s ease-in-out;
1870
- `;
1871
- overlay.appendChild(highlight);
1872
- }, { timeout: 500 });
1873
- }
1874
- catch (e) {
1875
- console.log('highlight: failed to run locator.evaluate()', e);
1955
+ static async getInstance(page, options) {
1956
+ proboLogger.debug(`NavTracker.getInstance called with options:`, options);
1957
+ if (!NavTracker.instance) {
1958
+ if (!page) {
1959
+ throw new Error('NavTracker: Page is required for first instance creation');
1960
+ }
1961
+ NavTracker.instance = new NavTracker(page, options);
1962
+ await NavTracker.instance.start();
1963
+ proboLogger.debug(`NavTracker: created new singleton instance with options:`, options);
1876
1964
  }
1877
- if (annotation) {
1878
- await locator.evaluate((el, annotation) => {
1879
- const overlay = el.ownerDocument.getElementById('highlight-overlay');
1880
- if (overlay) {
1881
- const bbox = el.getBoundingClientRect();
1882
- const annotationEl = el.ownerDocument.createElement('div');
1883
- annotationEl.style.cssText = `
1884
- position: fixed;
1885
- left: ${bbox.x}px;
1886
- top: ${bbox.y - 25}px;
1887
- padding: 2px 6px;
1888
- background-color: rgba(255, 255, 0, 0.6);
1889
- color: black;
1890
- font-size: 16px;
1891
- font-family: 'Courier New', Courier, monospace;
1892
- font-weight: bold;
1893
- border-radius: 3px;
1894
- pointer-events: none;
1895
- z-index: 2147483647;
1896
- `;
1897
- annotationEl.textContent = annotation;
1898
- // If element is too close to top of window, position annotation below
1899
- if (bbox.y < 30) {
1900
- annotationEl.style.top = `${bbox.y + bbox.height + 5}px`;
1901
- }
1902
- overlay.appendChild(annotationEl);
1965
+ else {
1966
+ // Update existing instance with new options
1967
+ if (options) {
1968
+ if (options.waitForStabilityQuietTimeout !== undefined) {
1969
+ NavTracker.instance.waitForStabilityQuietTimeout = options.waitForStabilityQuietTimeout;
1903
1970
  }
1904
- }, annotation, { timeout: 500 });
1971
+ if (options.waitForStabilityInitialDelay !== undefined) {
1972
+ NavTracker.instance.waitForStabilityInitialDelay = options.waitForStabilityInitialDelay;
1973
+ }
1974
+ if (options.waitForStabilityGlobalTimeout !== undefined) {
1975
+ NavTracker.instance.waitForStabilityGlobalTimeout = options.waitForStabilityGlobalTimeout;
1976
+ }
1977
+ if (options.pollMs !== undefined) {
1978
+ NavTracker.instance.pollMs = options.pollMs;
1979
+ }
1980
+ if (options.maxInflight !== undefined) {
1981
+ NavTracker.instance.maxInflight = options.maxInflight;
1982
+ }
1983
+ if (options.inflightGraceMs !== undefined) {
1984
+ NavTracker.instance.inflightGraceMs = options.inflightGraceMs;
1985
+ }
1986
+ if (options.waitForStabilityVerbose !== undefined) {
1987
+ NavTracker.instance.waitForStabilityVerbose = options.waitForStabilityVerbose;
1988
+ }
1989
+ proboLogger.debug(`NavTracker: updated existing instance with new values: quietTimeout=${NavTracker.instance.waitForStabilityQuietTimeout}, initialDelay=${NavTracker.instance.waitForStabilityInitialDelay}, globalTimeout=${NavTracker.instance.waitForStabilityGlobalTimeout}, verbose=${NavTracker.instance.waitForStabilityVerbose}`);
1990
+ }
1991
+ if (options === null || options === void 0 ? void 0 : options.waitForStabilityVerbose) {
1992
+ proboLogger.debug(`NavTracker: returning existing singleton instance`);
1993
+ }
1905
1994
  }
1995
+ return NavTracker.instance;
1906
1996
  }
1907
- ;
1908
1997
  /**
1909
- * Removes the highlight overlay from the target element.
1910
- * Cleans up the visual highlighting created by the highlight method.
1911
- *
1912
- * @param locator - The Playwright locator for the element to unhighlight
1998
+ * Reset the singleton instance (useful for testing or page changes)
1913
1999
  */
1914
- async unhighlight(locator) {
1915
- try {
1916
- await locator.evaluate((el) => {
1917
- const overlay = el.ownerDocument.getElementById('highlight-overlay');
1918
- if (overlay) {
1919
- overlay.remove();
1920
- }
1921
- }, { timeout: 500 });
1922
- }
1923
- catch (e) {
1924
- console.log('unhighlight: failed to run locator.evaluate()', e);
2000
+ static resetInstance() {
2001
+ if (NavTracker.instance) {
2002
+ NavTracker.instance.stop();
2003
+ NavTracker.instance = null;
2004
+ proboLogger.debug(`NavTracker: reset singleton instance`);
1925
2005
  }
1926
2006
  }
1927
- ;
2007
+ }
2008
+ NavTracker.instance = null;
2009
+
2010
+ class ProboPlaywright {
2011
+ constructor(timeoutConfig = {}, page = null) {
2012
+ this.page = null;
2013
+ // Merge provided config with defaults to ensure all properties are defined
2014
+ this.timeoutConfig = {
2015
+ ...DEFAULT_PLAYWRIGHT_TIMEOUT_CONFIG,
2016
+ ...timeoutConfig
2017
+ };
2018
+ this.setPage(page);
2019
+ }
1928
2020
  /**
1929
- * Attempts to fill a form field with the specified value using multiple fallback strategies.
1930
- * First tries the standard fill method, then falls back to click + type if needed.
2021
+ * Sets the Playwright page instance for this ProboPlaywright instance.
2022
+ * Also applies the configured default navigation and action timeouts to the page.
1931
2023
  *
1932
- * @param locator - The Playwright locator for the input element
1933
- * @param value - The text value to fill into the input field
2024
+ * @param page - The Playwright Page instance to use, or null to unset.
1934
2025
  */
1935
- async robustFill(locator, value) {
1936
- if (!this.page) {
1937
- throw new Error('ProboPlaywright: Page is not set');
1938
- }
1939
- try {
1940
- await locator.fill(value);
1941
- return;
1942
- }
1943
- catch (err) {
1944
- console.warn('robustFill: failed to run locator.fill()', err);
1945
- }
1946
- // fallback: click and type
1947
- try {
1948
- await this.robustClick(locator);
1949
- await this.page.keyboard.type(value);
1950
- return;
1951
- }
1952
- catch (err) {
1953
- console.warn('robustFill: failed to run locator.click() and page.keyboard.type()', err);
2026
+ setPage(page) {
2027
+ this.page = page;
2028
+ if (this.page) {
2029
+ this.page.setDefaultNavigationTimeout(this.timeoutConfig.playwrightNavigationTimeout);
2030
+ this.page.setDefaultTimeout(this.timeoutConfig.playwrightActionTimeout);
1954
2031
  }
1955
2032
  }
1956
- ;
1957
2033
  /**
1958
- * Performs a robust click operation using multiple fallback strategies.
1959
- * Attempts standard click first, then mouse click at center coordinates, and finally native DOM events.
2034
+ * Executes a single step in the test scenario with the specified action on the target element.
2035
+ * Handles iframe navigation, element highlighting, and various Playwright actions like click, fill, validate, etc.
1960
2036
  *
1961
- * @param locator - The Playwright locator for the element to click
1962
- * @throws Error if all click methods fail
2037
+ * @param params - Configuration object containing element selectors, action type, arguments, and display options
2038
+ * @returns Promise that resolves to a result object for extract actions, or void for other actions
2039
+ * @throws Error if element is not found or validation fails
1963
2040
  */
1964
- async robustClick(locator) {
2041
+ async runStep(params) {
2042
+ const { action, argument = '', iframeSelector = '', elementSelector = '', annotation = '', } = params;
2043
+ // 0. Check that page is set
1965
2044
  if (!this.page) {
1966
2045
  throw new Error('ProboPlaywright: Page is not set');
1967
2046
  }
1968
- // start with a standard click
1969
- try {
1970
- await locator.click({ noWaitAfter: false, timeout: this.config.playwrightActionTimeout });
2047
+ // 1. Check if we need to visit a url
2048
+ if (action === PlaywrightAction.VISIT_URL || action === PlaywrightAction.VISIT_BASE_URL) {
2049
+ try {
2050
+ await this.page.goto(argument, { timeout: this.timeoutConfig.playwrightNavigationTimeout });
2051
+ }
2052
+ catch (e) {
2053
+ throw new Error(`Failed to navigate to ${argument}`);
2054
+ }
1971
2055
  return;
1972
2056
  }
1973
- catch (err) {
1974
- console.warn('robustClick: failed to run locator.click(), trying mouse.click()');
1975
- }
1976
- // try clicking using mouse at the center of the element
1977
- try {
1978
- const bbox = await locator.boundingBox({ timeout: this.config.playwrightLocatorTimeout });
1979
- if (bbox) {
1980
- await this.page.mouse.click(bbox.x + bbox.width / 2, bbox.y + bbox.height / 2);
1981
- return;
1982
- }
1983
- else {
1984
- console.warn('robustClick: bounding box not found');
2057
+ // 2. Check if we need to assert the url
2058
+ if (action === PlaywrightAction.ASSERT_URL) {
2059
+ // wait for page to stabilize
2060
+ const navTracker = await NavTracker.getInstance(this.page, { waitForStabilityQuietTimeout: this.timeoutConfig.playwrightNavigationTimeout });
2061
+ await navTracker.waitForNavigationToStabilize();
2062
+ const currentUrl = await this.page.url();
2063
+ if (currentUrl !== argument) {
2064
+ throw new Error(`Assertion failed: Expected URL "${argument}" but got "${currentUrl}".`);
1985
2065
  }
2066
+ return;
1986
2067
  }
1987
- catch (err2) {
1988
- console.warn('robustClick: failed to run page.mouse.click()');
2068
+ // 3. Check if we need to type keys
2069
+ if (action === PlaywrightAction.TYPE_KEYS) {
2070
+ await this.robustTypeKeys(argument);
2071
+ return;
1989
2072
  }
1990
- // fallback: dispatch native mouse events manually
1991
- try {
1992
- await locator.evaluate((el) => {
1993
- ['mousedown', 'mouseup', 'click'].forEach(type => {
1994
- const event = new MouseEvent(type, {
1995
- bubbles: true,
1996
- cancelable: true,
1997
- view: window
1998
- });
1999
- el.dispatchEvent(event);
2000
- });
2001
- }, { timeout: this.config.playwrightActionTimeout });
2073
+ // 4. Get the locator (iframe or not)
2074
+ const startTime = Date.now();
2075
+ let locator;
2076
+ if (iframeSelector && iframeSelector.length > 0) {
2077
+ locator = this.page.frameLocator(iframeSelector).locator(elementSelector);
2002
2078
  }
2003
- catch (err3) {
2004
- console.error('robustClick: all click methods failed:', err3);
2005
- throw err3; // Re-throw final error if all fallbacks fail
2079
+ else {
2080
+ locator = this.page.locator(elementSelector);
2006
2081
  }
2007
- }
2008
- ;
2009
- /**
2010
- * Extracts text content from an element using multiple strategies.
2011
- * Tries textContent first, then inputValue, and finally looks for nested input elements.
2012
- * Returns normalized and trimmed text for consistent comparison.
2013
- *
2014
- * @param locator - The Playwright locator for the element to extract text from
2015
- * @returns Normalized text content with consistent whitespace handling
2016
- */
2017
- async getTextValue(locator) {
2018
- let textValue = await locator.textContent();
2019
- if (!textValue) {
2020
- try {
2021
- textValue = await locator.inputValue();
2022
- }
2023
- catch (err) {
2024
- console.warn('getTextValue: failed to run locator.inputValue()', err);
2025
- }
2082
+ // Fail fast: immediately validate that the element exists for non-wait actions
2083
+ const locator_timeout = (action === PlaywrightAction.WAIT_FOR) ? params.timeout || 10000 : this.timeoutConfig.playwrightLocatorTimeout;
2084
+ try {
2085
+ await locator.waitFor({ state: 'attached', timeout: locator_timeout });
2026
2086
  }
2027
- if (!textValue) {
2028
- try {
2029
- textValue = await locator.locator('input').inputValue();
2030
- }
2031
- catch (err) {
2032
- console.warn('getTextValue: failed to run locator.locator("input").inputValue()', err);
2087
+ catch (e) {
2088
+ throw new Error(`Element not found with selector: ${elementSelector}${iframeSelector ? ` in iframe: ${iframeSelector}` : ''} after ${locator_timeout}ms`);
2089
+ }
2090
+ if (action === PlaywrightAction.HOVER) {
2091
+ const visibleLocator = await findClosestVisibleElement(locator);
2092
+ if (visibleLocator) {
2093
+ locator = visibleLocator;
2033
2094
  }
2034
2095
  }
2035
- if (!textValue) {
2036
- textValue = '';
2096
+ // 5. Highlight, wait, unhighlight if highlightTimeout > 0
2097
+ if (this.timeoutConfig.highlightTimeout > 0) {
2098
+ await this.highlight(locator, annotation);
2099
+ await this.page.waitForTimeout(this.timeoutConfig.highlightTimeout);
2100
+ await this.unhighlight(locator);
2101
+ }
2102
+ // 6. Action logic
2103
+ switch (action) {
2104
+ case PlaywrightAction.CLICK:
2105
+ case PlaywrightAction.CHECK_CHECKBOX:
2106
+ case PlaywrightAction.SELECT_RADIO:
2107
+ await this.robustClick(locator);
2108
+ break;
2109
+ case PlaywrightAction.FILL_IN:
2110
+ await this.robustFill(locator, argument);
2111
+ break;
2112
+ case PlaywrightAction.SELECT_DROPDOWN:
2113
+ await locator.selectOption(argument);
2114
+ break;
2115
+ case PlaywrightAction.SET_SLIDER:
2116
+ await this.setSliderValue(locator, argument);
2117
+ break;
2118
+ case PlaywrightAction.WAIT_FOR_OTP:
2119
+ // till we figure out how to get the inbox name we will wait for ANY OTP in all inboxes
2120
+ const otp = await OTP.waitForOTP({ checkRecentMessagesSinceMs: 120000 });
2121
+ if (otp) {
2122
+ console.log(`✅ OTP found: ${otp}`);
2123
+ await locator.fill(otp);
2124
+ }
2125
+ else {
2126
+ console.log(`❌ OTP not found`);
2127
+ }
2128
+ break;
2129
+ case PlaywrightAction.ASSERT_CONTAINS_VALUE:
2130
+ const containerText = await this.getTextValue(locator);
2131
+ if (!matchRegex(containerText, argument)) {
2132
+ throw new Error(`Validation failed. Expected text "${containerText}" to match "${argument}".`);
2133
+ }
2134
+ break;
2135
+ case PlaywrightAction.ASSERT_EXACT_VALUE:
2136
+ const actualText = await this.getTextValue(locator);
2137
+ if (actualText !== argument) {
2138
+ throw new Error(`Validation failed. Expected text "${argument}", but got "${actualText}".`);
2139
+ }
2140
+ break;
2141
+ case PlaywrightAction.HOVER:
2142
+ //console.log('HOVER', locator);
2143
+ if (locator) {
2144
+ //console.log('executing HOVER on closest visible ancestor');
2145
+ await locator.hover();
2146
+ }
2147
+ break;
2148
+ case PlaywrightAction.SCROLL_TO_ELEMENT:
2149
+ // Restore exact scroll positions from recording
2150
+ const scrollData = JSON.parse(argument);
2151
+ try {
2152
+ console.log('🔄 Restoring scroll position for container:', locator, 'scrollTop:', scrollData.scrollTop, 'scrollLeft:', scrollData.scrollLeft);
2153
+ await locator.evaluate((el, scrollData) => {
2154
+ // el.scrollTop = scrollData.scrollTop;
2155
+ // el.scrollLeft = scrollData.scrollLeft;
2156
+ el.scrollTo({ left: scrollData.scrollLeft, top: scrollData.scrollTop, behavior: 'smooth' });
2157
+ }, { scrollTop: scrollData.scrollTop, scrollLeft: scrollData.scrollLeft }, { timeout: 2000 });
2158
+ }
2159
+ catch (e) {
2160
+ console.error('🔄 Failed to restore scroll position for container:', locator, 'scrollTop:', scrollData.scrollTop, 'scrollLeft:', scrollData.scrollLeft, 'error:', e);
2161
+ }
2162
+ await this.page.waitForTimeout(500);
2163
+ break;
2164
+ case PlaywrightAction.UPLOAD_FILES:
2165
+ await locator.setInputFiles(argument);
2166
+ break;
2167
+ case PlaywrightAction.EXTRACT_VALUE:
2168
+ let extractedText = await this.getTextValue(locator);
2169
+ return extractedText;
2170
+ case PlaywrightAction.WAIT_FOR:
2171
+ const expectedText = argument;
2172
+ const pollingInterval = params.pollingInterval || 500; // Default 500ms
2173
+ const timeout = params.timeout || 10000; // Default 10 seconds
2174
+ let textMatches = false;
2175
+ let currentText = '';
2176
+ while (!textMatches && (Date.now() - startTime) < timeout) {
2177
+ try {
2178
+ // Check if element is visible first
2179
+ const isVisible = await locator.isVisible();
2180
+ if (isVisible) {
2181
+ // Get the current text content only if element is visible
2182
+ currentText = await this.getTextValue(locator);
2183
+ // Check if the text matches (using the same logic as ASSERT_CONTAINS_VALUE)
2184
+ if (matchRegex(currentText, expectedText)) {
2185
+ textMatches = true;
2186
+ console.log(`✅ Wait for text completed successfully. Found: "${currentText}"`);
2187
+ }
2188
+ else {
2189
+ // Text doesn't match yet, wait for the polling interval
2190
+ if ((Date.now() - startTime) < timeout) {
2191
+ await this.page.waitForTimeout(pollingInterval);
2192
+ }
2193
+ }
2194
+ }
2195
+ else {
2196
+ // Element is not visible, wait for the polling interval
2197
+ if ((Date.now() - startTime) < timeout) {
2198
+ await this.page.waitForTimeout(pollingInterval);
2199
+ }
2200
+ }
2201
+ }
2202
+ catch (e) {
2203
+ throw new Error(`Wait for text failed while trying to extract text from selector: ${elementSelector}${iframeSelector ? ` in iframe: ${iframeSelector}` : ''}`);
2204
+ }
2205
+ }
2206
+ // Timeout reached without a match
2207
+ if (!textMatches) {
2208
+ throw new Error(`Wait for text failed. Expected "${expectedText}" to match "${currentText}" after ${timeout}ms of polling every ${pollingInterval}ms`);
2209
+ }
2210
+ break;
2211
+ default:
2212
+ throw new Error(`Unhandled action: ${action}`);
2037
2213
  }
2038
- // Trim and normalize whitespace to make comparison more robust
2039
- return textValue.trim().replace(/\s+/g, ' ');
2040
- }
2041
- ;
2042
- async setSliderValue(locator, value) {
2043
- await locator.evaluate((el, value) => {
2044
- el.value = value;
2045
- el.dispatchEvent(new Event('input', { bubbles: true }));
2046
- el.dispatchEvent(new Event('change', { bubbles: true }));
2047
- }, value, { timeout: this.config.playwrightActionTimeout });
2048
- }
2049
- } /* class ProboPlaywright */
2050
-
2051
- /**
2052
- * Global navigation tracker that monitors page navigation events and network activity
2053
- * using CDP (Chrome DevTools Protocol) for comprehensive network monitoring
2054
- *
2055
- * This is a singleton class - only one instance can exist at a time
2056
- */
2057
- class NavTracker {
2058
- /**
2059
- * Private constructor - use getInstance() to get the singleton instance
2060
- */
2061
- constructor(page, options = {}) {
2062
- var _a, _b, _c, _d, _f, _g, _h;
2063
- this.navigationCount = 0;
2064
- this.lastNavTime = null;
2065
- this.isListening = false;
2066
- this.client = null;
2067
- this.inflight = new Set();
2068
- this.lastHardNavAt = 0;
2069
- this.lastSoftNavAt = 0;
2070
- this.lastNetworkActivityAt = 0;
2071
- this.totalRequestsTracked = 0;
2072
- // Define relevant resource types and content types
2073
- this.RELEVANT_RESOURCE_TYPES = [
2074
- 'Document',
2075
- 'Stylesheet',
2076
- 'Image',
2077
- 'Font',
2078
- 'Script',
2079
- 'XHR',
2080
- 'Fetch'
2081
- ];
2082
- this.RELEVANT_CONTENT_TYPES = [
2083
- 'text/html',
2084
- 'text/css',
2085
- 'application/javascript',
2086
- 'image/',
2087
- 'font/',
2088
- 'application/json',
2089
- ];
2090
- // Additional patterns to filter out
2091
- this.IGNORED_URL_PATTERNS = [
2092
- // Analytics and tracking
2093
- 'analytics',
2094
- 'tracking',
2095
- 'telemetry',
2096
- 'beacon',
2097
- 'metrics',
2098
- // Ad-related
2099
- 'doubleclick',
2100
- 'adsystem',
2101
- 'adserver',
2102
- 'advertising',
2103
- // Social media widgets
2104
- 'facebook.com/plugins',
2105
- 'platform.twitter',
2106
- 'linkedin.com/embed',
2107
- // Live chat and support
2108
- 'livechat',
2109
- 'zendesk',
2110
- 'intercom',
2111
- 'crisp.chat',
2112
- 'hotjar',
2113
- // Push notifications
2114
- 'push-notifications',
2115
- 'onesignal',
2116
- 'pushwoosh',
2117
- // Background sync/heartbeat
2118
- 'heartbeat',
2119
- 'ping',
2120
- 'alive',
2121
- // WebRTC and streaming
2122
- 'webrtc',
2123
- 'rtmp://',
2124
- 'wss://',
2125
- // Common CDNs for dynamic content
2126
- 'cloudfront.net',
2127
- 'fastly.net',
2128
- ];
2129
- this.page = page;
2130
- this.waitForStabilityQuietTimeout = (_a = options.waitForStabilityQuietTimeout) !== null && _a !== void 0 ? _a : 2000;
2131
- this.waitForStabilityInitialDelay = (_b = options.waitForStabilityInitialDelay) !== null && _b !== void 0 ? _b : 500;
2132
- this.waitForStabilityGlobalTimeout = (_c = options.waitForStabilityGlobalTimeout) !== null && _c !== void 0 ? _c : 15000;
2133
- this.pollMs = (_d = options.pollMs) !== null && _d !== void 0 ? _d : 100;
2134
- this.maxInflight = (_f = options.maxInflight) !== null && _f !== void 0 ? _f : 0;
2135
- this.inflightGraceMs = (_g = options.inflightGraceMs) !== null && _g !== void 0 ? _g : 4000;
2136
- this.waitForStabilityVerbose = (_h = options.waitForStabilityVerbose) !== null && _h !== void 0 ? _h : false;
2137
- proboLogger.debug(`NavTracker constructor set values: quietTimeout=${this.waitForStabilityQuietTimeout}, initialDelay=${this.waitForStabilityInitialDelay}, globalTimeout=${this.waitForStabilityGlobalTimeout}, verbose=${this.waitForStabilityVerbose}`);
2138
- this.instanceId = Math.random().toString(36).substr(2, 9);
2139
- // Initialize timestamps
2140
- const now = Date.now();
2141
- this.lastHardNavAt = now;
2142
- this.lastSoftNavAt = now;
2143
- this.lastNetworkActivityAt = now;
2144
- // Note: start() is called asynchronously in getInstance() to ensure proper initialization
2145
2214
  }
2146
2215
  /**
2147
- * Start listening for navigation and network events using CDP (private method)
2216
+ * Creates a visual highlight overlay on the target element with optional annotation text.
2217
+ * The highlight appears as a red border around the element and can include descriptive text.
2218
+ *
2219
+ * @param locator - The Playwright locator for the element to highlight
2220
+ * @param annotation - Optional text annotation to display above/below the highlighted element
2148
2221
  */
2149
- async start() {
2150
- if (this.isListening) {
2151
- proboLogger.debug(`NavTracker[${this.instanceId}]: already listening, ignoring start()`);
2152
- return;
2153
- }
2222
+ async highlight(locator, annotation = null) {
2154
2223
  try {
2155
- // Set up CDP session
2156
- this.client = await this.page.context().newCDPSession(this.page);
2157
- await this.client.send('Page.enable');
2158
- await this.client.send('Network.enable');
2159
- await this.client.send('Runtime.enable');
2160
- await this.client.send('Page.setLifecycleEventsEnabled', { enabled: true });
2161
- // Set up navigation event handlers
2162
- this.client.on('Page.frameNavigated', (e) => {
2163
- var _a;
2164
- if (!((_a = e.frame) === null || _a === void 0 ? void 0 : _a.parentId)) { // main frame has no parentId
2165
- this.lastHardNavAt = Date.now();
2166
- this.navigationCount++;
2167
- this.lastNavTime = Date.now();
2168
- if (this.waitForStabilityVerbose) {
2169
- proboLogger.debug(`NavTracker[${this.instanceId}]: Hard navigation detected at ${this.lastHardNavAt}`);
2170
- }
2171
- }
2172
- });
2173
- this.client.on('Page.navigatedWithinDocument', (_e) => {
2174
- this.lastSoftNavAt = Date.now();
2175
- if (this.waitForStabilityVerbose) {
2176
- proboLogger.debug(`NavTracker[${this.instanceId}]: Soft navigation detected at ${this.lastSoftNavAt}`);
2177
- }
2178
- });
2179
- // Set up network event handlers
2180
- this.client.on('Network.requestWillBeSent', (e) => {
2181
- this.onNetworkRequest(e);
2182
- });
2183
- this.client.on('Network.loadingFinished', (e) => {
2184
- this.onNetworkResponse(e, 'finished');
2185
- });
2186
- this.client.on('Network.loadingFailed', (e) => {
2187
- this.onNetworkResponse(e, 'failed');
2188
- });
2189
- this.isListening = true;
2190
- proboLogger.debug(`NavTracker[${this.instanceId}]: started CDP-based monitoring`);
2224
+ await locator.evaluate((el) => {
2225
+ const overlay = el.ownerDocument.createElement('div');
2226
+ overlay.id = 'highlight-overlay';
2227
+ overlay.style.cssText = `
2228
+ position: fixed;
2229
+ top: 0;
2230
+ left: 0;
2231
+ width: 100%;
2232
+ height: 100%;
2233
+ pointer-events: none;
2234
+ z-index: 2147483647;
2235
+ `;
2236
+ el.ownerDocument.body.appendChild(overlay);
2237
+ const bbox = el.getBoundingClientRect();
2238
+ const highlight = el.ownerDocument.createElement('div');
2239
+ highlight.style.cssText = `
2240
+ position: fixed;
2241
+ left: ${bbox.x}px;
2242
+ top: ${bbox.y}px;
2243
+ width: ${bbox.width}px;
2244
+ height: ${bbox.height}px;
2245
+ border: 2px solid rgb(255, 0, 0);
2246
+ transition: all 0.2s ease-in-out;
2247
+ `;
2248
+ overlay.appendChild(highlight);
2249
+ }, { timeout: 500 });
2191
2250
  }
2192
- catch (error) {
2193
- proboLogger.error(`NavTracker[${this.instanceId}]: Failed to start CDP monitoring: ${error}`);
2194
- // Fall back to basic navigation tracking
2195
- this.page.on("framenavigated", (frame) => {
2196
- if (frame === this.page.mainFrame()) {
2197
- this.navigationCount++;
2198
- this.lastNavTime = Date.now();
2199
- this.lastHardNavAt = Date.now();
2251
+ catch (e) {
2252
+ console.log('highlight: failed to run locator.evaluate()', e);
2253
+ }
2254
+ if (annotation) {
2255
+ await locator.evaluate((el, annotation) => {
2256
+ const overlay = el.ownerDocument.getElementById('highlight-overlay');
2257
+ if (overlay) {
2258
+ const bbox = el.getBoundingClientRect();
2259
+ const annotationEl = el.ownerDocument.createElement('div');
2260
+ annotationEl.style.cssText = `
2261
+ position: fixed;
2262
+ left: ${bbox.x}px;
2263
+ top: ${bbox.y - 25}px;
2264
+ padding: 2px 6px;
2265
+ background-color: rgba(255, 255, 0, 0.6);
2266
+ color: black;
2267
+ font-size: 16px;
2268
+ font-family: 'Courier New', Courier, monospace;
2269
+ font-weight: bold;
2270
+ border-radius: 3px;
2271
+ pointer-events: none;
2272
+ z-index: 2147483647;
2273
+ `;
2274
+ annotationEl.textContent = annotation;
2275
+ // If element is too close to top of window, position annotation below
2276
+ if (bbox.y < 30) {
2277
+ annotationEl.style.top = `${bbox.y + bbox.height + 5}px`;
2278
+ }
2279
+ overlay.appendChild(annotationEl);
2200
2280
  }
2201
- });
2202
- this.isListening = true;
2281
+ }, annotation, { timeout: 500 });
2203
2282
  }
2204
2283
  }
2284
+ ;
2205
2285
  /**
2206
- * Stop listening for navigation and network events (private method)
2286
+ * Removes the highlight overlay from the target element.
2287
+ * Cleans up the visual highlighting created by the highlight method.
2288
+ *
2289
+ * @param locator - The Playwright locator for the element to unhighlight
2207
2290
  */
2208
- stop() {
2209
- if (!this.isListening) {
2210
- proboLogger.debug(`NavTracker[${this.instanceId}]: not listening, ignoring stop()`);
2211
- return;
2212
- }
2291
+ async unhighlight(locator) {
2213
2292
  try {
2214
- if (this.client) {
2215
- // Check if the page is still available before detaching
2216
- if (this.page && !this.page.isClosed()) {
2217
- this.client.detach();
2218
- }
2219
- else {
2220
- proboLogger.debug(`NavTracker[${this.instanceId}]: Page is closed, skipping CDP detach`);
2293
+ await locator.evaluate((el) => {
2294
+ const overlay = el.ownerDocument.getElementById('highlight-overlay');
2295
+ if (overlay) {
2296
+ overlay.remove();
2221
2297
  }
2222
- this.client = null;
2223
- }
2298
+ }, { timeout: 500 });
2224
2299
  }
2225
- catch (error) {
2226
- proboLogger.debug(`NavTracker[${this.instanceId}]: Error detaching CDP client: ${error}`);
2300
+ catch (e) {
2301
+ console.log('unhighlight: failed to run locator.evaluate()', e);
2227
2302
  }
2228
- this.isListening = false;
2229
- proboLogger.debug(`NavTracker[${this.instanceId}]: stopped CDP monitoring`);
2230
2303
  }
2304
+ ;
2231
2305
  /**
2232
- * Handle network request events
2306
+ * Attempts to fill a form field with the specified value using multiple fallback strategies.
2307
+ * First tries the standard fill method, then falls back to click + type if needed.
2308
+ *
2309
+ * @param locator - The Playwright locator for the input element
2310
+ * @param value - The text value to fill into the input field
2233
2311
  */
2234
- onNetworkRequest(e) {
2235
- var _a, _b, _c;
2236
- const requestType = e.type;
2237
- const url = (_b = (_a = e.request) === null || _a === void 0 ? void 0 : _a.url) !== null && _b !== void 0 ? _b : '';
2238
- // Filter by resource type
2239
- if (!this.RELEVANT_RESOURCE_TYPES.includes(requestType)) {
2240
- return;
2241
- }
2242
- // Filter out streaming, websocket, and other real-time requests
2243
- if (['WebSocket', 'EventSource', 'Media', 'Manifest', 'Other'].includes(requestType)) {
2244
- return;
2312
+ async robustFill(locator, value) {
2313
+ if (!this.page) {
2314
+ throw new Error('ProboPlaywright: Page is not set');
2245
2315
  }
2246
- // Filter out by URL patterns
2247
- const urlLower = url.toLowerCase();
2248
- if (this.IGNORED_URL_PATTERNS.some(pattern => urlLower.includes(pattern))) {
2316
+ try {
2317
+ await locator.fill(value);
2249
2318
  return;
2250
2319
  }
2251
- // Filter out data URLs and blob URLs
2252
- if (urlLower.startsWith('data:') || urlLower.startsWith('blob:')) {
2253
- return;
2320
+ catch (err) {
2321
+ console.warn('robustFill: failed to run locator.fill()', err);
2254
2322
  }
2255
- // Filter out requests with certain headers
2256
- const headers = ((_c = e.request) === null || _c === void 0 ? void 0 : _c.headers) || {};
2257
- if (headers['purpose'] === 'prefetch' || ['video', 'audio'].includes(headers['sec-fetch-dest'])) {
2323
+ // fallback: click and type
2324
+ try {
2325
+ await locator.focus();
2326
+ await this.page.keyboard.type(value);
2258
2327
  return;
2259
2328
  }
2260
- this.inflight.add(e.requestId);
2261
- this.totalRequestsTracked++;
2262
- this.lastNetworkActivityAt = Date.now();
2263
- if (this.waitForStabilityVerbose) {
2264
- proboLogger.debug(`NavTracker[${this.instanceId}]: Network request started: ${requestType} - ${url}`);
2265
- }
2266
- }
2267
- /**
2268
- * Handle network response events
2269
- */
2270
- onNetworkResponse(e, status) {
2271
- const requestId = e.requestId;
2272
- if (!this.inflight.has(requestId)) {
2329
+ catch (err) {
2330
+ console.warn('robustFill: failed to run locator.click() and page.keyboard.type()', err);
2331
+ }
2332
+ }
2333
+ ;
2334
+ async robustTypeKeys(value) {
2335
+ if (!this.page) {
2336
+ throw new Error('ProboPlaywright: Page is not set');
2337
+ }
2338
+ /* try {
2339
+ await locator.press(value);
2340
+ return;
2341
+ } catch (err) {
2342
+ console.warn('robustTypeKeys: failed to run locator.type()', err);
2343
+ } */
2344
+ // fallback: click and type
2345
+ try {
2346
+ // await locator.focus();
2347
+ await this.page.keyboard.press(value);
2273
2348
  return;
2274
2349
  }
2275
- this.inflight.delete(requestId);
2276
- this.lastNetworkActivityAt = Date.now();
2277
- if (this.waitForStabilityVerbose) {
2278
- proboLogger.debug(`NavTracker[${this.instanceId}]: Network request ${status} (${this.inflight.size} remaining)`);
2350
+ catch (err) {
2351
+ console.warn('robustTypeKeys: failed to run page.keyboard.type()', err);
2279
2352
  }
2280
2353
  }
2281
2354
  /**
2282
- * Check if navigation and network activity has stabilized
2355
+ * Performs a robust click operation using multiple fallback strategies.
2356
+ * Attempts standard click first, then mouse click at center coordinates, and finally native DOM events.
2357
+ *
2358
+ * @param locator - The Playwright locator for the element to click
2359
+ * @throws Error if all click methods fail
2283
2360
  */
2284
- hasNavigationStabilized() {
2285
- const now = Date.now();
2286
- // Use the most recent activity timestamp
2287
- const lastActivityAt = Math.max(this.lastHardNavAt, this.lastSoftNavAt, this.lastNetworkActivityAt);
2288
- const quietSinceMs = now - lastActivityAt;
2289
- const inflightOk = this.inflight.size <= this.maxInflight || (now - this.lastHardNavAt) > this.inflightGraceMs;
2290
- const isStabilized = quietSinceMs >= this.waitForStabilityQuietTimeout && inflightOk;
2291
- if (this.waitForStabilityVerbose) {
2292
- proboLogger.debug(`NavTracker[${this.instanceId}]: hasNavigationStabilized() - quietSinceMs=${quietSinceMs}ms, waitForStabilityQuietTimeout=${this.waitForStabilityQuietTimeout}ms, inflight=${this.inflight.size}, inflightOk=${inflightOk}, stabilized=${isStabilized}`);
2361
+ async robustClick(locator) {
2362
+ if (!this.page) {
2363
+ throw new Error('ProboPlaywright: Page is not set');
2293
2364
  }
2294
- return isStabilized;
2295
- }
2296
- /**
2297
- * Wait for navigation and network activity to stabilize
2298
- * Uses CDP-based monitoring for comprehensive network activity tracking
2299
- */
2300
- async waitForNavigationToStabilize() {
2301
- const now = Date.now();
2302
- const lastActivityAt = Math.max(this.lastHardNavAt, this.lastSoftNavAt, this.lastNetworkActivityAt);
2303
- const timeSinceLastActivity = now - lastActivityAt;
2304
- proboLogger.debug(`NavTracker[${this.instanceId}]: waiting for navigation and network to stabilize (quietTimeout: ${this.waitForStabilityQuietTimeout}ms, initialDelay: ${this.waitForStabilityInitialDelay}ms, globalTimeout: ${this.waitForStabilityGlobalTimeout}ms, verbose: ${this.waitForStabilityVerbose}, lastActivity: ${timeSinceLastActivity}ms ago)`);
2305
- // Ensure CDP monitoring is properly initialized
2306
- if (!this.isListening) {
2307
- proboLogger.warn(`NavTracker[${this.instanceId}]: CDP monitoring not initialized, initializing now...`);
2308
- await this.start();
2365
+ // start with a standard click
2366
+ try {
2367
+ await locator.click({ noWaitAfter: false, timeout: this.timeoutConfig.playwrightActionTimeout });
2368
+ return;
2309
2369
  }
2310
- const startTime = Date.now();
2311
- const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
2370
+ catch (err) {
2371
+ console.warn('robustClick: failed to run locator.click(), trying mouse.click()');
2372
+ }
2373
+ // try clicking using mouse at the center of the element
2312
2374
  try {
2313
- // Initial delay to catch any new network activity triggered by user actions
2314
- if (this.waitForStabilityInitialDelay > 0) {
2315
- if (this.waitForStabilityVerbose) {
2316
- proboLogger.debug(`NavTracker[${this.instanceId}]: initial delay of ${this.waitForStabilityInitialDelay}ms to catch new network activity`);
2317
- }
2318
- await sleep(this.waitForStabilityInitialDelay);
2375
+ const bbox = await locator.boundingBox({ timeout: this.timeoutConfig.playwrightLocatorTimeout });
2376
+ if (bbox) {
2377
+ await this.page.mouse.click(bbox.x + bbox.width / 2, bbox.y + bbox.height / 2);
2378
+ return;
2319
2379
  }
2320
- // Wait a short time to catch any missed events
2321
- await sleep(100);
2322
- // Main stabilization loop
2323
- while (true) {
2324
- const now = Date.now();
2325
- // Check for timeout
2326
- if (now - startTime > this.waitForStabilityGlobalTimeout) {
2327
- proboLogger.warn(`NavTracker[${this.instanceId}]: Timeout reached after ${this.waitForStabilityGlobalTimeout}ms with ${this.inflight.size} pending requests`);
2328
- break;
2329
- }
2330
- // Check if stabilized
2331
- if (this.hasNavigationStabilized()) {
2332
- const quietSinceMs = now - Math.max(this.lastHardNavAt, this.lastSoftNavAt, this.lastNetworkActivityAt);
2333
- proboLogger.debug(`NavTracker[${this.instanceId}]: Page stabilized after ${quietSinceMs}ms of quiet time`);
2334
- break;
2335
- }
2336
- // Log progress every 2 seconds in verbose mode
2337
- if (this.waitForStabilityVerbose && now % 2000 < this.pollMs) {
2338
- const quietSinceMs = now - Math.max(this.lastHardNavAt, this.lastSoftNavAt, this.lastNetworkActivityAt);
2339
- proboLogger.debug(`NavTracker[${this.instanceId}]: Status - quiet=${quietSinceMs}ms/${this.waitForStabilityQuietTimeout}ms, inflight=${this.inflight.size}/${this.maxInflight}`);
2340
- }
2341
- await sleep(this.pollMs);
2380
+ else {
2381
+ console.warn('robustClick: bounding box not found');
2342
2382
  }
2343
2383
  }
2344
- catch (error) {
2345
- proboLogger.error(`NavTracker[${this.instanceId}]: Error during stabilization: ${error}`);
2384
+ catch (err2) {
2385
+ console.warn('robustClick: failed to run page.mouse.click()');
2386
+ }
2387
+ // fallback: dispatch native mouse events manually
2388
+ try {
2389
+ await locator.evaluate((el) => {
2390
+ ['mousedown', 'mouseup', 'click'].forEach(type => {
2391
+ const event = new MouseEvent(type, {
2392
+ bubbles: true,
2393
+ cancelable: true,
2394
+ view: window
2395
+ });
2396
+ el.dispatchEvent(event);
2397
+ });
2398
+ }, { timeout: this.timeoutConfig.playwrightActionTimeout });
2399
+ }
2400
+ catch (err3) {
2401
+ console.error('robustClick: all click methods failed:', err3);
2402
+ throw err3; // Re-throw final error if all fallbacks fail
2346
2403
  }
2347
2404
  }
2348
- // ============================================================================
2349
- // SINGLETON METHODS
2350
- // ============================================================================
2405
+ ;
2351
2406
  /**
2352
- * Get the singleton instance of NavTracker
2353
- * @param page The page to track (required for first creation)
2354
- * @param options Optional configuration
2355
- * @returns The singleton NavTracker instance
2407
+ * Extracts text content from an element using multiple strategies.
2408
+ * Tries textContent first, then inputValue, and finally looks for nested input elements.
2409
+ * Returns normalized and trimmed text for consistent comparison.
2410
+ *
2411
+ * @param locator - The Playwright locator for the element to extract text from
2412
+ * @returns Normalized text content with consistent whitespace handling
2356
2413
  */
2357
- static async getInstance(page, options) {
2358
- proboLogger.debug(`NavTracker.getInstance called with options:`, options);
2359
- if (!NavTracker.instance) {
2360
- if (!page) {
2361
- throw new Error('NavTracker: Page is required for first instance creation');
2414
+ async getTextValue(locator) {
2415
+ let textValue = await locator.textContent();
2416
+ if (!textValue) {
2417
+ try {
2418
+ textValue = await locator.inputValue();
2419
+ }
2420
+ catch (err) {
2421
+ console.warn('getTextValue: failed to run locator.inputValue()', err);
2362
2422
  }
2363
- NavTracker.instance = new NavTracker(page, options);
2364
- await NavTracker.instance.start();
2365
- proboLogger.debug(`NavTracker: created new singleton instance with options:`, options);
2366
2423
  }
2367
- else {
2368
- // Update existing instance with new options
2369
- if (options) {
2370
- if (options.waitForStabilityQuietTimeout !== undefined) {
2371
- NavTracker.instance.waitForStabilityQuietTimeout = options.waitForStabilityQuietTimeout;
2372
- }
2373
- if (options.waitForStabilityInitialDelay !== undefined) {
2374
- NavTracker.instance.waitForStabilityInitialDelay = options.waitForStabilityInitialDelay;
2375
- }
2376
- if (options.waitForStabilityGlobalTimeout !== undefined) {
2377
- NavTracker.instance.waitForStabilityGlobalTimeout = options.waitForStabilityGlobalTimeout;
2378
- }
2379
- if (options.pollMs !== undefined) {
2380
- NavTracker.instance.pollMs = options.pollMs;
2381
- }
2382
- if (options.maxInflight !== undefined) {
2383
- NavTracker.instance.maxInflight = options.maxInflight;
2384
- }
2385
- if (options.inflightGraceMs !== undefined) {
2386
- NavTracker.instance.inflightGraceMs = options.inflightGraceMs;
2387
- }
2388
- if (options.waitForStabilityVerbose !== undefined) {
2389
- NavTracker.instance.waitForStabilityVerbose = options.waitForStabilityVerbose;
2390
- }
2391
- proboLogger.debug(`NavTracker: updated existing instance with new values: quietTimeout=${NavTracker.instance.waitForStabilityQuietTimeout}, initialDelay=${NavTracker.instance.waitForStabilityInitialDelay}, globalTimeout=${NavTracker.instance.waitForStabilityGlobalTimeout}, verbose=${NavTracker.instance.waitForStabilityVerbose}`);
2424
+ if (!textValue) {
2425
+ try {
2426
+ textValue = await locator.locator('input').inputValue();
2392
2427
  }
2393
- if (options === null || options === void 0 ? void 0 : options.waitForStabilityVerbose) {
2394
- proboLogger.debug(`NavTracker: returning existing singleton instance`);
2428
+ catch (err) {
2429
+ console.warn('getTextValue: failed to run locator.locator("input").inputValue()', err);
2395
2430
  }
2396
2431
  }
2397
- return NavTracker.instance;
2398
- }
2399
- /**
2400
- * Reset the singleton instance (useful for testing or page changes)
2401
- */
2402
- static resetInstance() {
2403
- if (NavTracker.instance) {
2404
- NavTracker.instance.stop();
2405
- NavTracker.instance = null;
2406
- proboLogger.debug(`NavTracker: reset singleton instance`);
2432
+ if (!textValue) {
2433
+ textValue = '';
2407
2434
  }
2435
+ // Trim and normalize whitespace to make comparison more robust
2436
+ return textValue.trim().replace(/\s+/g, ' ');
2408
2437
  }
2409
- }
2410
- NavTracker.instance = null;
2411
-
2412
- const retryOptions = {
2413
- retries: 3,
2414
- minTimeout: 1000,
2415
- onFailedAttempt: (error) => {
2416
- proboLogger.warn(`Page operation failed, attempt ${error.attemptNumber} of ${error.retriesLeft +
2417
- error.attemptNumber}...`);
2438
+ ;
2439
+ async setSliderValue(locator, value) {
2440
+ await locator.evaluate((el, value) => {
2441
+ el.value = value;
2442
+ el.dispatchEvent(new Event('input', { bubbles: true }));
2443
+ el.dispatchEvent(new Event('change', { bubbles: true }));
2444
+ }, value, { timeout: this.timeoutConfig.playwrightActionTimeout });
2418
2445
  }
2419
- };
2446
+ } /* class ProboPlaywright */
2447
+
2420
2448
  class Probo {
2421
2449
  constructor({ scenarioName, token = '', apiUrl = '', enableConsoleLogs = false, logToConsole = true, logToFile = false, debugLevel = ProboLogLevel.INFO, aiModel = AIModel.AZURE_GPT4_MINI, timeoutConfig = {} }) {
2422
2450
  // Configure logger transports and level
@@ -2443,167 +2471,142 @@ class Probo {
2443
2471
  proboLogger.info(`Initializing: scenario=${scenarioName}, apiUrl=${apiEndPoint}, ` +
2444
2472
  `enableConsoleLogs=${enableConsoleLogs}, debugLevel=${debugLevel}, aiModel=${aiModel}`);
2445
2473
  }
2446
- async askAI(page, question) {
2474
+ async askAI(page, question, options) {
2447
2475
  var _a, _b;
2448
- const response = await this.askAIHelper(page, question);
2476
+ const response = await this.askAIHelper(page, question, options);
2449
2477
  if ((_a = response === null || response === void 0 ? void 0 : response.result) === null || _a === void 0 ? void 0 : _a.error) {
2450
2478
  throw new Error(response.result.error);
2451
2479
  }
2452
2480
  return (_b = response === null || response === void 0 ? void 0 : response.result) === null || _b === void 0 ? void 0 : _b.answer;
2453
2481
  }
2454
- async runStep(page, stepPrompt, argument = null, options = { useCache: true, stepIdFromServer: undefined, aiModel: this.aiModel, timeoutConfig: this.timeoutConfig }) {
2455
- // Use the aiModel from options if provided, otherwise use the one from constructor
2456
- const aiModelToUse = options.aiModel !== undefined ? options.aiModel : this.aiModel;
2457
- proboLogger.log(`runStep: ${options.stepIdFromServer ? '#' + options.stepIdFromServer + ' - ' : ''}${stepPrompt}, aiModel: ${aiModelToUse}, pageUrl: ${page.url()}`);
2458
- this.setupConsoleLogs(page);
2459
- // initialize the NavTracker
2460
- await NavTracker.getInstance(page, {
2461
- waitForStabilityQuietTimeout: this.timeoutConfig.waitForStabilityQuietTimeout,
2462
- waitForStabilityInitialDelay: this.timeoutConfig.waitForStabilityInitialDelay,
2463
- waitForStabilityGlobalTimeout: this.timeoutConfig.waitForStabilityGlobalTimeout,
2464
- waitForStabilityVerbose: this.timeoutConfig.waitForStabilityVerbose
2465
- });
2466
- // First check if the step exists in the database
2467
- let stepId;
2468
- if (options.useCache) {
2469
- const [isCachedStep, returnValue] = await this._handleCachedStep(page, stepPrompt, argument);
2470
- if (isCachedStep) {
2471
- proboLogger.debug('performed cached step!');
2472
- return returnValue;
2473
- }
2474
- }
2475
- proboLogger.debug(`Cache disabled or step not found, creating new step`);
2476
- stepId = await this._handleStepCreation(page, stepPrompt, options.stepIdFromServer, options.useCache);
2477
- proboLogger.debug('Step ID:', stepId);
2478
- let instruction = null;
2479
- // Main execution loop
2480
- while (true) {
2481
- try {
2482
- // Get next instruction from server
2483
- const nextInstruction = await this.apiClient.resolveNextInstruction(stepId, instruction, aiModelToUse);
2484
- proboLogger.debug('Next Instruction from server:', nextInstruction);
2485
- // Exit conditions
2486
- if (nextInstruction.what_to_do === 'do_nothing') {
2487
- if (nextInstruction.args.success) {
2488
- proboLogger.info(`Reasoning: ${nextInstruction.args.message}`);
2489
- proboLogger.info('Step completed successfully');
2490
- return nextInstruction.args.return_value;
2491
- }
2492
- else {
2493
- throw new Error(nextInstruction.args.message);
2494
- }
2482
+ async runStep(page, stepPrompt, argument = null, options = { aiModel: this.aiModel, timeoutConfig: this.timeoutConfig }) {
2483
+ const runStepStartTime = Date.now();
2484
+ try {
2485
+ // Determine which AI model to use for this step
2486
+ const aiModelToUse = options.aiModel !== undefined ? options.aiModel : this.aiModel;
2487
+ let stepId = options.stepId;
2488
+ if (!stepId) {
2489
+ const stepByPrompt = await this.apiClient.findStepByPrompt(stepPrompt, this.scenarioName, page.url());
2490
+ if (!(stepByPrompt === null || stepByPrompt === void 0 ? void 0 : stepByPrompt.step)) {
2491
+ // Create new step before continuing
2492
+ stepId = Number(await this.apiClient.createStep({
2493
+ scenarioName: this.scenarioName,
2494
+ stepPrompt: stepPrompt,
2495
+ initial_screenshot_url: '',
2496
+ initial_html_content: '',
2497
+ argument: argument,
2498
+ use_cache: false,
2499
+ url: page.url(),
2500
+ }));
2495
2501
  }
2496
- // Handle different instruction types
2497
- switch (nextInstruction.what_to_do) {
2498
- case 'highlight_candidate_elements':
2499
- proboLogger.debug('Highlighting candidate elements:', nextInstruction.args.element_types);
2500
- const elementTags = nextInstruction.args.element_types;
2501
- const { candidates_screenshot_url, candidate_elements } = await this.findAndHighlightCandidateElements(page, elementTags);
2502
- const cleaned_candidate_elements = candidate_elements.map((elementInfo) => {
2503
- elementInfo.element = null;
2504
- return elementInfo;
2505
- });
2506
- proboLogger.debug(`Highlighted ${cleaned_candidate_elements.length} elements`);
2507
- const executed_instruction = {
2508
- what_to_do: 'highlight_candidate_elements',
2509
- args: {
2510
- element_types: nextInstruction.args.element_types
2511
- },
2512
- result: {
2513
- highlighted_elements: cleaned_candidate_elements,
2514
- candidate_elements_screenshot_url: candidates_screenshot_url
2515
- }
2516
- };
2517
- proboLogger.debug('Executed Instruction:', executed_instruction);
2518
- instruction = executed_instruction;
2519
- break;
2520
- case 'perform_action':
2521
- instruction = await this._handlePerformAction(page, nextInstruction, argument);
2522
- break;
2523
- default:
2524
- throw new Error(`Unknown instruction type: ${nextInstruction.what_to_do}`);
2502
+ else {
2503
+ stepId = Number(stepByPrompt.step.id);
2525
2504
  }
2526
2505
  }
2527
- catch (error) {
2528
- proboLogger.error(error.message);
2529
- throw Error(error.message);
2506
+ proboLogger.log(`runStep: ${'#' + stepId + ' - '}${stepPrompt}, aiModel: ${aiModelToUse}, pageUrl: ${page.url()}`);
2507
+ // Setup page monitoring and wait for navigation to stabilize
2508
+ this.setupConsoleLogs(page);
2509
+ proboLogger.debug(`🔍 [WaitForNavigationToStabilize] timeoutConfig values: quietTimeout=${this.timeoutConfig.waitForStabilityQuietTimeout}, initialDelay=${this.timeoutConfig.waitForStabilityInitialDelay}, globalTimeout=${this.timeoutConfig.waitForStabilityGlobalTimeout}, verbose=${this.timeoutConfig.waitForStabilityVerbose}`);
2510
+ const navTracker = await NavTracker.getInstance(page, {
2511
+ waitForStabilityQuietTimeout: this.timeoutConfig.waitForStabilityQuietTimeout,
2512
+ waitForStabilityInitialDelay: this.timeoutConfig.waitForStabilityInitialDelay,
2513
+ waitForStabilityGlobalTimeout: this.timeoutConfig.waitForStabilityGlobalTimeout,
2514
+ waitForStabilityVerbose: this.timeoutConfig.waitForStabilityVerbose
2515
+ });
2516
+ proboLogger.debug(`🔍 Calling waitForNavigationToStabilize: ${this.timeoutConfig.waitForStabilityQuietTimeout}ms, waitForStabilityInitialDelay: ${this.timeoutConfig.waitForStabilityInitialDelay}ms, waitForStabilityGlobalTimeout: ${this.timeoutConfig.waitForStabilityGlobalTimeout}ms, waitForStabilityVerbose: ${this.timeoutConfig.waitForStabilityVerbose}`);
2517
+ await navTracker.waitForNavigationToStabilize();
2518
+ // STEP 1: Check for cached step (if caching is enabled)
2519
+ let [isCachedStep, returnValue] = await this._handleCachedStep(page, stepId, stepPrompt, argument);
2520
+ if (isCachedStep) {
2521
+ const totalRunStepTime = Date.now() - runStepStartTime;
2522
+ proboLogger.debug(`⏱️ runStep total time: ${totalRunStepTime}ms (cached step)`);
2523
+ proboLogger.debug('Successfully executed cached step!');
2524
+ return returnValue;
2525
+ }
2526
+ // STEP 2: Create or update step in backend and capture initial page state
2527
+ proboLogger.log(`Execution failed or prompt changed, starting self-healing process...`);
2528
+ proboLogger.debug(`Taking initial screenshot from the page ${page.url()}`);
2529
+ const { base_screenshot_url, base_html_content } = await this.getInitialPageState(page);
2530
+ await this.apiClient.patchStep(stepId, {
2531
+ initial_screenshot: base_screenshot_url,
2532
+ pre_html_content: base_html_content !== '' ? base_html_content : undefined,
2533
+ prompt: stepPrompt
2534
+ });
2535
+ // STEP 4: Self-Healing Process - Get action from database or from AI
2536
+ const screenshotReasoningStartTime = Date.now();
2537
+ const action = await this.apiClient.screenshotReasoning(stepId, stepPrompt, this.aiModel);
2538
+ const screenshotReasoningTime = Date.now() - screenshotReasoningStartTime;
2539
+ proboLogger.debug(`⏱️ screenshotReasoning took ${screenshotReasoningTime}ms`);
2540
+ // STEP 5: Find and highlight candidate elements based on reasoning
2541
+ const elementTags = resolveElementTag(action);
2542
+ const { candidates_screenshot_url, candidate_elements } = await this.findAndHighlightCandidateElements(page, elementTags);
2543
+ proboLogger.debug(`Found ${candidate_elements.length} candidate elements`);
2544
+ // STEP 6: Clean up highlighting
2545
+ await this.unhighlightElements(page);
2546
+ // STEP 7: Select the best candidate element
2547
+ const findBestCandidateStartTime = Date.now();
2548
+ const index = await this.apiClient.findBestCandidateElement(stepId, candidates_screenshot_url, candidate_elements, this.aiModel);
2549
+ const findBestCandidateTime = Date.now() - findBestCandidateStartTime;
2550
+ proboLogger.debug(`⏱️ findBestCandidateElement took ${findBestCandidateTime}ms`);
2551
+ proboLogger.debug(`AI selected candidate element at index: ${index}`);
2552
+ // STEP 8: Find the actual element object from the candidates
2553
+ const actualElement = candidate_elements.find(element => Number(element.index) === index);
2554
+ if (!actualElement) {
2555
+ throw new Error(`No candidate element found with index ${index}. Available indices: ${candidate_elements.map(e => e.index).join(', ')}`);
2530
2556
  }
2557
+ proboLogger.debug(`Selected element: ${actualElement.css_selector} (index: ${actualElement.index})`);
2558
+ // STEP 9: Execute the interaction with the selected element
2559
+ const actionResult = await this._handlePerformAction(page, stepId, actualElement, action, argument);
2560
+ const totalRunStepTime = Date.now() - runStepStartTime;
2561
+ proboLogger.debug(`⏱️ runStep total time: ${totalRunStepTime}ms (screenshotReasoning: ${screenshotReasoningTime}ms, findBestCandidateElement: ${findBestCandidateTime}ms)`);
2562
+ proboLogger.debug('Step execution completed successfully');
2563
+ return actionResult;
2564
+ }
2565
+ catch (error) {
2566
+ proboLogger.error(`Error in runStep: ${error.message}`);
2567
+ proboLogger.error(`Step prompt: "${stepPrompt}"`);
2568
+ proboLogger.error(`Page URL: ${page.url()}`);
2569
+ // Re-throw the error to maintain the existing error handling behavior
2570
+ throw error;
2531
2571
  }
2532
2572
  }
2533
- async _handleCachedStep(page, stepPrompt, argument) {
2573
+ async _handleCachedStep(page, stepId, stepPrompt, argument) {
2534
2574
  proboLogger.debug(`Checking if step exists in database: ${stepPrompt}`);
2535
- const result = await this.apiClient.findStepByPrompt(stepPrompt, this.scenarioName, page.url());
2536
- if (result) {
2537
- const actionArgument = argument !== null && argument !== void 0 ? argument : result.step.argument;
2538
- proboLogger.log(`Found existing step with ID: ${result.step.id} going to perform action: ${result.step.action} with value: ${actionArgument}`);
2539
- proboLogger.debug(`Step in the DB: #${result.step.id} status: ${result.step.status} action: ${result.step.action} argument: ${actionArgument} locator: ${result.step.element_css_selector}`);
2540
- if (result.step.status !== 'EXECUTED') {
2541
- proboLogger.debug(`Step ${result.step.id} is not executed, returning false`);
2542
- return [false, false];
2543
- }
2544
- proboLogger.debug(`Step ${result.step.id} is in status executed, performing action directly with Playwright`);
2545
- const element_css_selector = result.step.element_css_selector;
2546
- const iframe_selector = result.step.iframe_selector;
2547
- let returnValue;
2548
- try {
2549
- // Create ProboPlaywright instance with the page
2550
- const proboPlaywright = new ProboPlaywright(this.timeoutConfig, page);
2551
- // Call runStep with the cached action
2552
- const runStepResult = await proboPlaywright.runStep({
2553
- iframeSelector: iframe_selector,
2554
- elementSelector: element_css_selector,
2555
- action: result.step.action,
2556
- argument: actionArgument,
2557
- // Disable highlighting for cached steps since they're already validated
2558
- highlightTimeout: 0
2559
- });
2560
- // Handle return value - runStep returns RunStepResult for EXTRACT_VALUE, void for others
2561
- if (runStepResult) {
2562
- returnValue = runStepResult;
2563
- }
2564
- else {
2565
- returnValue = true;
2566
- }
2567
- }
2568
- catch (error) {
2569
- proboLogger.error(`Error executing action for step ${result.step.id} going to reset the step`);
2570
- proboLogger.debug('Error details:', error);
2571
- await this.apiClient.resetStep(result.step.id);
2572
- return [false, false];
2573
- }
2574
- return [true, returnValue];
2575
+ const result = await this.apiClient.findStepById(stepId);
2576
+ if (!result) {
2577
+ proboLogger.error(`Step not found in database`);
2578
+ throw Error(`Step #${stepId} not found in database`);
2579
+ }
2580
+ else if (result.action == '' || !result.action) {
2581
+ proboLogger.error(`Step action is missing`);
2582
+ throw Error(`Step #${stepId} action is missing`);
2583
+ }
2584
+ if (result.prompt !== stepPrompt) {
2585
+ proboLogger.debug(`Step prompt changed, skipping execution and going to start self-healing process...`);
2586
+ return [false, undefined];
2587
+ }
2588
+ const actionArgument = argument !== null && argument !== void 0 ? argument : result.argument;
2589
+ proboLogger.log(`Found existing step with ID: ${result.id} going to perform action: ${result.action} with value: ${actionArgument}`);
2590
+ try {
2591
+ // Create ProboPlaywright instance with the page
2592
+ const proboPlaywright = new ProboPlaywright(this.timeoutConfig, page);
2593
+ // Call runStep with the cached action
2594
+ const runStepResult = await proboPlaywright.runStep({
2595
+ iframeSelector: result.iframe_selector,
2596
+ elementSelector: result.element_css_selector,
2597
+ action: result.action,
2598
+ argument: actionArgument,
2599
+ });
2600
+ return [true, runStepResult];
2601
+ }
2602
+ catch (error) {
2603
+ return [false, undefined];
2575
2604
  }
2576
- else {
2577
- proboLogger.debug(`Step not found in database, continuing with the normal flow`);
2578
- return [false, false];
2579
- }
2580
- }
2581
- async _handleStepCreation(page, stepPrompt, stepIdFromServer, useCache) {
2582
- proboLogger.debug(`Taking initial screenshot from the page ${page.url()}`);
2583
- const { base_screenshot_url, base_html_content } = await pRetry(() => this.getInitialPageState(page), retryOptions);
2584
- return await this.apiClient.createStep({
2585
- stepIdFromServer,
2586
- scenarioName: this.scenarioName,
2587
- stepPrompt: stepPrompt,
2588
- initial_screenshot_url: base_screenshot_url,
2589
- initial_html_content: base_html_content,
2590
- use_cache: useCache,
2591
- url: page.url()
2592
- });
2593
2605
  }
2594
2606
  setupConsoleLogs(page) {
2595
2607
  setupBrowserConsoleLogs(page, this.enableConsoleLogs, proboLogger);
2596
2608
  }
2597
2609
  async getInitialPageState(page) {
2598
- proboLogger.debug(`🔍 [getInitialPageState] timeoutConfig values: quietTimeout=${this.timeoutConfig.waitForStabilityQuietTimeout}, initialDelay=${this.timeoutConfig.waitForStabilityInitialDelay}, globalTimeout=${this.timeoutConfig.waitForStabilityGlobalTimeout}, verbose=${this.timeoutConfig.waitForStabilityVerbose}`);
2599
- const navTracker = await NavTracker.getInstance(page, {
2600
- waitForStabilityQuietTimeout: this.timeoutConfig.waitForStabilityQuietTimeout,
2601
- waitForStabilityInitialDelay: this.timeoutConfig.waitForStabilityInitialDelay,
2602
- waitForStabilityGlobalTimeout: this.timeoutConfig.waitForStabilityGlobalTimeout,
2603
- waitForStabilityVerbose: this.timeoutConfig.waitForStabilityVerbose
2604
- });
2605
- proboLogger.debug(`🔍 getting initial page state. calling waitForNavigationToStabilize: ${this.timeoutConfig.waitForStabilityQuietTimeout}ms, waitForStabilityInitialDelay: ${this.timeoutConfig.waitForStabilityInitialDelay}ms, waitForStabilityGlobalTimeout: ${this.timeoutConfig.waitForStabilityGlobalTimeout}ms, waitForStabilityVerbose: ${this.timeoutConfig.waitForStabilityVerbose}`);
2606
- await navTracker.waitForNavigationToStabilize();
2607
2610
  const baseScreenshot = await this.screenshot(page);
2608
2611
  proboLogger.debug(`🔍 baseScreenshot: ${baseScreenshot}`);
2609
2612
  const baseHtmlContent = '';
@@ -2651,13 +2654,12 @@ class Probo {
2651
2654
  const screenshot_url = await this.apiClient.uploadScreenshot(screenshot_bytes);
2652
2655
  return screenshot_url;
2653
2656
  }
2654
- async _handlePerformAction(page, nextInstruction, argument) {
2655
- proboLogger.debug('Handling perform action:', nextInstruction);
2656
- const action = nextInstruction.args.action;
2657
- const value = argument !== null && argument !== void 0 ? argument : nextInstruction.args.value;
2658
- const element_css_selector = nextInstruction.args.element_css_selector;
2659
- const iframe_selector = nextInstruction.args.iframe_selector;
2660
- const element_index = nextInstruction.args.element_index;
2657
+ async _handlePerformAction(page, stepId, actualElement, action, argument) {
2658
+ const value = argument !== null && argument !== void 0 ? argument : '';
2659
+ const element_css_selector = actualElement.css_selector;
2660
+ const iframe_selector = actualElement.iframe_selector;
2661
+ const element_index = actualElement.index;
2662
+ proboLogger.debug('Handling perform action:', action);
2661
2663
  if (action !== PlaywrightAction.VISIT_URL) {
2662
2664
  await this.unhighlightElements(page);
2663
2665
  proboLogger.debug('Unhighlighted elements');
@@ -2674,27 +2676,18 @@ class Probo {
2674
2676
  });
2675
2677
  await this.unhighlightElements(page);
2676
2678
  const post_action_screenshot_url = await this.screenshot(page);
2677
- const post_html_content = '';
2678
- const executed_instruction = {
2679
- what_to_do: 'perform_action',
2680
- args: {
2681
- action: action,
2682
- value: value,
2683
- element_css_selector: element_css_selector,
2684
- iframe_selector: iframe_selector
2685
- },
2686
- result: {
2687
- pre_action_screenshot_url: pre_action_screenshot_url,
2688
- post_action_screenshot_url: post_action_screenshot_url,
2689
- post_html_content: post_html_content,
2690
- validation_status: true,
2691
- return_value: returnValue
2692
- }
2693
- };
2694
- return executed_instruction;
2679
+ await this.apiClient.patchStep(stepId, {
2680
+ pre_screenshot: pre_action_screenshot_url,
2681
+ post_screenshot: post_action_screenshot_url,
2682
+ post_html_content: null ,
2683
+ });
2684
+ return returnValue;
2695
2685
  }
2696
- async askAIHelper(page, question) {
2697
- proboLogger.debug(`🔍 [askAI] Asking AI question: "${question}", scenarioName: ${this.scenarioName}, aiModel: ${this.aiModel} waitForStabilityQuietTimeout: ${this.timeoutConfig.waitForStabilityQuietTimeout}ms, waitForStabilityInitialDelay: ${this.timeoutConfig.waitForStabilityInitialDelay}ms, waitForStabilityGlobalTimeout: ${this.timeoutConfig.waitForStabilityGlobalTimeout}ms, waitForStabilityVerbose: ${this.timeoutConfig.waitForStabilityVerbose}`);
2686
+ async askAIHelper(page, question, options) {
2687
+ // Set default value for createStep to true if not provided
2688
+ const createStep = (options === null || options === void 0 ? void 0 : options.createStep) !== undefined ? options.createStep : true;
2689
+ const stepId = options === null || options === void 0 ? void 0 : options.stepId;
2690
+ proboLogger.debug(`🔍 [askAI] Asking AI question: "${question}", scenarioName: ${this.scenarioName}, aiModel: ${this.aiModel}, createStep: ${createStep}`);
2698
2691
  try {
2699
2692
  // Get current page and capture screenshot
2700
2693
  const navTracker = await NavTracker.getInstance(page, {
@@ -2710,6 +2703,53 @@ class Probo {
2710
2703
  // Use ApiClient to send request to backend API
2711
2704
  const serverResponse = await this.apiClient.askAI(question, this.scenarioName, screenshot, this.aiModel);
2712
2705
  proboLogger.debug(`✅ [askAI] Chat response received successfully`);
2706
+ // If stepId is provided, update the step
2707
+ if (stepId) {
2708
+ proboLogger.debug(`🔍 [askAI] Looking for step: ${stepId}`);
2709
+ const step = await this.apiClient.findStepById(stepId);
2710
+ if (!step) {
2711
+ proboLogger.error(`Step not found in database`);
2712
+ throw Error(`Step #${stepId} not found in database`);
2713
+ }
2714
+ if ((step === null || step === void 0 ? void 0 : step.prompt) === question) {
2715
+ proboLogger.debug(`✅ [askAI] Step already exists with question: ${question}, skipping update`);
2716
+ return serverResponse;
2717
+ }
2718
+ try {
2719
+ proboLogger.debug(`📝 [askAI] Updating step in backend for question: "${question}"`);
2720
+ // Get HTML content for the step
2721
+ const htmlContent = await page.content();
2722
+ await this.apiClient.patchStep(stepId, {
2723
+ stepPrompt: question,
2724
+ pre_html_content: htmlContent,
2725
+ initial_screenshot: screenshot,
2726
+ url: page.url(),
2727
+ action: PlaywrightAction.ASK_AI,
2728
+ vanilla_prompt: question,
2729
+ });
2730
+ proboLogger.debug(`✅ [askAI] Step updated successfully with ID: ${stepId}`);
2731
+ }
2732
+ catch (stepError) {
2733
+ proboLogger.error(`❌ [askAI] Error updating step: ${stepError}`);
2734
+ // Don't throw here, just log the error and continue
2735
+ }
2736
+ }
2737
+ else if (createStep) {
2738
+ proboLogger.debug(`🔍 [askAI] Creating new step in backend for question: "${question}"`);
2739
+ const stepId = await this.apiClient.createStep({
2740
+ scenarioName: this.scenarioName,
2741
+ stepPrompt: question,
2742
+ initial_screenshot_url: screenshot,
2743
+ argument: '',
2744
+ use_cache: false,
2745
+ url: page.url(),
2746
+ action: PlaywrightAction.ASK_AI,
2747
+ vanilla_prompt: question,
2748
+ is_vanilla_prompt_robust: true,
2749
+ target_element_name: 'AI Question',
2750
+ });
2751
+ proboLogger.debug(`✅ [askAI] Step created successfully with ID: ${stepId}`);
2752
+ }
2713
2753
  // Return the answer from the result, or the reasoning if no answer
2714
2754
  return serverResponse;
2715
2755
  }