@jarvis-agent/core 0.2.0 → 0.2.2

package/dist/agent/browser/build-dom-tree.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"build-dom-tree.d.ts","sourceRoot":"","sources":["../../../src/agent/browser/build-dom-tree.ts"],"names":[],"mappings":"AACA,wBAAgB,kBAAkB,SAoxBjC"}
+{"version":3,"file":"build-dom-tree.d.ts","sourceRoot":"","sources":["../../../src/agent/browser/build-dom-tree.ts"],"names":[],"mappings":"AACA,wBAAgB,kBAAkB,SAksBjC"}

package/dist/index.cjs

@@ -42557,30 +42557,19 @@ function run_build_dom_tree() {
     /**
      * Get clickable elements on the page
      *
-     * @param {*} markHighlightElements Is mark highlighted
+     * @param {*} doHighlightElements Is highlighted
      * @param {*} includeAttributes [attr_names...]
-     * @returns { element_str, client_rect, selector_map, area_map }
+     * @returns { element_str, selector_map }
      */
-    function get_clickable_elements(markHighlightElements = true, includeAttributes) {
+    function get_clickable_elements(doHighlightElements = true, includeAttributes) {
         window.clickable_elements = {};
         computedStyleCache = new WeakMap();
         document.querySelectorAll("[eko-user-highlight-id]").forEach(ele => ele.removeAttribute("eko-user-highlight-id"));
-        let page_tree = build_dom_tree(markHighlightElements);
+        let page_tree = build_dom_tree(doHighlightElements);
         let element_tree = parse_node(page_tree);
+        let selector_map = create_selector_map(element_tree);
         let element_str = clickable_elements_to_string(element_tree, includeAttributes);
-        let client_rect = {
-            width: window.innerWidth || document.documentElement.clientWidth,
-            height: window.innerHeight || document.documentElement.clientHeight,
-        };
-        if (markHighlightElements) {
-            let selector_map = {};
-            // selector_map = create_selector_map(element_tree);
-            return { element_str, client_rect, selector_map };
-        }
-        else {
-            let area_map = create_area_map(element_tree);
-            return { element_str, client_rect, area_map };
-        }
+        return { element_str, selector_map };
     }
     function get_highlight_element(highlightIndex) {
         let element = document.querySelector(`[eko-user-highlight-id="eko-highlight-${highlightIndex}"]`);
@@ -42679,13 +42668,12 @@ function run_build_dom_tree() {
         process_node(element_tree);
         return formatted_text.join('\n');
     }
-    function create_area_map(element_tree) {
-        let area_map = {};
+    function create_selector_map(element_tree) {
+        let selector_map = {};
         function process_node(node) {
             if (node.tagName) {
                 if (node.highlightIndex != null) {
-                    const element = window.clickable_elements[node.highlightIndex];
-                    area_map[node.highlightIndex] = get_element_real_bounding_rect(element);
+                    selector_map[node.highlightIndex] = node;
                 }
                 for (let i = 0; i < node.children.length; i++) {
                     process_node(node.children[i]);
@@ -42693,38 +42681,7 @@ function run_build_dom_tree() {
             }
         }
         process_node(element_tree);
-        return area_map;
-    }
-    function get_element_real_bounding_rect(element) {
-        if (!element || !(element instanceof Element)) {
-            return { x: 0, y: 0, width: 0, height: 0 };
-        }
-        let rect = element.getBoundingClientRect();
-        let x = rect.left;
-        let y = rect.top;
-        let width = rect.width;
-        let height = rect.height;
-        let win = element.ownerDocument.defaultView;
-        let maxDepth = 10;
-        let depth = 0;
-        while (win && win !== win.parent && depth < maxDepth) {
-            depth++;
-            const frameElement = win.frameElement;
-            if (!frameElement) {
-                break;
-            }
-            const frameRect = frameElement.getBoundingClientRect();
-            x += frameRect.left;
-            y += frameRect.top;
-            // Consider the border and padding of the iframe.
-            const frameStyle = getCachedComputedStyle(frameElement);
-            x += parseFloat(frameStyle.borderLeftWidth) || 0;
-            y += parseFloat(frameStyle.borderTopWidth) || 0;
-            x += parseFloat(frameStyle.paddingLeft) || 0;
-            y += parseFloat(frameStyle.paddingTop) || 0;
-            win = win.parent;
-        }
-        return { x, y, width, height };
+        return selector_map;
     }
     function parse_node(node_data, parent) {
         if (!node_data) {
@@ -42764,9 +42721,8 @@ function run_build_dom_tree() {
         }
         return element_node;
     }
-    function build_dom_tree(markHighlightElements) {
+    function build_dom_tree(doHighlightElements) {
         let highlightIndex = 0; // Reset highlight index
-        let duplicates = new Set();
         function highlightElement(element, index, parentIframe = null) {
             // Create or get highlight container
             let container = document.getElementById('eko-highlight-container');
@@ -42965,36 +42921,13 @@ function run_build_dom_tree() {
                 interactiveRoles.has(ariaRole) ||
                 (tabIndex !== null && tabIndex !== '-1') ||
                 element.getAttribute('data-action') === 'a-dropdown-select' ||
-                element.getAttribute('data-action') === 'a-dropdown-button' ||
-                element.getAttribute('contenteditable') === 'true';
+                element.getAttribute('data-action') === 'a-dropdown-button';
             if (hasInteractiveRole)
                 return true;
-            // const eventTypes = [
-            //   'click',
-            //   'mousedown',
-            //   'mouseup',
-            //   'touchstart',
-            //   'touchend',
-            //   'keydown',
-            //   'keyup',
-            //   'focus',
-            //   'blur',
-            // ];
-            const clickEventTypes = [
-                'click',
-                'mousedown',
-                'mouseup',
-                'touchstart',
-                'touchend',
-            ];
-            // Filter elements that have no real event listeners at all
-            if (window.getEventListeners) {
-                const listeners = window.getEventListeners(element);
-                const hasRealClickListeners = clickEventTypes.some((type) => listeners[type]?.length > 0);
-                if (!hasRealClickListeners) {
-                    return false;
-                }
-            }
+            // Get computed style
+            const style = getCachedComputedStyle(element);
+            // Check if element has click-like styling
+            const hasClickStyling = style.cursor === 'pointer' || element.style.cursor === 'pointer';
             // Check for event listeners
             const hasClickHandler = element.onclick !== null ||
                 element.getAttribute('onclick') !== null ||
@@ -43002,10 +42935,24 @@ function run_build_dom_tree() {
                 element.hasAttribute('@click') ||
                 element.hasAttribute('v-on:click');
             // Helper function to safely get event listeners
-            function getElementEventListeners(el) {
+            function getEventListeners(el) {
+                // if (window.getEventListeners) {
+                //   return window.getEventListeners?.(el) || {};
+                // }
                 // List of common event types to check
                 const listeners = {};
-                for (const type of clickEventTypes) {
+                const eventTypes = [
+                    'click',
+                    'mousedown',
+                    'mouseup',
+                    'touchstart',
+                    'touchend',
+                    'keydown',
+                    'keyup',
+                    'focus',
+                    'blur',
+                ];
+                for (const type of eventTypes) {
                     const handler = el[`on${type}`];
                     if (handler) {
                         listeners[type] = [
@@ -43019,34 +42966,30 @@ function run_build_dom_tree() {
                 return listeners;
             }
             // Check for click-related events on the element itself
-            const listeners = getElementEventListeners(element);
-            const hasClickListeners = clickEventTypes.some((type) => listeners[type]?.length > 0);
+            const listeners = getEventListeners(element);
+            const hasClickListeners = listeners &&
+                (listeners.click?.length > 0 ||
+                    listeners.mousedown?.length > 0 ||
+                    listeners.mouseup?.length > 0 ||
+                    listeners.touchstart?.length > 0 ||
+                    listeners.touchend?.length > 0);
             // Check for ARIA properties that suggest interactivity
             const hasAriaProps = element.hasAttribute('aria-expanded') ||
                 element.hasAttribute('aria-pressed') ||
                 element.hasAttribute('aria-selected') ||
                 element.hasAttribute('aria-checked');
+            // Check for form-related functionality
+            element.form !== undefined ||
+                element.hasAttribute('contenteditable') ||
+                (style && style.userSelect !== 'none');
             // Check if element is draggable
             const isDraggable = element.draggable || element.getAttribute('draggable') === 'true';
-            if (hasAriaProps || hasClickHandler || hasClickListeners || isDraggable) {
-                return true;
-            }
-            // Check if element has click-like styling
-            let hasClickStyling = element.style.cursor === 'pointer' || getCachedComputedStyle(element).cursor === 'pointer';
-            if (hasClickStyling) {
-                let count = 0;
-                let current = element.parentElement;
-                while (current && current !== document.documentElement) {
-                    hasClickStyling = current.style.cursor === 'pointer' || getCachedComputedStyle(current).cursor === 'pointer';
-                    if (hasClickStyling)
-                        return false;
-                    current = current.parentElement;
-                    if (++count > 10)
-                        break;
-                }
-                return true;
-            }
-            return false;
+            return (hasAriaProps ||
+                hasClickStyling ||
+                hasClickHandler ||
+                hasClickListeners ||
+                // isFormRelated ||
+                isDraggable);
         }
         // Helper function to check if element is visible
         function isElementVisible(element) {
@@ -43076,14 +43019,11 @@ function run_build_dom_tree() {
                     if (!topEl)
                         return false;
                     // Check if the element or any of its parents match our target element
-                    let count = 0;
                     let current = topEl;
                     while (current && current !== shadowRoot) {
                         if (current === element)
                             return true;
                         current = current.parentElement;
-                        if (++count > 15)
-                            break;
                     }
                     return false;
                 }
@@ -43098,14 +43038,11 @@ function run_build_dom_tree() {
                 const topEl = document.elementFromPoint(point.x, point.y);
                 if (!topEl)
                     return false;
-                let count = 0;
                 let current = topEl;
                 while (current && current !== document.documentElement) {
                     if (current === element)
                         return true;
                     current = current.parentElement;
-                    if (++count > 15)
-                        break;
                 }
                 return false;
             }
@@ -43129,10 +43066,8 @@ function run_build_dom_tree() {
         }
         // Function to traverse the DOM and create nested JSON
         function buildDomTree(node, parentIframe = null) {
-            if (!node || duplicates.has(node)) {
+            if (!node)
                 return null;
-            }
-            duplicates.add(node);
             // Special case for text nodes
             if (node.nodeType === Node.TEXT_NODE) {
                 const textContent = node.textContent.trim();
@@ -43177,7 +43112,7 @@ function run_build_dom_tree() {
                 if (shouldHighlight) {
                     nodeData.highlightIndex = highlightIndex++;
                     window.clickable_elements[nodeData.highlightIndex] = node;
-                    if (markHighlightElements) {
+                    if (doHighlightElements) {
                         highlightElement(node, nodeData.highlightIndex, parentIframe);
                     }
                 }
@@ -43192,7 +43127,7 @@ function run_build_dom_tree() {
             }
             // Handle shadow DOM
             if (node.shadowRoot) {
-                const shadowChildren = Array.from(node.shadowRoot.children).map((child) => buildDomTree(child, parentIframe)).filter(child => child !== null);
+                const shadowChildren = Array.from(node.shadowRoot.childNodes).map((child) => buildDomTree(child, parentIframe));
                 nodeData.children.push(...shadowChildren);
             }
             // Handle iframes
@@ -43200,7 +43135,7 @@ function run_build_dom_tree() {
                 try {
                     const iframeDoc = node.contentDocument || node.contentWindow.document;
                     if (iframeDoc) {
-                        const iframeChildren = Array.from(iframeDoc.body.children).map((child) => buildDomTree(child, node)).filter(child => child !== null);
+                        const iframeChildren = Array.from(iframeDoc.body.childNodes).map((child) => buildDomTree(child, node));
                         nodeData.children.push(...iframeChildren);
                     }
                 }
@@ -43209,9 +43144,8 @@ function run_build_dom_tree() {
                 }
             }
             else {
-                const style = getCachedComputedStyle(node);
-                if (style && style.display !== 'none') {
-                    const children = Array.from(node.children).map((child) => buildDomTree(child, parentIframe)).filter(child => child !== null);
+                if (nodeData.isVisible != false) {
+                    const children = Array.from(node.childNodes).map((child) => buildDomTree(child, parentIframe));
                     nodeData.children.push(...children);
                 }
             }