@eko-ai/eko 3.1.1 → 3.1.2-alpha.2

package/dist/index.esm.js

@@ -13,7 +13,7 @@ const config$1 = {
     maxDialogueImgFileNum: 1,
     toolResultMultimodal: true,
     parallelToolCalls: true,
-    markImageMode: "dom",
+    markImageMode: "draw",
     expertMode: false,
     expertModeTodoLoopNum: 10,
 };
@@ -35117,7 +35117,7 @@ function mark_screenshot_highlight_elements(screenshot, area_map, client_rect) {
                 .sort((a, b) => {
                 const areaA = a[1].width * a[1].height;
                 const areaB = b[1].width * b[1].height;
-                return areaA - areaB;
+                return areaB - areaA;
             });
             const colors = [
                 "#FF0000",
@@ -35135,6 +35135,11 @@ function mark_screenshot_highlight_elements(screenshot, area_map, client_rect) {
             ];
             sortedEntries.forEach(([id, area], index) => {
                 const color = colors[index % colors.length];
+                if (area.width * area.height < 40000) {
+                    // Draw a background color
+                    ctx.fillStyle = color + "1A";
+                    ctx.fillRect(area.x, area.y, area.width, area.height);
+                }
                 // Draw a border
                 ctx.strokeStyle = color;
                 ctx.lineWidth = 2;
@@ -35148,7 +35153,12 @@ function mark_screenshot_highlight_elements(screenshot, area_map, client_rect) {
                 const labelHeight = fontSize + padding * 2;
                 // The tag position is in the upper right corner.
                 const labelX = area.x + area.width - labelWidth;
-                const labelY = area.y;
+                let labelY = area.y;
+                // Adjust if box is too small
+                if (area.width < labelWidth + 4 || area.height < labelHeight + 4) {
+                    // Position outside the box if it's too small
+                    labelY = area.y - labelHeight;
+                }
                 // Draw label background
                 ctx.fillStyle = color;
                 ctx.fillRect(labelX, labelY, labelWidth, labelHeight);
@@ -35805,13 +35815,36 @@ function run_build_dom_tree() {
                 interactiveRoles.has(ariaRole) ||
                 (tabIndex !== null && tabIndex !== '-1') ||
                 element.getAttribute('data-action') === 'a-dropdown-select' ||
-                element.getAttribute('data-action') === 'a-dropdown-button';
+                element.getAttribute('data-action') === 'a-dropdown-button' ||
+                element.getAttribute('contenteditable') === 'true';
             if (hasInteractiveRole)
                 return true;
-            // Get computed style
-            const style = getCachedComputedStyle(element);
-            // Check if element has click-like styling
-            const hasClickStyling = style.cursor === 'pointer' || element.style.cursor === 'pointer';
+            // const eventTypes = [
+            //   'click',
+            //   'mousedown',
+            //   'mouseup',
+            //   'touchstart',
+            //   'touchend',
+            //   'keydown',
+            //   'keyup',
+            //   'focus',
+            //   'blur',
+            // ];
+            const clickEventTypes = [
+                'click',
+                'mousedown',
+                'mouseup',
+                'touchstart',
+                'touchend',
+            ];
+            // Filter elements that have no real event listeners at all
+            if (window.getEventListeners) {
+                const listeners = window.getEventListeners(element);
+                const hasRealClickListeners = clickEventTypes.some((type) => listeners[type]?.length > 0);
+                if (!hasRealClickListeners) {
+                    return false;
+                }
+            }
             // Check for event listeners
             const hasClickHandler = element.onclick !== null ||
                 element.getAttribute('onclick') !== null ||
@@ -35819,24 +35852,10 @@ function run_build_dom_tree() {
                 element.hasAttribute('@click') ||
                 element.hasAttribute('v-on:click');
             // Helper function to safely get event listeners
-            function getEventListeners(el) {
-                // if (window.getEventListeners) {
-                //   return window.getEventListeners?.(el) || {};
-                // }
+            function getElementEventListeners(el) {
                 // List of common event types to check
                 const listeners = {};
-                const eventTypes = [
-                    'click',
-                    'mousedown',
-                    'mouseup',
-                    'touchstart',
-                    'touchend',
-                    'keydown',
-                    'keyup',
-                    'focus',
-                    'blur',
-                ];
-                for (const type of eventTypes) {
+                for (const type of clickEventTypes) {
                     const handler = el[`on${type}`];
                     if (handler) {
                         listeners[type] = [
@@ -35850,39 +35869,47 @@ function run_build_dom_tree() {
                 return listeners;
             }
             // Check for click-related events on the element itself
-            const listeners = getEventListeners(element);
-            const hasClickListeners = listeners &&
-                (listeners.click?.length > 0 ||
-                    listeners.mousedown?.length > 0 ||
-                    listeners.mouseup?.length > 0 ||
-                    listeners.touchstart?.length > 0 ||
-                    listeners.touchend?.length > 0);
+            const listeners = getElementEventListeners(element);
+            const hasClickListeners = clickEventTypes.some((type) => listeners[type]?.length > 0);
             // Check for ARIA properties that suggest interactivity
             const hasAriaProps = element.hasAttribute('aria-expanded') ||
                 element.hasAttribute('aria-pressed') ||
                 element.hasAttribute('aria-selected') ||
                 element.hasAttribute('aria-checked');
-            // Check for form-related functionality
-            element.form !== undefined ||
-                element.hasAttribute('contenteditable') ||
-                (style && style.userSelect !== 'none');
             // Check if element is draggable
             const isDraggable = element.draggable || element.getAttribute('draggable') === 'true';
-            return (hasAriaProps ||
-                hasClickStyling ||
-                hasClickHandler ||
-                hasClickListeners ||
-                // isFormRelated ||
-                isDraggable);
+            if (hasAriaProps || hasClickHandler || hasClickListeners || isDraggable) {
+                return true;
+            }
+            // Check if element has click-like styling
+            let hasClickStyling = element.style.cursor === 'pointer' || getCachedComputedStyle(element).cursor === 'pointer';
+            if (hasClickStyling) {
+                let count = 0;
+                let current = element.parentElement;
+                while (current && current !== document.documentElement) {
+                    hasClickStyling = current.style.cursor === 'pointer' || getCachedComputedStyle(current).cursor === 'pointer';
+                    if (hasClickStyling)
+                        return false;
+                    current = current.parentElement;
+                    if (++count > 10)
+                        break;
+                }
+                return true;
+            }
+            return false;
+        }
+        // Helper function to check if element exists
+        function isElementExist(element) {
+            const style = getCachedComputedStyle(element);
+            return (style?.visibility !== 'hidden' &&
+                style?.display !== 'none');
         }
         // Helper function to check if element is visible
         function isElementVisible(element) {
             if (element.offsetWidth === 0 && element.offsetHeight === 0) {
                 return false;
             }
-            const style = getCachedComputedStyle(element);
-            return (style?.visibility !== 'hidden' &&
-                style?.display !== 'none');
+            return isElementExist(element);
         }
         // Helper function to check if element is the top element at its position
         function isTopElement(element) {
@@ -35903,11 +35930,14 @@ function run_build_dom_tree() {
                     if (!topEl)
                         return false;
                     // Check if the element or any of its parents match our target element
+                    let count = 0;
                     let current = topEl;
                     while (current && current !== shadowRoot) {
                         if (current === element)
                             return true;
                         current = current.parentElement;
+                        if (++count > 15)
+                            break;
                     }
                     return false;
                 }
@@ -35922,11 +35952,14 @@ function run_build_dom_tree() {
                 const topEl = document.elementFromPoint(point.x, point.y);
                 if (!topEl)
                     return false;
+                let count = 0;
                 let current = topEl;
                 while (current && current !== document.documentElement) {
                     if (current === element)
                         return true;
                     current = current.parentElement;
+                    if (++count > 15)
+                        break;
                 }
                 return false;
             }
@@ -36028,7 +36061,7 @@ function run_build_dom_tree() {
                 }
             }
             else {
-                if (nodeData.isVisible != false) {
+                if (isElementExist(node)) {
                     const children = Array.from(node.childNodes).map((child) => buildDomTree(child, parentIframe));
                     nodeData.children.push(...children);
                 }
@@ -36472,7 +36505,7 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
         ];
     }
     async double_screenshots(agentContext, messages, tools) {
-        return config$1.mode != "fast";
+        return config$1.mode == "expert";
     }
     async handleMessages(agentContext, messages, tools) {
         const pseudoHtmlDescription = "This is the environmental information after the operation, including the latest browser screenshot and page elements. Please perform the next operation based on the environmental information. Do not output the following elements and index information in your response.\n\nIndex and elements:\n";