@eko-ai/eko 3.1.2 → 3.1.3

package/dist/agent/browser/build_dom_tree.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"build_dom_tree.d.ts","sourceRoot":"","sources":["../../../src/agent/browser/build_dom_tree.ts"],"names":[],"mappings":"AACA,wBAAgB,kBAAkB,SAywBjC"}
+{"version":3,"file":"build_dom_tree.d.ts","sourceRoot":"","sources":["../../../src/agent/browser/build_dom_tree.ts"],"names":[],"mappings":"AACA,wBAAgB,kBAAkB,SAuxBjC"}

package/dist/agent/browser/utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browser/utils.ts"],"names":[],"mappings":"AAAA,wBAAgB,oBAAoB,CAClC,cAAc,SAAM,EACpB,kBAAkB,SAAQ,UAoH3B;AAED,wBAAgB,kCAAkC,CAChD,UAAU,EAAE;IACV,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,YAAY,GAAG,WAAW,CAAC;CACvC,EACD,QAAQ,EAAE,MAAM,CACd,MAAM,EACN;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CACxD,EACD,WAAW,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAC7C,OAAO,CAAC,MAAM,CAAC,CAyGjB;AAED,wBAAsB,cAAc,CAClC,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,YAAY,GAAG,WAAW,EACrC,QAAQ,EACJ;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,GACjB;IACE,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACtB,EACL,OAAO,GAAE,MAAU,GAClB,OAAO,CAAC;IACT,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,YAAY,GAAG,WAAW,CAAC;CACvC,CAAC,CA0CD"}
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browser/utils.ts"],"names":[],"mappings":"AAAA,wBAAgB,oBAAoB,CAClC,cAAc,SAAM,EACpB,kBAAkB,SAAQ,UAoH3B;AAED,wBAAgB,kCAAkC,CAChD,UAAU,EAAE;IACV,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,YAAY,GAAG,WAAW,CAAC;CACvC,EACD,QAAQ,EAAE,MAAM,CACd,MAAM,EACN;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CACxD,EACD,WAAW,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAC7C,OAAO,CAAC,MAAM,CAAC,CA+GjB;AAED,wBAAsB,cAAc,CAClC,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,YAAY,GAAG,WAAW,EACrC,QAAQ,EACJ;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,GACjB;IACE,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACtB,EACL,OAAO,GAAE,MAAU,GAClB,OAAO,CAAC;IACT,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,YAAY,GAAG,WAAW,CAAC;CACvC,CAAC,CA0CD"}

package/dist/index.cjs.js

@@ -19,7 +19,7 @@ const config$1 = {
     maxDialogueImgFileNum: 1,
     toolResultMultimodal: true,
     parallelToolCalls: true,
-    markImageMode: "dom",
+    markImageMode: "draw",
     expertMode: false,
     expertModeTodoLoopNum: 10,
 };
@@ -30777,6 +30777,15 @@ class RetryLanguageModel {
                 compatibility: llm.config?.compatibility,
             }).languageModel(llm.model);
         }
+        else if (llm.provider == "modelscope") {
+            return createOpenAICompatible({
+                name: llm.config?.name || llm.model.split("/")[0],
+                apiKey: apiKey,
+                baseURL: baseURL || "https://api-inference.modelscope.cn/v1",
+                fetch: llm.fetch,
+                headers: llm.config?.headers,
+            }).languageModel(llm.model);
+        }
         else {
             return llm.provider.languageModel(llm.model);
         }
@@ -35152,7 +35161,7 @@ function mark_screenshot_highlight_elements(screenshot, area_map, client_rect) {
                 .sort((a, b) => {
                 const areaA = a[1].width * a[1].height;
                 const areaB = b[1].width * b[1].height;
-                return areaA - areaB;
+                return areaB - areaA;
             });
             const colors = [
                 "#FF0000",
@@ -35170,6 +35179,11 @@ function mark_screenshot_highlight_elements(screenshot, area_map, client_rect) {
             ];
             sortedEntries.forEach(([id, area], index) => {
                 const color = colors[index % colors.length];
+                if (area.width * area.height < 40000) {
+                    // Draw a background color
+                    ctx.fillStyle = color + "1A";
+                    ctx.fillRect(area.x, area.y, area.width, area.height);
+                }
                 // Draw a border
                 ctx.strokeStyle = color;
                 ctx.lineWidth = 2;
@@ -35647,6 +35661,7 @@ function run_build_dom_tree() {
     }
     function build_dom_tree(markHighlightElements) {
         let highlightIndex = 0; // Reset highlight index
+        let duplicates = new Set();
         function highlightElement(element, index, parentIframe = null) {
             // Create or get highlight container
             let container = document.getElementById('eko-highlight-container');
@@ -35849,6 +35864,32 @@ function run_build_dom_tree() {
                 element.getAttribute('contenteditable') === 'true';
             if (hasInteractiveRole)
                 return true;
+            // const eventTypes = [
+            //   'click',
+            //   'mousedown',
+            //   'mouseup',
+            //   'touchstart',
+            //   'touchend',
+            //   'keydown',
+            //   'keyup',
+            //   'focus',
+            //   'blur',
+            // ];
+            const clickEventTypes = [
+                'click',
+                'mousedown',
+                'mouseup',
+                'touchstart',
+                'touchend',
+            ];
+            // Filter elements that have no real event listeners at all
+            if (window.getEventListeners) {
+                const listeners = window.getEventListeners(element);
+                const hasRealClickListeners = clickEventTypes.some((type) => listeners[type]?.length > 0);
+                if (!hasRealClickListeners) {
+                    return false;
+                }
+            }
             // Check for event listeners
             const hasClickHandler = element.onclick !== null ||
                 element.getAttribute('onclick') !== null ||
@@ -35857,26 +35898,9 @@ function run_build_dom_tree() {
                 element.hasAttribute('v-on:click');
             // Helper function to safely get event listeners
             function getElementEventListeners(el) {
-                if (window.getEventListeners) {
-                    const listeners = window.getEventListeners?.(el);
-                    if (listeners) {
-                        return listeners;
-                    }
-                }
                 // List of common event types to check
                 const listeners = {};
-                const eventTypes = [
-                    'click',
-                    'mousedown',
-                    'mouseup',
-                    'touchstart',
-                    'touchend',
-                    'keydown',
-                    'keyup',
-                    'focus',
-                    'blur',
-                ];
-                for (const type of eventTypes) {
+                for (const type of clickEventTypes) {
                     const handler = el[`on${type}`];
                     if (handler) {
                         listeners[type] = [
@@ -35891,12 +35915,7 @@ function run_build_dom_tree() {
             }
             // Check for click-related events on the element itself
             const listeners = getElementEventListeners(element);
-            const hasClickListeners = listeners &&
-                (listeners.click?.length > 0 ||
-                    listeners.mousedown?.length > 0 ||
-                    listeners.mouseup?.length > 0 ||
-                    listeners.touchstart?.length > 0 ||
-                    listeners.touchend?.length > 0);
+            const hasClickListeners = clickEventTypes.some((type) => listeners[type]?.length > 0);
             // Check for ARIA properties that suggest interactivity
             const hasAriaProps = element.hasAttribute('aria-expanded') ||
                 element.hasAttribute('aria-pressed') ||
@@ -35924,14 +35943,18 @@ function run_build_dom_tree() {
             }
             return false;
         }
+        // Helper function to check if element exists
+        function isElementExist(element) {
+            const style = getCachedComputedStyle(element);
+            return (style?.visibility !== 'hidden' &&
+                style?.display !== 'none');
+        }
         // Helper function to check if element is visible
         function isElementVisible(element) {
             if (element.offsetWidth === 0 && element.offsetHeight === 0) {
                 return false;
             }
-            const style = getCachedComputedStyle(element);
-            return (style?.visibility !== 'hidden' &&
-                style?.display !== 'none');
+            return isElementExist(element);
         }
         // Helper function to check if element is the top element at its position
         function isTopElement(element) {
@@ -36005,8 +36028,10 @@ function run_build_dom_tree() {
         }
         // Function to traverse the DOM and create nested JSON
         function buildDomTree(node, parentIframe = null) {
-            if (!node)
+            if (!node || duplicates.has(node)) {
                 return null;
+            }
+            duplicates.add(node);
             // Special case for text nodes
             if (node.nodeType === Node.TEXT_NODE) {
                 const textContent = node.textContent.trim();
@@ -36066,7 +36091,7 @@ function run_build_dom_tree() {
             }
             // Handle shadow DOM
             if (node.shadowRoot) {
-                const shadowChildren = Array.from(node.shadowRoot.childNodes).map((child) => buildDomTree(child, parentIframe));
+                const shadowChildren = Array.from(node.shadowRoot.children).map((child) => buildDomTree(child, parentIframe)).filter(child => child !== null);
                 nodeData.children.push(...shadowChildren);
             }
             // Handle iframes
@@ -36074,7 +36099,7 @@ function run_build_dom_tree() {
                 try {
                     const iframeDoc = node.contentDocument || node.contentWindow.document;
                     if (iframeDoc) {
-                        const iframeChildren = Array.from(iframeDoc.body.childNodes).map((child) => buildDomTree(child, node));
+                        const iframeChildren = Array.from(iframeDoc.body.children).map((child) => buildDomTree(child, node)).filter(child => child !== null);
                         nodeData.children.push(...iframeChildren);
                     }
                 }
@@ -36083,8 +36108,8 @@ function run_build_dom_tree() {
                 }
             }
             else {
-                if (nodeData.isVisible != false) {
-                    const children = Array.from(node.childNodes).map((child) => buildDomTree(child, parentIframe));
+                if (isElementExist(node)) {
+                    const children = Array.from(node.children).map((child) => buildDomTree(child, parentIframe)).filter(child => child !== null);
                     nodeData.children.push(...children);
                 }
             }
@@ -36527,7 +36552,7 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
         ];
     }
     async double_screenshots(agentContext, messages, tools) {
-        return config$1.mode != "fast";
+        return config$1.mode == "expert";
     }
     async handleMessages(agentContext, messages, tools) {
         const pseudoHtmlDescription = "This is the environmental information after the operation, including the latest browser screenshot and page elements. Please perform the next operation based on the environmental information. Do not output the following elements and index information in your response.\n\nIndex and elements:\n";