@eko-ai/eko 3.1.0 → 3.1.2

package/dist/index.esm.js

@@ -1,5 +1,6 @@
 const config$1 = {
     name: "Eko",
+    mode: "normal",
     platform: "mac",
     maxReactNum: 500,
     maxTokens: 16000,
@@ -12,6 +13,7 @@ const config$1 = {
     maxDialogueImgFileNum: 1,
     toolResultMultimodal: true,
     parallelToolCalls: true,
+    markImageMode: "dom",
     expertMode: false,
     expertModeTodoLoopNum: 10,
 };
@@ -31937,7 +31939,7 @@ class Eko {
                 results.push(agent_results.join("\n\n"));
             }
             context.conversation.splice(0, context.conversation.length);
-            if (config$1.expertMode &&
+            if ((config$1.mode == "expert" || config$1.expertMode) &&
                 !workflow.modified &&
                 agentTree.nextAgent &&
                 lastAgent?.AgentContext &&
@@ -34184,12 +34186,12 @@ class Agent {
             const finalResult = await this.handleCallResult(agentContext, messages, agentTools, results);
             loopNum++;
             if (!finalResult) {
-                if (config$1.expertMode && loopNum % config$1.expertModeTodoLoopNum == 0) {
+                if ((config$1.mode == "expert" || config$1.expertMode) && loopNum % config$1.expertModeTodoLoopNum == 0) {
                     await doTodoListManager(agentContext, rlm, messages, llm_tools);
                 }
                 continue;
             }
-            if (config$1.expertMode && checkNum == 0) {
+            if ((config$1.mode == "expert" || config$1.expertMode) && checkNum == 0) {
                 checkNum++;
                 const { completionStatus } = await doTaskResultCheck(agentContext, rlm, messages, llm_tools);
                 if (completionStatus == "incomplete") {
@@ -35081,10 +35083,145 @@ function extract_page_content(max_url_length = 200, max_content_length = 50000)
     result = result.replace(/\s*\n/g, "\n").replace(/\n+/g, "\n").trim();
     if (result.length > max_content_length) {
         // result = result.slice(0, max_content_length) + "...";
-        result = Array.from(result).slice(0, max_content_length).join('') + "...";
+        result = Array.from(result).slice(0, max_content_length).join("") + "...";
     }
     return result;
 }
+function mark_screenshot_highlight_elements(screenshot, area_map, client_rect) {
+    return new Promise(async (resolve, reject) => {
+        try {
+            // Convert base64 to Blob
+            const base64Data = screenshot.imageBase64;
+            const binaryString = atob(base64Data);
+            const bytes = new Uint8Array(binaryString.length);
+            for (let i = 0; i < binaryString.length; i++) {
+                bytes[i] = binaryString.charCodeAt(i);
+            }
+            const blob = new Blob([bytes], { type: screenshot.imageType });
+            const imageBitmap = await createImageBitmap(blob, {
+                resizeQuality: "high",
+                resizeWidth: client_rect.width,
+                resizeHeight: client_rect.height,
+            });
+            const canvas = new OffscreenCanvas(imageBitmap.width, imageBitmap.height);
+            const ctx = canvas.getContext("2d");
+            if (!ctx) {
+                reject(new Error("Failed to get canvas context"));
+                return;
+            }
+            ctx.imageSmoothingEnabled = true;
+            ctx.imageSmoothingQuality = "high";
+            ctx.drawImage(imageBitmap, 0, 0);
+            const sortedEntries = Object.entries(area_map)
+                .filter(([id, area]) => area.width > 0 && area.height > 0)
+                .sort((a, b) => {
+                const areaA = a[1].width * a[1].height;
+                const areaB = b[1].width * b[1].height;
+                return areaA - areaB;
+            });
+            const colors = [
+                "#FF0000",
+                "#00FF00",
+                "#0000FF",
+                "#FFA500",
+                "#800080",
+                "#008080",
+                "#FF69B4",
+                "#4B0082",
+                "#FF4500",
+                "#2E8B57",
+                "#DC143C",
+                "#4682B4",
+            ];
+            sortedEntries.forEach(([id, area], index) => {
+                const color = colors[index % colors.length];
+                // Draw a border
+                ctx.strokeStyle = color;
+                ctx.lineWidth = 2;
+                ctx.strokeRect(area.x, area.y, area.width, area.height);
+                // Draw ID tag background
+                const fontSize = Math.min(12, Math.max(8, area.height / 2));
+                ctx.font = `${fontSize}px sans-serif`;
+                const textMetrics = ctx.measureText(id);
+                const padding = 4;
+                const labelWidth = textMetrics.width + padding * 2;
+                const labelHeight = fontSize + padding * 2;
+                // The tag position is in the upper right corner.
+                const labelX = area.x + area.width - labelWidth;
+                let labelY = area.y;
+                // Adjust if box is too small
+                if (area.width < labelWidth + 4 || area.height < labelHeight + 4) {
+                    // Position outside the box if it's too small
+                    labelY = area.y - labelHeight;
+                }
+                // Draw label background
+                ctx.fillStyle = color;
+                ctx.fillRect(labelX, labelY, labelWidth, labelHeight);
+                // Draw ID text
+                ctx.fillStyle = "#FFFFFF";
+                ctx.textBaseline = "top";
+                ctx.fillText(id, labelX + padding, labelY + padding);
+            });
+            // Convert OffscreenCanvas to Blob, then to base64
+            const resultBlob = await canvas.convertToBlob({
+                type: screenshot.imageType,
+            });
+            const reader = new FileReader();
+            reader.onloadend = () => {
+                const resultBase64 = reader.result;
+                resolve(resultBase64);
+            };
+            reader.onerror = () => {
+                reject(new Error("Failed to convert blob to base64"));
+            };
+            reader.readAsDataURL(resultBlob);
+        }
+        catch (error) {
+            reject(error);
+        }
+    });
+}
+async function compress_image(imageBase64, imageType, compress, quality = 1) {
+    const base64Data = imageBase64;
+    const binaryString = atob(base64Data);
+    const bytes = new Uint8Array(binaryString.length);
+    for (let i = 0; i < binaryString.length; i++) {
+        bytes[i] = binaryString.charCodeAt(i);
+    }
+    const blob = new Blob([bytes], { type: imageType });
+    const bitmap = await createImageBitmap(blob);
+    const width = compress.scale
+        ? bitmap.width * compress.scale
+        : compress.resizeWidth;
+    const height = compress.scale
+        ? bitmap.height * compress.scale
+        : compress.resizeHeight;
+    if (bitmap.width == width && bitmap.height == height && quality == 1) {
+        return {
+            imageBase64: imageBase64,
+            imageType: imageType,
+        };
+    }
+    const canvas = new OffscreenCanvas(width, height);
+    const ctx = canvas.getContext("2d");
+    ctx.drawImage(bitmap, 0, 0, width, height);
+    const resultBlob = await canvas.convertToBlob({
+        type: "image/jpeg",
+        quality: quality,
+    });
+    return new Promise((resolve) => {
+        const reader = new FileReader();
+        reader.onloadend = () => {
+            let imageDataUrl = reader.result;
+            let imageBase64 = imageDataUrl.substring(imageDataUrl.indexOf("base64,") + 7);
+            resolve({
+                imageBase64: imageBase64,
+                imageType: "image/jpeg",
+            });
+        };
+        reader.readAsDataURL(resultBlob);
+    });
+}
 
 const AGENT_NAME = "Browser";
 class BaseBrowserAgent extends Agent {
@@ -35266,19 +35403,30 @@ function run_build_dom_tree() {
     /**
      * Get clickable elements on the page
      *
-     * @param {*} doHighlightElements Is highlighted
+     * @param {*} markHighlightElements Is mark highlighted
      * @param {*} includeAttributes [attr_names...]
-     * @returns { element_str, selector_map }
+     * @returns { element_str, client_rect, selector_map, area_map }
      */
-    function get_clickable_elements(doHighlightElements = true, includeAttributes) {
+    function get_clickable_elements(markHighlightElements = true, includeAttributes) {
         window.clickable_elements = {};
         computedStyleCache = new WeakMap();
         document.querySelectorAll("[eko-user-highlight-id]").forEach(ele => ele.removeAttribute("eko-user-highlight-id"));
-        let page_tree = build_dom_tree(doHighlightElements);
+        let page_tree = build_dom_tree(markHighlightElements);
         let element_tree = parse_node(page_tree);
-        let selector_map = create_selector_map(element_tree);
         let element_str = clickable_elements_to_string(element_tree, includeAttributes);
-        return { element_str, selector_map };
+        let client_rect = {
+            width: window.innerWidth || document.documentElement.clientWidth,
+            height: window.innerHeight || document.documentElement.clientHeight,
+        };
+        if (markHighlightElements) {
+            let selector_map = {};
+            // selector_map = create_selector_map(element_tree);
+            return { element_str, client_rect, selector_map };
+        }
+        else {
+            let area_map = create_area_map(element_tree);
+            return { element_str, client_rect, area_map };
+        }
     }
     function get_highlight_element(highlightIndex) {
         let element = document.querySelector(`[eko-user-highlight-id="eko-highlight-${highlightIndex}"]`);
@@ -35377,12 +35525,13 @@ function run_build_dom_tree() {
         process_node(element_tree);
         return formatted_text.join('\n');
     }
-    function create_selector_map(element_tree) {
-        let selector_map = {};
+    function create_area_map(element_tree) {
+        let area_map = {};
         function process_node(node) {
             if (node.tagName) {
                 if (node.highlightIndex != null) {
-                    selector_map[node.highlightIndex] = node;
+                    const element = window.clickable_elements[node.highlightIndex];
+                    area_map[node.highlightIndex] = get_element_real_bounding_rect(element);
                 }
                 for (let i = 0; i < node.children.length; i++) {
                     process_node(node.children[i]);
@@ -35390,7 +35539,38 @@ function run_build_dom_tree() {
             }
         }
         process_node(element_tree);
-        return selector_map;
+        return area_map;
+    }
+    function get_element_real_bounding_rect(element) {
+        if (!element || !(element instanceof Element)) {
+            return { x: 0, y: 0, width: 0, height: 0 };
+        }
+        let rect = element.getBoundingClientRect();
+        let x = rect.left;
+        let y = rect.top;
+        let width = rect.width;
+        let height = rect.height;
+        let win = element.ownerDocument.defaultView;
+        let maxDepth = 10;
+        let depth = 0;
+        while (win && win !== win.parent && depth < maxDepth) {
+            depth++;
+            const frameElement = win.frameElement;
+            if (!frameElement) {
+                break;
+            }
+            const frameRect = frameElement.getBoundingClientRect();
+            x += frameRect.left;
+            y += frameRect.top;
+            // Consider the border and padding of the iframe.
+            const frameStyle = getCachedComputedStyle(frameElement);
+            x += parseFloat(frameStyle.borderLeftWidth) || 0;
+            y += parseFloat(frameStyle.borderTopWidth) || 0;
+            x += parseFloat(frameStyle.paddingLeft) || 0;
+            y += parseFloat(frameStyle.paddingTop) || 0;
+            win = win.parent;
+        }
+        return { x, y, width, height };
     }
     function parse_node(node_data, parent) {
         if (!node_data) {
@@ -35430,7 +35610,7 @@ function run_build_dom_tree() {
         }
         return element_node;
     }
-    function build_dom_tree(doHighlightElements) {
+    function build_dom_tree(markHighlightElements) {
         let highlightIndex = 0; // Reset highlight index
         function highlightElement(element, index, parentIframe = null) {
             // Create or get highlight container
@@ -35630,13 +35810,10 @@ function run_build_dom_tree() {
                 interactiveRoles.has(ariaRole) ||
                 (tabIndex !== null && tabIndex !== '-1') ||
                 element.getAttribute('data-action') === 'a-dropdown-select' ||
-                element.getAttribute('data-action') === 'a-dropdown-button';
+                element.getAttribute('data-action') === 'a-dropdown-button' ||
+                element.getAttribute('contenteditable') === 'true';
             if (hasInteractiveRole)
                 return true;
-            // Get computed style
-            const style = getCachedComputedStyle(element);
-            // Check if element has click-like styling
-            const hasClickStyling = style.cursor === 'pointer' || element.style.cursor === 'pointer';
             // Check for event listeners
             const hasClickHandler = element.onclick !== null ||
                 element.getAttribute('onclick') !== null ||
@@ -35644,10 +35821,13 @@ function run_build_dom_tree() {
                 element.hasAttribute('@click') ||
                 element.hasAttribute('v-on:click');
             // Helper function to safely get event listeners
-            function getEventListeners(el) {
-                // if (window.getEventListeners) {
-                //   return window.getEventListeners?.(el) || {};
-                // }
+            function getElementEventListeners(el) {
+                if (window.getEventListeners) {
+                    const listeners = window.getEventListeners?.(el);
+                    if (listeners) {
+                        return listeners;
+                    }
+                }
                 // List of common event types to check
                 const listeners = {};
                 const eventTypes = [
@@ -35675,7 +35855,7 @@ function run_build_dom_tree() {
                 return listeners;
             }
             // Check for click-related events on the element itself
-            const listeners = getEventListeners(element);
+            const listeners = getElementEventListeners(element);
             const hasClickListeners = listeners &&
                 (listeners.click?.length > 0 ||
                     listeners.mousedown?.length > 0 ||
@@ -35687,18 +35867,27 @@ function run_build_dom_tree() {
                 element.hasAttribute('aria-pressed') ||
                 element.hasAttribute('aria-selected') ||
                 element.hasAttribute('aria-checked');
-            // Check for form-related functionality
-            element.form !== undefined ||
-                element.hasAttribute('contenteditable') ||
-                (style && style.userSelect !== 'none');
             // Check if element is draggable
             const isDraggable = element.draggable || element.getAttribute('draggable') === 'true';
-            return (hasAriaProps ||
-                hasClickStyling ||
-                hasClickHandler ||
-                hasClickListeners ||
-                // isFormRelated ||
-                isDraggable);
+            if (hasAriaProps || hasClickHandler || hasClickListeners || isDraggable) {
+                return true;
+            }
+            // Check if element has click-like styling
+            let hasClickStyling = element.style.cursor === 'pointer' || getCachedComputedStyle(element).cursor === 'pointer';
+            if (hasClickStyling) {
+                let count = 0;
+                let current = element.parentElement;
+                while (current && current !== document.documentElement) {
+                    hasClickStyling = current.style.cursor === 'pointer' || getCachedComputedStyle(current).cursor === 'pointer';
+                    if (hasClickStyling)
+                        return false;
+                    current = current.parentElement;
+                    if (++count > 10)
+                        break;
+                }
+                return true;
+            }
+            return false;
         }
         // Helper function to check if element is visible
         function isElementVisible(element) {
@@ -35728,11 +35917,14 @@ function run_build_dom_tree() {
                     if (!topEl)
                         return false;
                     // Check if the element or any of its parents match our target element
+                    let count = 0;
                     let current = topEl;
                     while (current && current !== shadowRoot) {
                         if (current === element)
                             return true;
                         current = current.parentElement;
+                        if (++count > 15)
+                            break;
                     }
                     return false;
                 }
@@ -35747,11 +35939,14 @@ function run_build_dom_tree() {
                 const topEl = document.elementFromPoint(point.x, point.y);
                 if (!topEl)
                     return false;
+                let count = 0;
                 let current = topEl;
                 while (current && current !== document.documentElement) {
                     if (current === element)
                         return true;
                     current = current.parentElement;
+                    if (++count > 15)
+                        break;
                 }
                 return false;
             }
@@ -35821,7 +36016,7 @@ function run_build_dom_tree() {
                 if (shouldHighlight) {
                     nodeData.highlightIndex = highlightIndex++;
                     window.clickable_elements[nodeData.highlightIndex] = node;
-                    if (doHighlightElements) {
+                    if (markHighlightElements) {
                         highlightElement(node, nodeData.highlightIndex, parentIframe);
                     }
                 }
@@ -35982,26 +36177,37 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
     }
     async screenshot_and_html(agentContext) {
         try {
-            let element_result = null;
+            let element_result;
+            let double_screenshots;
             for (let i = 0; i < 5; i++) {
                 await sleep(200);
                 await this.execute_script(agentContext, run_build_dom_tree, []);
                 await sleep(50);
-                element_result = (await this.execute_script(agentContext, () => {
-                    return window.get_clickable_elements(true);
-                }, []));
+                element_result = (await this.execute_script(agentContext, (markHighlightElements) => {
+                    return window.get_clickable_elements(markHighlightElements);
+                }, [config$1.mode != "fast" && config$1.markImageMode == "dom"]));
                 if (element_result) {
                     break;
                 }
             }
             await sleep(100);
-            let screenshot = await this.screenshot(agentContext);
-            // agentContext.variables.set("selector_map", element_result.selector_map);
-            let pseudoHtml = element_result?.element_str || "";
+            const screenshot = config$1.mode == "fast"
+                ? undefined
+                : await this.screenshot_and_compress(agentContext, element_result.client_rect);
+            if (config$1.markImageMode == "draw" &&
+                screenshot?.imageBase64 &&
+                element_result.area_map) {
+                double_screenshots = { ...screenshot };
+                const markImageBase64 = await mark_screenshot_highlight_elements(screenshot, element_result.area_map, element_result.client_rect);
+                screenshot.imageBase64 = markImageBase64;
+            }
+            const pseudoHtml = element_result.element_str || "";
             return {
-                imageBase64: screenshot.imageBase64,
-                imageType: screenshot.imageType,
+                double_screenshots: double_screenshots,
+                imageBase64: screenshot?.imageBase64,
+                imageType: screenshot?.imageType,
                 pseudoHtml: pseudoHtml,
+                client_rect: element_result.client_rect,
             };
         }
         finally {
@@ -36013,6 +36219,20 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
             catch (e) { }
         }
     }
+    async screenshot_and_compress(agentContext, client_rect) {
+        const screenshot = await this.screenshot(agentContext);
+        if (!client_rect || !screenshot) {
+            return screenshot;
+        }
+        const compressedImage = await compress_image(screenshot.imageBase64, screenshot.imageType, {
+            resizeWidth: client_rect.width,
+            resizeHeight: client_rect.height,
+        });
+        return {
+            imageBase64: compressedImage.imageBase64,
+            imageType: compressedImage.imageType,
+        };
+    }
     get_element_script(index) {
         return `window.get_highlight_element(${index});`;
     }
@@ -36272,7 +36492,7 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
         ];
     }
     async double_screenshots(agentContext, messages, tools) {
-        return true;
+        return config$1.mode != "fast";
     }
     async handleMessages(agentContext, messages, tools) {
         const pseudoHtmlDescription = "This is the environmental information after the operation, including the latest browser screenshot and page elements. Please perform the next operation based on the environmental information. Do not output the following elements and index information in your response.\n\nIndex and elements:\n";
@@ -36282,23 +36502,27 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
             lastTool.toolName !== "get_all_tabs" &&
             lastTool.toolName !== "variable_storage") {
             await sleep(300);
-            let image_contents = [];
+            const image_contents = [];
+            const result = await this.screenshot_and_html(agentContext);
             if (await this.double_screenshots(agentContext, messages, tools)) {
-                let imageResult = await this.screenshot(agentContext);
-                let image = toImage(imageResult.imageBase64);
+                const imageResult = result.double_screenshots
+                    ? result.double_screenshots
+                    : await this.screenshot_and_compress(agentContext, result.client_rect);
+                const image = toImage(imageResult.imageBase64);
                 image_contents.push({
                     type: "file",
                     data: image,
                     mediaType: imageResult.imageType,
                 });
             }
-            let result = await this.screenshot_and_html(agentContext);
-            let image = toImage(result.imageBase64);
-            image_contents.push({
-                type: "file",
-                data: image,
-                mediaType: result.imageType,
-            });
+            if (result.imageBase64) {
+                const image = toImage(result.imageBase64);
+                image_contents.push({
+                    type: "file",
+                    data: image,
+                    mediaType: result.imageType || "image/png",
+                });
+            }
             messages.push({
                 role: "user",
                 content: [
@@ -36447,7 +36671,7 @@ function do_click(params) {
                     cancelable: true,
                     button, // 0 left; 1 middle; 2 right
                 });
-                if (eventType === 'click' && element.click) {
+                if (eventType === "click" && element.click) {
                     // support shadow dom element
                     element.click();
                 }