@eko-ai/eko 1.0.2 → 1.0.4

package/dist/web.cjs.js

@@ -8,15 +8,16 @@
  * @returns { element_str, selector_map }
  */
 function get_clickable_elements(doHighlightElements = true, includeAttributes) {
+    window.clickable_elements = {};
     let page_tree = build_dom_tree(doHighlightElements);
     let element_tree = parse_node(page_tree);
     let selector_map = create_selector_map(element_tree);
     let element_str = clickable_elements_to_string(element_tree, includeAttributes);
     return { element_str, selector_map };
 }
-/**
- * Remove highlight
- */
+function get_highlight_element(highlightIndex) {
+    return window.clickable_elements[highlightIndex];
+}
 function remove_highlight() {
     let highlight = document.getElementById('playwright-highlight-container');
     if (highlight) {
@@ -539,6 +540,7 @@ function build_dom_tree(doHighlightElements) {
             // Highlight if element meets all criteria and highlighting is enabled
             if (isInteractive && isVisible && isTop) {
                 nodeData.highlightIndex = highlightIndex++;
+                window.clickable_elements[nodeData.highlightIndex] = node;
                 if (doHighlightElements) {
                     highlightElement(node, nodeData.highlightIndex, parentIframe);
                 }
@@ -579,198 +581,9 @@ function build_dom_tree(doHighlightElements) {
     return buildDomTree(document.body);
 }
 window.get_clickable_elements = get_clickable_elements;
+window.get_highlight_element = get_highlight_element;
 window.remove_highlight = remove_highlight;
 
-function exportFile(filename, type, content) {
-    const blob = new Blob([content], { type: type });
-    const link = document.createElement('a');
-    link.href = URL.createObjectURL(blob);
-    link.download = filename;
-    document.body.appendChild(link);
-    link.click();
-    document.body.removeChild(link);
-    URL.revokeObjectURL(link.href);
-}
-function xpath(element) {
-    if (element.id !== '') {
-        return '//*[@id=\"' + element.id + '\"]';
-    }
-    if (element == document.body) {
-        return '/html/' + element.tagName.toLowerCase();
-    }
-    var ix = 1, siblings = element.parentNode.childNodes;
-    for (var i = 0, l = siblings.length; i < l; i++) {
-        var sibling = siblings[i];
-        if (sibling == element) {
-            return xpath(element.parentNode) + '/' + element.tagName.toLowerCase() + '[' + ix + ']';
-        }
-        else if (sibling.nodeType == 1 && sibling.tagName == element.tagName) {
-            ix++;
-        }
-    }
-    return '';
-}
-function getDropdownOptions(xpath) {
-    const select = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null)
-        .singleNodeValue;
-    if (!select) {
-        return null;
-    }
-    return {
-        options: Array.from(select.options).map((opt) => ({
-            index: opt.index,
-            text: opt.text.trim(),
-            value: opt.value,
-        })),
-        id: select.id,
-        name: select.name,
-    };
-}
-function selectDropdownOption(xpath, text) {
-    const select = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null)
-        .singleNodeValue;
-    if (!select || select.tagName.toUpperCase() !== 'SELECT') {
-        return { success: false, error: 'Select not found or invalid element type' };
-    }
-    const option = Array.from(select.options).find((opt) => opt.text.trim() === text);
-    if (!option) {
-        return {
-            success: false,
-            error: 'Option not found',
-            availableOptions: Array.from(select.options).map((o) => o.text.trim()),
-        };
-    }
-    select.value = option.value;
-    select.dispatchEvent(new Event('change'));
-    return {
-        success: true,
-        selectedValue: option.value,
-        selectedText: option.text.trim(),
-    };
-}
-/**
- * Extract the elements related to html operability and wrap them into pseudo-html code.
- */
-function extractOperableElements() {
-    // visible
-    const isElementVisible = (element) => {
-        const style = window.getComputedStyle(element);
-        return (style.display !== 'none' &&
-            style.visibility !== 'hidden' &&
-            style.opacity !== '0' &&
-            element.offsetWidth > 0 &&
-            element.offsetHeight > 0);
-    };
-    // element original index
-    const getElementIndex = (element) => {
-        const xpath = document.evaluate('preceding::*', element, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
-        return xpath.snapshotLength;
-    };
-    // exclude
-    const addExclude = (excludes, children) => {
-        for (let i = 0; i < children.length; i++) {
-            excludes.push(children[i]);
-            if (children[i].children) {
-                addExclude(excludes, children[i].children);
-            }
-        }
-    };
-    // { pseudoId: element }
-    let elementMap = {};
-    let nextId = 1;
-    let elements = [];
-    let excludes = [];
-    // operable element
-    const operableSelectors = 'a, button, input, textarea, select';
-    document.querySelectorAll(operableSelectors).forEach((element) => {
-        if (isElementVisible(element) && excludes.indexOf(element) == -1) {
-            const id = nextId++;
-            elementMap[id.toString()] = element;
-            const tagName = element.tagName.toLowerCase();
-            const attributes = Array.from(element.attributes)
-                .filter((attr) => ['id', 'name', 'type', 'value', 'href', 'title', 'placeholder'].includes(attr.name))
-                .map((attr) => `${attr.name == 'id' ? 'target' : attr.name}="${attr.value}"`)
-                .join(' ');
-            elements.push({
-                originalIndex: getElementIndex(element),
-                id: id,
-                html: `<${tagName} id="${id}" ${attributes}>${tagName == 'select' ? element.innerHTML : element.innerText || ''}</${tagName}>`,
-            });
-            addExclude(excludes, element.children);
-        }
-    });
-    // short text element
-    const textWalker = document.createTreeWalker(document.body, NodeFilter.SHOW_ELEMENT, {
-        acceptNode: function (node) {
-            var _a;
-            if (node.matches(operableSelectors) || excludes.indexOf(node) != -1) {
-                // skip
-                return NodeFilter.FILTER_SKIP;
-            }
-            // text <= 100
-            const text = (_a = node.innerText) === null || _a === void 0 ? void 0 : _a.trim();
-            if (isElementVisible(node) &&
-                text &&
-                text.length <= 100 &&
-                text.length > 0 &&
-                node.children.length === 0) {
-                return NodeFilter.FILTER_ACCEPT;
-            }
-            // skip
-            return NodeFilter.FILTER_SKIP;
-        },
-    });
-    let currentNode;
-    while ((currentNode = textWalker.nextNode())) {
-        const id = nextId++;
-        elementMap[id.toString()] = currentNode;
-        const tagName = currentNode.tagName.toLowerCase();
-        elements.push({
-            originalIndex: getElementIndex(currentNode),
-            id: id,
-            html: `<${tagName} id="${id}">${currentNode.innerText.trim()}</${tagName}>`,
-        });
-    }
-    // element sort
-    elements.sort((a, b) => a.originalIndex - b.originalIndex);
-    // cache
-    window.operableElementMap = elementMap;
-    // pseudo html
-    return elements.map((e) => e.html).join('\n');
-}
-function clickOperableElement(id) {
-    let element = window.operableElementMap[id];
-    if (!element) {
-        return false;
-    }
-    if (element.click) {
-        element.click();
-    }
-    else {
-        element.dispatchEvent(new MouseEvent('click', {
-            view: window,
-            bubbles: true,
-            cancelable: true,
-        }));
-    }
-    return true;
-}
-function getOperableElementRect(id) {
-    let element = window.operableElementMap[id];
-    if (!element) {
-        return null;
-    }
-    const rect = element.getBoundingClientRect();
-    return {
-        left: rect.left + window.scrollX,
-        top: rect.top + window.scrollY,
-        right: rect.right + window.scrollX,
-        bottom: rect.bottom + window.scrollY,
-        width: rect.right - rect.left,
-        height: rect.bottom - rect.top,
-    };
-}
-
 /*!
  * html2canvas 1.4.1 <https://html2canvas.hertzen.com>
  * Copyright (c) 2022 Niklas von Hertzen <https://hertzen.com>
@@ -8591,20 +8404,20 @@ var parseBackgroundColor = function (context, element, backgroundColorOverride)
         : defaultBackgroundColor;
 };
 
-function type(xpath, text) {
-    return do_input(xpath, text);
+function type(text, xpath, highlightIndex) {
+    return do_input(text, xpath, highlightIndex);
 }
-function clear_input(xpath) {
-    return do_input(xpath, '');
+function clear_input(xpath, highlightIndex) {
+    return do_input('', xpath, highlightIndex);
 }
-function left_click(xpath) {
-    return simulateMouseEvent(xpath, ['mousedown', 'mouseup', 'click'], 0);
+function left_click(xpath, highlightIndex) {
+    return simulateMouseEvent(['mousedown', 'mouseup', 'click'], 0, xpath, highlightIndex);
 }
-function right_click(xpath) {
-    return simulateMouseEvent(xpath, ['mousedown', 'mouseup', 'contextmenu'], 2);
+function right_click(xpath, highlightIndex) {
+    return simulateMouseEvent(['mousedown', 'mouseup', 'contextmenu'], 2, xpath, highlightIndex);
 }
-function double_click(xpath) {
-    return simulateMouseEvent(xpath, ['mousedown', 'mouseup', 'click', 'mousedown', 'mouseup', 'click', 'dblclick'], 0);
+function double_click(xpath, highlightIndex) {
+    return simulateMouseEvent(['mousedown', 'mouseup', 'click', 'mousedown', 'mouseup', 'click', 'dblclick'], 0, xpath, highlightIndex);
 }
 async function screenshot() {
     const [width, height] = size();
@@ -8634,12 +8447,72 @@ async function screenshot() {
         },
     };
 }
-function scroll_to(xpath) {
-    let result = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
-    let element = result.singleNodeValue;
-    return element.scrollIntoView({
+function scroll_to(xpath, highlightIndex) {
+    let element = null;
+    if (highlightIndex != null) {
+        element = window.get_highlight_element(highlightIndex);
+    }
+    else if (xpath) {
+        element = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null)
+            .singleNodeValue;
+    }
+    if (!element) {
+        return false;
+    }
+    element.scrollIntoView({
         behavior: 'smooth',
     });
+    return true;
+}
+function get_dropdown_options(xpath, highlightIndex) {
+    let select = null;
+    if (highlightIndex != null) {
+        select = window.get_highlight_element(highlightIndex);
+    }
+    else if (xpath) {
+        select = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null)
+            .singleNodeValue;
+    }
+    if (!select) {
+        return null;
+    }
+    return {
+        options: Array.from(select.options).map((opt) => ({
+            index: opt.index,
+            text: opt.text.trim(),
+            value: opt.value,
+        })),
+        id: select.id,
+        name: select.name,
+    };
+}
+function select_dropdown_option(text, xpath, highlightIndex) {
+    let select = null;
+    if (highlightIndex != null) {
+        select = window.get_highlight_element(highlightIndex);
+    }
+    else if (xpath) {
+        select = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null)
+            .singleNodeValue;
+    }
+    if (!select || select.tagName.toUpperCase() !== 'SELECT') {
+        return { success: false, error: 'Select not found or invalid element type' };
+    }
+    const option = Array.from(select.options).find((opt) => opt.text.trim() === text);
+    if (!option) {
+        return {
+            success: false,
+            error: 'Option not found',
+            availableOptions: Array.from(select.options).map((o) => o.text.trim()),
+        };
+    }
+    select.value = option.value;
+    select.dispatchEvent(new Event('change'));
+    return {
+        success: true,
+        selectedValue: option.value,
+        selectedText: option.text.trim(),
+    };
 }
 function extractHtmlContent() {
     let element = document.body;
@@ -8676,11 +8549,26 @@ function size() {
         window.innerHeight || document.documentElement.clientHeight || document.body.clientHeight,
     ];
 }
-function do_input(xpath, text) {
-    let query_result = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
-    let element = query_result.singleNodeValue;
+function do_input(text, xpath, highlightIndex) {
+    let element = null;
+    if (highlightIndex != null) {
+        element = window.get_highlight_element(highlightIndex);
+    }
+    else if (xpath) {
+        element = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null)
+            .singleNodeValue;
+    }
     if (!element) {
-        return;
+        return false;
+    }
+    let enter = false;
+    if (text.endsWith('\\n')) {
+        enter = true;
+        text = text.substring(0, text.length - 2);
+    }
+    else if (text.endsWith('\n')) {
+        enter = true;
+        text = text.substring(0, text.length - 1);
     }
     let input;
     if (element.tagName == 'INPUT' ||
@@ -8699,13 +8587,33 @@ function do_input(xpath, text) {
         input.value += text;
     }
     let result = input.dispatchEvent(new Event('input', { bubbles: true }));
+    if (enter) {
+        ['keydown', 'keypress', 'keyup'].forEach((eventType) => {
+            const event = new KeyboardEvent(eventType, {
+                key: 'Enter',
+                code: 'Enter',
+                keyCode: 13,
+                bubbles: true,
+                cancelable: true,
+            });
+            input.dispatchEvent(event);
+        });
+    }
     console.log('type', input, result);
-    return result;
+    return true;
 }
-function simulateMouseEvent(xpath, eventTypes, button) {
-    let query_result = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
-    let element = query_result.singleNodeValue;
-    let result = false;
+function simulateMouseEvent(eventTypes, button, xpath, highlightIndex) {
+    let element = null;
+    if (highlightIndex != null) {
+        element = window.get_highlight_element(highlightIndex);
+    }
+    else if (xpath) {
+        element = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null)
+            .singleNodeValue;
+    }
+    if (!element) {
+        return false;
+    }
     for (let i = 0; i < eventTypes.length; i++) {
         const event = new MouseEvent(eventTypes[i], {
             view: window,
@@ -8713,10 +8621,10 @@ function simulateMouseEvent(xpath, eventTypes, button) {
             cancelable: true,
             button, // 0 left; 2 right
         });
-        result = element.dispatchEvent(event);
+        let result = element.dispatchEvent(event);
         console.log('simulateMouse', element, { xpath, eventTypes, button }, result);
     }
-    return result;
+    return true;
 }
 
 var browser = /*#__PURE__*/Object.freeze({
@@ -8724,10 +8632,12 @@ var browser = /*#__PURE__*/Object.freeze({
     clear_input: clear_input,
     double_click: double_click,
     extractHtmlContent: extractHtmlContent,
+    get_dropdown_options: get_dropdown_options,
     left_click: left_click,
     right_click: right_click,
     screenshot: screenshot,
     scroll_to: scroll_to,
+    select_dropdown_option: select_dropdown_option,
     size: size,
     type: type
 });
@@ -8817,7 +8727,7 @@ class BrowserUse {
             if (params === null || !params.action) {
                 throw new Error('Invalid parameters. Expected an object with a "action" property.');
             }
-            let selector_map = context.variables.get('selector_map');
+            let selector_map = context.selector_map;
             let selector_xpath;
             if (params.index != null && selector_map) {
                 selector_xpath = (_a = selector_map[params.index]) === null || _a === void 0 ? void 0 : _a.xpath;
@@ -8834,42 +8744,42 @@ class BrowserUse {
                     if (params.text == null) {
                         throw new Error('text parameter is required');
                     }
-                    result = await type(selector_xpath, params.text);
+                    result = await type(params.text, selector_xpath, params.index);
                     await sleep(200);
                     break;
                 case 'clear_text':
                     if (params.index == null) {
                         throw new Error('index parameter is required');
                     }
-                    result = await clear_input(selector_xpath);
+                    result = await clear_input(selector_xpath, params.index);
                     await sleep(100);
                     break;
                 case 'click':
                     if (params.index == null) {
                         throw new Error('index parameter is required');
                     }
-                    result = await left_click(selector_xpath);
+                    result = await left_click(selector_xpath, params.index);
                     await sleep(100);
                     break;
                 case 'right_click':
                     if (params.index == null) {
                         throw new Error('index parameter is required');
                     }
-                    result = await right_click(selector_xpath);
+                    result = await right_click(selector_xpath, params.index);
                     await sleep(100);
                     break;
                 case 'double_click':
                     if (params.index == null) {
                         throw new Error('index parameter is required');
                     }
-                    result = await double_click(selector_xpath);
+                    result = await double_click(selector_xpath, params.index);
                     await sleep(100);
                     break;
                 case 'scroll_to':
                     if (params.index == null) {
                         throw new Error('index parameter is required');
                     }
-                    result = await scroll_to(selector_xpath);
+                    result = await scroll_to(selector_xpath, params.index);
                     await sleep(500);
                     break;
                 case 'extract_content':
@@ -8884,7 +8794,7 @@ class BrowserUse {
                     if (params.index == null) {
                         throw new Error('index parameter is required');
                     }
-                    result = getDropdownOptions(selector_xpath);
+                    result = get_dropdown_options(selector_xpath, params.index);
                     break;
                 case 'select_dropdown_option':
                     if (params.index == null) {
@@ -8893,12 +8803,12 @@ class BrowserUse {
                     if (params.text == null) {
                         throw new Error('text parameter is required');
                     }
-                    result = selectDropdownOption(selector_xpath, params.text);
+                    result = select_dropdown_option(params.text, selector_xpath, params.index);
                     break;
                 case 'screenshot_extract_element':
                     await sleep(100);
                     let element_result = get_clickable_elements(true, null);
-                    context.variables.set('selector_map', element_result.selector_map);
+                    context.selector_map = element_result.selector_map;
                     let screenshot$1 = await screenshot();
                     remove_highlight();
                     result = { image: screenshot$1.image, text: element_result.element_str };
@@ -8917,11 +8827,187 @@ class BrowserUse {
             return { success: false, error: e === null || e === void 0 ? void 0 : e.message };
         }
     }
+    destroy(context) {
+        delete context.selector_map;
+    }
 }
 function sleep(time) {
     return new Promise((resolve) => setTimeout(() => resolve(), time));
 }
 
+function exportFile(filename, type, content) {
+    const blob = new Blob([content], { type: type });
+    const link = document.createElement('a');
+    link.href = URL.createObjectURL(blob);
+    link.download = filename;
+    document.body.appendChild(link);
+    link.click();
+    document.body.removeChild(link);
+    URL.revokeObjectURL(link.href);
+}
+function xpath(element) {
+    if (element == document.body) {
+        return '/html/' + element.tagName.toLowerCase();
+    }
+    if (element.parentNode instanceof ShadowRoot) {
+        let shadowRoot = element.parentNode;
+        let parent = shadowRoot.getRootNode().host;
+        return xpath(parent) + '//' + element.tagName.toLowerCase();
+    }
+    else {
+        let sp;
+        let parent;
+        if (element.parentNode instanceof ShadowRoot) {
+            sp = '//';
+            let shadowRoot = element.parentNode;
+            parent = shadowRoot.getRootNode().host;
+        }
+        else {
+            sp = '/';
+            parent = element.parentNode;
+        }
+        let siblings = parent.childNodes;
+        if (siblings.length == 1) {
+            return xpath(parent) + sp + element.tagName.toLowerCase();
+        }
+        else {
+            let ix = 1;
+            for (let i = 0, l = siblings.length; i < l; i++) {
+                let sibling = siblings[i];
+                if (sibling == element) {
+                    return xpath(parent) + sp + element.tagName.toLowerCase() + '[' + ix + ']';
+                }
+                else if (sibling.nodeType == 1 && sibling.tagName == element.tagName) {
+                    ix++;
+                }
+            }
+            return '';
+        }
+    }
+}
+/**
+ * Extract the elements related to html operability and wrap them into pseudo-html code.
+ */
+function extractOperableElements() {
+    // visible
+    const isElementVisible = (element) => {
+        const style = window.getComputedStyle(element);
+        return (style.display !== 'none' &&
+            style.visibility !== 'hidden' &&
+            style.opacity !== '0' &&
+            element.offsetWidth > 0 &&
+            element.offsetHeight > 0);
+    };
+    // element original index
+    const getElementIndex = (element) => {
+        const xpath = document.evaluate('preceding::*', element, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
+        return xpath.snapshotLength;
+    };
+    // exclude
+    const addExclude = (excludes, children) => {
+        for (let i = 0; i < children.length; i++) {
+            excludes.push(children[i]);
+            if (children[i].children) {
+                addExclude(excludes, children[i].children);
+            }
+        }
+    };
+    // { pseudoId: element }
+    let elementMap = {};
+    let nextId = 1;
+    let elements = [];
+    let excludes = [];
+    // operable element
+    const operableSelectors = 'a, button, input, textarea, select';
+    document.querySelectorAll(operableSelectors).forEach((element) => {
+        if (isElementVisible(element) && excludes.indexOf(element) == -1) {
+            const id = nextId++;
+            elementMap[id.toString()] = element;
+            const tagName = element.tagName.toLowerCase();
+            const attributes = Array.from(element.attributes)
+                .filter((attr) => ['id', 'name', 'type', 'value', 'href', 'title', 'placeholder'].includes(attr.name))
+                .map((attr) => `${attr.name == 'id' ? 'target' : attr.name}="${attr.value}"`)
+                .join(' ');
+            elements.push({
+                originalIndex: getElementIndex(element),
+                id: id,
+                html: `<${tagName} id="${id}" ${attributes}>${tagName == 'select' ? element.innerHTML : element.innerText || ''}</${tagName}>`,
+            });
+            addExclude(excludes, element.children);
+        }
+    });
+    // short text element
+    const textWalker = document.createTreeWalker(document.body, NodeFilter.SHOW_ELEMENT, {
+        acceptNode: function (node) {
+            var _a;
+            if (node.matches(operableSelectors) || excludes.indexOf(node) != -1) {
+                // skip
+                return NodeFilter.FILTER_SKIP;
+            }
+            // text <= 100
+            const text = (_a = node.innerText) === null || _a === void 0 ? void 0 : _a.trim();
+            if (isElementVisible(node) &&
+                text &&
+                text.length <= 100 &&
+                text.length > 0 &&
+                node.children.length === 0) {
+                return NodeFilter.FILTER_ACCEPT;
+            }
+            // skip
+            return NodeFilter.FILTER_SKIP;
+        },
+    });
+    let currentNode;
+    while ((currentNode = textWalker.nextNode())) {
+        const id = nextId++;
+        elementMap[id.toString()] = currentNode;
+        const tagName = currentNode.tagName.toLowerCase();
+        elements.push({
+            originalIndex: getElementIndex(currentNode),
+            id: id,
+            html: `<${tagName} id="${id}">${currentNode.innerText.trim()}</${tagName}>`,
+        });
+    }
+    // element sort
+    elements.sort((a, b) => a.originalIndex - b.originalIndex);
+    // cache
+    window.operableElementMap = elementMap;
+    // pseudo html
+    return elements.map((e) => e.html).join('\n');
+}
+function clickOperableElement(id) {
+    let element = window.operableElementMap[id];
+    if (!element) {
+        return false;
+    }
+    if (element.click) {
+        element.click();
+    }
+    else {
+        element.dispatchEvent(new MouseEvent('click', {
+            view: window,
+            bubbles: true,
+            cancelable: true,
+        }));
+    }
+    return true;
+}
+function getOperableElementRect(id) {
+    let element = window.operableElementMap[id];
+    if (!element) {
+        return null;
+    }
+    const rect = element.getBoundingClientRect();
+    return {
+        left: rect.left + window.scrollX,
+        top: rect.top + window.scrollY,
+        right: rect.right + window.scrollX,
+        bottom: rect.bottom + window.scrollY,
+        width: rect.right - rect.left,
+        height: rect.bottom - rect.top,
+    };
+}
+
 /**
  * Element click
  */