@eko-ai/eko 1.0.1 → 1.0.3

package/dist/extension.esm.js

@@ -103,7 +103,7 @@ async function executeScript(tabId, func, args) {
     });
     return frameResults[0].result;
 }
-async function waitForTabComplete(tabId, timeout = 30000) {
+async function waitForTabComplete(tabId, timeout = 15000) {
     return new Promise(async (resolve, reject) => {
         let tab = await chrome.tabs.get(tabId);
         if (tab.status === 'complete') {
@@ -233,100 +233,20 @@ var utils = /*#__PURE__*/Object.freeze({
     waitForTabComplete: waitForTabComplete
 });
 
-async function key(tabId, key, coordinate) {
-    if (!coordinate) {
-        coordinate = (await cursor_position(tabId)).coordinate;
-    }
-    await mouse_move(tabId, coordinate);
-    let mapping = {};
-    let keys = key.replace(/\s+/g, ' ').split(' ');
-    let result;
-    for (let i = 0; i < keys.length; i++) {
-        let _key = keys[i];
-        let keyEvents = {
-            key: '',
-            ctrlKey: false,
-            altKey: false,
-            shiftKey: false,
-            metaKey: false,
-        };
-        if (_key.indexOf('+') > -1) {
-            let mapped_keys = _key.split('+').map((k) => mapping[k] || k);
-            for (let i = 0; i < mapped_keys.length - 1; i++) {
-                let k = mapped_keys[i].toLowerCase();
-                if (k == 'ctrl' || k == 'control') {
-                    keyEvents.ctrlKey = true;
-                }
-                else if (k == 'alt' || k == 'option') {
-                    keyEvents.altKey = true;
-                }
-                else if (k == 'shift') {
-                    keyEvents.shiftKey = true;
-                }
-                else if (k == 'meta' || k == 'command') {
-                    keyEvents.metaKey = true;
-                }
-                else {
-                    console.log('Unknown Key: ' + k);
-                }
-            }
-            keyEvents.key = mapped_keys[mapped_keys.length - 1];
-        }
-        else {
-            keyEvents.key = mapping[_key] || _key;
-        }
-        if (!keyEvents.key) {
-            continue;
-        }
-        result = await chrome.tabs.sendMessage(tabId, {
-            type: 'computer:key',
-            coordinate,
-            ...keyEvents,
-        });
-        await sleep(100);
-    }
-    return result;
-}
-async function type(tabId, text, coordinate) {
-    if (!coordinate) {
-        coordinate = (await cursor_position(tabId)).coordinate;
-    }
-    await mouse_move(tabId, coordinate);
-    return await chrome.tabs.sendMessage(tabId, {
-        type: 'computer:type',
-        text,
-        coordinate,
-    });
-}
-async function type_by_xpath(tabId, text, xpath) {
+async function type_by(tabId, text, xpath, highlightIndex) {
     return await chrome.tabs.sendMessage(tabId, {
         type: 'computer:type',
         text,
         xpath,
+        highlightIndex,
     });
 }
-async function clear_input(tabId, coordinate) {
-    if (!coordinate) {
-        coordinate = (await cursor_position(tabId)).coordinate;
-    }
-    await mouse_move(tabId, coordinate);
-    return await chrome.tabs.sendMessage(tabId, {
-        type: 'computer:type',
-        text: '',
-        coordinate,
-    });
-}
-async function clear_input_by_xpath(tabId, xpath) {
+async function clear_input_by(tabId, xpath, highlightIndex) {
     return await chrome.tabs.sendMessage(tabId, {
         type: 'computer:type',
         text: '',
         xpath,
-    });
-}
-async function mouse_move(tabId, coordinate) {
-    return await chrome.tabs.sendMessage(tabId, {
-        type: 'computer:mouse_move',
-        coordinate,
+        highlightIndex,
     });
 }
 async function left_click(tabId, coordinate) {
@@ -338,48 +258,25 @@ async function left_click(tabId, coordinate) {
         coordinate,
     });
 }
-async function left_click_by_xpath(tabId, xpath) {
+async function left_click_by(tabId, xpath, highlightIndex) {
     return await chrome.tabs.sendMessage(tabId, {
         type: 'computer:left_click',
         xpath,
+        highlightIndex,
     });
 }
-async function left_click_drag(tabId, coordinate) {
-    let from_coordinate = (await cursor_position(tabId)).coordinate;
-    return await chrome.tabs.sendMessage(tabId, {
-        type: 'computer:left_click_drag',
-        from_coordinate,
-        to_coordinate: coordinate,
-    });
-}
-async function right_click(tabId, coordinate) {
-    if (!coordinate) {
-        coordinate = (await cursor_position(tabId)).coordinate;
-    }
-    return await chrome.tabs.sendMessage(tabId, {
-        type: 'computer:right_click',
-        coordinate,
-    });
-}
-async function right_click_by_xpath(tabId, xpath) {
+async function right_click_by(tabId, xpath, highlightIndex) {
     return await chrome.tabs.sendMessage(tabId, {
         type: 'computer:right_click',
         xpath,
+        highlightIndex,
     });
 }
-async function double_click(tabId, coordinate) {
-    if (!coordinate) {
-        coordinate = (await cursor_position(tabId)).coordinate;
-    }
-    return await chrome.tabs.sendMessage(tabId, {
-        type: 'computer:double_click',
-        coordinate,
-    });
-}
-async function double_click_by_xpath(tabId, xpath) {
+async function double_click_by(tabId, xpath, highlightIndex) {
     return await chrome.tabs.sendMessage(tabId, {
         type: 'computer:double_click',
         xpath,
+        highlightIndex,
     });
 }
 async function screenshot(windowId) {
@@ -396,18 +293,26 @@ async function screenshot(windowId) {
         },
     };
 }
-async function scroll_to(tabId, coordinate) {
-    let from_coordinate = (await cursor_position(tabId)).coordinate;
+async function scroll_to_by(tabId, xpath, highlightIndex) {
     return await chrome.tabs.sendMessage(tabId, {
         type: 'computer:scroll_to',
-        from_coordinate,
-        to_coordinate: coordinate,
+        xpath,
+        highlightIndex,
     });
 }
-async function scroll_to_xpath(tabId, xpath) {
+async function get_dropdown_options(tabId, xpath, highlightIndex) {
     return await chrome.tabs.sendMessage(tabId, {
-        type: 'computer:scroll_to',
+        type: 'computer:get_dropdown_options',
         xpath,
+        highlightIndex,
+    });
+}
+async function select_dropdown_option(tabId, text, xpath, highlightIndex) {
+    return await chrome.tabs.sendMessage(tabId, {
+        type: 'computer:select_dropdown_option',
+        text,
+        xpath,
+        highlightIndex,
     });
 }
 async function cursor_position(tabId) {
@@ -416,202 +321,6 @@ async function cursor_position(tabId) {
     });
     return { coordinate: result.coordinate };
 }
-async function size(tabId) {
-    return await getPageSize(tabId);
-}
-
-var browser = /*#__PURE__*/Object.freeze({
-    __proto__: null,
-    clear_input: clear_input,
-    clear_input_by_xpath: clear_input_by_xpath,
-    cursor_position: cursor_position,
-    double_click: double_click,
-    double_click_by_xpath: double_click_by_xpath,
-    key: key,
-    left_click: left_click,
-    left_click_by_xpath: left_click_by_xpath,
-    left_click_drag: left_click_drag,
-    mouse_move: mouse_move,
-    right_click: right_click,
-    right_click_by_xpath: right_click_by_xpath,
-    screenshot: screenshot,
-    scroll_to: scroll_to,
-    scroll_to_xpath: scroll_to_xpath,
-    size: size,
-    type: type,
-    type_by_xpath: type_by_xpath
-});
-
-function exportFile(filename, type, content) {
-    const blob = new Blob([content], { type: type });
-    const link = document.createElement('a');
-    link.href = URL.createObjectURL(blob);
-    link.download = filename;
-    document.body.appendChild(link);
-    link.click();
-    document.body.removeChild(link);
-    URL.revokeObjectURL(link.href);
-}
-function getDropdownOptions(xpath) {
-    const select = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null)
-        .singleNodeValue;
-    if (!select) {
-        return null;
-    }
-    return {
-        options: Array.from(select.options).map((opt) => ({
-            index: opt.index,
-            text: opt.text.trim(),
-            value: opt.value,
-        })),
-        id: select.id,
-        name: select.name,
-    };
-}
-function selectDropdownOption(xpath, text) {
-    const select = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null)
-        .singleNodeValue;
-    if (!select || select.tagName.toUpperCase() !== 'SELECT') {
-        return { success: false, error: 'Select not found or invalid element type' };
-    }
-    const option = Array.from(select.options).find((opt) => opt.text.trim() === text);
-    if (!option) {
-        return {
-            success: false,
-            error: 'Option not found',
-            availableOptions: Array.from(select.options).map((o) => o.text.trim()),
-        };
-    }
-    select.value = option.value;
-    select.dispatchEvent(new Event('change'));
-    return {
-        success: true,
-        selectedValue: option.value,
-        selectedText: option.text.trim(),
-    };
-}
-/**
- * Extract the elements related to html operability and wrap them into pseudo-html code.
- */
-function extractOperableElements() {
-    // visible
-    const isElementVisible = (element) => {
-        const style = window.getComputedStyle(element);
-        return (style.display !== 'none' &&
-            style.visibility !== 'hidden' &&
-            style.opacity !== '0' &&
-            element.offsetWidth > 0 &&
-            element.offsetHeight > 0);
-    };
-    // element original index
-    const getElementIndex = (element) => {
-        const xpath = document.evaluate('preceding::*', element, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
-        return xpath.snapshotLength;
-    };
-    // exclude
-    const addExclude = (excludes, children) => {
-        for (let i = 0; i < children.length; i++) {
-            excludes.push(children[i]);
-            if (children[i].children) {
-                addExclude(excludes, children[i].children);
-            }
-        }
-    };
-    // { pseudoId: element }
-    let elementMap = {};
-    let nextId = 1;
-    let elements = [];
-    let excludes = [];
-    // operable element
-    const operableSelectors = 'a, button, input, textarea, select';
-    document.querySelectorAll(operableSelectors).forEach((element) => {
-        if (isElementVisible(element) && excludes.indexOf(element) == -1) {
-            const id = nextId++;
-            elementMap[id.toString()] = element;
-            const tagName = element.tagName.toLowerCase();
-            const attributes = Array.from(element.attributes)
-                .filter((attr) => ['id', 'name', 'type', 'value', 'href', 'title', 'placeholder'].includes(attr.name))
-                .map((attr) => `${attr.name == 'id' ? 'target' : attr.name}="${attr.value}"`)
-                .join(' ');
-            elements.push({
-                originalIndex: getElementIndex(element),
-                id: id,
-                html: `<${tagName} id="${id}" ${attributes}>${tagName == 'select' ? element.innerHTML : element.innerText || ''}</${tagName}>`,
-            });
-            addExclude(excludes, element.children);
-        }
-    });
-    // short text element
-    const textWalker = document.createTreeWalker(document.body, NodeFilter.SHOW_ELEMENT, {
-        acceptNode: function (node) {
-            var _a;
-            if (node.matches(operableSelectors) || excludes.indexOf(node) != -1) {
-                // skip
-                return NodeFilter.FILTER_SKIP;
-            }
-            // text <= 100
-            const text = (_a = node.innerText) === null || _a === void 0 ? void 0 : _a.trim();
-            if (isElementVisible(node) &&
-                text &&
-                text.length <= 100 &&
-                text.length > 0 &&
-                node.children.length === 0) {
-                return NodeFilter.FILTER_ACCEPT;
-            }
-            // skip
-            return NodeFilter.FILTER_SKIP;
-        },
-    });
-    let currentNode;
-    while ((currentNode = textWalker.nextNode())) {
-        const id = nextId++;
-        elementMap[id.toString()] = currentNode;
-        const tagName = currentNode.tagName.toLowerCase();
-        elements.push({
-            originalIndex: getElementIndex(currentNode),
-            id: id,
-            html: `<${tagName} id="${id}">${currentNode.innerText.trim()}</${tagName}>`,
-        });
-    }
-    // element sort
-    elements.sort((a, b) => a.originalIndex - b.originalIndex);
-    // cache
-    window.operableElementMap = elementMap;
-    // pseudo html
-    return elements.map((e) => e.html).join('\n');
-}
-function clickOperableElement(id) {
-    let element = window.operableElementMap[id];
-    if (!element) {
-        return false;
-    }
-    if (element.click) {
-        element.click();
-    }
-    else {
-        element.dispatchEvent(new MouseEvent('click', {
-            view: window,
-            bubbles: true,
-            cancelable: true,
-        }));
-    }
-    return true;
-}
-function getOperableElementRect(id) {
-    let element = window.operableElementMap[id];
-    if (!element) {
-        return null;
-    }
-    const rect = element.getBoundingClientRect();
-    return {
-        left: rect.left + window.scrollX,
-        top: rect.top + window.scrollY,
-        right: rect.right + window.scrollX,
-        bottom: rect.bottom + window.scrollY,
-        width: rect.right - rect.left,
-        height: rect.bottom - rect.top,
-    };
-}
 
 /**
  * Browser Use for general
@@ -700,7 +409,7 @@ class BrowserUse {
             }
             let tabId = await getTabId(context);
             let windowId = await getWindowId(context);
-            let selector_map = context.variables.get('selector_map');
+            let selector_map = context.selector_map;
             let selector_xpath;
             if (params.index != null && selector_map) {
                 selector_xpath = (_a = selector_map[params.index]) === null || _a === void 0 ? void 0 : _a.xpath;
@@ -717,42 +426,42 @@ class BrowserUse {
                     if (params.text == null) {
                         throw new Error('text parameter is required');
                     }
-                    result = await type_by_xpath(tabId, params.text, selector_xpath);
+                    result = await type_by(tabId, params.text, selector_xpath, params.index);
                     await sleep(200);
                     break;
                 case 'clear_text':
                     if (params.index == null) {
                         throw new Error('index parameter is required');
                     }
-                    result = await clear_input_by_xpath(tabId, selector_xpath);
+                    result = await clear_input_by(tabId, selector_xpath, params.index);
                     await sleep(100);
                     break;
                 case 'click':
                     if (params.index == null) {
                         throw new Error('index parameter is required');
                     }
-                    result = await left_click_by_xpath(tabId, selector_xpath);
+                    result = await left_click_by(tabId, selector_xpath, params.index);
                     await sleep(100);
                     break;
                 case 'right_click':
                     if (params.index == null) {
                         throw new Error('index parameter is required');
                     }
-                    result = await right_click_by_xpath(tabId, selector_xpath);
+                    result = await right_click_by(tabId, selector_xpath, params.index);
                     await sleep(100);
                     break;
                 case 'double_click':
                     if (params.index == null) {
                         throw new Error('index parameter is required');
                     }
-                    result = await double_click_by_xpath(tabId, selector_xpath);
+                    result = await double_click_by(tabId, selector_xpath, params.index);
                     await sleep(100);
                     break;
                 case 'scroll_to':
                     if (params.index == null) {
                         throw new Error('index parameter is required');
                     }
-                    result = await scroll_to_xpath(tabId, selector_xpath);
+                    result = await scroll_to_by(tabId, selector_xpath, params.index);
                     await sleep(500);
                     break;
                 case 'extract_content':
@@ -772,7 +481,7 @@ class BrowserUse {
                     if (params.index == null) {
                         throw new Error('index parameter is required');
                     }
-                    result = await executeScript(tabId, getDropdownOptions, [selector_xpath]);
+                    result = await get_dropdown_options(tabId, selector_xpath, params.index);
                     break;
                 case 'select_dropdown_option':
                     if (params.index == null) {
@@ -781,7 +490,7 @@ class BrowserUse {
                     if (params.text == null) {
                         throw new Error('text parameter is required');
                     }
-                    result = await executeScript(tabId, selectDropdownOption, [selector_xpath, params.text]);
+                    result = await select_dropdown_option(tabId, params.text, selector_xpath, params.index);
                     break;
                 case 'screenshot_extract_element':
                     await sleep(100);
@@ -790,7 +499,7 @@ class BrowserUse {
                     let element_result = await executeScript(tabId, () => {
                         return window.get_clickable_elements(true);
                     }, []);
-                    context.variables.set('selector_map', element_result.selector_map);
+                    context.selector_map = element_result.selector_map;
                     let screenshot$1 = await screenshot(windowId);
                     await executeScript(tabId, () => {
                         return window.remove_highlight();
@@ -811,6 +520,142 @@ class BrowserUse {
             return { success: false, error: e === null || e === void 0 ? void 0 : e.message };
         }
     }
+    destroy(context) {
+        delete context.selector_map;
+    }
+}
+
+function exportFile(filename, type, content) {
+    const blob = new Blob([content], { type: type });
+    const link = document.createElement('a');
+    link.href = URL.createObjectURL(blob);
+    link.download = filename;
+    document.body.appendChild(link);
+    link.click();
+    document.body.removeChild(link);
+    URL.revokeObjectURL(link.href);
+}
+/**
+ * Extract the elements related to html operability and wrap them into pseudo-html code.
+ */
+function extractOperableElements() {
+    // visible
+    const isElementVisible = (element) => {
+        const style = window.getComputedStyle(element);
+        return (style.display !== 'none' &&
+            style.visibility !== 'hidden' &&
+            style.opacity !== '0' &&
+            element.offsetWidth > 0 &&
+            element.offsetHeight > 0);
+    };
+    // element original index
+    const getElementIndex = (element) => {
+        const xpath = document.evaluate('preceding::*', element, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
+        return xpath.snapshotLength;
+    };
+    // exclude
+    const addExclude = (excludes, children) => {
+        for (let i = 0; i < children.length; i++) {
+            excludes.push(children[i]);
+            if (children[i].children) {
+                addExclude(excludes, children[i].children);
+            }
+        }
+    };
+    // { pseudoId: element }
+    let elementMap = {};
+    let nextId = 1;
+    let elements = [];
+    let excludes = [];
+    // operable element
+    const operableSelectors = 'a, button, input, textarea, select';
+    document.querySelectorAll(operableSelectors).forEach((element) => {
+        if (isElementVisible(element) && excludes.indexOf(element) == -1) {
+            const id = nextId++;
+            elementMap[id.toString()] = element;
+            const tagName = element.tagName.toLowerCase();
+            const attributes = Array.from(element.attributes)
+                .filter((attr) => ['id', 'name', 'type', 'value', 'href', 'title', 'placeholder'].includes(attr.name))
+                .map((attr) => `${attr.name == 'id' ? 'target' : attr.name}="${attr.value}"`)
+                .join(' ');
+            elements.push({
+                originalIndex: getElementIndex(element),
+                id: id,
+                html: `<${tagName} id="${id}" ${attributes}>${tagName == 'select' ? element.innerHTML : element.innerText || ''}</${tagName}>`,
+            });
+            addExclude(excludes, element.children);
+        }
+    });
+    // short text element
+    const textWalker = document.createTreeWalker(document.body, NodeFilter.SHOW_ELEMENT, {
+        acceptNode: function (node) {
+            var _a;
+            if (node.matches(operableSelectors) || excludes.indexOf(node) != -1) {
+                // skip
+                return NodeFilter.FILTER_SKIP;
+            }
+            // text <= 100
+            const text = (_a = node.innerText) === null || _a === void 0 ? void 0 : _a.trim();
+            if (isElementVisible(node) &&
+                text &&
+                text.length <= 100 &&
+                text.length > 0 &&
+                node.children.length === 0) {
+                return NodeFilter.FILTER_ACCEPT;
+            }
+            // skip
+            return NodeFilter.FILTER_SKIP;
+        },
+    });
+    let currentNode;
+    while ((currentNode = textWalker.nextNode())) {
+        const id = nextId++;
+        elementMap[id.toString()] = currentNode;
+        const tagName = currentNode.tagName.toLowerCase();
+        elements.push({
+            originalIndex: getElementIndex(currentNode),
+            id: id,
+            html: `<${tagName} id="${id}">${currentNode.innerText.trim()}</${tagName}>`,
+        });
+    }
+    // element sort
+    elements.sort((a, b) => a.originalIndex - b.originalIndex);
+    // cache
+    window.operableElementMap = elementMap;
+    // pseudo html
+    return elements.map((e) => e.html).join('\n');
+}
+function clickOperableElement(id) {
+    let element = window.operableElementMap[id];
+    if (!element) {
+        return false;
+    }
+    if (element.click) {
+        element.click();
+    }
+    else {
+        element.dispatchEvent(new MouseEvent('click', {
+            view: window,
+            bubbles: true,
+            cancelable: true,
+        }));
+    }
+    return true;
+}
+function getOperableElementRect(id) {
+    let element = window.operableElementMap[id];
+    if (!element) {
+        return null;
+    }
+    const rect = element.getBoundingClientRect();
+    return {
+        left: rect.left + window.scrollX,
+        top: rect.top + window.scrollY,
+        right: rect.right + window.scrollX,
+        bottom: rect.bottom + window.scrollY,
+        width: rect.right - rect.left,
+        height: rect.bottom - rect.top,
+    };
 }
 
 /**
@@ -1183,24 +1028,6 @@ async function executeWithBrowserUse(context, task_prompt) {
     };
 }
 
-/**
- * Form Autofill
- */
-class FormAutofill {
-    constructor() {
-        this.name = 'form_autofill';
-        this.description = 'Automatically fill in form data on web pages';
-        this.input_schema = {
-            type: 'object',
-            properties: {}
-        };
-    }
-    async execute(context, params) {
-        // form -> input, textarea, select ...
-        throw new Error('Not implemented');
-    }
-}
-
 /**
  * Open Url
  */
@@ -1738,7 +1565,6 @@ var tools = /*#__PURE__*/Object.freeze({
     ExportFile: ExportFile,
     ExtractContent: ExtractContent,
     FindElementPosition: FindElementPosition,
-    FormAutofill: FormAutofill,
     OpenUrl: OpenUrl,
     Screenshot: Screenshot,
     TabManagement: TabManagement,
@@ -1773,4 +1599,4 @@ function getAllTools() {
     return toolsMap;
 }
 
-export { browser, getAllTools, getLLMConfig, pub, tools, utils };
+export { getAllTools, getLLMConfig, pub, tools, utils };