@eko-ai/eko 1.0.7 → 1.0.9

package/dist/extension.esm.js

@@ -28,12 +28,21 @@ async function getWindowId(context) {
 async function getTabId(context) {
     let tabId = context.variables.get('tabId');
     if (tabId) {
-        try {
-            await chrome.tabs.get(tabId);
-        }
-        catch (e) {
-            tabId = null;
+        // Ensure tabId is a number
+        tabId = Number(tabId);
+        // Check if it's a valid integer
+        if (!Number.isInteger(tabId)) {
             context.variables.delete('tabId');
+            tabId = null;
+        }
+        else {
+            try {
+                await chrome.tabs.get(tabId);
+            }
+            catch (e) {
+                tabId = null;
+                context.variables.delete('tabId');
+            }
         }
     }
     if (!tabId) {
@@ -50,28 +59,31 @@ async function getTabId(context) {
         else {
             tabId = await getCurrentTabId();
         }
+        if (!tabId) {
+            throw new Error('Could not find a valid tab');
+        }
+        context.variables.set('tabId', tabId);
     }
     return tabId;
 }
-function getCurrentTabId(windowId) {
-    return new Promise((resolve) => {
-        chrome.tabs.query({ windowId, active: true, lastFocusedWindow: true }, function (tabs) {
-            if (tabs.length > 0) {
-                resolve(tabs[0].id);
+async function getCurrentTabId(windowId) {
+    return new Promise((resolve, reject) => {
+        chrome.tabs.query({ active: true, currentWindow: true }, function (tabs) {
+            if (chrome.runtime.lastError) {
+                console.error('Chrome runtime error:', chrome.runtime.lastError);
+                reject(chrome.runtime.lastError);
+                return;
             }
-            else {
-                chrome.tabs.query({ windowId, active: true, currentWindow: true }, function (_tabs) {
-                    if (_tabs.length > 0) {
-                        resolve(_tabs[0].id);
-                        return;
-                    }
-                    else {
-                        chrome.tabs.query({ windowId, status: 'complete', currentWindow: true }, function (__tabs) {
-                            resolve(__tabs.length ? __tabs[__tabs.length - 1].id : undefined);
-                        });
-                    }
-                });
+            if (!tabs || tabs.length === 0) {
+                reject(new Error('No active tab found'));
+                return;
             }
+            const tabId = tabs[0].id;
+            if (typeof tabId !== 'number') {
+                reject(new Error('Invalid tab ID'));
+                return;
+            }
+            resolve(tabId);
         });
     });
 }
@@ -136,6 +148,19 @@ async function waitForTabComplete(tabId, timeout = 15000) {
         chrome.tabs.onUpdated.addListener(listener);
     });
 }
+async function doesTabExists(tabId) {
+    const tabExists = await new Promise((resolve) => {
+        chrome.tabs.get(tabId, (tab) => {
+            if (chrome.runtime.lastError) {
+                resolve(false);
+            }
+            else {
+                resolve(true);
+            }
+        });
+    });
+    return tabExists;
+}
 async function getPageSize(tabId) {
     if (!tabId) {
         tabId = await getCurrentTabId();
@@ -233,6 +258,7 @@ var utils = /*#__PURE__*/Object.freeze({
     __proto__: null,
     CountDownLatch: CountDownLatch,
     MsgEvent: MsgEvent,
+    doesTabExists: doesTabExists,
     executeScript: executeScript,
     getCurrentTabId: getCurrentTabId,
     getPageSize: getPageSize,
@@ -246,155 +272,355 @@ var utils = /*#__PURE__*/Object.freeze({
 });
 
 async function type(tabId, text, coordinate) {
-    if (!coordinate) {
-        coordinate = (await cursor_position(tabId)).coordinate;
+    console.log('Sending type message to tab:', tabId, { text, coordinate });
+    try {
+        if (!coordinate) {
+            coordinate = (await cursor_position(tabId)).coordinate;
+        }
+        await mouse_move(tabId, coordinate);
+        const response = await chrome.tabs.sendMessage(tabId, {
+            type: 'computer:type',
+            text,
+            coordinate,
+        });
+        console.log('Got response:', response);
+        return response;
+    }
+    catch (e) {
+        console.error('Failed to send type message:', e);
+        throw e;
     }
-    await mouse_move(tabId, coordinate);
-    return await chrome.tabs.sendMessage(tabId, {
-        type: 'computer:type',
-        text,
-        coordinate,
-    });
 }
 async function type_by(tabId, text, xpath, highlightIndex) {
-    return await chrome.tabs.sendMessage(tabId, {
-        type: 'computer:type',
-        text,
-        xpath,
-        highlightIndex,
-    });
+    console.log('Sending type message to tab:', tabId, { text, xpath, highlightIndex });
+    try {
+        const response = await chrome.tabs.sendMessage(tabId, {
+            type: 'computer:type',
+            text,
+            xpath,
+            highlightIndex,
+        });
+        console.log('Got response:', response);
+        return response;
+    }
+    catch (e) {
+        console.error('Failed to send type message:', e);
+        throw e;
+    }
 }
 async function clear_input(tabId, coordinate) {
-    if (!coordinate) {
-        coordinate = (await cursor_position(tabId)).coordinate;
+    console.log('Sending clear_input message to tab:', tabId, { coordinate });
+    try {
+        if (!coordinate) {
+            coordinate = (await cursor_position(tabId)).coordinate;
+        }
+        await mouse_move(tabId, coordinate);
+        const response = await chrome.tabs.sendMessage(tabId, {
+            type: 'computer:type',
+            text: '',
+            coordinate,
+        });
+        console.log('Got response:', response);
+        return response;
+    }
+    catch (e) {
+        console.error('Failed to send clear_input message:', e);
+        throw e;
     }
-    await mouse_move(tabId, coordinate);
-    return await chrome.tabs.sendMessage(tabId, {
-        type: 'computer:type',
-        text: '',
-        coordinate,
-    });
 }
 async function clear_input_by(tabId, xpath, highlightIndex) {
-    return await chrome.tabs.sendMessage(tabId, {
-        type: 'computer:type',
-        text: '',
-        xpath,
-        highlightIndex,
-    });
+    console.log('Sending clear_input_by message to tab:', tabId, { xpath, highlightIndex });
+    try {
+        const response = await chrome.tabs.sendMessage(tabId, {
+            type: 'computer:type',
+            text: '',
+            xpath,
+            highlightIndex,
+        });
+        console.log('Got response:', response);
+        return response;
+    }
+    catch (e) {
+        console.error('Failed to send clear_input_by message:', e);
+        throw e;
+    }
 }
 async function mouse_move(tabId, coordinate) {
-    return await chrome.tabs.sendMessage(tabId, {
-        type: 'computer:mouse_move',
-        coordinate,
-    });
+    console.log('Sending mouse_move message to tab:', tabId, { coordinate });
+    try {
+        const response = await chrome.tabs.sendMessage(tabId, {
+            type: 'computer:mouse_move',
+            coordinate,
+        });
+        console.log('Got response:', response);
+        return response;
+    }
+    catch (e) {
+        console.error('Failed to send mouse_move message:', e);
+        throw e;
+    }
 }
 async function left_click(tabId, coordinate) {
-    if (!coordinate) {
-        coordinate = (await cursor_position(tabId)).coordinate;
+    console.log('Sending left_click message to tab:', tabId, { coordinate });
+    try {
+        if (!coordinate) {
+            coordinate = (await cursor_position(tabId)).coordinate;
+        }
+        const response = await chrome.tabs.sendMessage(tabId, {
+            type: 'computer:left_click',
+            coordinate,
+        });
+        console.log('Got response:', response);
+        return response;
+    }
+    catch (e) {
+        console.error('Failed to send left_click message:', e);
+        throw e;
     }
-    return await chrome.tabs.sendMessage(tabId, {
-        type: 'computer:left_click',
-        coordinate,
-    });
 }
 async function left_click_by(tabId, xpath, highlightIndex) {
-    return await chrome.tabs.sendMessage(tabId, {
-        type: 'computer:left_click',
-        xpath,
-        highlightIndex,
-    });
+    console.log('Sending left_click_by message to tab:', tabId, { xpath, highlightIndex });
+    try {
+        const response = await chrome.tabs.sendMessage(tabId, {
+            type: 'computer:left_click',
+            xpath,
+            highlightIndex,
+        });
+        console.log('Got response:', response);
+        return response;
+    }
+    catch (e) {
+        console.error('Failed to send left_click_by message:', e);
+        throw e;
+    }
 }
 async function right_click(tabId, coordinate) {
-    if (!coordinate) {
-        coordinate = (await cursor_position(tabId)).coordinate;
+    console.log('Sending right_click message to tab:', tabId, { coordinate });
+    try {
+        if (!coordinate) {
+            coordinate = (await cursor_position(tabId)).coordinate;
+        }
+        const response = await chrome.tabs.sendMessage(tabId, {
+            type: 'computer:right_click',
+            coordinate,
+        });
+        console.log('Got response:', response);
+        return response;
+    }
+    catch (e) {
+        console.error('Failed to send right_click message:', e);
+        throw e;
     }
-    return await chrome.tabs.sendMessage(tabId, {
-        type: 'computer:right_click',
-        coordinate,
-    });
 }
 async function right_click_by(tabId, xpath, highlightIndex) {
-    return await chrome.tabs.sendMessage(tabId, {
-        type: 'computer:right_click',
-        xpath,
-        highlightIndex,
-    });
+    console.log('Sending right_click_by message to tab:', tabId, { xpath, highlightIndex });
+    try {
+        const response = await chrome.tabs.sendMessage(tabId, {
+            type: 'computer:right_click',
+            xpath,
+            highlightIndex,
+        });
+        console.log('Got response:', response);
+        return response;
+    }
+    catch (e) {
+        console.error('Failed to send right_click_by message:', e);
+        throw e;
+    }
 }
 async function double_click(tabId, coordinate) {
-    if (!coordinate) {
-        coordinate = (await cursor_position(tabId)).coordinate;
+    console.log('Sending double_click message to tab:', tabId, { coordinate });
+    try {
+        if (!coordinate) {
+            coordinate = (await cursor_position(tabId)).coordinate;
+        }
+        const response = await chrome.tabs.sendMessage(tabId, {
+            type: 'computer:double_click',
+            coordinate,
+        });
+        console.log('Got response:', response);
+        return response;
+    }
+    catch (e) {
+        console.error('Failed to send double_click message:', e);
+        throw e;
     }
-    return await chrome.tabs.sendMessage(tabId, {
-        type: 'computer:double_click',
-        coordinate,
-    });
 }
 async function double_click_by(tabId, xpath, highlightIndex) {
-    return await chrome.tabs.sendMessage(tabId, {
-        type: 'computer:double_click',
-        xpath,
-        highlightIndex,
-    });
+    console.log('Sending double_click_by message to tab:', tabId, { xpath, highlightIndex });
+    try {
+        const response = await chrome.tabs.sendMessage(tabId, {
+            type: 'computer:double_click',
+            xpath,
+            highlightIndex,
+        });
+        console.log('Got response:', response);
+        return response;
+    }
+    catch (e) {
+        console.error('Failed to send double_click_by message:', e);
+        throw e;
+    }
 }
-async function screenshot(windowId) {
-    let dataUrl = await chrome.tabs.captureVisibleTab(windowId, {
-        format: 'jpeg', // jpeg / png
-        quality: 50, // 0-100
-    });
-    let data = dataUrl.substring(dataUrl.indexOf('base64,') + 7);
-    return {
-        image: {
-            type: 'base64',
-            media_type: dataUrl.indexOf('image/png') > -1 ? 'image/png' : 'image/jpeg',
-            data: data,
-        },
-    };
+async function screenshot(windowId, compress) {
+    console.log('Taking screenshot of window:', windowId, { compress });
+    try {
+        let dataUrl;
+        if (compress) {
+            dataUrl = await chrome.tabs.captureVisibleTab(windowId, {
+                format: 'jpeg',
+                quality: 60, // 0-100
+            });
+            dataUrl = await compress_image(dataUrl, 0.7, 1);
+        }
+        else {
+            dataUrl = await chrome.tabs.captureVisibleTab(windowId, {
+                format: 'jpeg',
+                quality: 50,
+            });
+        }
+        let data = dataUrl.substring(dataUrl.indexOf('base64,') + 7);
+        const result = {
+            image: {
+                type: 'base64',
+                media_type: dataUrl.indexOf('image/png') > -1 ? 'image/png' : 'image/jpeg',
+                data: data,
+            },
+        };
+        console.log('Got screenshot result:', result);
+        return result;
+    }
+    catch (e) {
+        console.error('Failed to take screenshot:', e);
+        throw e;
+    }
+}
+async function compress_image(dataUrl, scale = 0.8, quality = 0.8) {
+    console.log('Compressing image', { scale, quality });
+    try {
+        const bitmap = await createImageBitmap(await (await fetch(dataUrl)).blob());
+        let width = bitmap.width * scale;
+        let height = bitmap.height * scale;
+        const canvas = new OffscreenCanvas(width, height);
+        const ctx = canvas.getContext('2d');
+        ctx.drawImage(bitmap, 0, 0, width, height);
+        const blob = await canvas.convertToBlob({
+            type: 'image/jpeg',
+            quality: quality,
+        });
+        return new Promise((resolve) => {
+            const reader = new FileReader();
+            reader.onloadend = () => {
+                const result = reader.result;
+                console.log('Got compressed image result:', result);
+                resolve(result);
+            };
+            reader.readAsDataURL(blob);
+        });
+    }
+    catch (e) {
+        console.error('Failed to compress image:', e);
+        throw e;
+    }
 }
 async function scroll_to(tabId, coordinate) {
-    let from_coordinate = (await cursor_position(tabId)).coordinate;
-    return await chrome.tabs.sendMessage(tabId, {
-        type: 'computer:scroll_to',
-        from_coordinate,
-        to_coordinate: coordinate,
-    });
+    console.log('Sending scroll_to message to tab:', tabId, { coordinate });
+    try {
+        let from_coordinate = (await cursor_position(tabId)).coordinate;
+        const response = await chrome.tabs.sendMessage(tabId, {
+            type: 'computer:scroll_to',
+            from_coordinate,
+            to_coordinate: coordinate,
+        });
+        console.log('Got response:', response);
+        return response;
+    }
+    catch (e) {
+        console.error('Failed to send scroll_to message:', e);
+        throw e;
+    }
 }
 async function scroll_to_by(tabId, xpath, highlightIndex) {
-    return await chrome.tabs.sendMessage(tabId, {
-        type: 'computer:scroll_to',
-        xpath,
-        highlightIndex,
-    });
+    console.log('Sending scroll_to_by message to tab:', tabId, { xpath, highlightIndex });
+    try {
+        const response = await chrome.tabs.sendMessage(tabId, {
+            type: 'computer:scroll_to',
+            xpath,
+            highlightIndex,
+        });
+        console.log('Got response:', response);
+        return response;
+    }
+    catch (e) {
+        console.error('Failed to send scroll_to_by message:', e);
+        throw e;
+    }
 }
 async function get_dropdown_options(tabId, xpath, highlightIndex) {
-    return await chrome.tabs.sendMessage(tabId, {
-        type: 'computer:get_dropdown_options',
-        xpath,
-        highlightIndex,
-    });
+    console.log('Sending get_dropdown_options message to tab:', tabId, { xpath, highlightIndex });
+    try {
+        const response = await chrome.tabs.sendMessage(tabId, {
+            type: 'computer:get_dropdown_options',
+            xpath,
+            highlightIndex,
+        });
+        console.log('Got response:', response);
+        return response;
+    }
+    catch (e) {
+        console.error('Failed to send get_dropdown_options message:', e);
+        throw e;
+    }
 }
 async function select_dropdown_option(tabId, text, xpath, highlightIndex) {
-    return await chrome.tabs.sendMessage(tabId, {
-        type: 'computer:select_dropdown_option',
-        text,
-        xpath,
-        highlightIndex,
-    });
+    console.log('Sending select_dropdown_option message to tab:', tabId, { text, xpath, highlightIndex });
+    try {
+        const response = await chrome.tabs.sendMessage(tabId, {
+            type: 'computer:select_dropdown_option',
+            text,
+            xpath,
+            highlightIndex,
+        });
+        console.log('Got response:', response);
+        return response;
+    }
+    catch (e) {
+        console.error('Failed to send select_dropdown_option message:', e);
+        throw e;
+    }
 }
 async function cursor_position(tabId) {
-    let result = await chrome.tabs.sendMessage(tabId, {
-        type: 'computer:cursor_position',
-    });
-    return { coordinate: result.coordinate };
+    console.log('Sending cursor_position message to tab:', tabId);
+    try {
+        let result = await chrome.tabs.sendMessage(tabId, {
+            type: 'computer:cursor_position',
+        });
+        console.log('Got cursor position:', result.coordinate);
+        return { coordinate: result.coordinate };
+    }
+    catch (e) {
+        console.error('Failed to send cursor_position message:', e);
+        throw e;
+    }
 }
 async function size(tabId) {
-    return await getPageSize(tabId);
+    console.log('Getting page size for tab:', tabId);
+    try {
+        const pageSize = await getPageSize(tabId);
+        console.log('Got page size:', pageSize);
+        return pageSize;
+    }
+    catch (e) {
+        console.error('Failed to get page size:', e);
+        throw e;
+    }
 }
 
 var browser = /*#__PURE__*/Object.freeze({
     __proto__: null,
     clear_input: clear_input,
     clear_input_by: clear_input_by,
+    compress_image: compress_image,
     cursor_position: cursor_position,
     double_click: double_click,
     double_click_by: double_click_by,
@@ -421,7 +647,6 @@ class BrowserUse {
         this.name = 'browser_use';
         this.description = `Use structured commands to interact with the browser, manipulating page elements through screenshots and webpage element extraction.
 * This is a browser GUI interface where you need to analyze webpages by taking screenshots and extracting page element structures, and specify action sequences to complete designated tasks.
-* Some operations may need time to process, so you might need to wait and continuously take screenshots and extract element structures to check the operation results.
 * Before any operation, you must first call the \`screenshot_extract_element\` command, which will return the browser page screenshot and structured element information, both specially processed.
 * ELEMENT INTERACTION:
    - Only use indexes that exist in the provided element list
@@ -431,17 +656,7 @@ class BrowserUse {
    - If no suitable elements exist, use other functions to complete the task
    - If stuck, try alternative approaches
    - Handle popups/cookies by accepting or closing them
-   - Use scroll to find elements you are looking for
-* Form filling:
-   - If you fill a input field and your action sequence is interrupted, most often a list with suggestions poped up under the field and you need to first select the right element from the suggestion list.
-* ACTION SEQUENCING:
-   - Actions are executed in the order they appear in the list 
-   - Each action should logically follow from the previous one
-   - If the page changes after an action, the sequence is interrupted and you get the new state.
-   - If content only disappears the sequence continues.
-   - Only provide the action sequence until you think the page will change.
-   - Try to be efficient, e.g. fill forms at once, or chain actions where nothing changes on the page like saving, extracting, checkboxes...
-   - only use multiple actions if it makes sense.`;
+   - Use scroll to find elements you are looking for`;
         this.input_schema = {
             type: 'object',
             properties: {
@@ -496,7 +711,17 @@ class BrowserUse {
             if (params === null || !params.action) {
                 throw new Error('Invalid parameters. Expected an object with a "action" property.');
             }
-            let tabId = await getTabId(context);
+            let tabId;
+            try {
+                tabId = await getTabId(context);
+                if (!tabId || !Number.isInteger(tabId)) {
+                    throw new Error('Could not get valid tab ID');
+                }
+            }
+            catch (e) {
+                console.error('Tab ID error:', e);
+                return { success: false, error: 'Could not access browser tab' };
+            }
             let windowId = await getWindowId(context);
             let selector_map = context.selector_map;
             let selector_xpath;
@@ -583,7 +808,7 @@ class BrowserUse {
                         return window.get_clickable_elements(true);
                     }, []);
                     context.selector_map = element_result.selector_map;
-                    let screenshot$1 = await screenshot(windowId);
+                    let screenshot$1 = await screenshot(windowId, true);
                     await executeScript(tabId, () => {
                         return window.remove_highlight();
                     }, []);
@@ -600,6 +825,7 @@ class BrowserUse {
             }
         }
         catch (e) {
+            console.error('Browser use error:', e);
             return { success: false, error: e === null || e === void 0 ? void 0 : e.message };
         }
     }
@@ -815,7 +1041,7 @@ ${pseudoHtml}
 async function executeWithBrowserUse$1(context, task_prompt) {
     let tabId = await getTabId(context);
     let windowId = await getWindowId(context);
-    let screenshot_result = await screenshot(windowId);
+    let screenshot_result = await screenshot(windowId, false);
     let messages = [
         {
             role: 'user',
@@ -895,6 +1121,7 @@ class ExportFile {
      * @returns > { success: true }
      */
     async execute(context, params) {
+        var _a, _b, _c;
         if (typeof params !== 'object' || params === null || !('content' in params)) {
             throw new Error('Invalid parameters. Expected an object with a "content" property.');
         }
@@ -929,8 +1156,8 @@ class ExportFile {
         else {
             filename = params.filename;
         }
-        let tabId = await getTabId(context);
         try {
+            let tabId = await getTabId(context);
             await chrome.scripting.executeScript({
                 target: { tabId: tabId },
                 func: exportFile,
@@ -939,13 +1166,14 @@ class ExportFile {
         }
         catch (e) {
             let tab = await open_new_tab('https://www.google.com', true);
-            tabId = tab.id;
+            (_c = (_b = (_a = context.callback) === null || _a === void 0 ? void 0 : _a.hooks) === null || _b === void 0 ? void 0 : _b.onTabCreated) === null || _c === void 0 ? void 0 : _c.call(_b, tab.id);
+            let tabId = tab.id;
             await chrome.scripting.executeScript({
                 target: { tabId: tabId },
                 func: exportFile,
                 args: [filename, type, params.content],
             });
-            await sleep(1000);
+            await sleep(5000);
             await chrome.tabs.remove(tabId);
         }
         return { success: true };
@@ -1063,7 +1291,7 @@ ${pseudoHtml}
 async function executeWithBrowserUse(context, task_prompt) {
     await getTabId(context);
     let windowId = await getWindowId(context);
-    let screenshot_result = await screenshot(windowId);
+    let screenshot_result = await screenshot(windowId, false);
     let messages = [
         {
             role: 'user',
@@ -1111,6 +1339,48 @@ async function executeWithBrowserUse(context, task_prompt) {
     };
 }
 
+class GetAllTabs {
+    constructor() {
+        this.name = 'get_all_tabs';
+        this.description = 'Get the tabId, title, url and content from current all tabs without opening new tab.';
+        this.input_schema = {
+            type: 'object',
+            properties: {},
+        };
+    }
+    async execute(context, params) {
+        const currentWindow = await chrome.windows.getCurrent();
+        const windowId = currentWindow.id;
+        const tabs = await chrome.tabs.query({ windowId });
+        const tabsInfo = [];
+        for (const tab of tabs) {
+            if (tab.id === undefined) {
+                console.warn(`Tab ID is undefined for tab with URL: ${tab.url}`);
+                continue;
+            }
+            await injectScript(tab.id);
+            await sleep(500);
+            let content = await executeScript(tab.id, () => {
+                return eko.extractHtmlContent();
+            }, []);
+            // Use title as description, but requirement may evolve
+            let description = tab.title ? tab.title : "No description available.";
+            const tabInfo = {
+                id: tab.id,
+                url: tab.url,
+                title: tab.title,
+                content: content,
+                description: description,
+            };
+            console.log("url: " + tab.url);
+            console.log("title: " + tab.title);
+            console.log("description: " + description);
+            tabsInfo.push(tabInfo);
+        }
+        return tabsInfo;
+    }
+}
+
 /**
  * Open Url
  */
@@ -1140,6 +1410,7 @@ class OpenUrl {
      * @returns > { tabId, windowId, title, success: true }
      */
     async execute(context, params) {
+        var _a, _b, _c, _d, _e, _f;
         if (typeof params !== 'object' || params === null || !params.url) {
             throw new Error('Invalid parameters. Expected an object with a "url" property.');
         }
@@ -1152,10 +1423,12 @@ class OpenUrl {
         let tab;
         if (newWindow) {
             tab = await open_new_tab(url, true);
+            (_c = (_b = (_a = context.callback) === null || _a === void 0 ? void 0 : _a.hooks) === null || _b === void 0 ? void 0 : _b.onTabCreated) === null || _c === void 0 ? void 0 : _c.call(_b, tab.id);
         }
         else {
             let windowId = await getWindowId(context);
             tab = await open_new_tab(url, false, windowId);
+            (_f = (_e = (_d = context.callback) === null || _d === void 0 ? void 0 : _d.hooks) === null || _e === void 0 ? void 0 : _e.onTabCreated) === null || _f === void 0 ? void 0 : _f.call(_e, tab.id);
         }
         let windowId = tab.windowId;
         let tabId = tab.id;
@@ -1212,9 +1485,9 @@ class TabManagement {
         this.input_schema = {
             type: 'object',
             properties: {
-                commond: {
+                command: {
                     type: 'string',
-                    description: `The commond to perform. The available commonds are:
+                    description: `The command to perform. The available commands are:
 * \`tab_all\`: View all tabs and return the tabId and title.
 * \`current_tab\`: Get current tab information (tabId, url, title).
 * \`go_back\`: Go back to the previous page in the current tab.
@@ -1224,29 +1497,30 @@ class TabManagement {
 * \`new_tab [url]\`: Open a new tab window and open the URL, eg: \`new_tab https://www.google.com\``,
                 },
             },
-            required: ['commond'],
+            required: ['command'],
         };
     }
     /**
      * Tab management
      *
-     * @param {*} params { commond: `new_tab [url]` | 'tab_all' | 'current_tab' | 'go_back' | 'close_tab' | 'switch_tab [tabId]' | `change_url [url]` }
+     * @param {*} params { command: `new_tab [url]` | 'tab_all' | 'current_tab' | 'go_back' | 'close_tab' | 'switch_tab [tabId]' | `change_url [url]` }
      * @returns > { result, success: true }
      */
     async execute(context, params) {
-        if (params === null || !params.commond) {
-            throw new Error('Invalid parameters. Expected an object with a "commond" property.');
+        var _a, _b, _c, _d, _e, _f;
+        if (params === null || !params.command) {
+            throw new Error('Invalid parameters. Expected an object with a "command" property.');
         }
         let windowId = await getWindowId(context);
-        let commond = params.commond.trim();
-        if (commond.startsWith('`')) {
-            commond = commond.substring(1);
+        let command = params.command.trim();
+        if (command.startsWith('`')) {
+            command = command.substring(1);
         }
-        if (commond.endsWith('`')) {
-            commond = commond.substring(0, commond.length - 1);
+        if (command.endsWith('`')) {
+            command = command.substring(0, command.length - 1);
         }
         let result;
-        if (commond == 'tab_all') {
+        if (command == 'tab_all') {
             result = [];
             let tabs = await chrome.tabs.query({ windowId: windowId });
             for (let i = 0; i < tabs.length; i++) {
@@ -1263,20 +1537,20 @@ class TabManagement {
                 result.push(tabInfo);
             }
         }
-        else if (commond == 'current_tab') {
+        else if (command == 'current_tab') {
             let tabId = await getTabId(context);
             let tab = await chrome.tabs.get(tabId);
             let tabInfo = { tabId, windowId: tab.windowId, title: tab.title, url: tab.url };
             result = tabInfo;
         }
-        else if (commond == 'go_back') {
+        else if (command == 'go_back') {
             let tabId = await getTabId(context);
             await chrome.tabs.goBack(tabId);
             let tab = await chrome.tabs.get(tabId);
             let tabInfo = { tabId, windowId: tab.windowId, title: tab.title, url: tab.url };
             result = tabInfo;
         }
-        else if (commond == 'close_tab') {
+        else if (command == 'close_tab') {
             let closedTabId = await getTabId(context);
             await chrome.tabs.remove(closedTabId);
             await sleep(100);
@@ -1294,16 +1568,16 @@ class TabManagement {
             let closeTabInfo = { closedTabId, newTabId, newTabTitle: tab.title };
             result = closeTabInfo;
         }
-        else if (commond.startsWith('switch_tab')) {
-            let tabId = parseInt(commond.replace('switch_tab', '').replace('[', '').replace(']', ''));
+        else if (command.startsWith('switch_tab')) {
+            let tabId = parseInt(command.replace('switch_tab', '').replace('[', '').replace(']', ''));
             let tab = await chrome.tabs.update(tabId, { active: true });
             context.variables.set('tabId', tab.id);
             context.variables.set('windowId', tab.windowId);
             let tabInfo = { tabId, windowId: tab.windowId, title: tab.title, url: tab.url };
             result = tabInfo;
         }
-        else if (commond.startsWith('change_url')) {
-            let url = commond.substring('change_url'.length).replace('[', '').replace(']', '').trim();
+        else if (command.startsWith('change_url')) {
+            let url = command.substring('change_url'.length).replace('[', '').replace(']', '').trim();
             let tabId = await getTabId(context);
             // await chrome.tabs.update(tabId, { url: url });
             await executeScript(tabId, () => {
@@ -1313,17 +1587,19 @@ class TabManagement {
             let tabInfo = { tabId, windowId: tab.windowId, title: tab.title, url: tab.url };
             result = tabInfo;
         }
-        else if (commond.startsWith('new_tab')) {
-            let url = commond.replace('new_tab', '').replace('[', '').replace(']', '').replace(/"/g, '');
+        else if (command.startsWith('new_tab')) {
+            let url = command.replace('new_tab', '').replace('[', '').replace(']', '').replace(/"/g, '');
             // First mandatory opening of a new window
             let newWindow = !context.variables.get('windowId') && !context.variables.get('tabId');
             let tab;
             if (newWindow) {
                 tab = await open_new_tab(url, true);
+                (_c = (_b = (_a = context.callback) === null || _a === void 0 ? void 0 : _a.hooks) === null || _b === void 0 ? void 0 : _b.onTabCreated) === null || _c === void 0 ? void 0 : _c.call(_b, tab.id);
             }
             else {
                 let windowId = await getWindowId(context);
                 tab = await open_new_tab(url, false, windowId);
+                (_f = (_e = (_d = context.callback) === null || _d === void 0 ? void 0 : _d.hooks) === null || _e === void 0 ? void 0 : _e.onTabCreated) === null || _f === void 0 ? void 0 : _f.call(_e, tab.id);
             }
             let windowId = tab.windowId;
             let tabId = tab.id;
@@ -1347,7 +1623,7 @@ class TabManagement {
             result = tabInfo;
         }
         else {
-            throw Error('Unknown commond: ' + commond);
+            throw Error('Unknown command: ' + command);
         }
         return result;
     }
@@ -1367,7 +1643,7 @@ class TabManagement {
 class WebSearch {
     constructor() {
         this.name = 'web_search';
-        this.description = 'Use web search to return search results';
+        this.description = 'Search the web based on keywords and return relevant extracted content from webpages.';
         this.input_schema = {
             type: 'object',
             properties: {
@@ -1402,7 +1678,7 @@ class WebSearch {
         }
         let taskId = new Date().getTime() + '';
         let searchs = [{ url: url, keyword: query }];
-        let searchInfo = await deepSearch(taskId, searchs, maxResults || 5);
+        let searchInfo = await deepSearch(context, taskId, searchs, maxResults || 5);
         let links = ((_a = searchInfo.result[0]) === null || _a === void 0 ? void 0 : _a.links) || [];
         return links.filter((s) => s.content);
     }
@@ -1475,7 +1751,7 @@ chrome.tabs.onUpdated.addListener(async function (tabId, changeInfo, tab) {
  * @param {array} searchs search list => [{ url: 'https://bing.com', keyword: 'ai' }]
  * @param {number} detailsMaxNum Maximum crawling quantity per search detail page
  */
-async function deepSearch(taskId, searchs, detailsMaxNum, window) {
+async function deepSearch(context, taskId, searchs, detailsMaxNum, window) {
     let closeWindow = false;
     if (!window) {
         // open new window
@@ -1488,9 +1764,9 @@ async function deepSearch(taskId, searchs, detailsMaxNum, window) {
     }
     // crawler the search page details page link
     // [{ links: [{ title, url }] }]
-    let detailLinkGroups = await doDetailLinkGroups(taskId, searchs, detailsMaxNum, window);
+    let detailLinkGroups = await doDetailLinkGroups(context, taskId, searchs, detailsMaxNum, window);
     // crawler all details page content and comments
-    let searchInfo = await doPageContent(taskId, detailLinkGroups, window);
+    let searchInfo = await doPageContent(context, taskId, detailLinkGroups, window);
     console.log('searchInfo: ', searchInfo);
     // close window
     closeWindow && chrome.windows.remove(window.id);
@@ -1505,7 +1781,8 @@ async function deepSearch(taskId, searchs, detailsMaxNum, window) {
  * @param {*} window
  * @returns [{ links: [{ title, url }] }]
  */
-async function doDetailLinkGroups(taskId, searchs, detailsMaxNum, window) {
+async function doDetailLinkGroups(context, taskId, searchs, detailsMaxNum, window) {
+    var _a, _b, _c;
     let detailLinkGroups = [];
     let countDownLatch = new CountDownLatch(searchs.length);
     for (let i = 0; i < searchs.length; i++) {
@@ -1517,6 +1794,7 @@ async function doDetailLinkGroups(taskId, searchs, detailsMaxNum, window) {
                 url: url,
                 windowId: window.id,
             });
+            (_c = (_b = (_a = context.callback) === null || _a === void 0 ? void 0 : _a.hooks) === null || _b === void 0 ? void 0 : _b.onTabCreated) === null || _c === void 0 ? void 0 : _c.call(_b, tab.id);
             let eventId = taskId + '_' + i;
             // monitor Tab status
             tabsUpdateEvent.addListener(async function (obj) {
@@ -1567,7 +1845,8 @@ async function doDetailLinkGroups(taskId, searchs, detailsMaxNum, window) {
  * @param {*} window
  * @returns search info
  */
-async function doPageContent(taskId, detailLinkGroups, window) {
+async function doPageContent(context, taskId, detailLinkGroups, window) {
+    var _a, _b, _c;
     const searchInfo = {
         total: 0,
         running: 0,
@@ -1591,65 +1870,374 @@ async function doPageContent(taskId, detailLinkGroups, window) {
                 url: link.url,
                 windowId: window.id,
             });
+            (_c = (_b = (_a = context.callback) === null || _a === void 0 ? void 0 : _a.hooks) === null || _b === void 0 ? void 0 : _b.onTabCreated) === null || _c === void 0 ? void 0 : _c.call(_b, tab.id);
             searchInfo.running++;
             let eventId = taskId + '_' + i + '_' + j;
-            // monitor Tab status
-            tabsUpdateEvent.addListener(async function (obj) {
-                if (obj.tabId != tab.id) {
-                    return;
-                }
-                if (obj.changeInfo.status === 'complete') {
-                    try {
+            // Create a timeout promise
+            const timeoutPromise = new Promise((_, reject) => {
+                setTimeout(() => reject(new Error('Page load timeout')), 10000); // Timeout after 10 seconds
+            });
+            // Create a tab monitoring promise
+            const monitorTabPromise = new Promise(async (resolve, reject) => {
+                tabsUpdateEvent.addListener(async function onTabUpdated(obj) {
+                    if (obj.tabId !== tab.id)
+                        return;
+                    if (obj.changeInfo.status === 'complete') {
                         tabsUpdateEvent.removeListener(eventId);
-                        // inject js
-                        await injectScript(tab.id, filename);
-                        await sleep(1000);
-                        // cralwer content and comments
-                        // { title, content }
-                        let result = await chrome.tabs.sendMessage(tab.id, {
-                            type: 'page:getContent',
-                        });
-                        if (!result) {
-                            throw Error('No Result');
+                        try {
+                            // Inject script and get page content
+                            await injectScript(tab.id, filename);
+                            await sleep(1000);
+                            let result = await chrome.tabs.sendMessage(tab.id, {
+                                type: 'page:getContent',
+                            });
+                            if (!result)
+                                throw new Error('No Result');
+                            link.content = result.content;
+                            link.page_title = result.title;
+                            searchInfo.succeed++;
+                            resolve(); // Resolve the promise if successful
+                        }
+                        catch (error) {
+                            searchInfo.failed++;
+                            searchInfo.failedLinks.push(link);
+                            reject(error); // Reject the promise on error
+                        }
+                        finally {
+                            searchInfo.running--;
+                            countDownLatch.countDown();
+                            chrome.tabs.remove(tab.id);
+                            tabsUpdateEvent.removeListener(eventId);
                         }
-                        link.content = result.content;
-                        link.page_title = result.title;
-                        searchInfo.succeed++;
-                    }
-                    catch (e) {
-                        searchInfo.failed++;
-                        searchInfo.failedLinks.push(link);
-                        console.error(link.title + ' crawler error', link.url, e);
                     }
-                    finally {
+                    else if (obj.changeInfo.status === 'unloaded') {
                         searchInfo.running--;
                         countDownLatch.countDown();
                         chrome.tabs.remove(tab.id);
                         tabsUpdateEvent.removeListener(eventId);
+                        reject(new Error('Tab unloaded')); // Reject if the tab is unloaded
                     }
-                }
-                else if (obj.changeInfo.status === 'unloaded') {
-                    searchInfo.running--;
-                    countDownLatch.countDown();
-                    chrome.tabs.remove(tab.id);
-                    tabsUpdateEvent.removeListener(eventId);
-                }
-            }, eventId);
+                }, eventId);
+            });
+            // Use Promise.race to enforce the timeout
+            try {
+                await Promise.race([monitorTabPromise, timeoutPromise]);
+            }
+            catch (e) {
+                console.error(`${link.title} failed:`, e);
+                searchInfo.running--;
+                searchInfo.failed++;
+                searchInfo.failedLinks.push(link);
+                countDownLatch.countDown();
+                chrome.tabs.remove(tab.id); // Clean up tab on failure
+            }
         }
     }
     await countDownLatch.await(60000);
     return searchInfo;
 }
 
+class RequestLogin {
+    constructor() {
+        this.name = 'request_login';
+        this.description =
+            'Login to this website, assist with identity verification when manual intervention is needed, guide users through the login process, and wait for their confirmation of successful login.';
+        this.input_schema = {
+            type: 'object',
+            properties: {},
+        };
+    }
+    async execute(context, params) {
+        if (!params.force && await this.isLoginIn(context)) {
+            return true;
+        }
+        let tabId = await getTabId(context);
+        let task_id = 'login_required_' + tabId;
+        const request_user_help = async () => {
+            await chrome.tabs.sendMessage(tabId, {
+                type: 'request_user_help',
+                task_id,
+                failure_type: 'login_required',
+                failure_message: 'Access page require user authentication.',
+            });
+        };
+        const login_interval = setInterval(async () => {
+            try {
+                request_user_help();
+            }
+            catch (e) {
+                clearInterval(login_interval);
+            }
+        }, 2000);
+        try {
+            return await this.awaitLogin(tabId, task_id);
+        }
+        finally {
+            clearInterval(login_interval);
+        }
+    }
+    async awaitLogin(tabId, task_id) {
+        return new Promise((resolve) => {
+            const checkTabClosedInterval = setInterval(async () => {
+                const tabExists = await doesTabExists(tabId);
+                if (!tabExists) {
+                    clearInterval(checkTabClosedInterval);
+                    resolve(false);
+                    chrome.runtime.onMessage.removeListener(listener);
+                }
+            }, 1000);
+            const listener = (message) => {
+                if (message.type === 'issue_resolved' && message.task_id === task_id) {
+                    resolve(true);
+                    clearInterval(checkTabClosedInterval);
+                }
+            };
+            chrome.runtime.onMessage.addListener(listener);
+        });
+    }
+    async isLoginIn(context) {
+        let windowId = await getWindowId(context);
+        let screenshot_result = await screenshot(windowId, true);
+        let messages = [
+            {
+                role: 'user',
+                content: [
+                    {
+                        type: 'image',
+                        source: screenshot_result.image,
+                    },
+                    {
+                        type: 'text',
+                        text: 'Check if the current website is logged in. If not logged in, output `NOT_LOGIN`. If logged in, output `LOGGED_IN`. Output directly without explanation.',
+                    },
+                ],
+            },
+        ];
+        let response = await context.llmProvider.generateText(messages, { maxTokens: 256 });
+        let text = response.textContent;
+        if (!text) {
+            text = JSON.stringify(response.content);
+        }
+        return text.indexOf('LOGGED_IN') > -1;
+    }
+}
+
+class CancelWorkflow {
+    constructor() {
+        this.name = 'cancel_workflow';
+        this.description = 'Cancel the workflow. If any tool consistently encounters exceptions, invoke this tool to cancel the workflow.';
+        this.input_schema = {
+            type: 'object',
+            properties: {
+                reason: {
+                    type: 'string',
+                    description: 'Why the workflow should be cancelled.',
+                },
+            },
+            required: ['reason'],
+        };
+    }
+    async execute(context, params) {
+        var _a;
+        if (typeof params !== 'object' || params === null || !params.reason) {
+            throw new Error('Invalid parameters. Expected an object with a "reason" property.');
+        }
+        const reason = params.reason;
+        console.log("The workflow has been cancelled because: " + reason);
+        await ((_a = context.workflow) === null || _a === void 0 ? void 0 : _a.cancel());
+        return;
+    }
+}
+
+class HumanInputText {
+    constructor() {
+        this.name = 'human_input_text';
+        this.description = 'When you are unsure about the details of your next action, call me and ask the user for details in the "question" field. The user will provide you with a text as an answer.';
+        this.input_schema = {
+            type: 'object',
+            properties: {
+                question: {
+                    type: 'string',
+                    description: 'Ask the user here.',
+                },
+            },
+            required: ['question'],
+        };
+    }
+    async execute(context, params) {
+        var _a, _b, _c;
+        if (typeof params !== 'object' || params === null || !params.question) {
+            throw new Error('Invalid parameters. Expected an object with a "question" property.');
+        }
+        const question = params.question;
+        console.log("question: " + question);
+        let answer = await ((_c = (_a = context.callback) === null || _a === void 0 ? void 0 : (_b = _a.hooks).onHumanInputText) === null || _c === void 0 ? void 0 : _c.call(_b, question));
+        if (!answer) {
+            console.error("Cannot get user's answer.");
+            return { status: "Error: Cannot get user's answer.", answer: "" };
+        }
+        else {
+            console.log("answer: " + answer);
+            return { status: "OK", answer: answer };
+        }
+    }
+}
+class HumanInputSingleChoice {
+    constructor() {
+        this.name = 'human_input_single_choice';
+        this.description = 'When you are unsure about the details of your next action, call me and ask the user for details in the "question" field with at least 2 choices. The user will provide you with ONE choice as an answer.';
+        this.input_schema = {
+            type: 'object',
+            properties: {
+                question: {
+                    type: 'string',
+                    description: 'Ask the user here.',
+                },
+                choices: {
+                    type: 'array',
+                    description: 'All of the choices.',
+                }
+            },
+            required: ['question', 'choices'],
+        };
+    }
+    async execute(context, params) {
+        var _a, _b, _c;
+        if (typeof params !== 'object' || params === null || !params.question || !params.choices) {
+            throw new Error('Invalid parameters. Expected an object with a "question" and "choices" property.');
+        }
+        const question = params.question;
+        const choices = params.choices;
+        console.log("question: " + question);
+        console.log("choices: " + choices);
+        let answer = await ((_c = (_a = context.callback) === null || _a === void 0 ? void 0 : (_b = _a.hooks).onHumanInputSingleChoice) === null || _c === void 0 ? void 0 : _c.call(_b, question, choices));
+        if (!answer) {
+            console.error("Cannot get user's answer.");
+            return { status: "Error: Cannot get user's answer.", answer: "" };
+        }
+        else {
+            console.log("answer: " + answer);
+            return { status: "OK", answer: answer };
+        }
+    }
+}
+class HumanInputMultipleChoice {
+    constructor() {
+        this.name = 'human_input_multiple_choice';
+        this.description = 'When you are unsure about the details of your next action, call me and ask the user for details in the "question" field with at least 2 choices. The user will provide you with ONE or MORE choice as an answer.';
+        this.input_schema = {
+            type: 'object',
+            properties: {
+                question: {
+                    type: 'string',
+                    description: 'Ask the user here.',
+                },
+                choices: {
+                    type: 'array',
+                    description: 'All of the choices.',
+                }
+            },
+            required: ['question', 'choices'],
+        };
+    }
+    async execute(context, params) {
+        var _a, _b, _c;
+        if (typeof params !== 'object' || params === null || !params.question || !params.choices) {
+            throw new Error('Invalid parameters. Expected an object with a "question" and "choices" property.');
+        }
+        const question = params.question;
+        const choices = params.choices;
+        console.log("question: " + question);
+        console.log("choices: " + choices);
+        let answer = await ((_c = (_a = context.callback) === null || _a === void 0 ? void 0 : (_b = _a.hooks).onHumanInputMultipleChoice) === null || _c === void 0 ? void 0 : _c.call(_b, question, choices));
+        if (!answer) {
+            console.error("Cannot get user's answer.");
+            return { status: "Error: Cannot get user's answer.", answer: [] };
+        }
+        else {
+            console.log("answer: " + answer);
+            return { status: "OK", answer: answer };
+        }
+    }
+}
+class HumanOperate {
+    constructor() {
+        this.name = 'human_operate';
+        this.description = 'When you encounter operations that require login, CAPTCHA verification, or other tasks that you cannot complete, please call this tool, transfer control to the user, and explain why.';
+        this.input_schema = {
+            type: 'object',
+            properties: {
+                reason: {
+                    type: 'string',
+                    description: 'The reason why you need to transfer control.',
+                },
+            },
+            required: ['reason'],
+        };
+    }
+    async execute(context, params) {
+        var _a, _b, _c;
+        if (typeof params !== 'object' || params === null || !params.reason) {
+            throw new Error('Invalid parameters. Expected an object with a "reason" property.');
+        }
+        const reason = params.reason;
+        console.log("reason: " + reason);
+        let userOperation = await ((_c = (_a = context.callback) === null || _a === void 0 ? void 0 : (_b = _a.hooks).onHumanOperate) === null || _c === void 0 ? void 0 : _c.call(_b, reason));
+        if (!userOperation) {
+            console.error("Cannot get user's operation.");
+            return { status: "Error: Cannot get user's operation.", userOperation: "" };
+        }
+        else {
+            console.log("userOperation: " + userOperation);
+            return { status: "OK", userOperation: userOperation };
+        }
+    }
+}
+
+class SummaryWorkflow {
+    constructor() {
+        this.name = 'summary_workflow';
+        this.description = 'Summarize what this workflow has done from start to finish using an ordered list .';
+        this.input_schema = {
+            type: 'object',
+            properties: {
+                summary: {
+                    type: 'string',
+                    description: 'Your summary in markdown format.',
+                },
+            },
+            required: ['summary'],
+        };
+    }
+    async execute(context, params) {
+        var _a, _b, _c;
+        if (typeof params !== 'object' || params === null || !params.summary) {
+            throw new Error('Invalid parameters. Expected an object with a "summary" property.');
+        }
+        const summary = params.summary;
+        console.log("summary: " + summary);
+        await ((_c = (_a = context.callback) === null || _a === void 0 ? void 0 : (_b = _a.hooks).onSummaryWorkflow) === null || _c === void 0 ? void 0 : _c.call(_b, summary));
+        return { status: "OK" };
+    }
+}
+
 var tools = /*#__PURE__*/Object.freeze({
     __proto__: null,
     BrowserUse: BrowserUse,
+    CancelWorkflow: CancelWorkflow,
     ElementClick: ElementClick,
     ExportFile: ExportFile,
     ExtractContent: ExtractContent,
     FindElementPosition: FindElementPosition,
+    GetAllTabs: GetAllTabs,
+    HumanInputMultipleChoice: HumanInputMultipleChoice,
+    HumanInputSingleChoice: HumanInputSingleChoice,
+    HumanInputText: HumanInputText,
+    HumanOperate: HumanOperate,
     OpenUrl: OpenUrl,
+    RequestLogin: RequestLogin,
     Screenshot: Screenshot,
+    SummaryWorkflow: SummaryWorkflow,
     TabManagement: TabManagement,
     WebSearch: WebSearch
 });