@eko-ai/eko 1.0.8 → 1.0.10

package/dist/src/extension/tools/element_click.js

@@ -0,0 +1,123 @@
+import { executeScript, getTabId, getWindowId } from '../utils';
+import { extractOperableElements, clickOperableElement } from './html_script';
+import { left_click, screenshot } from './browser';
+/**
+ * Element click
+ */
+export class ElementClick {
+    constructor() {
+        this.name = 'element_click';
+        this.description = 'Click the element through task prompts';
+        this.input_schema = {
+            type: 'object',
+            properties: {
+                task_prompt: {
+                    type: 'string',
+                    description: 'Task prompt, eg: click search button',
+                },
+            },
+            required: ['task_prompt'],
+        };
+    }
+    async execute(context, params) {
+        if (typeof params !== 'object' || params === null || !params.task_prompt) {
+            throw new Error('Invalid parameters. Expected an object with a "task_prompt" property.');
+        }
+        let result;
+        let task_prompt = params.task_prompt;
+        try {
+            result = await executeWithHtmlElement(context, task_prompt);
+        }
+        catch (e) {
+            console.log(e);
+            result = false;
+        }
+        if (!result) {
+            result = await executeWithBrowserUse(context, task_prompt);
+        }
+        return result;
+    }
+}
+async function executeWithHtmlElement(context, task_prompt) {
+    let tabId = await getTabId(context);
+    let pseudoHtml = await executeScript(tabId, extractOperableElements, []);
+    let messages = [
+        {
+            role: 'user',
+            content: `# Task
+Determine the operation intent based on user input, find the element ID that the user needs to operate on in the webpage HTML, and if the element does not exist, do nothing.
+Output JSON format, no explanation required.
+
+# User input
+${task_prompt}
+
+# Output example (when the element exists)
+{"elementId": "1", "operationType": "click"}
+
+# Output example (when the element does not exist)
+{"elementId": null, "operationType": "unknown"}
+
+# HTML
+${pseudoHtml}
+`,
+        },
+    ];
+    let llm_params = { maxTokens: 1024 };
+    let response = await context.llmProvider.generateText(messages, llm_params);
+    let content = typeof response.content == 'string' ? response.content : response.content[0].text;
+    let json = content.substring(content.indexOf('{'), content.indexOf('}') + 1);
+    let elementId = JSON.parse(json).elementId;
+    if (elementId) {
+        return await executeScript(tabId, clickOperableElement, [elementId]);
+    }
+    return false;
+}
+async function executeWithBrowserUse(context, task_prompt) {
+    let tabId = await getTabId(context);
+    let windowId = await getWindowId(context);
+    let screenshot_result = await screenshot(windowId, false);
+    let messages = [
+        {
+            role: 'user',
+            content: [
+                {
+                    type: 'image',
+                    source: screenshot_result.image,
+                },
+                {
+                    type: 'text',
+                    text: 'click: ' + task_prompt,
+                },
+            ],
+        },
+    ];
+    let llm_params = {
+        maxTokens: 1024,
+        toolChoice: {
+            type: 'tool',
+            name: 'left_click',
+        },
+        tools: [
+            {
+                name: 'left_click',
+                description: 'click element',
+                input_schema: {
+                    type: 'object',
+                    properties: {
+                        coordinate: {
+                            type: 'array',
+                            description: '(x, y): The x (pixels from the left edge) and y (pixels from the top edge) coordinates.',
+                        },
+                    },
+                    required: ['coordinate'],
+                },
+            },
+        ],
+    };
+    let response = await context.llmProvider.generateText(messages, llm_params);
+    let input = response.toolCalls[0].input;
+    let coordinate = input.coordinate;
+    let click_result = await left_click(tabId, coordinate);
+    return click_result;
+}
+//# sourceMappingURL=element_click.js.map

package/dist/src/extension/tools/element_click.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"element_click.js","sourceRoot":"","sources":["../../../../src/extension/tools/element_click.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAChE,OAAO,EAAE,uBAAuB,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAC9E,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAGnD;;GAEG;AACH,MAAM,OAAO,YAAY;IAKvB;QACE,IAAI,CAAC,IAAI,GAAG,eAAe,CAAC;QAC5B,IAAI,CAAC,WAAW,GAAG,wCAAwC,CAAC;QAC5D,IAAI,CAAC,YAAY,GAAG;YAClB,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,WAAW,EAAE;oBACX,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,sCAAsC;iBACpD;aACF;YACD,QAAQ,EAAE,CAAC,aAAa,CAAC;SAC1B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,OAAyB,EAAE,MAAkB;QACzD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YACzE,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAC;QAC3F,CAAC;QACD,IAAI,MAAM,CAAC;QACX,IAAI,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACrC,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,sBAAsB,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QAC9D,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACf,MAAM,GAAG,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,GAAG,MAAM,qBAAqB,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QAC7D,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAED,KAAK,UAAU,sBAAsB,CACnC,OAAyB,EACzB,WAAmB;IAEnB,IAAI,KAAK,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;IACpC,IAAI,UAAU,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,uBAAuB,EAAE,EAAE,CAAC,CAAC;IACzE,IAAI,QAAQ,GAAc;QACxB;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;;;;;EAKb,WAAW;;;;;;;;;EASX,UAAU;CACX;SACI;KACF,CAAC;IACF,IAAI,UAAU,GAAkB,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IACpD,IAAI,QAAQ,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,YAAY,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC5E,IAAI,OAAO,GAAG,OAAO,QAAQ,CAAC,OAAO,IAAI,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAE,QAAQ,CAAC,OAAiB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC3G,IAAI,IAAI,GAAG,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7E,IAAI,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC;IAC3C,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,MAAM,aAAa,CAAC,KAAK,EAAE,oBAAoB,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC;IACvE,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,qBAAqB,CAClC,OAAyB,EACzB,WAAmB;IAEnB,IAAI,KAAK,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;IACpC,IAAI,QAAQ,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,CAAC;IAC1C,IAAI,iBAAiB,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IAC1D,IAAI,QAAQ,GAAc;QACxB;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,OAAO;oBACb,MAAM,EAAE,iBAAiB,CAAC,KAAK;iBAChC;gBACD;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,SAAS,GAAG,WAAW;iBAC9B;aACF;SACF;KACF,CAAC;IACF,IAAI,UAAU,GAAkB;QAC9B,SAAS,EAAE,IAAI;QACf,UAAU,EAAE;YACV,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,YAAY;SACnB;QACD,KAAK,EAAE;YACL;gBACE,IAAI,EAAE,YAAY;gBAClB,WAAW,EAAE,eAAe;gBAC5B,YAAY,EAAE;oBACZ,IAAI,EAAE,QAAQ;oBACd,UAAU,EAAE;wBACV,UAAU,EAAE;4BACV,IAAI,EAAE,OAAO;4BACb,WAAW,EACT,yFAAyF;yBAC5F;qBACF;oBACD,QAAQ,EAAE,CAAC,YAAY,CAAC;iBACzB;aACF;SACF;KACF,CAAC;IACF,IAAI,QAAQ,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,YAAY,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC5E,IAAI,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACxC,IAAI,UAAU,GAAG,KAAK,CAAC,UAA8B,CAAC;IACtD,IAAI,YAAY,GAAG,MAAM,UAAU,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;IACvD,OAAO,YAAY,CAAC;AACtB,CAAC"}

package/dist/src/extension/tools/export_file.js

@@ -0,0 +1,93 @@
+import { getTabId, open_new_tab, sleep } from '../utils';
+import { exportFile } from './html_script';
+/**
+ * Export file
+ */
+export class ExportFile {
+    constructor() {
+        this.name = 'export_file';
+        this.description = 'Content exported as a file, support text format';
+        this.input_schema = {
+            type: 'object',
+            properties: {
+                fileType: {
+                    type: 'string',
+                    description: 'File format type',
+                    enum: ['txt', 'csv', 'md', 'html', 'js', 'xml', 'json', 'yml', 'sql'],
+                },
+                content: {
+                    type: 'string',
+                    description: 'Export file content',
+                },
+                filename: {
+                    type: 'string',
+                    description: 'File name',
+                },
+            },
+            required: ['fileType', 'content'],
+        };
+    }
+    /**
+     * export
+     *
+     * @param {*} params { fileType: 'csv', content: 'field1,field2\ndata1,data2' }
+     * @returns > { success: true }
+     */
+    async execute(context, params) {
+        if (typeof params !== 'object' || params === null || !('content' in params)) {
+            throw new Error('Invalid parameters. Expected an object with a "content" property.');
+        }
+        let type = 'text/plain';
+        switch (params.fileType) {
+            case 'csv':
+                type = 'text/csv';
+                break;
+            case 'md':
+                type = 'text/markdown';
+                break;
+            case 'html':
+                type = 'text/html';
+                break;
+            case 'js':
+                type = 'application/javascript';
+                break;
+            case 'xml':
+                type = 'text/xml';
+                break;
+            case 'json':
+                type = 'application/json';
+                break;
+        }
+        let filename;
+        if (!params.filename) {
+            filename = new Date().getTime() + '.' + params.fileType;
+        }
+        else if (!(params.filename + '').endsWith(params.fileType)) {
+            filename = params.filename + '.' + params.fileType;
+        }
+        else {
+            filename = params.filename;
+        }
+        let tabId = await getTabId(context);
+        try {
+            await chrome.scripting.executeScript({
+                target: { tabId: tabId },
+                func: exportFile,
+                args: [filename, type, params.content],
+            });
+        }
+        catch (e) {
+            let tab = await open_new_tab('https://www.google.com', true);
+            tabId = tab.id;
+            await chrome.scripting.executeScript({
+                target: { tabId: tabId },
+                func: exportFile,
+                args: [filename, type, params.content],
+            });
+            await sleep(1000);
+            await chrome.tabs.remove(tabId);
+        }
+        return { success: true };
+    }
+}
+//# sourceMappingURL=export_file.js.map

package/dist/src/extension/tools/export_file.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"export_file.js","sourceRoot":"","sources":["../../../../src/extension/tools/export_file.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AACzD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAE3C;;GAEG;AACH,MAAM,OAAO,UAAU;IAKrB;QACE,IAAI,CAAC,IAAI,GAAG,aAAa,CAAC;QAC1B,IAAI,CAAC,WAAW,GAAG,iDAAiD,CAAC;QACrE,IAAI,CAAC,YAAY,GAAG;YAClB,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,QAAQ,EAAE;oBACR,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,kBAAkB;oBAC/B,IAAI,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC;iBACtE;gBACD,OAAO,EAAE;oBACP,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,qBAAqB;iBACnC;gBACD,QAAQ,EAAE;oBACR,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,WAAW;iBACzB;aACF;YACD,QAAQ,EAAE,CAAC,UAAU,EAAE,SAAS,CAAC;SAClC,CAAC;IACJ,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,OAAO,CAAC,OAAyB,EAAE,MAAuB;QAC9D,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,IAAI,CAAC,CAAC,SAAS,IAAI,MAAM,CAAC,EAAE,CAAC;YAC5E,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAC;QACvF,CAAC;QACD,IAAI,IAAI,GAAG,YAAY,CAAC;QACxB,QAAQ,MAAM,CAAC,QAAQ,EAAE,CAAC;YACxB,KAAK,KAAK;gBACR,IAAI,GAAG,UAAU,CAAC;gBAClB,MAAM;YACR,KAAK,IAAI;gBACP,IAAI,GAAG,eAAe,CAAC;gBACvB,MAAM;YACR,KAAK,MAAM;gBACT,IAAI,GAAG,WAAW,CAAC;gBACnB,MAAM;YACR,KAAK,IAAI;gBACP,IAAI,GAAG,wBAAwB,CAAC;gBAChC,MAAM;YACR,KAAK,KAAK;gBACR,IAAI,GAAG,UAAU,CAAC;gBAClB,MAAM;YACR,KAAK,MAAM;gBACT,IAAI,GAAG,kBAAkB,CAAC;gBAC1B,MAAM;QACV,CAAC;QACD,IAAI,QAAgB,CAAC;QACrB,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YACrB,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC;QAC1D,CAAC;aAAM,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ,GAAG,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC7D,QAAQ,GAAG,MAAM,CAAC,QAAQ,GAAG,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC;QACrD,CAAC;aAAM,CAAC;YACN,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QAC7B,CAAC;QACD,IAAI,KAAK,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;QACpC,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC;gBACnC,MAAM,EAAE,EAAE,KAAK,EAAE,KAAe,EAAE;gBAClC,IAAI,EAAE,UAAU;gBAChB,IAAI,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC;aACvC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,GAAG,GAAG,MAAM,YAAY,CAAC,wBAAwB,EAAE,IAAI,CAAC,CAAC;YAC7D,KAAK,GAAG,GAAG,CAAC,EAAY,CAAC;YACzB,MAAM,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC;gBACnC,MAAM,EAAE,EAAE,KAAK,EAAE,KAAe,EAAE;gBAClC,IAAI,EAAE,UAAU;gBAChB,IAAI,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC;aACvC,CAAC,CAAC;YACH,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;YAClB,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAClC,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;CACF"}

package/dist/src/extension/tools/extract_content.js

@@ -0,0 +1,38 @@
+import { getTabId, executeScript, injectScript, sleep } from '../utils';
+/**
+ * Extract Page Content
+ */
+export class ExtractContent {
+    constructor() {
+        this.name = 'extract_content';
+        this.description = 'Extract the text content of the current webpage';
+        this.input_schema = {
+            type: 'object',
+            properties: {},
+        };
+    }
+    /**
+     * Extract Page Content
+     *
+     * @param {*} params {}
+     * @returns > { tabId, result: { title, url, content }, success: true }
+     */
+    async execute(context, params) {
+        let tabId = await getTabId(context);
+        let tab = await chrome.tabs.get(tabId);
+        await injectScript(tabId);
+        await sleep(500);
+        let content = await executeScript(tabId, () => {
+            return eko.extractHtmlContent();
+        }, []);
+        return {
+            tabId,
+            result: {
+                title: tab.title,
+                url: tab.url,
+                content: content,
+            }
+        };
+    }
+}
+//# sourceMappingURL=extract_content.js.map

package/dist/src/extension/tools/extract_content.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract_content.js","sourceRoot":"","sources":["../../../../src/extension/tools/extract_content.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AAExE;;GAEG;AACH,MAAM,OAAO,cAAc;IAKzB;QACE,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;QAC9B,IAAI,CAAC,WAAW,GAAG,iDAAiD,CAAC;QACrE,IAAI,CAAC,YAAY,GAAG;YAClB,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE,EAAE;SACf,CAAC;IACJ,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,OAAO,CAAC,OAAyB,EAAE,MAAW;QAClD,IAAI,KAAK,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;QACpC,IAAI,GAAG,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACvC,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;QAC1B,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QACjB,IAAI,OAAO,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,GAAG,EAAE;YAC5C,OAAO,GAAG,CAAC,kBAAkB,EAAE,CAAC;QAClC,CAAC,EAAE,EAAE,CAAC,CAAC;QACP,OAAO;YACL,KAAK;YACL,MAAM,EAAE;gBACN,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,OAAO,EAAE,OAAO;aACjB;SACsB,CAAC;IAC5B,CAAC;CACF"}

package/dist/src/extension/tools/find_element_position.js

@@ -0,0 +1,125 @@
+import { executeScript, getTabId, getWindowId } from '../utils';
+import { extractOperableElements, getOperableElementRect } from './html_script';
+import { screenshot } from './browser';
+/**
+ * Find Element Position
+ */
+export class FindElementPosition {
+    constructor() {
+        this.name = 'find_element_position';
+        this.description = 'Locate Element Coordinates through Task Prompts';
+        this.input_schema = {
+            type: 'object',
+            properties: {
+                task_prompt: {
+                    type: 'string',
+                    description: 'Task prompt, eg: find the search input box',
+                },
+            },
+            required: ['task_prompt'],
+        };
+    }
+    async execute(context, params) {
+        if (typeof params !== 'object' || params === null || !params.task_prompt) {
+            throw new Error('Invalid parameters. Expected an object with a "task_prompt" property.');
+        }
+        let result;
+        let task_prompt = params.task_prompt;
+        try {
+            result = await executeWithHtmlElement(context, task_prompt);
+        }
+        catch (e) {
+            console.log(e);
+            result = null;
+        }
+        if (!result) {
+            result = await executeWithBrowserUse(context, task_prompt);
+        }
+        return result;
+    }
+}
+async function executeWithHtmlElement(context, task_prompt) {
+    let tabId = await getTabId(context);
+    let pseudoHtml = await executeScript(tabId, extractOperableElements, []);
+    let messages = [
+        {
+            role: 'user',
+            content: `# Task
+Find the element ID that the user needs to operate on in the webpage HTML, and if the element does not exist, do nothing.
+Output JSON format, no explanation required.
+
+# User input
+${task_prompt}
+
+# Output example (when the element exists)
+{"elementId": "1"}
+
+# Output example (when the element does not exist)
+{"elementId": null}
+
+# HTML
+${pseudoHtml}
+`,
+        },
+    ];
+    let llm_params = { maxTokens: 1024 };
+    let response = await context.llmProvider.generateText(messages, llm_params);
+    let content = typeof response.content == 'string' ? response.content : response.content[0].text;
+    let json = content.substring(content.indexOf('{'), content.indexOf('}') + 1);
+    let elementId = JSON.parse(json).elementId;
+    if (elementId) {
+        return await executeScript(tabId, getOperableElementRect, [elementId]);
+    }
+    return null;
+}
+async function executeWithBrowserUse(context, task_prompt) {
+    let tabId = await getTabId(context);
+    let windowId = await getWindowId(context);
+    let screenshot_result = await screenshot(windowId, false);
+    let messages = [
+        {
+            role: 'user',
+            content: [
+                {
+                    type: 'image',
+                    source: screenshot_result.image,
+                },
+                {
+                    type: 'text',
+                    text: 'Find the element: ' + task_prompt,
+                },
+            ],
+        },
+    ];
+    let llm_params = {
+        maxTokens: 1024,
+        toolChoice: {
+            type: 'tool',
+            name: 'get_element_by_coordinate',
+        },
+        tools: [
+            {
+                name: 'get_element_by_coordinate',
+                description: 'Retrieve element information based on coordinate',
+                input_schema: {
+                    type: 'object',
+                    properties: {
+                        coordinate: {
+                            type: 'array',
+                            description: '(x, y): The x (pixels from the left edge) and y (pixels from the top edge) coordinates.',
+                        },
+                    },
+                    required: ['coordinate'],
+                },
+            },
+        ],
+    };
+    let response = await context.llmProvider.generateText(messages, llm_params);
+    let input = response.toolCalls[0].input;
+    let coordinate = input.coordinate;
+    return {
+        left: coordinate[0],
+        top: coordinate[1],
+    };
+}
+//# sourceMappingURL=find_element_position.js.map

package/dist/src/extension/tools/find_element_position.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"find_element_position.js","sourceRoot":"","sources":["../../../../src/extension/tools/find_element_position.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAChE,OAAO,EAAE,uBAAuB,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAChF,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEvC;;GAEG;AACH,MAAM,OAAO,mBAAmB;IAK9B;QACE,IAAI,CAAC,IAAI,GAAG,uBAAuB,CAAC;QACpC,IAAI,CAAC,WAAW,GAAG,iDAAiD,CAAC;QACrE,IAAI,CAAC,YAAY,GAAG;YAClB,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,WAAW,EAAE;oBACX,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,4CAA4C;iBAC1D;aACF;YACD,QAAQ,EAAE,CAAC,aAAa,CAAC;SAC1B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,OAAyB,EAAE,MAAkB;QACzD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;YACzE,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAC;QAC3F,CAAC;QACD,IAAI,MAA0B,CAAC;QAC/B,IAAI,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACrC,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,sBAAsB,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QAC9D,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACf,MAAM,GAAG,IAAI,CAAC;QAChB,CAAC;QACD,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,GAAG,MAAM,qBAAqB,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;QAC7D,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAED,KAAK,UAAU,sBAAsB,CACnC,OAAyB,EACzB,WAAmB;IAEnB,IAAI,KAAK,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;IACpC,IAAI,UAAU,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,uBAAuB,EAAE,EAAE,CAAC,CAAC;IACzE,IAAI,QAAQ,GAAc;QACxB;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;;;;;EAKb,WAAW;;;;;;;;;EASX,UAAU;CACX;SACI;KACF,CAAC;IACF,IAAI,UAAU,GAAkB,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;IACpD,IAAI,QAAQ,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,YAAY,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC5E,IAAI,OAAO,GACT,OAAO,QAAQ,CAAC,OAAO,IAAI,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAE,QAAQ,CAAC,OAAiB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC/F,IAAI,IAAI,GAAG,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7E,IAAI,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC;IAC3C,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,MAAM,aAAa,CAAC,KAAK,EAAE,sBAAsB,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC;IACzE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,qBAAqB,CAClC,OAAyB,EACzB,WAAmB;IAEnB,IAAI,KAAK,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;IACpC,IAAI,QAAQ,GAAG,MAAM,WAAW,CAAC,OAAO,CAAC,CAAC;IAC1C,IAAI,iBAAiB,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IAC1D,IAAI,QAAQ,GAAc;QACxB;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,OAAO;oBACb,MAAM,EAAE,iBAAiB,CAAC,KAAK;iBAChC;gBACD;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,oBAAoB,GAAG,WAAW;iBACzC;aACF;SACF;KACF,CAAC;IACF,IAAI,UAAU,GAAkB;QAC9B,SAAS,EAAE,IAAI;QACf,UAAU,EAAE;YACV,IAAI,EAAE,MAAM;YACZ,IAAI,EAAE,2BAA2B;SAClC;QACD,KAAK,EAAE;YACL;gBACE,IAAI,EAAE,2BAA2B;gBACjC,WAAW,EAAE,kDAAkD;gBAC/D,YAAY,EAAE;oBACZ,IAAI,EAAE,QAAQ;oBACd,UAAU,EAAE;wBACV,UAAU,EAAE;4BACV,IAAI,EAAE,OAAO;4BACb,WAAW,EACT,yFAAyF;yBAC5F;qBACF;oBACD,QAAQ,EAAE,CAAC,YAAY,CAAC;iBACzB;aACF;SACF;KACF,CAAC;IACF,IAAI,QAAQ,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,YAAY,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IAC5E,IAAI,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACxC,IAAI,UAAU,GAAG,KAAK,CAAC,UAA8B,CAAC;IAEtD,OAAO;QACL,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC;QACnB,GAAG,EAAE,UAAU,CAAC,CAAC,CAAC;KACJ,CAAC;AACnB,CAAC"}

package/dist/src/extension/tools/html_script.js

@@ -0,0 +1,219 @@
+export function exportFile(filename, type, content) {
+    const blob = new Blob([content], { type: type });
+    const link = document.createElement('a');
+    link.href = URL.createObjectURL(blob);
+    link.download = filename;
+    document.body.appendChild(link);
+    link.click();
+    document.body.removeChild(link);
+    URL.revokeObjectURL(link.href);
+}
+export function xpath(element) {
+    if (element == document.body) {
+        return '/html/' + element.tagName.toLowerCase();
+    }
+    if (element.parentNode instanceof ShadowRoot) {
+        let shadowRoot = element.parentNode;
+        let parent = shadowRoot.getRootNode().host;
+        return xpath(parent) + '//' + element.tagName.toLowerCase();
+    }
+    else {
+        let sp;
+        let parent;
+        if (element.parentNode instanceof ShadowRoot) {
+            sp = '//';
+            let shadowRoot = element.parentNode;
+            parent = shadowRoot.getRootNode().host;
+        }
+        else {
+            sp = '/';
+            parent = element.parentNode;
+        }
+        let siblings = parent.childNodes;
+        if (siblings.length == 1) {
+            return xpath(parent) + sp + element.tagName.toLowerCase();
+        }
+        else {
+            let ix = 1;
+            for (let i = 0, l = siblings.length; i < l; i++) {
+                let sibling = siblings[i];
+                if (sibling == element) {
+                    return xpath(parent) + sp + element.tagName.toLowerCase() + '[' + ix + ']';
+                }
+                else if (sibling.nodeType == 1 && sibling.tagName == element.tagName) {
+                    ix++;
+                }
+            }
+            return '';
+        }
+    }
+}
+export function queryWithXpath(xpath) {
+    let xpaths = xpath.split('//');
+    if (xpaths.length == 1) {
+        return document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null)
+            .singleNodeValue;
+    }
+    let element = document;
+    for (let i = 0; i < xpaths.length; i++) {
+        let _element = null;
+        if (element instanceof ShadowRoot) {
+            let _xpaths = xpaths[i].split('/');
+            let current = _xpaths[0].toLowerCase();
+            let ix = 1;
+            for (let j = 0; j < element.childNodes.length; j++) {
+                let tagName = element.childNodes[j].tagName;
+                if (!tagName) {
+                    ix++;
+                    continue;
+                }
+                tagName = tagName.toLowerCase();
+                if (current == tagName || current == tagName + '[' + ix + ']') {
+                    element = element.childNodes[j];
+                    let _xpath = _xpaths.slice(1).join('/');
+                    _element = document.evaluate(_xpath, element, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
+                    break;
+                }
+                else {
+                    ix++;
+                }
+            }
+        }
+        else {
+            _element = document.evaluate(xpaths[i], element, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
+        }
+        if (!_element) {
+            return null;
+        }
+        if (_element.shadowRoot) {
+            element = _element.shadowRoot;
+        }
+        else {
+            element = _element;
+        }
+    }
+    return element != document ? element : null;
+}
+/**
+ * Extract the elements related to html operability and wrap them into pseudo-html code.
+ */
+export function extractOperableElements() {
+    // visible
+    const isElementVisible = (element) => {
+        const style = window.getComputedStyle(element);
+        return (style.display !== 'none' &&
+            style.visibility !== 'hidden' &&
+            style.opacity !== '0' &&
+            element.offsetWidth > 0 &&
+            element.offsetHeight > 0);
+    };
+    // element original index
+    const getElementIndex = (element) => {
+        const xpath = document.evaluate('preceding::*', element, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
+        return xpath.snapshotLength;
+    };
+    // exclude
+    const addExclude = (excludes, children) => {
+        for (let i = 0; i < children.length; i++) {
+            excludes.push(children[i]);
+            if (children[i].children) {
+                addExclude(excludes, children[i].children);
+            }
+        }
+    };
+    // { pseudoId: element }
+    let elementMap = {};
+    let nextId = 1;
+    let elements = [];
+    let excludes = [];
+    // operable element
+    const operableSelectors = 'a, button, input, textarea, select';
+    document.querySelectorAll(operableSelectors).forEach((element) => {
+        if (isElementVisible(element) && excludes.indexOf(element) == -1) {
+            const id = nextId++;
+            elementMap[id.toString()] = element;
+            const tagName = element.tagName.toLowerCase();
+            const attributes = Array.from(element.attributes)
+                .filter((attr) => ['id', 'name', 'type', 'value', 'href', 'title', 'placeholder'].includes(attr.name))
+                .map((attr) => `${attr.name == 'id' ? 'target' : attr.name}="${attr.value}"`)
+                .join(' ');
+            elements.push({
+                originalIndex: getElementIndex(element),
+                id: id,
+                html: `<${tagName} id="${id}" ${attributes}>${tagName == 'select' ? element.innerHTML : element.innerText || ''}</${tagName}>`,
+            });
+            addExclude(excludes, element.children);
+        }
+    });
+    // short text element
+    const textWalker = document.createTreeWalker(document.body, NodeFilter.SHOW_ELEMENT, {
+        acceptNode: function (node) {
+            var _a;
+            if (node.matches(operableSelectors) || excludes.indexOf(node) != -1) {
+                // skip
+                return NodeFilter.FILTER_SKIP;
+            }
+            // text <= 100
+            const text = (_a = node.innerText) === null || _a === void 0 ? void 0 : _a.trim();
+            if (isElementVisible(node) &&
+                text &&
+                text.length <= 100 &&
+                text.length > 0 &&
+                node.children.length === 0) {
+                return NodeFilter.FILTER_ACCEPT;
+            }
+            // skip
+            return NodeFilter.FILTER_SKIP;
+        },
+    });
+    let currentNode;
+    while ((currentNode = textWalker.nextNode())) {
+        const id = nextId++;
+        elementMap[id.toString()] = currentNode;
+        const tagName = currentNode.tagName.toLowerCase();
+        elements.push({
+            originalIndex: getElementIndex(currentNode),
+            id: id,
+            html: `<${tagName} id="${id}">${currentNode.innerText.trim()}</${tagName}>`,
+        });
+    }
+    // element sort
+    elements.sort((a, b) => a.originalIndex - b.originalIndex);
+    // cache
+    window.operableElementMap = elementMap;
+    // pseudo html
+    return elements.map((e) => e.html).join('\n');
+}
+export function clickOperableElement(id) {
+    let element = window.operableElementMap[id];
+    if (!element) {
+        return false;
+    }
+    if (element.click) {
+        element.click();
+    }
+    else {
+        element.dispatchEvent(new MouseEvent('click', {
+            view: window,
+            bubbles: true,
+            cancelable: true,
+        }));
+    }
+    return true;
+}
+export function getOperableElementRect(id) {
+    let element = window.operableElementMap[id];
+    if (!element) {
+        return null;
+    }
+    const rect = element.getBoundingClientRect();
+    return {
+        left: rect.left + window.scrollX,
+        top: rect.top + window.scrollY,
+        right: rect.right + window.scrollX,
+        bottom: rect.bottom + window.scrollY,
+        width: rect.right - rect.left,
+        height: rect.bottom - rect.top,
+    };
+}
+//# sourceMappingURL=html_script.js.map

package/dist/src/extension/tools/html_script.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"html_script.js","sourceRoot":"","sources":["../../../../src/extension/tools/html_script.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,UAAU,CAAC,QAAgB,EAAE,IAAY,EAAE,OAAe;IACxE,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IACjD,MAAM,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,CAAC,IAAI,GAAG,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IACtC,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;IACzB,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,CAAC,KAAK,EAAE,CAAC;IACb,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAChC,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,UAAU,KAAK,CAAC,OAAY;IAChC,IAAI,OAAO,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC7B,OAAO,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IAClD,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,YAAY,UAAU,EAAE,CAAC;QAC7C,IAAI,UAAU,GAAG,OAAO,CAAC,UAAwB,CAAC;QAClD,IAAI,MAAM,GAAI,UAAU,CAAC,WAAW,EAAU,CAAC,IAAI,CAAC;QACpD,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IAC9D,CAAC;SAAM,CAAC;QACN,IAAI,EAAE,CAAC;QACP,IAAI,MAAM,CAAC;QACX,IAAI,OAAO,CAAC,UAAU,YAAY,UAAU,EAAE,CAAC;YAC7C,EAAE,GAAG,IAAI,CAAC;YACV,IAAI,UAAU,GAAG,OAAO,CAAC,UAAwB,CAAC;YAClD,MAAM,GAAI,UAAU,CAAC,WAAW,EAAU,CAAC,IAAI,CAAC;QAClD,CAAC;aAAM,CAAC;YACN,EAAE,GAAG,GAAG,CAAC;YACT,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;QAC9B,CAAC;QACD,IAAI,QAAQ,GAAG,MAAM,CAAC,UAAU,CAAC;QACjC,IAAI,QAAQ,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACzB,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAC5D,CAAC;aAAM,CAAC;YACN,IAAI,EAAE,GAAG,CAAC,CAAC;YACX,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChD,IAAI,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;gBAC1B,IAAI,OAAO,IAAI,OAAO,EAAE,CAAC;oBACvB,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,GAAG,GAAG,EAAE,GAAG,GAAG,CAAC;gBAC7E,CAAC;qBAAM,IAAI,OAAO,CAAC,QAAQ,IAAI,CAAC,IAAI,OAAO,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;oBACvE,EAAE,EAAE,CAAC;gBACP,CAAC;YACH,CAAC;YACD,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,IAAI,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACvB,OAAO,QAAQ,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,CAAC,uBAAuB,EAAE,IAAI,CAAC;aACvF,eAAe,CAAC;IACrB,CAAC;IACD,IAAI,OAAO,GAAQ,QAAQ,CAAC;IAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,IAAI,QAAQ,GAAG,IAAI,CAAC;QACpB,IAAI,OAAO,YAAY,UAAU,EAAE,CAAC;YAClC,IAAI,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACnC,IAAI,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YACvC,IAAI,EAAE,GAAG,CAAC,CAAC;YACX,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACnD,IAAI,OAAO,GAAI,OAAO,CAAC,UAAU,CAAC,CAAC,CAAS,CAAC,OAAO,CAAC;gBACrD,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,EAAE,EAAE,CAAC;oBACL,SAAS;gBACX,CAAC;gBACD,OAAO,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;gBAChC,IAAI,OAAO,IAAI,OAAO,IAAI,OAAO,IAAI,OAAO,GAAG,GAAG,GAAG,EAAE,GAAG,GAAG,EAAE,CAAC;oBAC9D,OAAO,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;oBAChC,IAAI,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBACxC,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAC1B,MAAM,EACN,OAAO,EACP,IAAI,EACJ,WAAW,CAAC,uBAAuB,EACnC,IAAI,CACL,CAAC,eAAsB,CAAC;oBACzB,MAAM;gBACR,CAAC;qBAAM,CAAC;oBACN,EAAE,EAAE,CAAC;gBACP,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAC1B,MAAM,CAAC,CAAC,CAAC,EACT,OAAO,EACP,IAAI,EACJ,WAAW,CAAC,uBAAuB,EACnC,IAAI,CACL,CAAC,eAAsB,CAAC;QAC3B,CAAC;QACD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,OAAO,IAAI,CAAC;QACd,CAAC;QACD,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;YACxB,OAAO,GAAG,QAAQ,CAAC,UAAU,CAAC;QAChC,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,QAAQ,CAAC;QACrB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,IAAI,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;AAC9C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB;IACrC,UAAU;IACV,MAAM,gBAAgB,GAAG,CAAC,OAAY,EAAE,EAAE;QACxC,MAAM,KAAK,GAAG,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;QAC/C,OAAO,CACL,KAAK,CAAC,OAAO,KAAK,MAAM;YACxB,KAAK,CAAC,UAAU,KAAK,QAAQ;YAC7B,KAAK,CAAC,OAAO,KAAK,GAAG;YACrB,OAAO,CAAC,WAAW,GAAG,CAAC;YACvB,OAAO,CAAC,YAAY,GAAG,CAAC,CACzB,CAAC;IACJ,CAAC,CAAC;IAEF,yBAAyB;IACzB,MAAM,eAAe,GAAG,CAAC,OAAY,EAAE,EAAE;QACvC,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAC7B,cAAc,EACd,OAAO,EACP,IAAI,EACJ,WAAW,CAAC,0BAA0B,EACtC,IAAI,CACL,CAAC;QACF,OAAO,KAAK,CAAC,cAAc,CAAC;IAC9B,CAAC,CAAC;IAEF,UAAU;IACV,MAAM,UAAU,GAAG,CAAC,QAAa,EAAE,QAAa,EAAE,EAAE;QAClD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACzC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3B,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;gBACzB,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IAEF,wBAAwB;IACxB,IAAI,UAAU,GAAQ,EAAE,CAAC;IACzB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,QAAQ,GAAG,EAAW,CAAC;IAC3B,IAAI,QAAQ,GAAG,EAAW,CAAC;IAE3B,mBAAmB;IACnB,MAAM,iBAAiB,GAAG,oCAAoC,CAAC;IAC/D,QAAQ,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC,CAAC,OAAY,EAAE,EAAE;QACpE,IAAI,gBAAgB,CAAC,OAAO,CAAC,IAAI,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACjE,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC;YACpB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,OAAO,CAAC;YAEpC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YAC9C,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC;iBAC9C,MAAM,CAAC,CAAC,IAAS,EAAE,EAAE,CACpB,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CACpF;iBACA,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,KAAK,GAAG,CAAC;iBACjF,IAAI,CAAC,GAAG,CAAC,CAAC;YAEb,QAAQ,CAAC,IAAI,CAAC;gBACZ,aAAa,EAAE,eAAe,CAAC,OAAO,CAAC;gBACvC,EAAE,EAAE,EAAE;gBACN,IAAI,EAAE,IAAI,OAAO,QAAQ,EAAE,KAAK,UAAU,IAAI,OAAO,IAAI,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,IAAI,EAAE,KAAK,OAAO,GAAG;aAC/H,CAAC,CAAC;YAEH,UAAU,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzC,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,qBAAqB;IACrB,MAAM,UAAU,GAAG,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC,YAAY,EAAE;QACnF,UAAU,EAAE,UAAU,IAAS;;YAC7B,IAAI,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,IAAI,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;gBACpE,OAAO;gBACP,OAAO,UAAU,CAAC,WAAW,CAAC;YAChC,CAAC;YAED,cAAc;YACd,MAAM,IAAI,GAAG,MAAA,IAAI,CAAC,SAAS,0CAAE,IAAI,EAAE,CAAC;YACpC,IACE,gBAAgB,CAAC,IAAI,CAAC;gBACtB,IAAI;gBACJ,IAAI,CAAC,MAAM,IAAI,GAAG;gBAClB,IAAI,CAAC,MAAM,GAAG,CAAC;gBACf,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAC1B,CAAC;gBACD,OAAO,UAAU,CAAC,aAAa,CAAC;YAClC,CAAC;YAED,OAAO;YACP,OAAO,UAAU,CAAC,WAAW,CAAC;QAChC,CAAC;KACF,CAAC,CAAC;IAEH,IAAI,WAAgB,CAAC;IACrB,OAAO,CAAC,WAAW,GAAG,UAAU,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC;QAC7C,MAAM,EAAE,GAAG,MAAM,EAAE,CAAC;QACpB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,WAAW,CAAC;QAExC,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAClD,QAAQ,CAAC,IAAI,CAAC;YACZ,aAAa,EAAE,eAAe,CAAC,WAAW,CAAC;YAC3C,EAAE,EAAE,EAAE;YACN,IAAI,EAAE,IAAI,OAAO,QAAQ,EAAE,KAAK,WAAW,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,OAAO,GAAG;SAC5E,CAAC,CAAC;IACL,CAAC;IAED,eAAe;IACf,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,aAAa,CAAC,CAAC;IAE3D,QAAQ;IACP,MAAc,CAAC,kBAAkB,GAAG,UAAU,CAAC;IAChD,cAAc;IACd,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,EAAO;IAC1C,IAAI,OAAO,GAAI,MAAc,CAAC,kBAAkB,CAAC,EAAE,CAAC,CAAC;IACrD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,OAAO,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,aAAa,CACnB,IAAI,UAAU,CAAC,OAAO,EAAE;YACtB,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE,IAAI;YACb,UAAU,EAAE,IAAI;SACjB,CAAC,CACH,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,EAAO;IAC5C,IAAI,OAAO,GAAI,MAAc,CAAC,kBAAkB,CAAC,EAAE,CAAC,CAAC;IACrD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,IAAI,GAAG,OAAO,CAAC,qBAAqB,EAAE,CAAC;IAC7C,OAAO;QACL,IAAI,EAAE,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,OAAO;QAChC,GAAG,EAAE,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC,OAAO;QAC9B,KAAK,EAAE,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,OAAO;QAClC,MAAM,EAAE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,OAAO;QACpC,KAAK,EAAE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI;QAC7B,MAAM,EAAE,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG;KAChB,CAAC;AACnB,CAAC"}

package/dist/src/extension/tools/index.js

@@ -0,0 +1,12 @@
+import { BrowserUse } from './browser_use';
+import { ElementClick } from './element_click';
+import { ExportFile } from './export_file';
+import { ExtractContent } from './extract_content';
+import { FindElementPosition } from './find_element_position';
+import { OpenUrl } from './open_url';
+import { Screenshot } from './screenshot';
+import { TabManagement } from './tab_management';
+import { WebSearch } from './web_search';
+import { RequestLogin } from './request_login';
+export { BrowserUse, ElementClick, ExportFile, ExtractContent, FindElementPosition, OpenUrl, Screenshot, TabManagement, WebSearch, RequestLogin, };
+//# sourceMappingURL=index.js.map

package/dist/src/extension/tools/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/extension/tools/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAC9D,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,EACL,UAAU,EACV,YAAY,EACZ,UAAU,EACV,cAAc,EACd,mBAAmB,EACnB,OAAO,EACP,UAAU,EACV,aAAa,EACb,SAAS,EACT,YAAY,GACb,CAAC"}