@eko-ai/eko 1.0.9 → 1.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +139 -139
- package/dist/core/eko.d.ts +3 -2
- package/dist/extension/script/bing.js +0 -0
- package/dist/extension/script/build_dom_tree.js +661 -661
- package/dist/extension/script/common.js +12 -4
- package/dist/extension/script/duckduckgo.js +0 -0
- package/dist/extension/script/google.js +0 -0
- package/dist/extension/tools/cancel_workflow.d.ts +9 -0
- package/dist/extension/utils.d.ts +1 -1
- package/dist/extension.cjs.js +117 -111
- package/dist/extension.esm.js +117 -111
- package/dist/extension_content_script.js +79 -61
- package/dist/fellou.cjs.js +18 -18
- package/dist/fellou.esm.js +18 -18
- package/dist/index.cjs.js +262 -172
- package/dist/index.esm.js +262 -172
- package/dist/jest.config.js +10 -0
- package/dist/jest.config.js.map +1 -0
- package/dist/models/workflow.d.ts +3 -1
- package/dist/nodejs.cjs.js +1651 -1633
- package/dist/nodejs.esm.js +1651 -1633
- package/dist/rollup.config.js +171 -0
- package/dist/rollup.config.js.map +1 -0
- package/dist/script.js +10 -0
- package/dist/script.js.map +1 -0
- package/dist/services/llm/deepseek-provider.d.ts +13 -0
- package/dist/services/llm/provider-factory.d.ts +4 -0
- package/dist/services/llm/siliconflow-provider.d.ts +13 -0
- package/dist/services/parser/workflow-parser.d.ts +2 -1
- package/dist/services/workflow/generator.d.ts +3 -2
- package/dist/src/core/eko.js +99 -0
- package/dist/src/core/eko.js.map +1 -0
- package/dist/src/core/tool-registry.js +51 -0
- package/dist/src/core/tool-registry.js.map +1 -0
- package/dist/src/extension/content/index.js +409 -0
- package/dist/src/extension/content/index.js.map +1 -0
- package/dist/src/extension/core.js +29 -0
- package/dist/src/extension/core.js.map +1 -0
- package/dist/src/extension/index.js +12 -0
- package/dist/src/extension/index.js.map +1 -0
- package/dist/src/extension/script/bing.js +25 -0
- package/dist/src/extension/script/bing.js.map +1 -0
- package/dist/src/extension/script/build_dom_tree.js +585 -0
- package/dist/src/extension/script/build_dom_tree.js.map +1 -0
- package/dist/src/extension/script/common.js +203 -0
- package/dist/src/extension/script/common.js.map +1 -0
- package/dist/src/extension/script/duckduckgo.js +25 -0
- package/dist/src/extension/script/duckduckgo.js.map +1 -0
- package/dist/src/extension/script/google.js +26 -0
- package/dist/src/extension/script/google.js.map +1 -0
- package/dist/src/extension/tools/browser.js +174 -0
- package/dist/src/extension/tools/browser.js.map +1 -0
- package/dist/src/extension/tools/browser_use.js +186 -0
- package/dist/src/extension/tools/browser_use.js.map +1 -0
- package/dist/src/extension/tools/element_click.js +123 -0
- package/dist/src/extension/tools/element_click.js.map +1 -0
- package/dist/src/extension/tools/export_file.js +93 -0
- package/dist/src/extension/tools/export_file.js.map +1 -0
- package/dist/src/extension/tools/extract_content.js +38 -0
- package/dist/src/extension/tools/extract_content.js.map +1 -0
- package/dist/src/extension/tools/find_element_position.js +125 -0
- package/dist/src/extension/tools/find_element_position.js.map +1 -0
- package/dist/src/extension/tools/html_script.js +219 -0
- package/dist/src/extension/tools/html_script.js.map +1 -0
- package/dist/src/extension/tools/index.js +12 -0
- package/dist/src/extension/tools/index.js.map +1 -0
- package/dist/src/extension/tools/open_url.js +68 -0
- package/dist/src/extension/tools/open_url.js.map +1 -0
- package/dist/src/extension/tools/request_login.js +87 -0
- package/dist/src/extension/tools/request_login.js.map +1 -0
- package/dist/src/extension/tools/screenshot.js +26 -0
- package/dist/src/extension/tools/screenshot.js.map +1 -0
- package/dist/src/extension/tools/tab_management.js +160 -0
- package/dist/src/extension/tools/tab_management.js.map +1 -0
- package/dist/src/extension/tools/web_search.js +281 -0
- package/dist/src/extension/tools/web_search.js.map +1 -0
- package/dist/src/extension/utils.js +244 -0
- package/dist/src/extension/utils.js.map +1 -0
- package/dist/src/fellou/computer.js +104 -0
- package/dist/src/fellou/computer.js.map +1 -0
- package/dist/src/fellou/index.js +7 -0
- package/dist/src/fellou/index.js.map +1 -0
- package/dist/src/fellou/tools/computer_use.js +111 -0
- package/dist/src/fellou/tools/computer_use.js.map +1 -0
- package/dist/src/index.js +9 -0
- package/dist/src/index.js.map +1 -0
- package/dist/src/models/action.js +364 -0
- package/dist/src/models/action.js.map +1 -0
- package/dist/src/models/workflow.js +120 -0
- package/dist/src/models/workflow.js.map +1 -0
- package/dist/src/nodejs/core.js +18 -0
- package/dist/src/nodejs/core.js.map +1 -0
- package/dist/src/nodejs/index.js +6 -0
- package/dist/src/nodejs/index.js.map +1 -0
- package/dist/src/nodejs/script/build_dom_tree.js +586 -0
- package/dist/src/nodejs/script/build_dom_tree.js.map +1 -0
- package/dist/src/nodejs/tools/browser_use.js +458 -0
- package/dist/src/nodejs/tools/browser_use.js.map +1 -0
- package/dist/src/nodejs/tools/command_execute.js +65 -0
- package/dist/src/nodejs/tools/command_execute.js.map +1 -0
- package/dist/src/nodejs/tools/file_read.js +45 -0
- package/dist/src/nodejs/tools/file_read.js.map +1 -0
- package/dist/src/nodejs/tools/file_write.js +95 -0
- package/dist/src/nodejs/tools/file_write.js.map +1 -0
- package/dist/src/nodejs/tools/index.js +5 -0
- package/dist/src/nodejs/tools/index.js.map +1 -0
- package/dist/src/schemas/workflow.schema.js +64 -0
- package/dist/src/schemas/workflow.schema.js.map +1 -0
- package/dist/src/services/llm/claude-provider.js +140 -0
- package/dist/src/services/llm/claude-provider.js.map +1 -0
- package/dist/src/services/llm/deepseek-provider.js +432 -0
- package/dist/src/services/llm/deepseek-provider.js.map +1 -0
- package/dist/src/services/llm/glm-provider.js +317 -0
- package/dist/src/services/llm/glm-provider.js.map +1 -0
- package/dist/src/services/llm/openai-provider copy.js +317 -0
- package/dist/src/services/llm/openai-provider copy.js.map +1 -0
- package/dist/src/services/llm/openai-provider.js +317 -0
- package/dist/src/services/llm/openai-provider.js.map +1 -0
- package/dist/src/services/parser/workflow-parser.js +208 -0
- package/dist/src/services/parser/workflow-parser.js.map +1 -0
- package/dist/src/services/workflow/generator.js +105 -0
- package/dist/src/services/workflow/generator.js.map +1 -0
- package/dist/src/services/workflow/templates.js +42 -0
- package/dist/src/services/workflow/templates.js.map +1 -0
- package/dist/src/types/action.types.js +2 -0
- package/dist/src/types/action.types.js.map +1 -0
- package/dist/src/types/eko.types.js +2 -0
- package/dist/src/types/eko.types.js.map +1 -0
- package/dist/src/types/index.js +6 -0
- package/dist/src/types/index.js.map +1 -0
- package/dist/src/types/llm.types.js +2 -0
- package/dist/src/types/llm.types.js.map +1 -0
- package/dist/src/types/parser.types.js +2 -0
- package/dist/src/types/parser.types.js.map +1 -0
- package/dist/src/types/tools.types.js +2 -0
- package/dist/src/types/tools.types.js.map +1 -0
- package/dist/src/types/workflow.types.js +3 -0
- package/dist/src/types/workflow.types.js.map +1 -0
- package/dist/src/web/core.js +18 -0
- package/dist/src/web/core.js.map +1 -0
- package/dist/src/web/index.js +9 -0
- package/dist/src/web/index.js.map +1 -0
- package/dist/src/web/script/build_dom_tree.js +584 -0
- package/dist/src/web/script/build_dom_tree.js.map +1 -0
- package/dist/src/web/tools/browser.js +249 -0
- package/dist/src/web/tools/browser.js.map +1 -0
- package/dist/src/web/tools/browser_use.js +176 -0
- package/dist/src/web/tools/browser_use.js.map +1 -0
- package/dist/src/web/tools/element_click.js +121 -0
- package/dist/src/web/tools/element_click.js.map +1 -0
- package/dist/src/web/tools/export_file.js +74 -0
- package/dist/src/web/tools/export_file.js.map +1 -0
- package/dist/src/web/tools/extract_content.js +24 -0
- package/dist/src/web/tools/extract_content.js.map +1 -0
- package/dist/src/web/tools/find_element_position.js +121 -0
- package/dist/src/web/tools/find_element_position.js.map +1 -0
- package/dist/src/web/tools/html_script.js +219 -0
- package/dist/src/web/tools/html_script.js.map +1 -0
- package/dist/src/web/tools/index.js +8 -0
- package/dist/src/web/tools/index.js.map +1 -0
- package/dist/src/web/tools/screenshot.js +24 -0
- package/dist/src/web/tools/screenshot.js.map +1 -0
- package/dist/test/integration/claude-provider.test.js +170 -0
- package/dist/test/integration/claude-provider.test.js.map +1 -0
- package/dist/test/integration/deepseek-provider.test.js +171 -0
- package/dist/test/integration/deepseek-provider.test.js.map +1 -0
- package/dist/test/integration/glm-provider.test.js +173 -0
- package/dist/test/integration/glm-provider.test.js.map +1 -0
- package/dist/test/integration/openai-provider.test copy.js +170 -0
- package/dist/test/integration/openai-provider.test copy.js.map +1 -0
- package/dist/test/integration/openai-provider.test.js +170 -0
- package/dist/test/integration/openai-provider.test.js.map +1 -0
- package/dist/test/integration/qwen-provider.js +170 -0
- package/dist/test/integration/qwen-provider.js.map +1 -0
- package/dist/test/integration/qwen-provider.test copy.js +170 -0
- package/dist/test/integration/qwen-provider.test copy.js.map +1 -0
- package/dist/test/integration/qwen-provider.test.js +170 -0
- package/dist/test/integration/qwen-provider.test.js.map +1 -0
- package/dist/test/integration/workflow.execution.test.js +152 -0
- package/dist/test/integration/workflow.execution.test.js.map +1 -0
- package/dist/test/integration/workflow.generation-and-execution.test.js +131 -0
- package/dist/test/integration/workflow.generation-and-execution.test.js.map +1 -0
- package/dist/test/integration/workflow.generator.test.js +207 -0
- package/dist/test/integration/workflow.generator.test.js.map +1 -0
- package/dist/test/unit/action.test.js +186 -0
- package/dist/test/unit/action.test.js.map +1 -0
- package/dist/test/unit/tool-registry.test.js +99 -0
- package/dist/test/unit/tool-registry.test.js.map +1 -0
- package/dist/test/unit/workflow-parser.test.js +189 -0
- package/dist/test/unit/workflow-parser.test.js.map +1 -0
- package/dist/test/unit/workflow.test.js +102 -0
- package/dist/test/unit/workflow.test.js.map +1 -0
- package/dist/types/action.types.d.ts +2 -0
- package/dist/types/eko.types.d.ts +4 -1
- package/dist/types/jest.config.d.ts +10 -0
- package/dist/types/llm.types.d.ts +0 -2
- package/dist/types/rollup.config.d.ts +16 -0
- package/dist/types/script.d.ts +1 -0
- package/dist/types/src/core/eko.d.ts +20 -0
- package/dist/types/src/core/tool-registry.d.ts +13 -0
- package/dist/types/src/extension/content/index.d.ts +16 -0
- package/dist/types/src/extension/core.d.ts +11 -0
- package/dist/types/src/extension/index.d.ts +7 -0
- package/dist/types/src/extension/script/bing.d.ts +0 -0
- package/dist/types/src/extension/script/build_dom_tree.d.ts +38 -0
- package/dist/types/src/extension/script/common.d.ts +0 -0
- package/dist/types/src/extension/script/duckduckgo.d.ts +0 -0
- package/dist/types/src/extension/script/google.d.ts +0 -0
- package/dist/types/src/extension/tools/browser.d.ts +22 -0
- package/dist/types/src/extension/tools/browser_use.d.ts +19 -0
- package/dist/types/src/extension/tools/element_click.d.ts +12 -0
- package/dist/types/src/extension/tools/export_file.d.ts +18 -0
- package/dist/types/src/extension/tools/extract_content.d.ts +18 -0
- package/dist/types/src/extension/tools/find_element_position.d.ts +12 -0
- package/dist/types/src/extension/tools/html_script.d.ts +10 -0
- package/dist/types/src/extension/tools/index.d.ts +11 -0
- package/dist/types/src/extension/tools/open_url.d.ts +18 -0
- package/dist/{extension/tools/form_autofill.d.ts → types/src/extension/tools/request_login.d.ts} +3 -4
- package/dist/types/src/extension/tools/screenshot.d.ts +18 -0
- package/dist/types/src/extension/tools/tab_management.d.ts +19 -0
- package/dist/types/src/extension/tools/web_search.d.ts +18 -0
- package/dist/types/src/extension/utils.d.ts +31 -0
- package/dist/types/src/fellou/computer.d.ts +20 -0
- package/dist/types/src/fellou/index.d.ts +6 -0
- package/dist/types/src/fellou/tools/computer_use.d.ts +18 -0
- package/dist/types/src/index.d.ts +8 -0
- package/dist/types/src/models/action.d.ts +22 -0
- package/dist/types/src/models/workflow.d.ts +16 -0
- package/dist/types/src/nodejs/core.d.ts +2 -0
- package/dist/types/src/nodejs/index.d.ts +3 -0
- package/dist/types/src/nodejs/script/build_dom_tree.d.ts +1 -0
- package/dist/types/src/nodejs/tools/browser_use.d.ts +28 -0
- package/dist/types/src/nodejs/tools/command_execute.d.ts +12 -0
- package/dist/types/src/nodejs/tools/file_read.d.ts +11 -0
- package/dist/types/src/nodejs/tools/file_write.d.ts +15 -0
- package/dist/types/src/nodejs/tools/index.d.ts +4 -0
- package/dist/types/src/schemas/workflow.schema.d.ts +88 -0
- package/dist/types/src/services/llm/claude-provider.d.ts +11 -0
- package/dist/types/src/services/llm/deepseek-provider.d.ts +15 -0
- package/dist/types/src/services/llm/glm-provider.d.ts +11 -0
- package/dist/types/src/services/llm/openai-provider copy.d.ts +11 -0
- package/dist/types/src/services/llm/openai-provider.d.ts +11 -0
- package/dist/types/src/services/parser/workflow-parser.d.ts +29 -0
- package/dist/types/src/services/workflow/generator.d.ts +13 -0
- package/dist/types/src/services/workflow/templates.d.ts +8 -0
- package/dist/types/src/types/action.types.d.ts +38 -0
- package/dist/types/src/types/eko.types.d.ts +21 -0
- package/dist/types/src/types/index.d.ts +5 -0
- package/dist/types/src/types/llm.types.d.ts +54 -0
- package/dist/types/src/types/parser.types.d.ts +9 -0
- package/dist/types/src/types/tools.types.d.ts +88 -0
- package/dist/types/src/types/workflow.types.d.ts +39 -0
- package/dist/types/src/web/core.d.ts +2 -0
- package/dist/types/src/web/index.d.ts +5 -0
- package/dist/types/src/web/script/build_dom_tree.d.ts +10 -0
- package/dist/types/src/web/tools/browser.d.ts +21 -0
- package/dist/types/src/web/tools/browser_use.d.ts +19 -0
- package/dist/types/src/web/tools/element_click.d.ts +12 -0
- package/dist/types/src/web/tools/export_file.d.ts +18 -0
- package/dist/types/src/web/tools/extract_content.d.ts +17 -0
- package/dist/types/src/web/tools/find_element_position.d.ts +12 -0
- package/dist/types/src/web/tools/html_script.d.ts +10 -0
- package/dist/types/src/web/tools/index.d.ts +7 -0
- package/dist/types/src/web/tools/screenshot.d.ts +18 -0
- package/dist/types/test/integration/claude-provider.test.d.ts +1 -0
- package/dist/types/test/integration/deepseek-provider.test.d.ts +1 -0
- package/dist/types/test/integration/glm-provider.test.d.ts +1 -0
- package/dist/types/test/integration/openai-provider.test copy.d.ts +1 -0
- package/dist/types/test/integration/openai-provider.test.d.ts +1 -0
- package/dist/types/test/integration/qwen-provider.d.ts +1 -0
- package/dist/types/test/integration/qwen-provider.test copy.d.ts +1 -0
- package/dist/types/test/integration/qwen-provider.test.d.ts +1 -0
- package/dist/types/test/integration/workflow.execution.test.d.ts +1 -0
- package/dist/types/test/integration/workflow.generation-and-execution.test.d.ts +1 -0
- package/dist/types/test/integration/workflow.generator.test.d.ts +1 -0
- package/dist/types/test/unit/action.test.d.ts +1 -0
- package/dist/types/test/unit/tool-registry.test.d.ts +1 -0
- package/dist/types/test/unit/workflow-parser.test.d.ts +1 -0
- package/dist/types/test/unit/workflow.test.d.ts +1 -0
- package/dist/universal_tools/human/text.d.ts +9 -0
- package/dist/web.cjs.js +128 -121
- package/dist/web.esm.js +128 -121
- package/package.json +106 -106
- package/dist/fellou/tools/index.d.ts +0 -2
- package/dist/types/framework.types.d.ts +0 -11
package/dist/extension.esm.js
CHANGED
|
@@ -28,21 +28,12 @@ async function getWindowId(context) {
|
|
|
28
28
|
async function getTabId(context) {
|
|
29
29
|
let tabId = context.variables.get('tabId');
|
|
30
30
|
if (tabId) {
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
// Check if it's a valid integer
|
|
34
|
-
if (!Number.isInteger(tabId)) {
|
|
35
|
-
context.variables.delete('tabId');
|
|
36
|
-
tabId = null;
|
|
31
|
+
try {
|
|
32
|
+
await chrome.tabs.get(tabId);
|
|
37
33
|
}
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
}
|
|
42
|
-
catch (e) {
|
|
43
|
-
tabId = null;
|
|
44
|
-
context.variables.delete('tabId');
|
|
45
|
-
}
|
|
34
|
+
catch (e) {
|
|
35
|
+
tabId = null;
|
|
36
|
+
context.variables.delete('tabId');
|
|
46
37
|
}
|
|
47
38
|
}
|
|
48
39
|
if (!tabId) {
|
|
@@ -66,24 +57,30 @@ async function getTabId(context) {
|
|
|
66
57
|
}
|
|
67
58
|
return tabId;
|
|
68
59
|
}
|
|
69
|
-
|
|
60
|
+
function getCurrentTabId(windowId) {
|
|
70
61
|
return new Promise((resolve, reject) => {
|
|
71
|
-
chrome.tabs.query({ active: true,
|
|
62
|
+
chrome.tabs.query({ windowId, active: true, lastFocusedWindow: true }, function (tabs) {
|
|
72
63
|
if (chrome.runtime.lastError) {
|
|
73
64
|
console.error('Chrome runtime error:', chrome.runtime.lastError);
|
|
74
65
|
reject(chrome.runtime.lastError);
|
|
75
66
|
return;
|
|
76
67
|
}
|
|
77
|
-
if (
|
|
78
|
-
|
|
79
|
-
return;
|
|
68
|
+
if (tabs.length > 0) {
|
|
69
|
+
resolve(tabs[0].id);
|
|
80
70
|
}
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
71
|
+
else {
|
|
72
|
+
chrome.tabs.query({ windowId, active: true, currentWindow: true }, function (_tabs) {
|
|
73
|
+
if (_tabs.length > 0) {
|
|
74
|
+
resolve(_tabs[0].id);
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
else {
|
|
78
|
+
chrome.tabs.query({ windowId, status: 'complete', currentWindow: true }, function (__tabs) {
|
|
79
|
+
resolve(__tabs.length ? __tabs[__tabs.length - 1].id : undefined);
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
});
|
|
85
83
|
}
|
|
86
|
-
resolve(tabId);
|
|
87
84
|
});
|
|
88
85
|
});
|
|
89
86
|
}
|
|
@@ -645,35 +642,35 @@ var browser = /*#__PURE__*/Object.freeze({
|
|
|
645
642
|
class BrowserUse {
|
|
646
643
|
constructor() {
|
|
647
644
|
this.name = 'browser_use';
|
|
648
|
-
this.description = `Use structured commands to interact with the browser, manipulating page elements through screenshots and webpage element extraction.
|
|
649
|
-
* This is a browser GUI interface where you need to analyze webpages by taking screenshots and extracting page element structures, and specify action sequences to complete designated tasks.
|
|
650
|
-
* Before any operation, you must first call the \`screenshot_extract_element\` command, which will return the browser page screenshot and structured element information, both specially processed.
|
|
651
|
-
* ELEMENT INTERACTION:
|
|
652
|
-
- Only use indexes that exist in the provided element list
|
|
653
|
-
- Each element has a unique index number (e.g., "[33]:<button>")
|
|
654
|
-
- Elements marked with "[]:" are non-interactive (for context only)
|
|
655
|
-
* NAVIGATION & ERROR HANDLING:
|
|
656
|
-
- If no suitable elements exist, use other functions to complete the task
|
|
657
|
-
- If stuck, try alternative approaches
|
|
658
|
-
- Handle popups/cookies by accepting or closing them
|
|
645
|
+
this.description = `Use structured commands to interact with the browser, manipulating page elements through screenshots and webpage element extraction.
|
|
646
|
+
* This is a browser GUI interface where you need to analyze webpages by taking screenshots and extracting page element structures, and specify action sequences to complete designated tasks.
|
|
647
|
+
* Before any operation, you must first call the \`screenshot_extract_element\` command, which will return the browser page screenshot and structured element information, both specially processed.
|
|
648
|
+
* ELEMENT INTERACTION:
|
|
649
|
+
- Only use indexes that exist in the provided element list
|
|
650
|
+
- Each element has a unique index number (e.g., "[33]:<button>")
|
|
651
|
+
- Elements marked with "[]:" are non-interactive (for context only)
|
|
652
|
+
* NAVIGATION & ERROR HANDLING:
|
|
653
|
+
- If no suitable elements exist, use other functions to complete the task
|
|
654
|
+
- If stuck, try alternative approaches
|
|
655
|
+
- Handle popups/cookies by accepting or closing them
|
|
659
656
|
- Use scroll to find elements you are looking for`;
|
|
660
657
|
this.input_schema = {
|
|
661
658
|
type: 'object',
|
|
662
659
|
properties: {
|
|
663
660
|
action: {
|
|
664
661
|
type: 'string',
|
|
665
|
-
description: `The action to perform. The available actions are:
|
|
666
|
-
* \`screenshot_extract_element\`: Take a screenshot of the web page and extract operable elements.
|
|
667
|
-
- Screenshots are used to understand page layouts, with labeled bounding boxes corresponding to element indexes. Each bounding box and its label share the same color, with labels typically positioned in the top-right corner of the box.
|
|
668
|
-
- Screenshots help verify element positions and relationships. Labels may sometimes overlap, so extracted elements are used to verify the correct elements.
|
|
669
|
-
- In addition to screenshots, simplified information about interactive elements is returned, with element indexes corresponding to those in the screenshots.
|
|
670
|
-
* \`input_text\`: Enter a string in the interactive element.
|
|
671
|
-
* \`click\`: Click to element.
|
|
672
|
-
* \`right_click\`: Right-click on the element.
|
|
673
|
-
* \`double_click\`: Double-click on the element.
|
|
674
|
-
* \`scroll_to\`: Scroll to the specified element.
|
|
675
|
-
* \`extract_content\`: Extract the text content of the current webpage.
|
|
676
|
-
* \`get_dropdown_options\`: Get all options from a native dropdown element.
|
|
662
|
+
description: `The action to perform. The available actions are:
|
|
663
|
+
* \`screenshot_extract_element\`: Take a screenshot of the web page and extract operable elements.
|
|
664
|
+
- Screenshots are used to understand page layouts, with labeled bounding boxes corresponding to element indexes. Each bounding box and its label share the same color, with labels typically positioned in the top-right corner of the box.
|
|
665
|
+
- Screenshots help verify element positions and relationships. Labels may sometimes overlap, so extracted elements are used to verify the correct elements.
|
|
666
|
+
- In addition to screenshots, simplified information about interactive elements is returned, with element indexes corresponding to those in the screenshots.
|
|
667
|
+
* \`input_text\`: Enter a string in the interactive element.
|
|
668
|
+
* \`click\`: Click to element.
|
|
669
|
+
* \`right_click\`: Right-click on the element.
|
|
670
|
+
* \`double_click\`: Double-click on the element.
|
|
671
|
+
* \`scroll_to\`: Scroll to the specified element.
|
|
672
|
+
* \`extract_content\`: Extract the text content of the current webpage.
|
|
673
|
+
* \`get_dropdown_options\`: Get all options from a native dropdown element.
|
|
677
674
|
* \`select_dropdown_option\`: Select dropdown option for interactive element index by the text of the option you want to select.`,
|
|
678
675
|
enum: [
|
|
679
676
|
'screenshot_extract_element',
|
|
@@ -826,7 +823,7 @@ class BrowserUse {
|
|
|
826
823
|
}
|
|
827
824
|
catch (e) {
|
|
828
825
|
console.error('Browser use error:', e);
|
|
829
|
-
return { success: false, error: e === null || e ===
|
|
826
|
+
return { success: false, error: e === null || e === undefined ? undefined : e.message };
|
|
830
827
|
}
|
|
831
828
|
}
|
|
832
829
|
destroy(context) {
|
|
@@ -904,7 +901,7 @@ function extractOperableElements() {
|
|
|
904
901
|
return NodeFilter.FILTER_SKIP;
|
|
905
902
|
}
|
|
906
903
|
// text <= 100
|
|
907
|
-
const text = (_a = node.innerText) === null || _a ===
|
|
904
|
+
const text = (_a = node.innerText) === null || _a === undefined ? undefined : _a.trim();
|
|
908
905
|
if (isElementVisible(node) &&
|
|
909
906
|
text &&
|
|
910
907
|
text.length <= 100 &&
|
|
@@ -1010,21 +1007,21 @@ async function executeWithHtmlElement$1(context, task_prompt) {
|
|
|
1010
1007
|
let messages = [
|
|
1011
1008
|
{
|
|
1012
1009
|
role: 'user',
|
|
1013
|
-
content: `# Task
|
|
1014
|
-
Determine the operation intent based on user input, find the element ID that the user needs to operate on in the webpage HTML, and if the element does not exist, do nothing.
|
|
1015
|
-
Output JSON format, no explanation required.
|
|
1016
|
-
|
|
1017
|
-
# User input
|
|
1018
|
-
${task_prompt}
|
|
1019
|
-
|
|
1020
|
-
# Output example (when the element exists)
|
|
1021
|
-
{"elementId": "1", "operationType": "click"}
|
|
1022
|
-
|
|
1023
|
-
# Output example (when the element does not exist)
|
|
1024
|
-
{"elementId": null, "operationType": "unknown"}
|
|
1025
|
-
|
|
1026
|
-
# HTML
|
|
1027
|
-
${pseudoHtml}
|
|
1010
|
+
content: `# Task
|
|
1011
|
+
Determine the operation intent based on user input, find the element ID that the user needs to operate on in the webpage HTML, and if the element does not exist, do nothing.
|
|
1012
|
+
Output JSON format, no explanation required.
|
|
1013
|
+
|
|
1014
|
+
# User input
|
|
1015
|
+
${task_prompt}
|
|
1016
|
+
|
|
1017
|
+
# Output example (when the element exists)
|
|
1018
|
+
{"elementId": "1", "operationType": "click"}
|
|
1019
|
+
|
|
1020
|
+
# Output example (when the element does not exist)
|
|
1021
|
+
{"elementId": null, "operationType": "unknown"}
|
|
1022
|
+
|
|
1023
|
+
# HTML
|
|
1024
|
+
${pseudoHtml}
|
|
1028
1025
|
`,
|
|
1029
1026
|
},
|
|
1030
1027
|
];
|
|
@@ -1165,8 +1162,15 @@ class ExportFile {
|
|
|
1165
1162
|
});
|
|
1166
1163
|
}
|
|
1167
1164
|
catch (e) {
|
|
1168
|
-
let tab
|
|
1169
|
-
|
|
1165
|
+
let tab;
|
|
1166
|
+
const url = 'https://www.google.com';
|
|
1167
|
+
if (context.ekoConfig.workingWindowId) {
|
|
1168
|
+
tab = await open_new_tab(url, false, context.ekoConfig.workingWindowId);
|
|
1169
|
+
}
|
|
1170
|
+
else {
|
|
1171
|
+
tab = await open_new_tab(url, true);
|
|
1172
|
+
}
|
|
1173
|
+
(_c = (_b = (_a = context.callback) === null || _a === undefined ? undefined : _a.hooks) === null || _b === undefined ? undefined : _b.onTabCreated) === null || _c === undefined ? undefined : _c.call(_b, tab.id);
|
|
1170
1174
|
let tabId = tab.id;
|
|
1171
1175
|
await chrome.scripting.executeScript({
|
|
1172
1176
|
target: { tabId: tabId },
|
|
@@ -1260,21 +1264,21 @@ async function executeWithHtmlElement(context, task_prompt) {
|
|
|
1260
1264
|
let messages = [
|
|
1261
1265
|
{
|
|
1262
1266
|
role: 'user',
|
|
1263
|
-
content: `# Task
|
|
1264
|
-
Find the element ID that the user needs to operate on in the webpage HTML, and if the element does not exist, do nothing.
|
|
1265
|
-
Output JSON format, no explanation required.
|
|
1266
|
-
|
|
1267
|
-
# User input
|
|
1268
|
-
${task_prompt}
|
|
1269
|
-
|
|
1270
|
-
# Output example (when the element exists)
|
|
1271
|
-
{"elementId": "1"}
|
|
1272
|
-
|
|
1273
|
-
# Output example (when the element does not exist)
|
|
1274
|
-
{"elementId": null}
|
|
1275
|
-
|
|
1276
|
-
# HTML
|
|
1277
|
-
${pseudoHtml}
|
|
1267
|
+
content: `# Task
|
|
1268
|
+
Find the element ID that the user needs to operate on in the webpage HTML, and if the element does not exist, do nothing.
|
|
1269
|
+
Output JSON format, no explanation required.
|
|
1270
|
+
|
|
1271
|
+
# User input
|
|
1272
|
+
${task_prompt}
|
|
1273
|
+
|
|
1274
|
+
# Output example (when the element exists)
|
|
1275
|
+
{"elementId": "1"}
|
|
1276
|
+
|
|
1277
|
+
# Output example (when the element does not exist)
|
|
1278
|
+
{"elementId": null}
|
|
1279
|
+
|
|
1280
|
+
# HTML
|
|
1281
|
+
${pseudoHtml}
|
|
1278
1282
|
`,
|
|
1279
1283
|
},
|
|
1280
1284
|
];
|
|
@@ -1423,12 +1427,12 @@ class OpenUrl {
|
|
|
1423
1427
|
let tab;
|
|
1424
1428
|
if (newWindow) {
|
|
1425
1429
|
tab = await open_new_tab(url, true);
|
|
1426
|
-
(_c = (_b = (_a = context.callback) === null || _a ===
|
|
1430
|
+
(_c = (_b = (_a = context.callback) === null || _a === undefined ? undefined : _a.hooks) === null || _b === undefined ? undefined : _b.onTabCreated) === null || _c === undefined ? undefined : _c.call(_b, tab.id);
|
|
1427
1431
|
}
|
|
1428
1432
|
else {
|
|
1429
1433
|
let windowId = await getWindowId(context);
|
|
1430
1434
|
tab = await open_new_tab(url, false, windowId);
|
|
1431
|
-
(_f = (_e = (_d = context.callback) === null || _d ===
|
|
1435
|
+
(_f = (_e = (_d = context.callback) === null || _d === undefined ? undefined : _d.hooks) === null || _e === undefined ? undefined : _e.onTabCreated) === null || _f === undefined ? undefined : _f.call(_e, tab.id);
|
|
1432
1436
|
}
|
|
1433
1437
|
let windowId = tab.windowId;
|
|
1434
1438
|
let tabId = tab.id;
|
|
@@ -1487,13 +1491,13 @@ class TabManagement {
|
|
|
1487
1491
|
properties: {
|
|
1488
1492
|
command: {
|
|
1489
1493
|
type: 'string',
|
|
1490
|
-
description: `The command to perform. The available commands are:
|
|
1491
|
-
* \`tab_all\`: View all tabs and return the tabId and title.
|
|
1492
|
-
* \`current_tab\`: Get current tab information (tabId, url, title).
|
|
1493
|
-
* \`go_back\`: Go back to the previous page in the current tab.
|
|
1494
|
-
* \`change_url [url]\`: open URL in the current tab, eg: \`change_url https://www.google.com\`.
|
|
1495
|
-
* \`close_tab\`: Close the current tab.
|
|
1496
|
-
* \`switch_tab [tabId]\`: Switch to the specified tab using tabId, eg: \`switch_tab 1000\`.
|
|
1494
|
+
description: `The command to perform. The available commands are:
|
|
1495
|
+
* \`tab_all\`: View all tabs and return the tabId and title.
|
|
1496
|
+
* \`current_tab\`: Get current tab information (tabId, url, title).
|
|
1497
|
+
* \`go_back\`: Go back to the previous page in the current tab.
|
|
1498
|
+
* \`change_url [url]\`: open URL in the current tab, eg: \`change_url https://www.google.com\`.
|
|
1499
|
+
* \`close_tab\`: Close the current tab.
|
|
1500
|
+
* \`switch_tab [tabId]\`: Switch to the specified tab using tabId, eg: \`switch_tab 1000\`.
|
|
1497
1501
|
* \`new_tab [url]\`: Open a new tab window and open the URL, eg: \`new_tab https://www.google.com\``,
|
|
1498
1502
|
},
|
|
1499
1503
|
},
|
|
@@ -1594,12 +1598,12 @@ class TabManagement {
|
|
|
1594
1598
|
let tab;
|
|
1595
1599
|
if (newWindow) {
|
|
1596
1600
|
tab = await open_new_tab(url, true);
|
|
1597
|
-
(_c = (_b = (_a = context.callback) === null || _a ===
|
|
1601
|
+
(_c = (_b = (_a = context.callback) === null || _a === undefined ? undefined : _a.hooks) === null || _b === undefined ? undefined : _b.onTabCreated) === null || _c === undefined ? undefined : _c.call(_b, tab.id);
|
|
1598
1602
|
}
|
|
1599
1603
|
else {
|
|
1600
1604
|
let windowId = await getWindowId(context);
|
|
1601
1605
|
tab = await open_new_tab(url, false, windowId);
|
|
1602
|
-
(_f = (_e = (_d = context.callback) === null || _d ===
|
|
1606
|
+
(_f = (_e = (_d = context.callback) === null || _d === undefined ? undefined : _d.hooks) === null || _e === undefined ? undefined : _e.onTabCreated) === null || _f === undefined ? undefined : _f.call(_e, tab.id);
|
|
1603
1607
|
}
|
|
1604
1608
|
let windowId = tab.windowId;
|
|
1605
1609
|
let tabId = tab.id;
|
|
@@ -1678,8 +1682,8 @@ class WebSearch {
|
|
|
1678
1682
|
}
|
|
1679
1683
|
let taskId = new Date().getTime() + '';
|
|
1680
1684
|
let searchs = [{ url: url, keyword: query }];
|
|
1681
|
-
let searchInfo = await deepSearch(context, taskId, searchs, maxResults || 5);
|
|
1682
|
-
let links = ((_a = searchInfo.result[0]) === null || _a ===
|
|
1685
|
+
let searchInfo = await deepSearch(context, taskId, searchs, maxResults || 5, context.ekoConfig.workingWindowId);
|
|
1686
|
+
let links = ((_a = searchInfo.result[0]) === null || _a === undefined ? undefined : _a.links) || [];
|
|
1683
1687
|
return links.filter((s) => s.content);
|
|
1684
1688
|
}
|
|
1685
1689
|
}
|
|
@@ -1751,25 +1755,27 @@ chrome.tabs.onUpdated.addListener(async function (tabId, changeInfo, tab) {
|
|
|
1751
1755
|
* @param {array} searchs search list => [{ url: 'https://bing.com', keyword: 'ai' }]
|
|
1752
1756
|
* @param {number} detailsMaxNum Maximum crawling quantity per search detail page
|
|
1753
1757
|
*/
|
|
1754
|
-
async function deepSearch(context, taskId, searchs, detailsMaxNum,
|
|
1758
|
+
async function deepSearch(context, taskId, searchs, detailsMaxNum, windowId) {
|
|
1755
1759
|
let closeWindow = false;
|
|
1756
|
-
if (!
|
|
1760
|
+
if (!windowId) {
|
|
1757
1761
|
// open new window
|
|
1758
|
-
window = await chrome.windows.create({
|
|
1762
|
+
let window = await chrome.windows.create({
|
|
1759
1763
|
type: 'normal',
|
|
1760
1764
|
state: 'maximized',
|
|
1761
1765
|
url: null,
|
|
1762
1766
|
});
|
|
1767
|
+
windowId = window.id;
|
|
1763
1768
|
closeWindow = true;
|
|
1764
1769
|
}
|
|
1770
|
+
windowId = windowId;
|
|
1765
1771
|
// crawler the search page details page link
|
|
1766
1772
|
// [{ links: [{ title, url }] }]
|
|
1767
|
-
let detailLinkGroups = await doDetailLinkGroups(context, taskId, searchs, detailsMaxNum,
|
|
1773
|
+
let detailLinkGroups = await doDetailLinkGroups(context, taskId, searchs, detailsMaxNum, windowId);
|
|
1768
1774
|
// crawler all details page content and comments
|
|
1769
|
-
let searchInfo = await doPageContent(context, taskId, detailLinkGroups,
|
|
1775
|
+
let searchInfo = await doPageContent(context, taskId, detailLinkGroups, windowId);
|
|
1770
1776
|
console.log('searchInfo: ', searchInfo);
|
|
1771
1777
|
// close window
|
|
1772
|
-
closeWindow && chrome.windows.remove(
|
|
1778
|
+
closeWindow && chrome.windows.remove(windowId);
|
|
1773
1779
|
return searchInfo;
|
|
1774
1780
|
}
|
|
1775
1781
|
/**
|
|
@@ -1781,7 +1787,7 @@ async function deepSearch(context, taskId, searchs, detailsMaxNum, window) {
|
|
|
1781
1787
|
* @param {*} window
|
|
1782
1788
|
* @returns [{ links: [{ title, url }] }]
|
|
1783
1789
|
*/
|
|
1784
|
-
async function doDetailLinkGroups(context, taskId, searchs, detailsMaxNum,
|
|
1790
|
+
async function doDetailLinkGroups(context, taskId, searchs, detailsMaxNum, windowId) {
|
|
1785
1791
|
var _a, _b, _c;
|
|
1786
1792
|
let detailLinkGroups = [];
|
|
1787
1793
|
let countDownLatch = new CountDownLatch(searchs.length);
|
|
@@ -1792,7 +1798,7 @@ async function doDetailLinkGroups(context, taskId, searchs, detailsMaxNum, windo
|
|
|
1792
1798
|
// open new Tab
|
|
1793
1799
|
let tab = await chrome.tabs.create({
|
|
1794
1800
|
url: url,
|
|
1795
|
-
windowId
|
|
1801
|
+
windowId,
|
|
1796
1802
|
});
|
|
1797
1803
|
(_c = (_b = (_a = context.callback) === null || _a === void 0 ? void 0 : _a.hooks) === null || _b === void 0 ? void 0 : _b.onTabCreated) === null || _c === void 0 ? void 0 : _c.call(_b, tab.id);
|
|
1798
1804
|
let eventId = taskId + '_' + i;
|
|
@@ -1845,7 +1851,7 @@ async function doDetailLinkGroups(context, taskId, searchs, detailsMaxNum, windo
|
|
|
1845
1851
|
* @param {*} window
|
|
1846
1852
|
* @returns search info
|
|
1847
1853
|
*/
|
|
1848
|
-
async function doPageContent(context, taskId, detailLinkGroups,
|
|
1854
|
+
async function doPageContent(context, taskId, detailLinkGroups, windowId) {
|
|
1849
1855
|
var _a, _b, _c;
|
|
1850
1856
|
const searchInfo = {
|
|
1851
1857
|
total: 0,
|
|
@@ -1868,9 +1874,9 @@ async function doPageContent(context, taskId, detailLinkGroups, window) {
|
|
|
1868
1874
|
// open new tab
|
|
1869
1875
|
let tab = await chrome.tabs.create({
|
|
1870
1876
|
url: link.url,
|
|
1871
|
-
windowId
|
|
1877
|
+
windowId,
|
|
1872
1878
|
});
|
|
1873
|
-
(_c = (_b = (_a = context.callback) === null || _a ===
|
|
1879
|
+
(_c = (_b = (_a = context.callback) === null || _a === undefined ? undefined : _a.hooks) === null || _b === undefined ? undefined : _b.onTabCreated) === null || _c === undefined ? undefined : _c.call(_b, tab.id);
|
|
1874
1880
|
searchInfo.running++;
|
|
1875
1881
|
let eventId = taskId + '_' + i + '_' + j;
|
|
1876
1882
|
// Create a timeout promise
|
|
@@ -2044,7 +2050,7 @@ class CancelWorkflow {
|
|
|
2044
2050
|
}
|
|
2045
2051
|
const reason = params.reason;
|
|
2046
2052
|
console.log("The workflow has been cancelled because: " + reason);
|
|
2047
|
-
await ((_a = context.workflow) === null || _a ===
|
|
2053
|
+
await ((_a = context.workflow) === null || _a === undefined ? undefined : _a.cancel());
|
|
2048
2054
|
return;
|
|
2049
2055
|
}
|
|
2050
2056
|
}
|
|
@@ -2071,7 +2077,7 @@ class HumanInputText {
|
|
|
2071
2077
|
}
|
|
2072
2078
|
const question = params.question;
|
|
2073
2079
|
console.log("question: " + question);
|
|
2074
|
-
let answer = await ((_c = (_a = context.callback) === null || _a ===
|
|
2080
|
+
let answer = await ((_c = (_a = context.callback) === null || _a === undefined ? undefined : (_b = _a.hooks).onHumanInputText) === null || _c === undefined ? undefined : _c.call(_b, question));
|
|
2075
2081
|
if (!answer) {
|
|
2076
2082
|
console.error("Cannot get user's answer.");
|
|
2077
2083
|
return { status: "Error: Cannot get user's answer.", answer: "" };
|
|
@@ -2110,7 +2116,7 @@ class HumanInputSingleChoice {
|
|
|
2110
2116
|
const choices = params.choices;
|
|
2111
2117
|
console.log("question: " + question);
|
|
2112
2118
|
console.log("choices: " + choices);
|
|
2113
|
-
let answer = await ((_c = (_a = context.callback) === null || _a ===
|
|
2119
|
+
let answer = await ((_c = (_a = context.callback) === null || _a === undefined ? undefined : (_b = _a.hooks).onHumanInputSingleChoice) === null || _c === undefined ? undefined : _c.call(_b, question, choices));
|
|
2114
2120
|
if (!answer) {
|
|
2115
2121
|
console.error("Cannot get user's answer.");
|
|
2116
2122
|
return { status: "Error: Cannot get user's answer.", answer: "" };
|
|
@@ -2149,7 +2155,7 @@ class HumanInputMultipleChoice {
|
|
|
2149
2155
|
const choices = params.choices;
|
|
2150
2156
|
console.log("question: " + question);
|
|
2151
2157
|
console.log("choices: " + choices);
|
|
2152
|
-
let answer = await ((_c = (_a = context.callback) === null || _a ===
|
|
2158
|
+
let answer = await ((_c = (_a = context.callback) === null || _a === undefined ? undefined : (_b = _a.hooks).onHumanInputMultipleChoice) === null || _c === undefined ? undefined : _c.call(_b, question, choices));
|
|
2153
2159
|
if (!answer) {
|
|
2154
2160
|
console.error("Cannot get user's answer.");
|
|
2155
2161
|
return { status: "Error: Cannot get user's answer.", answer: [] };
|
|
@@ -2182,7 +2188,7 @@ class HumanOperate {
|
|
|
2182
2188
|
}
|
|
2183
2189
|
const reason = params.reason;
|
|
2184
2190
|
console.log("reason: " + reason);
|
|
2185
|
-
let userOperation = await ((_c = (_a = context.callback) === null || _a ===
|
|
2191
|
+
let userOperation = await ((_c = (_a = context.callback) === null || _a === undefined ? undefined : (_b = _a.hooks).onHumanOperate) === null || _c === undefined ? undefined : _c.call(_b, reason));
|
|
2186
2192
|
if (!userOperation) {
|
|
2187
2193
|
console.error("Cannot get user's operation.");
|
|
2188
2194
|
return { status: "Error: Cannot get user's operation.", userOperation: "" };
|
|
@@ -2216,7 +2222,7 @@ class SummaryWorkflow {
|
|
|
2216
2222
|
}
|
|
2217
2223
|
const summary = params.summary;
|
|
2218
2224
|
console.log("summary: " + summary);
|
|
2219
|
-
await ((_c = (_a = context.callback) === null || _a ===
|
|
2225
|
+
await ((_c = (_a = context.callback) === null || _a === undefined ? undefined : (_b = _a.hooks).onSummaryWorkflow) === null || _c === undefined ? undefined : _c.call(_b, summary));
|
|
2220
2226
|
return { status: "OK" };
|
|
2221
2227
|
}
|
|
2222
2228
|
}
|
|
@@ -118,23 +118,41 @@ function type(request) {
|
|
|
118
118
|
return false;
|
|
119
119
|
}
|
|
120
120
|
let input;
|
|
121
|
-
if (element.tagName == '
|
|
121
|
+
if (element.tagName == 'IFRAME') {
|
|
122
|
+
let iframeDoc = element.contentDocument || element.contentWindow.document;
|
|
123
|
+
input =
|
|
124
|
+
iframeDoc.querySelector('textarea') ||
|
|
125
|
+
iframeDoc.querySelector('*[contenteditable="true"]') ||
|
|
126
|
+
iframeDoc.querySelector('input');
|
|
127
|
+
}
|
|
128
|
+
else if (element.tagName == 'INPUT' ||
|
|
122
129
|
element.tagName == 'TEXTAREA' ||
|
|
123
130
|
element.childElementCount == 0) {
|
|
124
131
|
input = element;
|
|
125
132
|
}
|
|
126
133
|
else {
|
|
127
|
-
input =
|
|
134
|
+
input =
|
|
135
|
+
element.querySelector('input') ||
|
|
136
|
+
element.querySelector('textarea') ||
|
|
137
|
+
element.querySelector('*[contenteditable="true"]') ||
|
|
138
|
+
element;
|
|
128
139
|
}
|
|
129
140
|
input.focus && input.focus();
|
|
130
141
|
if (!text) {
|
|
131
|
-
if (input.value ==
|
|
132
|
-
|
|
142
|
+
if (input.value == undefined) {
|
|
143
|
+
input.textContent = '';
|
|
144
|
+
}
|
|
145
|
+
else {
|
|
146
|
+
input.value = '';
|
|
133
147
|
}
|
|
134
|
-
input.value = '';
|
|
135
148
|
}
|
|
136
149
|
else {
|
|
137
|
-
input.value
|
|
150
|
+
if (input.value == undefined) {
|
|
151
|
+
input.textContent += text;
|
|
152
|
+
}
|
|
153
|
+
else {
|
|
154
|
+
input.value += text;
|
|
155
|
+
}
|
|
138
156
|
}
|
|
139
157
|
let result = input.dispatchEvent(new Event('input', { bubbles: true }));
|
|
140
158
|
if (enter) {
|
|
@@ -296,23 +314,23 @@ function request_user_help(task_id, failure_type, failure_message) {
|
|
|
296
314
|
};
|
|
297
315
|
const notification = document.createElement('div');
|
|
298
316
|
notification.id = domId;
|
|
299
|
-
notification.style.cssText = `
|
|
300
|
-
position: fixed;
|
|
301
|
-
top: 5px;
|
|
302
|
-
left: 18px;
|
|
303
|
-
z-index: 9999;
|
|
304
|
-
background-color: #FEF0ED;
|
|
305
|
-
color: white;
|
|
306
|
-
padding: 16px;
|
|
307
|
-
border-radius: 12px;
|
|
308
|
-
border: 1px solid #FBB8A5;
|
|
309
|
-
font-family: Arial, sans-serif;
|
|
310
|
-
width: 350px;
|
|
311
|
-
display: flex;
|
|
312
|
-
flex-direction: row;
|
|
313
|
-
gap: 10px;
|
|
314
|
-
cursor: move;
|
|
315
|
-
user-select: none;
|
|
317
|
+
notification.style.cssText = `
|
|
318
|
+
position: fixed;
|
|
319
|
+
top: 5px;
|
|
320
|
+
left: 18px;
|
|
321
|
+
z-index: 9999;
|
|
322
|
+
background-color: #FEF0ED;
|
|
323
|
+
color: white;
|
|
324
|
+
padding: 16px;
|
|
325
|
+
border-radius: 12px;
|
|
326
|
+
border: 1px solid #FBB8A5;
|
|
327
|
+
font-family: Arial, sans-serif;
|
|
328
|
+
width: 350px;
|
|
329
|
+
display: flex;
|
|
330
|
+
flex-direction: row;
|
|
331
|
+
gap: 10px;
|
|
332
|
+
cursor: move;
|
|
333
|
+
user-select: none;
|
|
316
334
|
`;
|
|
317
335
|
let isDragging = false;
|
|
318
336
|
let xOffset = 0;
|
|
@@ -338,58 +356,58 @@ function request_user_help(task_id, failure_type, failure_message) {
|
|
|
338
356
|
isDragging = false;
|
|
339
357
|
});
|
|
340
358
|
const leftContainer = document.createElement('div');
|
|
341
|
-
leftContainer.style.cssText = `
|
|
342
|
-
width: 28px;
|
|
343
|
-
height: 28px;
|
|
344
|
-
display: flex;
|
|
345
|
-
flex-direction: column;
|
|
346
|
-
align-items: center;
|
|
347
|
-
border-radius: 99px;
|
|
348
|
-
background: #FDCCCC;
|
|
349
|
-
justify-content: center;
|
|
359
|
+
leftContainer.style.cssText = `
|
|
360
|
+
width: 28px;
|
|
361
|
+
height: 28px;
|
|
362
|
+
display: flex;
|
|
363
|
+
flex-direction: column;
|
|
364
|
+
align-items: center;
|
|
365
|
+
border-radius: 99px;
|
|
366
|
+
background: #FDCCCC;
|
|
367
|
+
justify-content: center;
|
|
350
368
|
`;
|
|
351
369
|
leftContainer.innerHTML = ``;
|
|
352
370
|
const rightContainer = document.createElement('div');
|
|
353
|
-
rightContainer.style.cssText = `
|
|
354
|
-
flex: 1;
|
|
355
|
-
display: flex;
|
|
356
|
-
flex-direction: column;
|
|
371
|
+
rightContainer.style.cssText = `
|
|
372
|
+
flex: 1;
|
|
373
|
+
display: flex;
|
|
374
|
+
flex-direction: column;
|
|
357
375
|
`;
|
|
358
376
|
const title = document.createElement('div');
|
|
359
|
-
title.style.cssText = `
|
|
360
|
-
font-size: 16px;
|
|
361
|
-
font-weight: 700;
|
|
362
|
-
line-height: 22px;
|
|
363
|
-
color: #DD342D;
|
|
364
|
-
text-align: left;
|
|
377
|
+
title.style.cssText = `
|
|
378
|
+
font-size: 16px;
|
|
379
|
+
font-weight: 700;
|
|
380
|
+
line-height: 22px;
|
|
381
|
+
color: #DD342D;
|
|
382
|
+
text-align: left;
|
|
365
383
|
`;
|
|
366
384
|
title.innerText = failureTitleMap[failure_type] || failure_type;
|
|
367
385
|
const message2 = document.createElement('div');
|
|
368
|
-
message2.style.cssText = `
|
|
369
|
-
font-size: 16px;
|
|
370
|
-
font-weight: 400;
|
|
371
|
-
line-height: 22px;
|
|
372
|
-
color: #DD342D;
|
|
373
|
-
text-align: left;
|
|
386
|
+
message2.style.cssText = `
|
|
387
|
+
font-size: 16px;
|
|
388
|
+
font-weight: 400;
|
|
389
|
+
line-height: 22px;
|
|
390
|
+
color: #DD342D;
|
|
391
|
+
text-align: left;
|
|
374
392
|
`;
|
|
375
393
|
message2.innerText = failure_message + '\nWhen you resolve the issue, click the button below.';
|
|
376
394
|
const buttonDiv = document.createElement('div');
|
|
377
|
-
buttonDiv.style.cssText = `
|
|
378
|
-
margin-top: 16px;
|
|
379
|
-
display: flex;
|
|
380
|
-
flex-direction: row-reverse;
|
|
381
|
-
justify-content: flex-start;
|
|
382
|
-
align-items: center;
|
|
395
|
+
buttonDiv.style.cssText = `
|
|
396
|
+
margin-top: 16px;
|
|
397
|
+
display: flex;
|
|
398
|
+
flex-direction: row-reverse;
|
|
399
|
+
justify-content: flex-start;
|
|
400
|
+
align-items: center;
|
|
383
401
|
`;
|
|
384
402
|
const resolvedBut = document.createElement('div');
|
|
385
403
|
resolvedBut.innerText = 'Resolved';
|
|
386
|
-
resolvedBut.style.cssText = `
|
|
387
|
-
border-radius: 8px;
|
|
388
|
-
background: #DD342D;
|
|
389
|
-
color: white;
|
|
390
|
-
padding: 10px;
|
|
391
|
-
border: none;
|
|
392
|
-
cursor: pointer;
|
|
404
|
+
resolvedBut.style.cssText = `
|
|
405
|
+
border-radius: 8px;
|
|
406
|
+
background: #DD342D;
|
|
407
|
+
color: white;
|
|
408
|
+
padding: 10px;
|
|
409
|
+
border: none;
|
|
410
|
+
cursor: pointer;
|
|
393
411
|
`;
|
|
394
412
|
resolvedBut.onclick = () => {
|
|
395
413
|
chrome.runtime.sendMessage({ type: 'issue_resolved', task_id, failure_type }, () => {
|
package/dist/fellou.cjs.js
CHANGED
|
@@ -126,30 +126,30 @@ var computer = /*#__PURE__*/Object.freeze({
|
|
|
126
126
|
class ComputerUse {
|
|
127
127
|
constructor() {
|
|
128
128
|
this.name = 'computer_use';
|
|
129
|
-
this.description = `Use a mouse and keyboard to interact with a computer, and take screenshots.
|
|
130
|
-
* This is a browser GUI interface where you do not have access to the address bar or bookmarks. You must operate the browser using inputs like screenshots, mouse, keyboard, etc.
|
|
131
|
-
* Some operations may take time to process, so you may need to wait and take successive screenshots to see the results of your actions. E.g. if you clicked submit button, but it didn't work, try taking another screenshot.
|
|
132
|
-
* Whenever you intend to move the cursor to click on an element, you should consult a screenshot to determine the coordinates of the element before moving the cursor.
|
|
133
|
-
* If you tried clicking on a button or link but it failed to load, even after waiting, try adjusting your cursor position so that the tip of the cursor visually falls on the element that you want to click.
|
|
129
|
+
this.description = `Use a mouse and keyboard to interact with a computer, and take screenshots.
|
|
130
|
+
* This is a browser GUI interface where you do not have access to the address bar or bookmarks. You must operate the browser using inputs like screenshots, mouse, keyboard, etc.
|
|
131
|
+
* Some operations may take time to process, so you may need to wait and take successive screenshots to see the results of your actions. E.g. if you clicked submit button, but it didn't work, try taking another screenshot.
|
|
132
|
+
* Whenever you intend to move the cursor to click on an element, you should consult a screenshot to determine the coordinates of the element before moving the cursor.
|
|
133
|
+
* If you tried clicking on a button or link but it failed to load, even after waiting, try adjusting your cursor position so that the tip of the cursor visually falls on the element that you want to click.
|
|
134
134
|
* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element.`;
|
|
135
135
|
this.input_schema = {
|
|
136
136
|
type: 'object',
|
|
137
137
|
properties: {
|
|
138
138
|
action: {
|
|
139
139
|
type: 'string',
|
|
140
|
-
description: `The action to perform. The available actions are:
|
|
141
|
-
* \`key\`: Press a key or key-combination on the keyboard.
|
|
142
|
-
- This supports robotgo hotkey syntax.
|
|
143
|
-
- Multiple keys are combined using the "+" symbol.
|
|
144
|
-
- Examples: "a", "enter", "ctrl+s", "command+shift+a", "num0".
|
|
145
|
-
* \`type\`: Type a string of text on the keyboard.
|
|
146
|
-
* \`cursor_position\`: Get the current (x, y) pixel coordinate of the cursor on the screen.
|
|
147
|
-
* \`mouse_move\`: Move the cursor to a specified (x, y) pixel coordinate on the screen.
|
|
148
|
-
* \`left_click\`: Click the left mouse button.
|
|
149
|
-
* \`left_click_drag\`: Click and drag the cursor to a specified (x, y) pixel coordinate on the screen.
|
|
150
|
-
* \`right_click\`: Click the right mouse button.
|
|
151
|
-
* \`double_click\`: Double-click the left mouse button.
|
|
152
|
-
* \`screenshot\`: Take a screenshot of the screen.
|
|
140
|
+
description: `The action to perform. The available actions are:
|
|
141
|
+
* \`key\`: Press a key or key-combination on the keyboard.
|
|
142
|
+
- This supports robotgo hotkey syntax.
|
|
143
|
+
- Multiple keys are combined using the "+" symbol.
|
|
144
|
+
- Examples: "a", "enter", "ctrl+s", "command+shift+a", "num0".
|
|
145
|
+
* \`type\`: Type a string of text on the keyboard.
|
|
146
|
+
* \`cursor_position\`: Get the current (x, y) pixel coordinate of the cursor on the screen.
|
|
147
|
+
* \`mouse_move\`: Move the cursor to a specified (x, y) pixel coordinate on the screen.
|
|
148
|
+
* \`left_click\`: Click the left mouse button.
|
|
149
|
+
* \`left_click_drag\`: Click and drag the cursor to a specified (x, y) pixel coordinate on the screen.
|
|
150
|
+
* \`right_click\`: Click the right mouse button.
|
|
151
|
+
* \`double_click\`: Double-click the left mouse button.
|
|
152
|
+
* \`screenshot\`: Take a screenshot of the screen.
|
|
153
153
|
* \`scroll\`: Scroll to the specified (x, y) pixel coordinates on the screen.`,
|
|
154
154
|
enum: [
|
|
155
155
|
'key',
|