@eko-ai/eko 1.0.8 → 1.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +139 -131
- package/dist/core/eko.d.ts +3 -2
- package/dist/extension/script/bing.js +0 -0
- package/dist/extension/script/build_dom_tree.js +661 -661
- package/dist/extension/script/common.js +0 -0
- package/dist/extension/script/duckduckgo.js +0 -0
- package/dist/extension/script/google.js +0 -0
- package/dist/extension/tools/cancel_workflow.d.ts +9 -0
- package/dist/extension/tools/get_all_tabs.d.ts +9 -0
- package/dist/extension/tools/index.d.ts +3 -1
- package/dist/extension/tools/tab_management.d.ts +1 -1
- package/dist/extension/utils.d.ts +1 -1
- package/dist/extension.cjs.js +749 -278
- package/dist/extension.esm.js +749 -278
- package/dist/extension_content_script.js +55 -55
- package/dist/fellou.cjs.js +18 -18
- package/dist/fellou.esm.js +18 -18
- package/dist/index.cjs.js +2034 -1609
- package/dist/index.d.ts +3 -1
- package/dist/index.esm.js +2033 -1610
- package/dist/jest.config.js +10 -0
- package/dist/jest.config.js.map +1 -0
- package/dist/models/action.d.ts +8 -4
- package/dist/models/workflow.d.ts +8 -3
- package/dist/nodejs/tools/index.d.ts +1 -0
- package/dist/nodejs.cjs.js +1832 -1623
- package/dist/nodejs.esm.js +1832 -1623
- package/dist/rollup.config.js +171 -0
- package/dist/rollup.config.js.map +1 -0
- package/dist/schemas/workflow.schema.d.ts +2 -13
- package/dist/script.js +10 -0
- package/dist/script.js.map +1 -0
- package/dist/services/llm/claude-provider.d.ts +2 -1
- package/dist/services/llm/deepseek-provider.d.ts +13 -0
- package/dist/services/llm/openai-provider.d.ts +2 -1
- package/dist/services/llm/provider-factory.d.ts +4 -0
- package/dist/services/llm/siliconflow-provider.d.ts +13 -0
- package/dist/services/parser/workflow-parser.d.ts +0 -7
- package/dist/services/workflow/generator.d.ts +1 -0
- package/dist/src/core/eko.js +99 -0
- package/dist/src/core/eko.js.map +1 -0
- package/dist/src/core/tool-registry.js +51 -0
- package/dist/src/core/tool-registry.js.map +1 -0
- package/dist/src/extension/content/index.js +409 -0
- package/dist/src/extension/content/index.js.map +1 -0
- package/dist/src/extension/core.js +29 -0
- package/dist/src/extension/core.js.map +1 -0
- package/dist/src/extension/index.js +12 -0
- package/dist/src/extension/index.js.map +1 -0
- package/dist/src/extension/script/bing.js +25 -0
- package/dist/src/extension/script/bing.js.map +1 -0
- package/dist/src/extension/script/build_dom_tree.js +585 -0
- package/dist/src/extension/script/build_dom_tree.js.map +1 -0
- package/dist/src/extension/script/common.js +203 -0
- package/dist/src/extension/script/common.js.map +1 -0
- package/dist/src/extension/script/duckduckgo.js +25 -0
- package/dist/src/extension/script/duckduckgo.js.map +1 -0
- package/dist/src/extension/script/google.js +26 -0
- package/dist/src/extension/script/google.js.map +1 -0
- package/dist/src/extension/tools/browser.js +174 -0
- package/dist/src/extension/tools/browser.js.map +1 -0
- package/dist/src/extension/tools/browser_use.js +186 -0
- package/dist/src/extension/tools/browser_use.js.map +1 -0
- package/dist/src/extension/tools/element_click.js +123 -0
- package/dist/src/extension/tools/element_click.js.map +1 -0
- package/dist/src/extension/tools/export_file.js +93 -0
- package/dist/src/extension/tools/export_file.js.map +1 -0
- package/dist/src/extension/tools/extract_content.js +38 -0
- package/dist/src/extension/tools/extract_content.js.map +1 -0
- package/dist/src/extension/tools/find_element_position.js +125 -0
- package/dist/src/extension/tools/find_element_position.js.map +1 -0
- package/dist/src/extension/tools/html_script.js +219 -0
- package/dist/src/extension/tools/html_script.js.map +1 -0
- package/dist/src/extension/tools/index.js +12 -0
- package/dist/src/extension/tools/index.js.map +1 -0
- package/dist/src/extension/tools/open_url.js +68 -0
- package/dist/src/extension/tools/open_url.js.map +1 -0
- package/dist/src/extension/tools/request_login.js +87 -0
- package/dist/src/extension/tools/request_login.js.map +1 -0
- package/dist/src/extension/tools/screenshot.js +26 -0
- package/dist/src/extension/tools/screenshot.js.map +1 -0
- package/dist/src/extension/tools/tab_management.js +160 -0
- package/dist/src/extension/tools/tab_management.js.map +1 -0
- package/dist/src/extension/tools/web_search.js +281 -0
- package/dist/src/extension/tools/web_search.js.map +1 -0
- package/dist/src/extension/utils.js +244 -0
- package/dist/src/extension/utils.js.map +1 -0
- package/dist/src/fellou/computer.js +104 -0
- package/dist/src/fellou/computer.js.map +1 -0
- package/dist/src/fellou/index.js +7 -0
- package/dist/src/fellou/index.js.map +1 -0
- package/dist/src/fellou/tools/computer_use.js +111 -0
- package/dist/src/fellou/tools/computer_use.js.map +1 -0
- package/dist/src/index.js +9 -0
- package/dist/src/index.js.map +1 -0
- package/dist/src/models/action.js +364 -0
- package/dist/src/models/action.js.map +1 -0
- package/dist/src/models/workflow.js +120 -0
- package/dist/src/models/workflow.js.map +1 -0
- package/dist/src/nodejs/core.js +18 -0
- package/dist/src/nodejs/core.js.map +1 -0
- package/dist/src/nodejs/index.js +6 -0
- package/dist/src/nodejs/index.js.map +1 -0
- package/dist/src/nodejs/script/build_dom_tree.js +586 -0
- package/dist/src/nodejs/script/build_dom_tree.js.map +1 -0
- package/dist/src/nodejs/tools/browser_use.js +458 -0
- package/dist/src/nodejs/tools/browser_use.js.map +1 -0
- package/dist/src/nodejs/tools/command_execute.js +65 -0
- package/dist/src/nodejs/tools/command_execute.js.map +1 -0
- package/dist/src/nodejs/tools/file_read.js +45 -0
- package/dist/src/nodejs/tools/file_read.js.map +1 -0
- package/dist/src/nodejs/tools/file_write.js +95 -0
- package/dist/src/nodejs/tools/file_write.js.map +1 -0
- package/dist/src/nodejs/tools/index.js +5 -0
- package/dist/src/nodejs/tools/index.js.map +1 -0
- package/dist/src/schemas/workflow.schema.js +64 -0
- package/dist/src/schemas/workflow.schema.js.map +1 -0
- package/dist/src/services/llm/claude-provider.js +140 -0
- package/dist/src/services/llm/claude-provider.js.map +1 -0
- package/dist/src/services/llm/deepseek-provider.js +432 -0
- package/dist/src/services/llm/deepseek-provider.js.map +1 -0
- package/dist/src/services/llm/glm-provider.js +317 -0
- package/dist/src/services/llm/glm-provider.js.map +1 -0
- package/dist/src/services/llm/openai-provider copy.js +317 -0
- package/dist/src/services/llm/openai-provider copy.js.map +1 -0
- package/dist/src/services/llm/openai-provider.js +317 -0
- package/dist/src/services/llm/openai-provider.js.map +1 -0
- package/dist/src/services/parser/workflow-parser.js +208 -0
- package/dist/src/services/parser/workflow-parser.js.map +1 -0
- package/dist/src/services/workflow/generator.js +105 -0
- package/dist/src/services/workflow/generator.js.map +1 -0
- package/dist/src/services/workflow/templates.js +42 -0
- package/dist/src/services/workflow/templates.js.map +1 -0
- package/dist/src/types/action.types.js +2 -0
- package/dist/src/types/action.types.js.map +1 -0
- package/dist/src/types/eko.types.js +2 -0
- package/dist/src/types/eko.types.js.map +1 -0
- package/dist/src/types/index.js +6 -0
- package/dist/src/types/index.js.map +1 -0
- package/dist/src/types/llm.types.js +2 -0
- package/dist/src/types/llm.types.js.map +1 -0
- package/dist/src/types/parser.types.js +2 -0
- package/dist/src/types/parser.types.js.map +1 -0
- package/dist/src/types/tools.types.js +2 -0
- package/dist/src/types/tools.types.js.map +1 -0
- package/dist/src/types/workflow.types.js +3 -0
- package/dist/src/types/workflow.types.js.map +1 -0
- package/dist/src/web/core.js +18 -0
- package/dist/src/web/core.js.map +1 -0
- package/dist/src/web/index.js +9 -0
- package/dist/src/web/index.js.map +1 -0
- package/dist/src/web/script/build_dom_tree.js +584 -0
- package/dist/src/web/script/build_dom_tree.js.map +1 -0
- package/dist/src/web/tools/browser.js +249 -0
- package/dist/src/web/tools/browser.js.map +1 -0
- package/dist/src/web/tools/browser_use.js +176 -0
- package/dist/src/web/tools/browser_use.js.map +1 -0
- package/dist/src/web/tools/element_click.js +121 -0
- package/dist/src/web/tools/element_click.js.map +1 -0
- package/dist/src/web/tools/export_file.js +74 -0
- package/dist/src/web/tools/export_file.js.map +1 -0
- package/dist/src/web/tools/extract_content.js +24 -0
- package/dist/src/web/tools/extract_content.js.map +1 -0
- package/dist/src/web/tools/find_element_position.js +121 -0
- package/dist/src/web/tools/find_element_position.js.map +1 -0
- package/dist/src/web/tools/html_script.js +219 -0
- package/dist/src/web/tools/html_script.js.map +1 -0
- package/dist/src/web/tools/index.js +8 -0
- package/dist/src/web/tools/index.js.map +1 -0
- package/dist/src/web/tools/screenshot.js +24 -0
- package/dist/src/web/tools/screenshot.js.map +1 -0
- package/dist/test/integration/claude-provider.test.js +170 -0
- package/dist/test/integration/claude-provider.test.js.map +1 -0
- package/dist/test/integration/deepseek-provider.test.js +171 -0
- package/dist/test/integration/deepseek-provider.test.js.map +1 -0
- package/dist/test/integration/glm-provider.test.js +173 -0
- package/dist/test/integration/glm-provider.test.js.map +1 -0
- package/dist/test/integration/openai-provider.test copy.js +170 -0
- package/dist/test/integration/openai-provider.test copy.js.map +1 -0
- package/dist/test/integration/openai-provider.test.js +170 -0
- package/dist/test/integration/openai-provider.test.js.map +1 -0
- package/dist/test/integration/qwen-provider.js +170 -0
- package/dist/test/integration/qwen-provider.js.map +1 -0
- package/dist/test/integration/qwen-provider.test copy.js +170 -0
- package/dist/test/integration/qwen-provider.test copy.js.map +1 -0
- package/dist/test/integration/qwen-provider.test.js +170 -0
- package/dist/test/integration/qwen-provider.test.js.map +1 -0
- package/dist/test/integration/workflow.execution.test.js +152 -0
- package/dist/test/integration/workflow.execution.test.js.map +1 -0
- package/dist/test/integration/workflow.generation-and-execution.test.js +131 -0
- package/dist/test/integration/workflow.generation-and-execution.test.js.map +1 -0
- package/dist/test/integration/workflow.generator.test.js +207 -0
- package/dist/test/integration/workflow.generator.test.js.map +1 -0
- package/dist/test/unit/action.test.js +186 -0
- package/dist/test/unit/action.test.js.map +1 -0
- package/dist/test/unit/tool-registry.test.js +99 -0
- package/dist/test/unit/tool-registry.test.js.map +1 -0
- package/dist/test/unit/workflow-parser.test.js +189 -0
- package/dist/test/unit/workflow-parser.test.js.map +1 -0
- package/dist/test/unit/workflow.test.js +102 -0
- package/dist/test/unit/workflow.test.js.map +1 -0
- package/dist/types/action.types.d.ts +8 -3
- package/dist/types/jest.config.d.ts +10 -0
- package/dist/types/rollup.config.d.ts +16 -0
- package/dist/types/script.d.ts +1 -0
- package/dist/types/src/core/eko.d.ts +20 -0
- package/dist/types/src/core/tool-registry.d.ts +13 -0
- package/dist/types/src/extension/content/index.d.ts +16 -0
- package/dist/types/src/extension/core.d.ts +11 -0
- package/dist/types/src/extension/index.d.ts +7 -0
- package/dist/types/src/extension/script/bing.d.ts +0 -0
- package/dist/types/src/extension/script/build_dom_tree.d.ts +38 -0
- package/dist/types/src/extension/script/common.d.ts +0 -0
- package/dist/types/src/extension/script/duckduckgo.d.ts +0 -0
- package/dist/types/src/extension/script/google.d.ts +0 -0
- package/dist/types/src/extension/tools/browser.d.ts +22 -0
- package/dist/types/src/extension/tools/browser_use.d.ts +19 -0
- package/dist/types/src/extension/tools/element_click.d.ts +12 -0
- package/dist/types/src/extension/tools/export_file.d.ts +18 -0
- package/dist/types/src/extension/tools/extract_content.d.ts +18 -0
- package/dist/types/src/extension/tools/find_element_position.d.ts +12 -0
- package/dist/types/src/extension/tools/html_script.d.ts +10 -0
- package/dist/types/src/extension/tools/index.d.ts +11 -0
- package/dist/types/src/extension/tools/open_url.d.ts +18 -0
- package/dist/{extension/tools/form_autofill.d.ts → types/src/extension/tools/request_login.d.ts} +3 -4
- package/dist/types/src/extension/tools/screenshot.d.ts +18 -0
- package/dist/types/src/extension/tools/tab_management.d.ts +19 -0
- package/dist/types/src/extension/tools/web_search.d.ts +18 -0
- package/dist/types/src/extension/utils.d.ts +31 -0
- package/dist/types/src/fellou/computer.d.ts +20 -0
- package/dist/types/src/fellou/index.d.ts +6 -0
- package/dist/types/src/fellou/tools/computer_use.d.ts +18 -0
- package/dist/types/src/index.d.ts +8 -0
- package/dist/types/src/models/action.d.ts +22 -0
- package/dist/types/src/models/workflow.d.ts +16 -0
- package/dist/types/src/nodejs/core.d.ts +2 -0
- package/dist/types/src/nodejs/index.d.ts +3 -0
- package/dist/types/src/nodejs/script/build_dom_tree.d.ts +1 -0
- package/dist/types/src/nodejs/tools/browser_use.d.ts +28 -0
- package/dist/types/src/nodejs/tools/command_execute.d.ts +12 -0
- package/dist/types/src/nodejs/tools/file_read.d.ts +11 -0
- package/dist/types/src/nodejs/tools/file_write.d.ts +15 -0
- package/dist/types/src/nodejs/tools/index.d.ts +4 -0
- package/dist/types/src/schemas/workflow.schema.d.ts +88 -0
- package/dist/types/src/services/llm/claude-provider.d.ts +11 -0
- package/dist/types/src/services/llm/deepseek-provider.d.ts +15 -0
- package/dist/types/src/services/llm/glm-provider.d.ts +11 -0
- package/dist/types/src/services/llm/openai-provider copy.d.ts +11 -0
- package/dist/types/src/services/llm/openai-provider.d.ts +11 -0
- package/dist/types/src/services/parser/workflow-parser.d.ts +29 -0
- package/dist/types/src/services/workflow/generator.d.ts +13 -0
- package/dist/types/src/services/workflow/templates.d.ts +8 -0
- package/dist/types/src/types/action.types.d.ts +38 -0
- package/dist/types/src/types/eko.types.d.ts +21 -0
- package/dist/types/src/types/index.d.ts +5 -0
- package/dist/types/src/types/llm.types.d.ts +54 -0
- package/dist/types/src/types/parser.types.d.ts +9 -0
- package/dist/types/src/types/tools.types.d.ts +88 -0
- package/dist/types/src/types/workflow.types.d.ts +39 -0
- package/dist/types/src/web/core.d.ts +2 -0
- package/dist/types/src/web/index.d.ts +5 -0
- package/dist/types/src/web/script/build_dom_tree.d.ts +10 -0
- package/dist/types/src/web/tools/browser.d.ts +21 -0
- package/dist/types/src/web/tools/browser_use.d.ts +19 -0
- package/dist/types/src/web/tools/element_click.d.ts +12 -0
- package/dist/types/src/web/tools/export_file.d.ts +18 -0
- package/dist/types/src/web/tools/extract_content.d.ts +17 -0
- package/dist/types/src/web/tools/find_element_position.d.ts +12 -0
- package/dist/types/src/web/tools/html_script.d.ts +10 -0
- package/dist/types/src/web/tools/index.d.ts +7 -0
- package/dist/types/src/web/tools/screenshot.d.ts +18 -0
- package/dist/types/test/integration/claude-provider.test.d.ts +1 -0
- package/dist/types/test/integration/deepseek-provider.test.d.ts +1 -0
- package/dist/types/test/integration/glm-provider.test.d.ts +1 -0
- package/dist/types/test/integration/openai-provider.test copy.d.ts +1 -0
- package/dist/types/test/integration/openai-provider.test.d.ts +1 -0
- package/dist/types/test/integration/qwen-provider.d.ts +1 -0
- package/dist/types/test/integration/qwen-provider.test copy.d.ts +1 -0
- package/dist/types/test/integration/qwen-provider.test.d.ts +1 -0
- package/dist/types/test/integration/workflow.execution.test.d.ts +1 -0
- package/dist/types/test/integration/workflow.generation-and-execution.test.d.ts +1 -0
- package/dist/types/test/integration/workflow.generator.test.d.ts +1 -0
- package/dist/types/test/unit/action.test.d.ts +1 -0
- package/dist/types/test/unit/tool-registry.test.d.ts +1 -0
- package/dist/types/test/unit/workflow-parser.test.d.ts +1 -0
- package/dist/types/test/unit/workflow.test.d.ts +1 -0
- package/dist/types/tools.types.d.ts +44 -1
- package/dist/types/workflow.types.d.ts +22 -9
- package/dist/universal_tools/cancel_workflow.d.ts +9 -0
- package/dist/universal_tools/human/text.d.ts +9 -0
- package/dist/universal_tools/human.d.ts +30 -0
- package/dist/universal_tools/index.d.ts +4 -0
- package/dist/universal_tools/summary_workflow.d.ts +9 -0
- package/dist/utils/execution-logger.d.ts +69 -0
- package/dist/web.cjs.js +117 -117
- package/dist/web.esm.js +117 -117
- package/package.json +106 -107
- package/dist/fellou/tools/index.d.ts +0 -2
- package/dist/types/framework.types.d.ts +0 -11
package/dist/extension.esm.js
CHANGED
|
@@ -28,12 +28,21 @@ async function getWindowId(context) {
|
|
|
28
28
|
async function getTabId(context) {
|
|
29
29
|
let tabId = context.variables.get('tabId');
|
|
30
30
|
if (tabId) {
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
tabId = null;
|
|
31
|
+
// Ensure tabId is a number
|
|
32
|
+
tabId = Number(tabId);
|
|
33
|
+
// Check if it's a valid integer
|
|
34
|
+
if (!Number.isInteger(tabId)) {
|
|
36
35
|
context.variables.delete('tabId');
|
|
36
|
+
tabId = null;
|
|
37
|
+
}
|
|
38
|
+
else {
|
|
39
|
+
try {
|
|
40
|
+
await chrome.tabs.get(tabId);
|
|
41
|
+
}
|
|
42
|
+
catch (e) {
|
|
43
|
+
tabId = null;
|
|
44
|
+
context.variables.delete('tabId');
|
|
45
|
+
}
|
|
37
46
|
}
|
|
38
47
|
}
|
|
39
48
|
if (!tabId) {
|
|
@@ -50,28 +59,31 @@ async function getTabId(context) {
|
|
|
50
59
|
else {
|
|
51
60
|
tabId = await getCurrentTabId();
|
|
52
61
|
}
|
|
62
|
+
if (!tabId) {
|
|
63
|
+
throw new Error('Could not find a valid tab');
|
|
64
|
+
}
|
|
65
|
+
context.variables.set('tabId', tabId);
|
|
53
66
|
}
|
|
54
67
|
return tabId;
|
|
55
68
|
}
|
|
56
|
-
function getCurrentTabId(windowId) {
|
|
57
|
-
return new Promise((resolve) => {
|
|
58
|
-
chrome.tabs.query({
|
|
59
|
-
if (
|
|
60
|
-
|
|
69
|
+
async function getCurrentTabId(windowId) {
|
|
70
|
+
return new Promise((resolve, reject) => {
|
|
71
|
+
chrome.tabs.query({ active: true, currentWindow: true }, function (tabs) {
|
|
72
|
+
if (chrome.runtime.lastError) {
|
|
73
|
+
console.error('Chrome runtime error:', chrome.runtime.lastError);
|
|
74
|
+
reject(chrome.runtime.lastError);
|
|
75
|
+
return;
|
|
61
76
|
}
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
resolve(_tabs[0].id);
|
|
66
|
-
return;
|
|
67
|
-
}
|
|
68
|
-
else {
|
|
69
|
-
chrome.tabs.query({ windowId, status: 'complete', currentWindow: true }, function (__tabs) {
|
|
70
|
-
resolve(__tabs.length ? __tabs[__tabs.length - 1].id : undefined);
|
|
71
|
-
});
|
|
72
|
-
}
|
|
73
|
-
});
|
|
77
|
+
if (!tabs || tabs.length === 0) {
|
|
78
|
+
reject(new Error('No active tab found'));
|
|
79
|
+
return;
|
|
74
80
|
}
|
|
81
|
+
const tabId = tabs[0].id;
|
|
82
|
+
if (typeof tabId !== 'number') {
|
|
83
|
+
reject(new Error('Invalid tab ID'));
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
resolve(tabId);
|
|
75
87
|
});
|
|
76
88
|
});
|
|
77
89
|
}
|
|
@@ -260,176 +272,348 @@ var utils = /*#__PURE__*/Object.freeze({
|
|
|
260
272
|
});
|
|
261
273
|
|
|
262
274
|
async function type(tabId, text, coordinate) {
|
|
263
|
-
|
|
264
|
-
|
|
275
|
+
console.log('Sending type message to tab:', tabId, { text, coordinate });
|
|
276
|
+
try {
|
|
277
|
+
if (!coordinate) {
|
|
278
|
+
coordinate = (await cursor_position(tabId)).coordinate;
|
|
279
|
+
}
|
|
280
|
+
await mouse_move(tabId, coordinate);
|
|
281
|
+
const response = await chrome.tabs.sendMessage(tabId, {
|
|
282
|
+
type: 'computer:type',
|
|
283
|
+
text,
|
|
284
|
+
coordinate,
|
|
285
|
+
});
|
|
286
|
+
console.log('Got response:', response);
|
|
287
|
+
return response;
|
|
288
|
+
}
|
|
289
|
+
catch (e) {
|
|
290
|
+
console.error('Failed to send type message:', e);
|
|
291
|
+
throw e;
|
|
265
292
|
}
|
|
266
|
-
await mouse_move(tabId, coordinate);
|
|
267
|
-
return await chrome.tabs.sendMessage(tabId, {
|
|
268
|
-
type: 'computer:type',
|
|
269
|
-
text,
|
|
270
|
-
coordinate,
|
|
271
|
-
});
|
|
272
293
|
}
|
|
273
294
|
async function type_by(tabId, text, xpath, highlightIndex) {
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
295
|
+
console.log('Sending type message to tab:', tabId, { text, xpath, highlightIndex });
|
|
296
|
+
try {
|
|
297
|
+
const response = await chrome.tabs.sendMessage(tabId, {
|
|
298
|
+
type: 'computer:type',
|
|
299
|
+
text,
|
|
300
|
+
xpath,
|
|
301
|
+
highlightIndex,
|
|
302
|
+
});
|
|
303
|
+
console.log('Got response:', response);
|
|
304
|
+
return response;
|
|
305
|
+
}
|
|
306
|
+
catch (e) {
|
|
307
|
+
console.error('Failed to send type message:', e);
|
|
308
|
+
throw e;
|
|
309
|
+
}
|
|
280
310
|
}
|
|
281
311
|
async function clear_input(tabId, coordinate) {
|
|
282
|
-
|
|
283
|
-
|
|
312
|
+
console.log('Sending clear_input message to tab:', tabId, { coordinate });
|
|
313
|
+
try {
|
|
314
|
+
if (!coordinate) {
|
|
315
|
+
coordinate = (await cursor_position(tabId)).coordinate;
|
|
316
|
+
}
|
|
317
|
+
await mouse_move(tabId, coordinate);
|
|
318
|
+
const response = await chrome.tabs.sendMessage(tabId, {
|
|
319
|
+
type: 'computer:type',
|
|
320
|
+
text: '',
|
|
321
|
+
coordinate,
|
|
322
|
+
});
|
|
323
|
+
console.log('Got response:', response);
|
|
324
|
+
return response;
|
|
325
|
+
}
|
|
326
|
+
catch (e) {
|
|
327
|
+
console.error('Failed to send clear_input message:', e);
|
|
328
|
+
throw e;
|
|
284
329
|
}
|
|
285
|
-
await mouse_move(tabId, coordinate);
|
|
286
|
-
return await chrome.tabs.sendMessage(tabId, {
|
|
287
|
-
type: 'computer:type',
|
|
288
|
-
text: '',
|
|
289
|
-
coordinate,
|
|
290
|
-
});
|
|
291
330
|
}
|
|
292
331
|
async function clear_input_by(tabId, xpath, highlightIndex) {
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
332
|
+
console.log('Sending clear_input_by message to tab:', tabId, { xpath, highlightIndex });
|
|
333
|
+
try {
|
|
334
|
+
const response = await chrome.tabs.sendMessage(tabId, {
|
|
335
|
+
type: 'computer:type',
|
|
336
|
+
text: '',
|
|
337
|
+
xpath,
|
|
338
|
+
highlightIndex,
|
|
339
|
+
});
|
|
340
|
+
console.log('Got response:', response);
|
|
341
|
+
return response;
|
|
342
|
+
}
|
|
343
|
+
catch (e) {
|
|
344
|
+
console.error('Failed to send clear_input_by message:', e);
|
|
345
|
+
throw e;
|
|
346
|
+
}
|
|
299
347
|
}
|
|
300
348
|
async function mouse_move(tabId, coordinate) {
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
349
|
+
console.log('Sending mouse_move message to tab:', tabId, { coordinate });
|
|
350
|
+
try {
|
|
351
|
+
const response = await chrome.tabs.sendMessage(tabId, {
|
|
352
|
+
type: 'computer:mouse_move',
|
|
353
|
+
coordinate,
|
|
354
|
+
});
|
|
355
|
+
console.log('Got response:', response);
|
|
356
|
+
return response;
|
|
357
|
+
}
|
|
358
|
+
catch (e) {
|
|
359
|
+
console.error('Failed to send mouse_move message:', e);
|
|
360
|
+
throw e;
|
|
361
|
+
}
|
|
305
362
|
}
|
|
306
363
|
async function left_click(tabId, coordinate) {
|
|
307
|
-
|
|
308
|
-
|
|
364
|
+
console.log('Sending left_click message to tab:', tabId, { coordinate });
|
|
365
|
+
try {
|
|
366
|
+
if (!coordinate) {
|
|
367
|
+
coordinate = (await cursor_position(tabId)).coordinate;
|
|
368
|
+
}
|
|
369
|
+
const response = await chrome.tabs.sendMessage(tabId, {
|
|
370
|
+
type: 'computer:left_click',
|
|
371
|
+
coordinate,
|
|
372
|
+
});
|
|
373
|
+
console.log('Got response:', response);
|
|
374
|
+
return response;
|
|
375
|
+
}
|
|
376
|
+
catch (e) {
|
|
377
|
+
console.error('Failed to send left_click message:', e);
|
|
378
|
+
throw e;
|
|
309
379
|
}
|
|
310
|
-
return await chrome.tabs.sendMessage(tabId, {
|
|
311
|
-
type: 'computer:left_click',
|
|
312
|
-
coordinate,
|
|
313
|
-
});
|
|
314
380
|
}
|
|
315
381
|
async function left_click_by(tabId, xpath, highlightIndex) {
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
382
|
+
console.log('Sending left_click_by message to tab:', tabId, { xpath, highlightIndex });
|
|
383
|
+
try {
|
|
384
|
+
const response = await chrome.tabs.sendMessage(tabId, {
|
|
385
|
+
type: 'computer:left_click',
|
|
386
|
+
xpath,
|
|
387
|
+
highlightIndex,
|
|
388
|
+
});
|
|
389
|
+
console.log('Got response:', response);
|
|
390
|
+
return response;
|
|
391
|
+
}
|
|
392
|
+
catch (e) {
|
|
393
|
+
console.error('Failed to send left_click_by message:', e);
|
|
394
|
+
throw e;
|
|
395
|
+
}
|
|
321
396
|
}
|
|
322
397
|
async function right_click(tabId, coordinate) {
|
|
323
|
-
|
|
324
|
-
|
|
398
|
+
console.log('Sending right_click message to tab:', tabId, { coordinate });
|
|
399
|
+
try {
|
|
400
|
+
if (!coordinate) {
|
|
401
|
+
coordinate = (await cursor_position(tabId)).coordinate;
|
|
402
|
+
}
|
|
403
|
+
const response = await chrome.tabs.sendMessage(tabId, {
|
|
404
|
+
type: 'computer:right_click',
|
|
405
|
+
coordinate,
|
|
406
|
+
});
|
|
407
|
+
console.log('Got response:', response);
|
|
408
|
+
return response;
|
|
409
|
+
}
|
|
410
|
+
catch (e) {
|
|
411
|
+
console.error('Failed to send right_click message:', e);
|
|
412
|
+
throw e;
|
|
325
413
|
}
|
|
326
|
-
return await chrome.tabs.sendMessage(tabId, {
|
|
327
|
-
type: 'computer:right_click',
|
|
328
|
-
coordinate,
|
|
329
|
-
});
|
|
330
414
|
}
|
|
331
415
|
async function right_click_by(tabId, xpath, highlightIndex) {
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
416
|
+
console.log('Sending right_click_by message to tab:', tabId, { xpath, highlightIndex });
|
|
417
|
+
try {
|
|
418
|
+
const response = await chrome.tabs.sendMessage(tabId, {
|
|
419
|
+
type: 'computer:right_click',
|
|
420
|
+
xpath,
|
|
421
|
+
highlightIndex,
|
|
422
|
+
});
|
|
423
|
+
console.log('Got response:', response);
|
|
424
|
+
return response;
|
|
425
|
+
}
|
|
426
|
+
catch (e) {
|
|
427
|
+
console.error('Failed to send right_click_by message:', e);
|
|
428
|
+
throw e;
|
|
429
|
+
}
|
|
337
430
|
}
|
|
338
431
|
async function double_click(tabId, coordinate) {
|
|
339
|
-
|
|
340
|
-
|
|
432
|
+
console.log('Sending double_click message to tab:', tabId, { coordinate });
|
|
433
|
+
try {
|
|
434
|
+
if (!coordinate) {
|
|
435
|
+
coordinate = (await cursor_position(tabId)).coordinate;
|
|
436
|
+
}
|
|
437
|
+
const response = await chrome.tabs.sendMessage(tabId, {
|
|
438
|
+
type: 'computer:double_click',
|
|
439
|
+
coordinate,
|
|
440
|
+
});
|
|
441
|
+
console.log('Got response:', response);
|
|
442
|
+
return response;
|
|
443
|
+
}
|
|
444
|
+
catch (e) {
|
|
445
|
+
console.error('Failed to send double_click message:', e);
|
|
446
|
+
throw e;
|
|
341
447
|
}
|
|
342
|
-
return await chrome.tabs.sendMessage(tabId, {
|
|
343
|
-
type: 'computer:double_click',
|
|
344
|
-
coordinate,
|
|
345
|
-
});
|
|
346
448
|
}
|
|
347
449
|
async function double_click_by(tabId, xpath, highlightIndex) {
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
450
|
+
console.log('Sending double_click_by message to tab:', tabId, { xpath, highlightIndex });
|
|
451
|
+
try {
|
|
452
|
+
const response = await chrome.tabs.sendMessage(tabId, {
|
|
453
|
+
type: 'computer:double_click',
|
|
454
|
+
xpath,
|
|
455
|
+
highlightIndex,
|
|
456
|
+
});
|
|
457
|
+
console.log('Got response:', response);
|
|
458
|
+
return response;
|
|
459
|
+
}
|
|
460
|
+
catch (e) {
|
|
461
|
+
console.error('Failed to send double_click_by message:', e);
|
|
462
|
+
throw e;
|
|
463
|
+
}
|
|
353
464
|
}
|
|
354
465
|
async function screenshot(windowId, compress) {
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
dataUrl
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
466
|
+
console.log('Taking screenshot of window:', windowId, { compress });
|
|
467
|
+
try {
|
|
468
|
+
let dataUrl;
|
|
469
|
+
if (compress) {
|
|
470
|
+
dataUrl = await chrome.tabs.captureVisibleTab(windowId, {
|
|
471
|
+
format: 'jpeg',
|
|
472
|
+
quality: 60, // 0-100
|
|
473
|
+
});
|
|
474
|
+
dataUrl = await compress_image(dataUrl, 0.7, 1);
|
|
475
|
+
}
|
|
476
|
+
else {
|
|
477
|
+
dataUrl = await chrome.tabs.captureVisibleTab(windowId, {
|
|
478
|
+
format: 'jpeg',
|
|
479
|
+
quality: 50,
|
|
480
|
+
});
|
|
481
|
+
}
|
|
482
|
+
let data = dataUrl.substring(dataUrl.indexOf('base64,') + 7);
|
|
483
|
+
const result = {
|
|
484
|
+
image: {
|
|
485
|
+
type: 'base64',
|
|
486
|
+
media_type: dataUrl.indexOf('image/png') > -1 ? 'image/png' : 'image/jpeg',
|
|
487
|
+
data: data,
|
|
488
|
+
},
|
|
489
|
+
};
|
|
490
|
+
console.log('Got screenshot result:', result);
|
|
491
|
+
return result;
|
|
362
492
|
}
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
quality: 50,
|
|
367
|
-
});
|
|
493
|
+
catch (e) {
|
|
494
|
+
console.error('Failed to take screenshot:', e);
|
|
495
|
+
throw e;
|
|
368
496
|
}
|
|
369
|
-
let data = dataUrl.substring(dataUrl.indexOf('base64,') + 7);
|
|
370
|
-
return {
|
|
371
|
-
image: {
|
|
372
|
-
type: 'base64',
|
|
373
|
-
media_type: dataUrl.indexOf('image/png') > -1 ? 'image/png' : 'image/jpeg',
|
|
374
|
-
data: data,
|
|
375
|
-
},
|
|
376
|
-
};
|
|
377
497
|
}
|
|
378
498
|
async function compress_image(dataUrl, scale = 0.8, quality = 0.8) {
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
499
|
+
console.log('Compressing image', { scale, quality });
|
|
500
|
+
try {
|
|
501
|
+
const bitmap = await createImageBitmap(await (await fetch(dataUrl)).blob());
|
|
502
|
+
let width = bitmap.width * scale;
|
|
503
|
+
let height = bitmap.height * scale;
|
|
504
|
+
const canvas = new OffscreenCanvas(width, height);
|
|
505
|
+
const ctx = canvas.getContext('2d');
|
|
506
|
+
ctx.drawImage(bitmap, 0, 0, width, height);
|
|
507
|
+
const blob = await canvas.convertToBlob({
|
|
508
|
+
type: 'image/jpeg',
|
|
509
|
+
quality: quality,
|
|
510
|
+
});
|
|
511
|
+
return new Promise((resolve) => {
|
|
512
|
+
const reader = new FileReader();
|
|
513
|
+
reader.onloadend = () => {
|
|
514
|
+
const result = reader.result;
|
|
515
|
+
console.log('Got compressed image result:', result);
|
|
516
|
+
resolve(result);
|
|
517
|
+
};
|
|
518
|
+
reader.readAsDataURL(blob);
|
|
519
|
+
});
|
|
520
|
+
}
|
|
521
|
+
catch (e) {
|
|
522
|
+
console.error('Failed to compress image:', e);
|
|
523
|
+
throw e;
|
|
524
|
+
}
|
|
394
525
|
}
|
|
395
526
|
async function scroll_to(tabId, coordinate) {
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
527
|
+
console.log('Sending scroll_to message to tab:', tabId, { coordinate });
|
|
528
|
+
try {
|
|
529
|
+
let from_coordinate = (await cursor_position(tabId)).coordinate;
|
|
530
|
+
const response = await chrome.tabs.sendMessage(tabId, {
|
|
531
|
+
type: 'computer:scroll_to',
|
|
532
|
+
from_coordinate,
|
|
533
|
+
to_coordinate: coordinate,
|
|
534
|
+
});
|
|
535
|
+
console.log('Got response:', response);
|
|
536
|
+
return response;
|
|
537
|
+
}
|
|
538
|
+
catch (e) {
|
|
539
|
+
console.error('Failed to send scroll_to message:', e);
|
|
540
|
+
throw e;
|
|
541
|
+
}
|
|
402
542
|
}
|
|
403
543
|
async function scroll_to_by(tabId, xpath, highlightIndex) {
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
544
|
+
console.log('Sending scroll_to_by message to tab:', tabId, { xpath, highlightIndex });
|
|
545
|
+
try {
|
|
546
|
+
const response = await chrome.tabs.sendMessage(tabId, {
|
|
547
|
+
type: 'computer:scroll_to',
|
|
548
|
+
xpath,
|
|
549
|
+
highlightIndex,
|
|
550
|
+
});
|
|
551
|
+
console.log('Got response:', response);
|
|
552
|
+
return response;
|
|
553
|
+
}
|
|
554
|
+
catch (e) {
|
|
555
|
+
console.error('Failed to send scroll_to_by message:', e);
|
|
556
|
+
throw e;
|
|
557
|
+
}
|
|
409
558
|
}
|
|
410
559
|
async function get_dropdown_options(tabId, xpath, highlightIndex) {
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
560
|
+
console.log('Sending get_dropdown_options message to tab:', tabId, { xpath, highlightIndex });
|
|
561
|
+
try {
|
|
562
|
+
const response = await chrome.tabs.sendMessage(tabId, {
|
|
563
|
+
type: 'computer:get_dropdown_options',
|
|
564
|
+
xpath,
|
|
565
|
+
highlightIndex,
|
|
566
|
+
});
|
|
567
|
+
console.log('Got response:', response);
|
|
568
|
+
return response;
|
|
569
|
+
}
|
|
570
|
+
catch (e) {
|
|
571
|
+
console.error('Failed to send get_dropdown_options message:', e);
|
|
572
|
+
throw e;
|
|
573
|
+
}
|
|
416
574
|
}
|
|
417
575
|
async function select_dropdown_option(tabId, text, xpath, highlightIndex) {
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
576
|
+
console.log('Sending select_dropdown_option message to tab:', tabId, { text, xpath, highlightIndex });
|
|
577
|
+
try {
|
|
578
|
+
const response = await chrome.tabs.sendMessage(tabId, {
|
|
579
|
+
type: 'computer:select_dropdown_option',
|
|
580
|
+
text,
|
|
581
|
+
xpath,
|
|
582
|
+
highlightIndex,
|
|
583
|
+
});
|
|
584
|
+
console.log('Got response:', response);
|
|
585
|
+
return response;
|
|
586
|
+
}
|
|
587
|
+
catch (e) {
|
|
588
|
+
console.error('Failed to send select_dropdown_option message:', e);
|
|
589
|
+
throw e;
|
|
590
|
+
}
|
|
424
591
|
}
|
|
425
592
|
async function cursor_position(tabId) {
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
593
|
+
console.log('Sending cursor_position message to tab:', tabId);
|
|
594
|
+
try {
|
|
595
|
+
let result = await chrome.tabs.sendMessage(tabId, {
|
|
596
|
+
type: 'computer:cursor_position',
|
|
597
|
+
});
|
|
598
|
+
console.log('Got cursor position:', result.coordinate);
|
|
599
|
+
return { coordinate: result.coordinate };
|
|
600
|
+
}
|
|
601
|
+
catch (e) {
|
|
602
|
+
console.error('Failed to send cursor_position message:', e);
|
|
603
|
+
throw e;
|
|
604
|
+
}
|
|
430
605
|
}
|
|
431
606
|
async function size(tabId) {
|
|
432
|
-
|
|
607
|
+
console.log('Getting page size for tab:', tabId);
|
|
608
|
+
try {
|
|
609
|
+
const pageSize = await getPageSize(tabId);
|
|
610
|
+
console.log('Got page size:', pageSize);
|
|
611
|
+
return pageSize;
|
|
612
|
+
}
|
|
613
|
+
catch (e) {
|
|
614
|
+
console.error('Failed to get page size:', e);
|
|
615
|
+
throw e;
|
|
616
|
+
}
|
|
433
617
|
}
|
|
434
618
|
|
|
435
619
|
var browser = /*#__PURE__*/Object.freeze({
|
|
@@ -461,35 +645,35 @@ var browser = /*#__PURE__*/Object.freeze({
|
|
|
461
645
|
class BrowserUse {
|
|
462
646
|
constructor() {
|
|
463
647
|
this.name = 'browser_use';
|
|
464
|
-
this.description = `Use structured commands to interact with the browser, manipulating page elements through screenshots and webpage element extraction.
|
|
465
|
-
* This is a browser GUI interface where you need to analyze webpages by taking screenshots and extracting page element structures, and specify action sequences to complete designated tasks.
|
|
466
|
-
* Before any operation, you must first call the \`screenshot_extract_element\` command, which will return the browser page screenshot and structured element information, both specially processed.
|
|
467
|
-
* ELEMENT INTERACTION:
|
|
468
|
-
- Only use indexes that exist in the provided element list
|
|
469
|
-
- Each element has a unique index number (e.g., "[33]:<button>")
|
|
470
|
-
- Elements marked with "[]:" are non-interactive (for context only)
|
|
471
|
-
* NAVIGATION & ERROR HANDLING:
|
|
472
|
-
- If no suitable elements exist, use other functions to complete the task
|
|
473
|
-
- If stuck, try alternative approaches
|
|
474
|
-
- Handle popups/cookies by accepting or closing them
|
|
648
|
+
this.description = `Use structured commands to interact with the browser, manipulating page elements through screenshots and webpage element extraction.
|
|
649
|
+
* This is a browser GUI interface where you need to analyze webpages by taking screenshots and extracting page element structures, and specify action sequences to complete designated tasks.
|
|
650
|
+
* Before any operation, you must first call the \`screenshot_extract_element\` command, which will return the browser page screenshot and structured element information, both specially processed.
|
|
651
|
+
* ELEMENT INTERACTION:
|
|
652
|
+
- Only use indexes that exist in the provided element list
|
|
653
|
+
- Each element has a unique index number (e.g., "[33]:<button>")
|
|
654
|
+
- Elements marked with "[]:" are non-interactive (for context only)
|
|
655
|
+
* NAVIGATION & ERROR HANDLING:
|
|
656
|
+
- If no suitable elements exist, use other functions to complete the task
|
|
657
|
+
- If stuck, try alternative approaches
|
|
658
|
+
- Handle popups/cookies by accepting or closing them
|
|
475
659
|
- Use scroll to find elements you are looking for`;
|
|
476
660
|
this.input_schema = {
|
|
477
661
|
type: 'object',
|
|
478
662
|
properties: {
|
|
479
663
|
action: {
|
|
480
664
|
type: 'string',
|
|
481
|
-
description: `The action to perform. The available actions are:
|
|
482
|
-
* \`screenshot_extract_element\`: Take a screenshot of the web page and extract operable elements.
|
|
483
|
-
- Screenshots are used to understand page layouts, with labeled bounding boxes corresponding to element indexes. Each bounding box and its label share the same color, with labels typically positioned in the top-right corner of the box.
|
|
484
|
-
- Screenshots help verify element positions and relationships. Labels may sometimes overlap, so extracted elements are used to verify the correct elements.
|
|
485
|
-
- In addition to screenshots, simplified information about interactive elements is returned, with element indexes corresponding to those in the screenshots.
|
|
486
|
-
* \`input_text\`: Enter a string in the interactive element.
|
|
487
|
-
* \`click\`: Click to element.
|
|
488
|
-
* \`right_click\`: Right-click on the element.
|
|
489
|
-
* \`double_click\`: Double-click on the element.
|
|
490
|
-
* \`scroll_to\`: Scroll to the specified element.
|
|
491
|
-
* \`extract_content\`: Extract the text content of the current webpage.
|
|
492
|
-
* \`get_dropdown_options\`: Get all options from a native dropdown element.
|
|
665
|
+
description: `The action to perform. The available actions are:
|
|
666
|
+
* \`screenshot_extract_element\`: Take a screenshot of the web page and extract operable elements.
|
|
667
|
+
- Screenshots are used to understand page layouts, with labeled bounding boxes corresponding to element indexes. Each bounding box and its label share the same color, with labels typically positioned in the top-right corner of the box.
|
|
668
|
+
- Screenshots help verify element positions and relationships. Labels may sometimes overlap, so extracted elements are used to verify the correct elements.
|
|
669
|
+
- In addition to screenshots, simplified information about interactive elements is returned, with element indexes corresponding to those in the screenshots.
|
|
670
|
+
* \`input_text\`: Enter a string in the interactive element.
|
|
671
|
+
* \`click\`: Click to element.
|
|
672
|
+
* \`right_click\`: Right-click on the element.
|
|
673
|
+
* \`double_click\`: Double-click on the element.
|
|
674
|
+
* \`scroll_to\`: Scroll to the specified element.
|
|
675
|
+
* \`extract_content\`: Extract the text content of the current webpage.
|
|
676
|
+
* \`get_dropdown_options\`: Get all options from a native dropdown element.
|
|
493
677
|
* \`select_dropdown_option\`: Select dropdown option for interactive element index by the text of the option you want to select.`,
|
|
494
678
|
enum: [
|
|
495
679
|
'screenshot_extract_element',
|
|
@@ -527,7 +711,17 @@ class BrowserUse {
|
|
|
527
711
|
if (params === null || !params.action) {
|
|
528
712
|
throw new Error('Invalid parameters. Expected an object with a "action" property.');
|
|
529
713
|
}
|
|
530
|
-
let tabId
|
|
714
|
+
let tabId;
|
|
715
|
+
try {
|
|
716
|
+
tabId = await getTabId(context);
|
|
717
|
+
if (!tabId || !Number.isInteger(tabId)) {
|
|
718
|
+
throw new Error('Could not get valid tab ID');
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
catch (e) {
|
|
722
|
+
console.error('Tab ID error:', e);
|
|
723
|
+
return { success: false, error: 'Could not access browser tab' };
|
|
724
|
+
}
|
|
531
725
|
let windowId = await getWindowId(context);
|
|
532
726
|
let selector_map = context.selector_map;
|
|
533
727
|
let selector_xpath;
|
|
@@ -631,7 +825,8 @@ class BrowserUse {
|
|
|
631
825
|
}
|
|
632
826
|
}
|
|
633
827
|
catch (e) {
|
|
634
|
-
|
|
828
|
+
console.error('Browser use error:', e);
|
|
829
|
+
return { success: false, error: e === null || e === undefined ? undefined : e.message };
|
|
635
830
|
}
|
|
636
831
|
}
|
|
637
832
|
destroy(context) {
|
|
@@ -709,7 +904,7 @@ function extractOperableElements() {
|
|
|
709
904
|
return NodeFilter.FILTER_SKIP;
|
|
710
905
|
}
|
|
711
906
|
// text <= 100
|
|
712
|
-
const text = (_a = node.innerText) === null || _a ===
|
|
907
|
+
const text = (_a = node.innerText) === null || _a === undefined ? undefined : _a.trim();
|
|
713
908
|
if (isElementVisible(node) &&
|
|
714
909
|
text &&
|
|
715
910
|
text.length <= 100 &&
|
|
@@ -815,21 +1010,21 @@ async function executeWithHtmlElement$1(context, task_prompt) {
|
|
|
815
1010
|
let messages = [
|
|
816
1011
|
{
|
|
817
1012
|
role: 'user',
|
|
818
|
-
content: `# Task
|
|
819
|
-
Determine the operation intent based on user input, find the element ID that the user needs to operate on in the webpage HTML, and if the element does not exist, do nothing.
|
|
820
|
-
Output JSON format, no explanation required.
|
|
821
|
-
|
|
822
|
-
# User input
|
|
823
|
-
${task_prompt}
|
|
824
|
-
|
|
825
|
-
# Output example (when the element exists)
|
|
826
|
-
{"elementId": "1", "operationType": "click"}
|
|
827
|
-
|
|
828
|
-
# Output example (when the element does not exist)
|
|
829
|
-
{"elementId": null, "operationType": "unknown"}
|
|
830
|
-
|
|
831
|
-
# HTML
|
|
832
|
-
${pseudoHtml}
|
|
1013
|
+
content: `# Task
|
|
1014
|
+
Determine the operation intent based on user input, find the element ID that the user needs to operate on in the webpage HTML, and if the element does not exist, do nothing.
|
|
1015
|
+
Output JSON format, no explanation required.
|
|
1016
|
+
|
|
1017
|
+
# User input
|
|
1018
|
+
${task_prompt}
|
|
1019
|
+
|
|
1020
|
+
# Output example (when the element exists)
|
|
1021
|
+
{"elementId": "1", "operationType": "click"}
|
|
1022
|
+
|
|
1023
|
+
# Output example (when the element does not exist)
|
|
1024
|
+
{"elementId": null, "operationType": "unknown"}
|
|
1025
|
+
|
|
1026
|
+
# HTML
|
|
1027
|
+
${pseudoHtml}
|
|
833
1028
|
`,
|
|
834
1029
|
},
|
|
835
1030
|
];
|
|
@@ -926,6 +1121,7 @@ class ExportFile {
|
|
|
926
1121
|
* @returns > { success: true }
|
|
927
1122
|
*/
|
|
928
1123
|
async execute(context, params) {
|
|
1124
|
+
var _a, _b, _c;
|
|
929
1125
|
if (typeof params !== 'object' || params === null || !('content' in params)) {
|
|
930
1126
|
throw new Error('Invalid parameters. Expected an object with a "content" property.');
|
|
931
1127
|
}
|
|
@@ -960,8 +1156,8 @@ class ExportFile {
|
|
|
960
1156
|
else {
|
|
961
1157
|
filename = params.filename;
|
|
962
1158
|
}
|
|
963
|
-
let tabId = await getTabId(context);
|
|
964
1159
|
try {
|
|
1160
|
+
let tabId = await getTabId(context);
|
|
965
1161
|
await chrome.scripting.executeScript({
|
|
966
1162
|
target: { tabId: tabId },
|
|
967
1163
|
func: exportFile,
|
|
@@ -970,13 +1166,14 @@ class ExportFile {
|
|
|
970
1166
|
}
|
|
971
1167
|
catch (e) {
|
|
972
1168
|
let tab = await open_new_tab('https://www.google.com', true);
|
|
973
|
-
|
|
1169
|
+
(_c = (_b = (_a = context.callback) === null || _a === undefined ? undefined : _a.hooks) === null || _b === undefined ? undefined : _b.onTabCreated) === null || _c === undefined ? undefined : _c.call(_b, tab.id);
|
|
1170
|
+
let tabId = tab.id;
|
|
974
1171
|
await chrome.scripting.executeScript({
|
|
975
1172
|
target: { tabId: tabId },
|
|
976
1173
|
func: exportFile,
|
|
977
1174
|
args: [filename, type, params.content],
|
|
978
1175
|
});
|
|
979
|
-
await sleep(
|
|
1176
|
+
await sleep(5000);
|
|
980
1177
|
await chrome.tabs.remove(tabId);
|
|
981
1178
|
}
|
|
982
1179
|
return { success: true };
|
|
@@ -1063,21 +1260,21 @@ async function executeWithHtmlElement(context, task_prompt) {
|
|
|
1063
1260
|
let messages = [
|
|
1064
1261
|
{
|
|
1065
1262
|
role: 'user',
|
|
1066
|
-
content: `# Task
|
|
1067
|
-
Find the element ID that the user needs to operate on in the webpage HTML, and if the element does not exist, do nothing.
|
|
1068
|
-
Output JSON format, no explanation required.
|
|
1069
|
-
|
|
1070
|
-
# User input
|
|
1071
|
-
${task_prompt}
|
|
1072
|
-
|
|
1073
|
-
# Output example (when the element exists)
|
|
1074
|
-
{"elementId": "1"}
|
|
1075
|
-
|
|
1076
|
-
# Output example (when the element does not exist)
|
|
1077
|
-
{"elementId": null}
|
|
1078
|
-
|
|
1079
|
-
# HTML
|
|
1080
|
-
${pseudoHtml}
|
|
1263
|
+
content: `# Task
|
|
1264
|
+
Find the element ID that the user needs to operate on in the webpage HTML, and if the element does not exist, do nothing.
|
|
1265
|
+
Output JSON format, no explanation required.
|
|
1266
|
+
|
|
1267
|
+
# User input
|
|
1268
|
+
${task_prompt}
|
|
1269
|
+
|
|
1270
|
+
# Output example (when the element exists)
|
|
1271
|
+
{"elementId": "1"}
|
|
1272
|
+
|
|
1273
|
+
# Output example (when the element does not exist)
|
|
1274
|
+
{"elementId": null}
|
|
1275
|
+
|
|
1276
|
+
# HTML
|
|
1277
|
+
${pseudoHtml}
|
|
1081
1278
|
`,
|
|
1082
1279
|
},
|
|
1083
1280
|
];
|
|
@@ -1142,6 +1339,48 @@ async function executeWithBrowserUse(context, task_prompt) {
|
|
|
1142
1339
|
};
|
|
1143
1340
|
}
|
|
1144
1341
|
|
|
1342
|
+
class GetAllTabs {
|
|
1343
|
+
constructor() {
|
|
1344
|
+
this.name = 'get_all_tabs';
|
|
1345
|
+
this.description = 'Get the tabId, title, url and content from current all tabs without opening new tab.';
|
|
1346
|
+
this.input_schema = {
|
|
1347
|
+
type: 'object',
|
|
1348
|
+
properties: {},
|
|
1349
|
+
};
|
|
1350
|
+
}
|
|
1351
|
+
async execute(context, params) {
|
|
1352
|
+
const currentWindow = await chrome.windows.getCurrent();
|
|
1353
|
+
const windowId = currentWindow.id;
|
|
1354
|
+
const tabs = await chrome.tabs.query({ windowId });
|
|
1355
|
+
const tabsInfo = [];
|
|
1356
|
+
for (const tab of tabs) {
|
|
1357
|
+
if (tab.id === undefined) {
|
|
1358
|
+
console.warn(`Tab ID is undefined for tab with URL: ${tab.url}`);
|
|
1359
|
+
continue;
|
|
1360
|
+
}
|
|
1361
|
+
await injectScript(tab.id);
|
|
1362
|
+
await sleep(500);
|
|
1363
|
+
let content = await executeScript(tab.id, () => {
|
|
1364
|
+
return eko.extractHtmlContent();
|
|
1365
|
+
}, []);
|
|
1366
|
+
// Use title as description, but requirement may evolve
|
|
1367
|
+
let description = tab.title ? tab.title : "No description available.";
|
|
1368
|
+
const tabInfo = {
|
|
1369
|
+
id: tab.id,
|
|
1370
|
+
url: tab.url,
|
|
1371
|
+
title: tab.title,
|
|
1372
|
+
content: content,
|
|
1373
|
+
description: description,
|
|
1374
|
+
};
|
|
1375
|
+
console.log("url: " + tab.url);
|
|
1376
|
+
console.log("title: " + tab.title);
|
|
1377
|
+
console.log("description: " + description);
|
|
1378
|
+
tabsInfo.push(tabInfo);
|
|
1379
|
+
}
|
|
1380
|
+
return tabsInfo;
|
|
1381
|
+
}
|
|
1382
|
+
}
|
|
1383
|
+
|
|
1145
1384
|
/**
|
|
1146
1385
|
* Open Url
|
|
1147
1386
|
*/
|
|
@@ -1171,6 +1410,7 @@ class OpenUrl {
|
|
|
1171
1410
|
* @returns > { tabId, windowId, title, success: true }
|
|
1172
1411
|
*/
|
|
1173
1412
|
async execute(context, params) {
|
|
1413
|
+
var _a, _b, _c, _d, _e, _f;
|
|
1174
1414
|
if (typeof params !== 'object' || params === null || !params.url) {
|
|
1175
1415
|
throw new Error('Invalid parameters. Expected an object with a "url" property.');
|
|
1176
1416
|
}
|
|
@@ -1183,10 +1423,12 @@ class OpenUrl {
|
|
|
1183
1423
|
let tab;
|
|
1184
1424
|
if (newWindow) {
|
|
1185
1425
|
tab = await open_new_tab(url, true);
|
|
1426
|
+
(_c = (_b = (_a = context.callback) === null || _a === undefined ? undefined : _a.hooks) === null || _b === undefined ? undefined : _b.onTabCreated) === null || _c === undefined ? undefined : _c.call(_b, tab.id);
|
|
1186
1427
|
}
|
|
1187
1428
|
else {
|
|
1188
1429
|
let windowId = await getWindowId(context);
|
|
1189
1430
|
tab = await open_new_tab(url, false, windowId);
|
|
1431
|
+
(_f = (_e = (_d = context.callback) === null || _d === undefined ? undefined : _d.hooks) === null || _e === undefined ? undefined : _e.onTabCreated) === null || _f === undefined ? undefined : _f.call(_e, tab.id);
|
|
1190
1432
|
}
|
|
1191
1433
|
let windowId = tab.windowId;
|
|
1192
1434
|
let tabId = tab.id;
|
|
@@ -1243,41 +1485,42 @@ class TabManagement {
|
|
|
1243
1485
|
this.input_schema = {
|
|
1244
1486
|
type: 'object',
|
|
1245
1487
|
properties: {
|
|
1246
|
-
|
|
1488
|
+
command: {
|
|
1247
1489
|
type: 'string',
|
|
1248
|
-
description: `The
|
|
1249
|
-
* \`tab_all\`: View all tabs and return the tabId and title.
|
|
1250
|
-
* \`current_tab\`: Get current tab information (tabId, url, title).
|
|
1251
|
-
* \`go_back\`: Go back to the previous page in the current tab.
|
|
1252
|
-
* \`change_url [url]\`: open URL in the current tab, eg: \`change_url https://www.google.com\`.
|
|
1253
|
-
* \`close_tab\`: Close the current tab.
|
|
1254
|
-
* \`switch_tab [tabId]\`: Switch to the specified tab using tabId, eg: \`switch_tab 1000\`.
|
|
1490
|
+
description: `The command to perform. The available commands are:
|
|
1491
|
+
* \`tab_all\`: View all tabs and return the tabId and title.
|
|
1492
|
+
* \`current_tab\`: Get current tab information (tabId, url, title).
|
|
1493
|
+
* \`go_back\`: Go back to the previous page in the current tab.
|
|
1494
|
+
* \`change_url [url]\`: open URL in the current tab, eg: \`change_url https://www.google.com\`.
|
|
1495
|
+
* \`close_tab\`: Close the current tab.
|
|
1496
|
+
* \`switch_tab [tabId]\`: Switch to the specified tab using tabId, eg: \`switch_tab 1000\`.
|
|
1255
1497
|
* \`new_tab [url]\`: Open a new tab window and open the URL, eg: \`new_tab https://www.google.com\``,
|
|
1256
1498
|
},
|
|
1257
1499
|
},
|
|
1258
|
-
required: ['
|
|
1500
|
+
required: ['command'],
|
|
1259
1501
|
};
|
|
1260
1502
|
}
|
|
1261
1503
|
/**
|
|
1262
1504
|
* Tab management
|
|
1263
1505
|
*
|
|
1264
|
-
* @param {*} params {
|
|
1506
|
+
* @param {*} params { command: `new_tab [url]` | 'tab_all' | 'current_tab' | 'go_back' | 'close_tab' | 'switch_tab [tabId]' | `change_url [url]` }
|
|
1265
1507
|
* @returns > { result, success: true }
|
|
1266
1508
|
*/
|
|
1267
1509
|
async execute(context, params) {
|
|
1268
|
-
|
|
1269
|
-
|
|
1510
|
+
var _a, _b, _c, _d, _e, _f;
|
|
1511
|
+
if (params === null || !params.command) {
|
|
1512
|
+
throw new Error('Invalid parameters. Expected an object with a "command" property.');
|
|
1270
1513
|
}
|
|
1271
1514
|
let windowId = await getWindowId(context);
|
|
1272
|
-
let
|
|
1273
|
-
if (
|
|
1274
|
-
|
|
1515
|
+
let command = params.command.trim();
|
|
1516
|
+
if (command.startsWith('`')) {
|
|
1517
|
+
command = command.substring(1);
|
|
1275
1518
|
}
|
|
1276
|
-
if (
|
|
1277
|
-
|
|
1519
|
+
if (command.endsWith('`')) {
|
|
1520
|
+
command = command.substring(0, command.length - 1);
|
|
1278
1521
|
}
|
|
1279
1522
|
let result;
|
|
1280
|
-
if (
|
|
1523
|
+
if (command == 'tab_all') {
|
|
1281
1524
|
result = [];
|
|
1282
1525
|
let tabs = await chrome.tabs.query({ windowId: windowId });
|
|
1283
1526
|
for (let i = 0; i < tabs.length; i++) {
|
|
@@ -1294,20 +1537,20 @@ class TabManagement {
|
|
|
1294
1537
|
result.push(tabInfo);
|
|
1295
1538
|
}
|
|
1296
1539
|
}
|
|
1297
|
-
else if (
|
|
1540
|
+
else if (command == 'current_tab') {
|
|
1298
1541
|
let tabId = await getTabId(context);
|
|
1299
1542
|
let tab = await chrome.tabs.get(tabId);
|
|
1300
1543
|
let tabInfo = { tabId, windowId: tab.windowId, title: tab.title, url: tab.url };
|
|
1301
1544
|
result = tabInfo;
|
|
1302
1545
|
}
|
|
1303
|
-
else if (
|
|
1546
|
+
else if (command == 'go_back') {
|
|
1304
1547
|
let tabId = await getTabId(context);
|
|
1305
1548
|
await chrome.tabs.goBack(tabId);
|
|
1306
1549
|
let tab = await chrome.tabs.get(tabId);
|
|
1307
1550
|
let tabInfo = { tabId, windowId: tab.windowId, title: tab.title, url: tab.url };
|
|
1308
1551
|
result = tabInfo;
|
|
1309
1552
|
}
|
|
1310
|
-
else if (
|
|
1553
|
+
else if (command == 'close_tab') {
|
|
1311
1554
|
let closedTabId = await getTabId(context);
|
|
1312
1555
|
await chrome.tabs.remove(closedTabId);
|
|
1313
1556
|
await sleep(100);
|
|
@@ -1325,16 +1568,16 @@ class TabManagement {
|
|
|
1325
1568
|
let closeTabInfo = { closedTabId, newTabId, newTabTitle: tab.title };
|
|
1326
1569
|
result = closeTabInfo;
|
|
1327
1570
|
}
|
|
1328
|
-
else if (
|
|
1329
|
-
let tabId = parseInt(
|
|
1571
|
+
else if (command.startsWith('switch_tab')) {
|
|
1572
|
+
let tabId = parseInt(command.replace('switch_tab', '').replace('[', '').replace(']', ''));
|
|
1330
1573
|
let tab = await chrome.tabs.update(tabId, { active: true });
|
|
1331
1574
|
context.variables.set('tabId', tab.id);
|
|
1332
1575
|
context.variables.set('windowId', tab.windowId);
|
|
1333
1576
|
let tabInfo = { tabId, windowId: tab.windowId, title: tab.title, url: tab.url };
|
|
1334
1577
|
result = tabInfo;
|
|
1335
1578
|
}
|
|
1336
|
-
else if (
|
|
1337
|
-
let url =
|
|
1579
|
+
else if (command.startsWith('change_url')) {
|
|
1580
|
+
let url = command.substring('change_url'.length).replace('[', '').replace(']', '').trim();
|
|
1338
1581
|
let tabId = await getTabId(context);
|
|
1339
1582
|
// await chrome.tabs.update(tabId, { url: url });
|
|
1340
1583
|
await executeScript(tabId, () => {
|
|
@@ -1344,17 +1587,19 @@ class TabManagement {
|
|
|
1344
1587
|
let tabInfo = { tabId, windowId: tab.windowId, title: tab.title, url: tab.url };
|
|
1345
1588
|
result = tabInfo;
|
|
1346
1589
|
}
|
|
1347
|
-
else if (
|
|
1348
|
-
let url =
|
|
1590
|
+
else if (command.startsWith('new_tab')) {
|
|
1591
|
+
let url = command.replace('new_tab', '').replace('[', '').replace(']', '').replace(/"/g, '');
|
|
1349
1592
|
// First mandatory opening of a new window
|
|
1350
1593
|
let newWindow = !context.variables.get('windowId') && !context.variables.get('tabId');
|
|
1351
1594
|
let tab;
|
|
1352
1595
|
if (newWindow) {
|
|
1353
1596
|
tab = await open_new_tab(url, true);
|
|
1597
|
+
(_c = (_b = (_a = context.callback) === null || _a === undefined ? undefined : _a.hooks) === null || _b === undefined ? undefined : _b.onTabCreated) === null || _c === undefined ? undefined : _c.call(_b, tab.id);
|
|
1354
1598
|
}
|
|
1355
1599
|
else {
|
|
1356
1600
|
let windowId = await getWindowId(context);
|
|
1357
1601
|
tab = await open_new_tab(url, false, windowId);
|
|
1602
|
+
(_f = (_e = (_d = context.callback) === null || _d === undefined ? undefined : _d.hooks) === null || _e === undefined ? undefined : _e.onTabCreated) === null || _f === undefined ? undefined : _f.call(_e, tab.id);
|
|
1358
1603
|
}
|
|
1359
1604
|
let windowId = tab.windowId;
|
|
1360
1605
|
let tabId = tab.id;
|
|
@@ -1378,7 +1623,7 @@ class TabManagement {
|
|
|
1378
1623
|
result = tabInfo;
|
|
1379
1624
|
}
|
|
1380
1625
|
else {
|
|
1381
|
-
throw Error('Unknown
|
|
1626
|
+
throw Error('Unknown command: ' + command);
|
|
1382
1627
|
}
|
|
1383
1628
|
return result;
|
|
1384
1629
|
}
|
|
@@ -1398,7 +1643,7 @@ class TabManagement {
|
|
|
1398
1643
|
class WebSearch {
|
|
1399
1644
|
constructor() {
|
|
1400
1645
|
this.name = 'web_search';
|
|
1401
|
-
this.description = '
|
|
1646
|
+
this.description = 'Search the web based on keywords and return relevant extracted content from webpages.';
|
|
1402
1647
|
this.input_schema = {
|
|
1403
1648
|
type: 'object',
|
|
1404
1649
|
properties: {
|
|
@@ -1433,8 +1678,8 @@ class WebSearch {
|
|
|
1433
1678
|
}
|
|
1434
1679
|
let taskId = new Date().getTime() + '';
|
|
1435
1680
|
let searchs = [{ url: url, keyword: query }];
|
|
1436
|
-
let searchInfo = await deepSearch(taskId, searchs, maxResults || 5);
|
|
1437
|
-
let links = ((_a = searchInfo.result[0]) === null || _a ===
|
|
1681
|
+
let searchInfo = await deepSearch(context, taskId, searchs, maxResults || 5);
|
|
1682
|
+
let links = ((_a = searchInfo.result[0]) === null || _a === undefined ? undefined : _a.links) || [];
|
|
1438
1683
|
return links.filter((s) => s.content);
|
|
1439
1684
|
}
|
|
1440
1685
|
}
|
|
@@ -1506,7 +1751,7 @@ chrome.tabs.onUpdated.addListener(async function (tabId, changeInfo, tab) {
|
|
|
1506
1751
|
* @param {array} searchs search list => [{ url: 'https://bing.com', keyword: 'ai' }]
|
|
1507
1752
|
* @param {number} detailsMaxNum Maximum crawling quantity per search detail page
|
|
1508
1753
|
*/
|
|
1509
|
-
async function deepSearch(taskId, searchs, detailsMaxNum, window) {
|
|
1754
|
+
async function deepSearch(context, taskId, searchs, detailsMaxNum, window) {
|
|
1510
1755
|
let closeWindow = false;
|
|
1511
1756
|
if (!window) {
|
|
1512
1757
|
// open new window
|
|
@@ -1519,9 +1764,9 @@ async function deepSearch(taskId, searchs, detailsMaxNum, window) {
|
|
|
1519
1764
|
}
|
|
1520
1765
|
// crawler the search page details page link
|
|
1521
1766
|
// [{ links: [{ title, url }] }]
|
|
1522
|
-
let detailLinkGroups = await doDetailLinkGroups(taskId, searchs, detailsMaxNum, window);
|
|
1767
|
+
let detailLinkGroups = await doDetailLinkGroups(context, taskId, searchs, detailsMaxNum, window);
|
|
1523
1768
|
// crawler all details page content and comments
|
|
1524
|
-
let searchInfo = await doPageContent(taskId, detailLinkGroups, window);
|
|
1769
|
+
let searchInfo = await doPageContent(context, taskId, detailLinkGroups, window);
|
|
1525
1770
|
console.log('searchInfo: ', searchInfo);
|
|
1526
1771
|
// close window
|
|
1527
1772
|
closeWindow && chrome.windows.remove(window.id);
|
|
@@ -1536,7 +1781,8 @@ async function deepSearch(taskId, searchs, detailsMaxNum, window) {
|
|
|
1536
1781
|
* @param {*} window
|
|
1537
1782
|
* @returns [{ links: [{ title, url }] }]
|
|
1538
1783
|
*/
|
|
1539
|
-
async function doDetailLinkGroups(taskId, searchs, detailsMaxNum, window) {
|
|
1784
|
+
async function doDetailLinkGroups(context, taskId, searchs, detailsMaxNum, window) {
|
|
1785
|
+
var _a, _b, _c;
|
|
1540
1786
|
let detailLinkGroups = [];
|
|
1541
1787
|
let countDownLatch = new CountDownLatch(searchs.length);
|
|
1542
1788
|
for (let i = 0; i < searchs.length; i++) {
|
|
@@ -1548,6 +1794,7 @@ async function doDetailLinkGroups(taskId, searchs, detailsMaxNum, window) {
|
|
|
1548
1794
|
url: url,
|
|
1549
1795
|
windowId: window.id,
|
|
1550
1796
|
});
|
|
1797
|
+
(_c = (_b = (_a = context.callback) === null || _a === void 0 ? void 0 : _a.hooks) === null || _b === void 0 ? void 0 : _b.onTabCreated) === null || _c === void 0 ? void 0 : _c.call(_b, tab.id);
|
|
1551
1798
|
let eventId = taskId + '_' + i;
|
|
1552
1799
|
// monitor Tab status
|
|
1553
1800
|
tabsUpdateEvent.addListener(async function (obj) {
|
|
@@ -1598,7 +1845,8 @@ async function doDetailLinkGroups(taskId, searchs, detailsMaxNum, window) {
|
|
|
1598
1845
|
* @param {*} window
|
|
1599
1846
|
* @returns search info
|
|
1600
1847
|
*/
|
|
1601
|
-
async function doPageContent(taskId, detailLinkGroups, window) {
|
|
1848
|
+
async function doPageContent(context, taskId, detailLinkGroups, window) {
|
|
1849
|
+
var _a, _b, _c;
|
|
1602
1850
|
const searchInfo = {
|
|
1603
1851
|
total: 0,
|
|
1604
1852
|
running: 0,
|
|
@@ -1622,50 +1870,67 @@ async function doPageContent(taskId, detailLinkGroups, window) {
|
|
|
1622
1870
|
url: link.url,
|
|
1623
1871
|
windowId: window.id,
|
|
1624
1872
|
});
|
|
1873
|
+
(_c = (_b = (_a = context.callback) === null || _a === undefined ? undefined : _a.hooks) === null || _b === undefined ? undefined : _b.onTabCreated) === null || _c === undefined ? undefined : _c.call(_b, tab.id);
|
|
1625
1874
|
searchInfo.running++;
|
|
1626
1875
|
let eventId = taskId + '_' + i + '_' + j;
|
|
1627
|
-
//
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1876
|
+
// Create a timeout promise
|
|
1877
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
1878
|
+
setTimeout(() => reject(new Error('Page load timeout')), 10000); // Timeout after 10 seconds
|
|
1879
|
+
});
|
|
1880
|
+
// Create a tab monitoring promise
|
|
1881
|
+
const monitorTabPromise = new Promise(async (resolve, reject) => {
|
|
1882
|
+
tabsUpdateEvent.addListener(async function onTabUpdated(obj) {
|
|
1883
|
+
if (obj.tabId !== tab.id)
|
|
1884
|
+
return;
|
|
1885
|
+
if (obj.changeInfo.status === 'complete') {
|
|
1634
1886
|
tabsUpdateEvent.removeListener(eventId);
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1887
|
+
try {
|
|
1888
|
+
// Inject script and get page content
|
|
1889
|
+
await injectScript(tab.id, filename);
|
|
1890
|
+
await sleep(1000);
|
|
1891
|
+
let result = await chrome.tabs.sendMessage(tab.id, {
|
|
1892
|
+
type: 'page:getContent',
|
|
1893
|
+
});
|
|
1894
|
+
if (!result)
|
|
1895
|
+
throw new Error('No Result');
|
|
1896
|
+
link.content = result.content;
|
|
1897
|
+
link.page_title = result.title;
|
|
1898
|
+
searchInfo.succeed++;
|
|
1899
|
+
resolve(); // Resolve the promise if successful
|
|
1900
|
+
}
|
|
1901
|
+
catch (error) {
|
|
1902
|
+
searchInfo.failed++;
|
|
1903
|
+
searchInfo.failedLinks.push(link);
|
|
1904
|
+
reject(error); // Reject the promise on error
|
|
1905
|
+
}
|
|
1906
|
+
finally {
|
|
1907
|
+
searchInfo.running--;
|
|
1908
|
+
countDownLatch.countDown();
|
|
1909
|
+
chrome.tabs.remove(tab.id);
|
|
1910
|
+
tabsUpdateEvent.removeListener(eventId);
|
|
1645
1911
|
}
|
|
1646
|
-
link.content = result.content;
|
|
1647
|
-
link.page_title = result.title;
|
|
1648
|
-
searchInfo.succeed++;
|
|
1649
|
-
}
|
|
1650
|
-
catch (e) {
|
|
1651
|
-
searchInfo.failed++;
|
|
1652
|
-
searchInfo.failedLinks.push(link);
|
|
1653
|
-
console.error(link.title + ' crawler error', link.url, e);
|
|
1654
1912
|
}
|
|
1655
|
-
|
|
1913
|
+
else if (obj.changeInfo.status === 'unloaded') {
|
|
1656
1914
|
searchInfo.running--;
|
|
1657
1915
|
countDownLatch.countDown();
|
|
1658
1916
|
chrome.tabs.remove(tab.id);
|
|
1659
1917
|
tabsUpdateEvent.removeListener(eventId);
|
|
1918
|
+
reject(new Error('Tab unloaded')); // Reject if the tab is unloaded
|
|
1660
1919
|
}
|
|
1661
|
-
}
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1920
|
+
}, eventId);
|
|
1921
|
+
});
|
|
1922
|
+
// Use Promise.race to enforce the timeout
|
|
1923
|
+
try {
|
|
1924
|
+
await Promise.race([monitorTabPromise, timeoutPromise]);
|
|
1925
|
+
}
|
|
1926
|
+
catch (e) {
|
|
1927
|
+
console.error(`${link.title} failed:`, e);
|
|
1928
|
+
searchInfo.running--;
|
|
1929
|
+
searchInfo.failed++;
|
|
1930
|
+
searchInfo.failedLinks.push(link);
|
|
1931
|
+
countDownLatch.countDown();
|
|
1932
|
+
chrome.tabs.remove(tab.id); // Clean up tab on failure
|
|
1933
|
+
}
|
|
1669
1934
|
}
|
|
1670
1935
|
}
|
|
1671
1936
|
await countDownLatch.await(60000);
|
|
@@ -1757,16 +2022,222 @@ class RequestLogin {
|
|
|
1757
2022
|
}
|
|
1758
2023
|
}
|
|
1759
2024
|
|
|
2025
|
+
class CancelWorkflow {
|
|
2026
|
+
constructor() {
|
|
2027
|
+
this.name = 'cancel_workflow';
|
|
2028
|
+
this.description = 'Cancel the workflow. If any tool consistently encounters exceptions, invoke this tool to cancel the workflow.';
|
|
2029
|
+
this.input_schema = {
|
|
2030
|
+
type: 'object',
|
|
2031
|
+
properties: {
|
|
2032
|
+
reason: {
|
|
2033
|
+
type: 'string',
|
|
2034
|
+
description: 'Why the workflow should be cancelled.',
|
|
2035
|
+
},
|
|
2036
|
+
},
|
|
2037
|
+
required: ['reason'],
|
|
2038
|
+
};
|
|
2039
|
+
}
|
|
2040
|
+
async execute(context, params) {
|
|
2041
|
+
var _a;
|
|
2042
|
+
if (typeof params !== 'object' || params === null || !params.reason) {
|
|
2043
|
+
throw new Error('Invalid parameters. Expected an object with a "reason" property.');
|
|
2044
|
+
}
|
|
2045
|
+
const reason = params.reason;
|
|
2046
|
+
console.log("The workflow has been cancelled because: " + reason);
|
|
2047
|
+
await ((_a = context.workflow) === null || _a === undefined ? undefined : _a.cancel());
|
|
2048
|
+
return;
|
|
2049
|
+
}
|
|
2050
|
+
}
|
|
2051
|
+
|
|
2052
|
+
class HumanInputText {
|
|
2053
|
+
constructor() {
|
|
2054
|
+
this.name = 'human_input_text';
|
|
2055
|
+
this.description = 'When you are unsure about the details of your next action, call me and ask the user for details in the "question" field. The user will provide you with a text as an answer.';
|
|
2056
|
+
this.input_schema = {
|
|
2057
|
+
type: 'object',
|
|
2058
|
+
properties: {
|
|
2059
|
+
question: {
|
|
2060
|
+
type: 'string',
|
|
2061
|
+
description: 'Ask the user here.',
|
|
2062
|
+
},
|
|
2063
|
+
},
|
|
2064
|
+
required: ['question'],
|
|
2065
|
+
};
|
|
2066
|
+
}
|
|
2067
|
+
async execute(context, params) {
|
|
2068
|
+
var _a, _b, _c;
|
|
2069
|
+
if (typeof params !== 'object' || params === null || !params.question) {
|
|
2070
|
+
throw new Error('Invalid parameters. Expected an object with a "question" property.');
|
|
2071
|
+
}
|
|
2072
|
+
const question = params.question;
|
|
2073
|
+
console.log("question: " + question);
|
|
2074
|
+
let answer = await ((_c = (_a = context.callback) === null || _a === undefined ? undefined : (_b = _a.hooks).onHumanInputText) === null || _c === undefined ? undefined : _c.call(_b, question));
|
|
2075
|
+
if (!answer) {
|
|
2076
|
+
console.error("Cannot get user's answer.");
|
|
2077
|
+
return { status: "Error: Cannot get user's answer.", answer: "" };
|
|
2078
|
+
}
|
|
2079
|
+
else {
|
|
2080
|
+
console.log("answer: " + answer);
|
|
2081
|
+
return { status: "OK", answer: answer };
|
|
2082
|
+
}
|
|
2083
|
+
}
|
|
2084
|
+
}
|
|
2085
|
+
class HumanInputSingleChoice {
|
|
2086
|
+
constructor() {
|
|
2087
|
+
this.name = 'human_input_single_choice';
|
|
2088
|
+
this.description = 'When you are unsure about the details of your next action, call me and ask the user for details in the "question" field with at least 2 choices. The user will provide you with ONE choice as an answer.';
|
|
2089
|
+
this.input_schema = {
|
|
2090
|
+
type: 'object',
|
|
2091
|
+
properties: {
|
|
2092
|
+
question: {
|
|
2093
|
+
type: 'string',
|
|
2094
|
+
description: 'Ask the user here.',
|
|
2095
|
+
},
|
|
2096
|
+
choices: {
|
|
2097
|
+
type: 'array',
|
|
2098
|
+
description: 'All of the choices.',
|
|
2099
|
+
}
|
|
2100
|
+
},
|
|
2101
|
+
required: ['question', 'choices'],
|
|
2102
|
+
};
|
|
2103
|
+
}
|
|
2104
|
+
async execute(context, params) {
|
|
2105
|
+
var _a, _b, _c;
|
|
2106
|
+
if (typeof params !== 'object' || params === null || !params.question || !params.choices) {
|
|
2107
|
+
throw new Error('Invalid parameters. Expected an object with a "question" and "choices" property.');
|
|
2108
|
+
}
|
|
2109
|
+
const question = params.question;
|
|
2110
|
+
const choices = params.choices;
|
|
2111
|
+
console.log("question: " + question);
|
|
2112
|
+
console.log("choices: " + choices);
|
|
2113
|
+
let answer = await ((_c = (_a = context.callback) === null || _a === undefined ? undefined : (_b = _a.hooks).onHumanInputSingleChoice) === null || _c === undefined ? undefined : _c.call(_b, question, choices));
|
|
2114
|
+
if (!answer) {
|
|
2115
|
+
console.error("Cannot get user's answer.");
|
|
2116
|
+
return { status: "Error: Cannot get user's answer.", answer: "" };
|
|
2117
|
+
}
|
|
2118
|
+
else {
|
|
2119
|
+
console.log("answer: " + answer);
|
|
2120
|
+
return { status: "OK", answer: answer };
|
|
2121
|
+
}
|
|
2122
|
+
}
|
|
2123
|
+
}
|
|
2124
|
+
class HumanInputMultipleChoice {
|
|
2125
|
+
constructor() {
|
|
2126
|
+
this.name = 'human_input_multiple_choice';
|
|
2127
|
+
this.description = 'When you are unsure about the details of your next action, call me and ask the user for details in the "question" field with at least 2 choices. The user will provide you with ONE or MORE choice as an answer.';
|
|
2128
|
+
this.input_schema = {
|
|
2129
|
+
type: 'object',
|
|
2130
|
+
properties: {
|
|
2131
|
+
question: {
|
|
2132
|
+
type: 'string',
|
|
2133
|
+
description: 'Ask the user here.',
|
|
2134
|
+
},
|
|
2135
|
+
choices: {
|
|
2136
|
+
type: 'array',
|
|
2137
|
+
description: 'All of the choices.',
|
|
2138
|
+
}
|
|
2139
|
+
},
|
|
2140
|
+
required: ['question', 'choices'],
|
|
2141
|
+
};
|
|
2142
|
+
}
|
|
2143
|
+
async execute(context, params) {
|
|
2144
|
+
var _a, _b, _c;
|
|
2145
|
+
if (typeof params !== 'object' || params === null || !params.question || !params.choices) {
|
|
2146
|
+
throw new Error('Invalid parameters. Expected an object with a "question" and "choices" property.');
|
|
2147
|
+
}
|
|
2148
|
+
const question = params.question;
|
|
2149
|
+
const choices = params.choices;
|
|
2150
|
+
console.log("question: " + question);
|
|
2151
|
+
console.log("choices: " + choices);
|
|
2152
|
+
let answer = await ((_c = (_a = context.callback) === null || _a === undefined ? undefined : (_b = _a.hooks).onHumanInputMultipleChoice) === null || _c === undefined ? undefined : _c.call(_b, question, choices));
|
|
2153
|
+
if (!answer) {
|
|
2154
|
+
console.error("Cannot get user's answer.");
|
|
2155
|
+
return { status: "Error: Cannot get user's answer.", answer: [] };
|
|
2156
|
+
}
|
|
2157
|
+
else {
|
|
2158
|
+
console.log("answer: " + answer);
|
|
2159
|
+
return { status: "OK", answer: answer };
|
|
2160
|
+
}
|
|
2161
|
+
}
|
|
2162
|
+
}
|
|
2163
|
+
class HumanOperate {
|
|
2164
|
+
constructor() {
|
|
2165
|
+
this.name = 'human_operate';
|
|
2166
|
+
this.description = 'When you encounter operations that require login, CAPTCHA verification, or other tasks that you cannot complete, please call this tool, transfer control to the user, and explain why.';
|
|
2167
|
+
this.input_schema = {
|
|
2168
|
+
type: 'object',
|
|
2169
|
+
properties: {
|
|
2170
|
+
reason: {
|
|
2171
|
+
type: 'string',
|
|
2172
|
+
description: 'The reason why you need to transfer control.',
|
|
2173
|
+
},
|
|
2174
|
+
},
|
|
2175
|
+
required: ['reason'],
|
|
2176
|
+
};
|
|
2177
|
+
}
|
|
2178
|
+
async execute(context, params) {
|
|
2179
|
+
var _a, _b, _c;
|
|
2180
|
+
if (typeof params !== 'object' || params === null || !params.reason) {
|
|
2181
|
+
throw new Error('Invalid parameters. Expected an object with a "reason" property.');
|
|
2182
|
+
}
|
|
2183
|
+
const reason = params.reason;
|
|
2184
|
+
console.log("reason: " + reason);
|
|
2185
|
+
let userOperation = await ((_c = (_a = context.callback) === null || _a === undefined ? undefined : (_b = _a.hooks).onHumanOperate) === null || _c === undefined ? undefined : _c.call(_b, reason));
|
|
2186
|
+
if (!userOperation) {
|
|
2187
|
+
console.error("Cannot get user's operation.");
|
|
2188
|
+
return { status: "Error: Cannot get user's operation.", userOperation: "" };
|
|
2189
|
+
}
|
|
2190
|
+
else {
|
|
2191
|
+
console.log("userOperation: " + userOperation);
|
|
2192
|
+
return { status: "OK", userOperation: userOperation };
|
|
2193
|
+
}
|
|
2194
|
+
}
|
|
2195
|
+
}
|
|
2196
|
+
|
|
2197
|
+
class SummaryWorkflow {
|
|
2198
|
+
constructor() {
|
|
2199
|
+
this.name = 'summary_workflow';
|
|
2200
|
+
this.description = 'Summarize what this workflow has done from start to finish using an ordered list .';
|
|
2201
|
+
this.input_schema = {
|
|
2202
|
+
type: 'object',
|
|
2203
|
+
properties: {
|
|
2204
|
+
summary: {
|
|
2205
|
+
type: 'string',
|
|
2206
|
+
description: 'Your summary in markdown format.',
|
|
2207
|
+
},
|
|
2208
|
+
},
|
|
2209
|
+
required: ['summary'],
|
|
2210
|
+
};
|
|
2211
|
+
}
|
|
2212
|
+
async execute(context, params) {
|
|
2213
|
+
var _a, _b, _c;
|
|
2214
|
+
if (typeof params !== 'object' || params === null || !params.summary) {
|
|
2215
|
+
throw new Error('Invalid parameters. Expected an object with a "summary" property.');
|
|
2216
|
+
}
|
|
2217
|
+
const summary = params.summary;
|
|
2218
|
+
console.log("summary: " + summary);
|
|
2219
|
+
await ((_c = (_a = context.callback) === null || _a === undefined ? undefined : (_b = _a.hooks).onSummaryWorkflow) === null || _c === undefined ? undefined : _c.call(_b, summary));
|
|
2220
|
+
return { status: "OK" };
|
|
2221
|
+
}
|
|
2222
|
+
}
|
|
2223
|
+
|
|
1760
2224
|
var tools = /*#__PURE__*/Object.freeze({
|
|
1761
2225
|
__proto__: null,
|
|
1762
2226
|
BrowserUse: BrowserUse,
|
|
2227
|
+
CancelWorkflow: CancelWorkflow,
|
|
1763
2228
|
ElementClick: ElementClick,
|
|
1764
2229
|
ExportFile: ExportFile,
|
|
1765
2230
|
ExtractContent: ExtractContent,
|
|
1766
2231
|
FindElementPosition: FindElementPosition,
|
|
2232
|
+
GetAllTabs: GetAllTabs,
|
|
2233
|
+
HumanInputMultipleChoice: HumanInputMultipleChoice,
|
|
2234
|
+
HumanInputSingleChoice: HumanInputSingleChoice,
|
|
2235
|
+
HumanInputText: HumanInputText,
|
|
2236
|
+
HumanOperate: HumanOperate,
|
|
1767
2237
|
OpenUrl: OpenUrl,
|
|
1768
2238
|
RequestLogin: RequestLogin,
|
|
1769
2239
|
Screenshot: Screenshot,
|
|
2240
|
+
SummaryWorkflow: SummaryWorkflow,
|
|
1770
2241
|
TabManagement: TabManagement,
|
|
1771
2242
|
WebSearch: WebSearch
|
|
1772
2243
|
});
|