@eko-ai/eko 3.1.0 → 3.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (13) hide show
  1. package/README.md +42 -16
  2. package/dist/agent/browser/browser_labels.d.ts +16 -1
  3. package/dist/agent/browser/browser_labels.d.ts.map +1 -1
  4. package/dist/agent/browser/build_dom_tree.d.ts.map +1 -1
  5. package/dist/agent/browser/utils.d.ts +21 -0
  6. package/dist/agent/browser/utils.d.ts.map +1 -1
  7. package/dist/config/index.d.ts +3 -0
  8. package/dist/config/index.d.ts.map +1 -1
  9. package/dist/index.cjs.js +241 -37
  10. package/dist/index.cjs.js.map +1 -1
  11. package/dist/index.esm.js +241 -37
  12. package/dist/index.esm.js.map +1 -1
  13. package/package.json +1 -1
package/README.md CHANGED
@@ -87,7 +87,25 @@ const llms: LLMs = {
87
87
  provider: "openai",
88
88
  model: "gpt-5",
89
89
  apiKey: "your-api-key"
90
+ },
91
+ // OpenAI-compatible models (Qwen, Doubao, etc.)
92
+ qwen: {
93
+ provider: "openai",
94
+ model: "qwen-plus",
95
+ apiKey: "your-qwen-api-key",
96
+ config: {
97
+ baseURL: "https://dashscope-intl.aliyuncs.com/compatible-mode/v1"
98
+ }
99
+ },
100
+ doubao: {
101
+ provider: "openai", // Use OpenAI provider for compatibility
102
+ model: "doubao-seed-1-6-250615", // or other Doubao model
103
+ apiKey: "your-volcengine-api-key",
104
+ config: {
105
+ baseURL: "https://ark.cn-beijing.volces.com/api/v3" // Volcengine endpoint
106
+ }
90
107
  }
108
+
91
109
  };
92
110
 
93
111
  let agents: Agent[] = [new BrowserAgent(), new FileAgent()];
@@ -101,26 +119,36 @@ $ pnpm install @eko-ai/eko
101
119
 
102
120
  ## Example Projects
103
121
 
104
- The repository ships with three workspace examples under the `example/` folder. After running the
105
- core install (`pnpm install`) you can launch any of them with the commands below. Each example
106
- consumes the local 3.0 packages, so rebuilding the main workspace automatically refreshes them.
122
+ The repository ships with three workspace examples under the `example/` folder.
123
+
124
+ ### Prerequisites
125
+
126
+ Before running any example, install dependencies and build the core packages from the root directory:
127
+
128
+ ```bash
129
+ pnpm install
130
+ pnpm build
131
+ ```
107
132
 
108
133
  ### Browser Extension (`example/extension`)
109
134
 
110
135
  ```bash
111
- pnpm --filter @eko-ai/eko-extension build
112
- pnpm --filter @eko-ai/eko-extension-example run build
136
+ cd example/extension
137
+ pnpm install
138
+ pnpm run build
113
139
  ```
114
140
 
115
- Load the generated `example/extension/dist` directory via `chrome://extensions` → Developer Mode →
116
- Load unpacked. Configure your API key in the extension options before running the automation task.
141
+ Load the generated `dist` directory via `chrome://extensions` → Developer Mode → Load unpacked.
142
+ Configure your API key in the extension options before running the automation task.
117
143
 
118
144
  ### Node.js Automation (`example/nodejs`)
119
145
 
120
146
  ```bash
121
- pnpm --filter @eko-ai/eko-nodejs-example run build
122
- pnpm --filter @eko-ai/eko-nodejs-example run playwright # first time only, installs browsers
123
- OPENAI_API_KEY=... ANTHROPIC_API_KEY=... pnpm --filter @eko-ai/eko-nodejs-example run start
147
+ cd example/nodejs
148
+ pnpm install
149
+ pnpm run playwright # first time only, installs browsers
150
+ pnpm run build
151
+ OPENAI_API_KEY=... ANTHROPIC_API_KEY=... pnpm run start
124
152
  ```
125
153
 
126
154
  The Node.js demo drives Playwright through Eko; provide at least one model API key before running it.
@@ -128,7 +156,9 @@ The Node.js demo drives Playwright through Eko; provide at least one model API k
128
156
  ### Web Login Demo (`example/web`)
129
157
 
130
158
  ```bash
131
- pnpm --filter @eko-ai/eko-web-example run start
159
+ cd example/web
160
+ pnpm install
161
+ pnpm run start
132
162
  ```
133
163
 
134
164
  This starts a React dev server on the default port with a simple login flow that you can automate
@@ -173,12 +203,8 @@ Eko can be used in multiple environments:
173
203
  - **Career Co-Pilot**: https://github.com/wangwangbobo/career_skill_learnig.git
174
204
  - **Slides Agent by Eko**: https://github.com/MICAHFANG/slides-agent-by-eko
175
205
  - **Universal Sidebar Assistant**: https://github.com/San12341/eko-broser-extension.git
176
- - **48 Hour Browser Challenge**: https://github.com/MoonIRL/eko
177
206
  - **Orbit X Smart Terminal**: https://github.com/Skywang16/OrbitX/tree/main
178
- - **EkoMeet Recommender**: https://github.com/JasonRobertDestiny/EkoMeet
179
- - **Vision-Eko Plugin**: https://github.com/dcpwilliam/vision-eko/tree/main/serviceNowPlugin
180
- - **IC Cafe Platform**: https://cnb.cool/giggle_giraffe/ic-info-wxapp
181
- - **AI Monitoring Assistant**: https://github.com/yuhoudecheqiancao/eko-ai-monitoring
207
+ - **48 Hour Browser Challenge**: https://github.com/MoonIRL/eko
182
208
 
183
209
  ## License
184
210
 
package/dist/agent/browser/browser_labels.d.ts CHANGED
@@ -12,9 +12,24 @@ export default abstract class BaseBrowserLabelsAgent extends BaseBrowserAgent {
12
12
  protected get_select_options(agentContext: AgentContext, index: number): Promise<any>;
13
13
  protected select_option(agentContext: AgentContext, index: number, option: string): Promise<any>;
14
14
  protected screenshot_and_html(agentContext: AgentContext): Promise<{
15
+ imageBase64?: string;
16
+ imageType?: "image/jpeg" | "image/png";
17
+ pseudoHtml: string;
18
+ double_screenshots?: {
19
+ imageBase64: string;
20
+ imageType: "image/jpeg" | "image/png";
21
+ };
22
+ client_rect: {
23
+ width: number;
24
+ height: number;
25
+ };
26
+ }>;
27
+ protected screenshot_and_compress(agentContext: AgentContext, client_rect?: {
28
+ width: number;
29
+ height: number;
30
+ }): Promise<{
15
31
  imageBase64: string;
16
32
  imageType: "image/jpeg" | "image/png";
17
- pseudoHtml: string;
18
33
  }>;
19
34
  protected get_element_script(index: number): string;
20
35
  canParallelToolCalls(toolCalls?: LanguageModelV2ToolCallPart[]): boolean;
package/dist/agent/browser/browser_labels.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"browser_labels.d.ts","sourceRoot":"","sources":["../../../src/agent/browser/browser_labels.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,OAAO,EAAE,gBAAgB,EAAc,MAAM,gBAAgB,CAAC;AAC9D,OAAO,EACL,qBAAqB,EAErB,2BAA2B,EAC5B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,IAAI,EAAc,UAAU,EAAE,MAAM,aAAa,CAAC;AAG3D,MAAM,CAAC,OAAO,CAAC,QAAQ,OAAO,sBAAuB,SAAQ,gBAAgB;gBAC/D,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE,SAAS,CAAC,EAAE,UAAU;cAoDvD,UAAU,CACxB,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,OAAO,GACb,OAAO,CAAC,GAAG,CAAC;cAOC,aAAa,CAC3B,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,GAClC,OAAO,CAAC,GAAG,CAAC;cAMC,iBAAiB,CAC/B,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,IAAI,CAAC;cAaA,kBAAkB,CAChC,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,oBAAoB,EAAE,OAAO,GAC5B,OAAO,CAAC,GAAG,CAAC;cAkCC,gBAAgB,CAC9B,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,IAAI,CAAC;cAIA,kBAAkB,CAChC,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,GAAG,CAAC;cAMC,aAAa,CAC3B,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,GAAG,CAAC;cAMC,mBAAmB,CAAC,YAAY,EAAE,YAAY,GAAG,OAAO,CAAC;QACvE,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,YAAY,GAAG,WAAW,CAAC;QACtC,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IAwCF,SAAS,CAAC,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAI5C,oBAAoB,CAAC,SAAS,CAAC,EAAE,2BAA2B,EAAE,GAAG,OAAO;IAiB/E,OAAO,CAAC,cAAc;cAuUN,kBAAkB,CAChC,YAAY,EAAE,YAAY,EAC1B,QAAQ,EAAE,qBAAqB,EAC/B,KAAK,EAAE,IAAI,EAAE,GACZ,OAAO,CAAC,OAAO,CAAC;cAIH,cAAc,CAC5B,YAAY,EAAE,YAAY,EAC1B,QAAQ,EAAE,qBAAqB,EAC/B,KAAK,EAAE,IAAI,EAAE,GACZ,OAAO,CAAC,IAAI,CAAC;IA2ChB,OAAO,CAAC,oBAAoB;IAkC5B,OAAO,CAAC,oBAAoB;CA0B7B;AA4SD,OAAO,EAAE,sBAAsB,EAAE,CAAC"}
1
+ {"version":3,"file":"browser_labels.d.ts","sourceRoot":"","sources":["../../../src/agent/browser/browser_labels.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,OAAO,EAAE,gBAAgB,EAAc,MAAM,gBAAgB,CAAC;AAC9D,OAAO,EACL,qBAAqB,EAErB,2BAA2B,EAC5B,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,IAAI,EAAc,UAAU,EAAE,MAAM,aAAa,CAAC;AAI3D,MAAM,CAAC,OAAO,CAAC,QAAQ,OAAO,sBAAuB,SAAQ,gBAAgB;gBAC/D,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE,SAAS,CAAC,EAAE,UAAU;cAoDvD,UAAU,CACxB,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,OAAO,GACb,OAAO,CAAC,GAAG,CAAC;cAOC,aAAa,CAC3B,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,GAClC,OAAO,CAAC,GAAG,CAAC;cAMC,iBAAiB,CAC/B,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,IAAI,CAAC;cAaA,kBAAkB,CAChC,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,EACd,oBAAoB,EAAE,OAAO,GAC5B,OAAO,CAAC,GAAG,CAAC;cAkCC,gBAAgB,CAC9B,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,IAAI,CAAC;cAIA,kBAAkB,CAChC,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,GAAG,CAAC;cAMC,aAAa,CAC3B,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,GAAG,CAAC;cAMC,mBAAmB,CAAC,YAAY,EAAE,YAAY,GAAG,OAAO,CAAC;QACvE,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,YAAY,GAAG,WAAW,CAAC;QACvC,UAAU,EAAE,MAAM,CAAC;QACnB,kBAAkB,CAAC,EAAE;YACnB,WAAW,EAAE,MAAM,CAAC;YACpB,SAAS,EAAE,YAAY,GAAG,WAAW,CAAC;SACvC,CAAC;QACF,WAAW,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAA;SAAE,CAAC;KAChD,CAAC;cA+Dc,uBAAuB,CACrC,YAAY,EAAE,YAAY,EAC1B,WAAW,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAC9C,OAAO,CAAC;QACT,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,YAAY,GAAG,WAAW,CAAC;KACvC,CAAC;IAmBF,SAAS,CAAC,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAI5C,oBAAoB,CACzB,SAAS,CAAC,EAAE,2BAA2B,EAAE,GACxC,OAAO;IAiBV,OAAO,CAAC,cAAc;cA6UN,kBAAkB,CAChC,YAAY,EAAE,YAAY,EAC1B,QAAQ,EAAE,qBAAqB,EAC/B,KAAK,EAAE,IAAI,EAAE,GACZ,OAAO,CAAC,OAAO,CAAC;cAIH,cAAc,CAC5B,YAAY,EAAE,YAAY,EAC1B,QAAQ,EAAE,qBAAqB,EAC/B,KAAK,EAAE,IAAI,EAAE,GACZ,OAAO,CAAC,IAAI,CAAC;IAmDhB,OAAO,CAAC,oBAAoB;IAkC5B,OAAO,CAAC,oBAAoB;CA0B7B;AA4SD,OAAO,EAAE,sBAAsB,EAAE,CAAC"}
package/dist/agent/browser/build_dom_tree.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"build_dom_tree.d.ts","sourceRoot":"","sources":["../../../src/agent/browser/build_dom_tree.ts"],"names":[],"mappings":"AACA,wBAAgB,kBAAkB,SAksBjC"}
1
+ {"version":3,"file":"build_dom_tree.d.ts","sourceRoot":"","sources":["../../../src/agent/browser/build_dom_tree.ts"],"names":[],"mappings":"AACA,wBAAgB,kBAAkB,SAkwBjC"}
package/dist/agent/browser/utils.d.ts CHANGED
@@ -1,2 +1,23 @@
1
1
  export declare function extract_page_content(max_url_length?: number, max_content_length?: number): string;
2
+ export declare function mark_screenshot_highlight_elements(screenshot: {
3
+ imageBase64: string;
4
+ imageType: "image/jpeg" | "image/png";
5
+ }, area_map: Record<string, {
6
+ x: number;
7
+ y: number;
8
+ width: number;
9
+ height: number;
10
+ }>, client_rect: {
11
+ width: number;
12
+ height: number;
13
+ }): Promise<string>;
14
+ export declare function compress_image(imageBase64: string, imageType: "image/jpeg" | "image/png", compress: {
15
+ scale: number;
16
+ } | {
17
+ resizeWidth: number;
18
+ resizeHeight: number;
19
+ }, quality?: number): Promise<{
20
+ imageBase64: string;
21
+ imageType: "image/jpeg" | "image/png";
22
+ }>;
2
23
  //# sourceMappingURL=utils.d.ts.map
package/dist/agent/browser/utils.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browser/utils.ts"],"names":[],"mappings":"AAAA,wBAAgB,oBAAoB,CAAC,cAAc,SAAM,EAAE,kBAAkB,SAAQ,UAmHpF"}
1
+ {"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browser/utils.ts"],"names":[],"mappings":"AAAA,wBAAgB,oBAAoB,CAClC,cAAc,SAAM,EACpB,kBAAkB,SAAQ,UAoH3B;AAED,wBAAgB,kCAAkC,CAChD,UAAU,EAAE;IACV,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,YAAY,GAAG,WAAW,CAAC;CACvC,EACD,QAAQ,EAAE,MAAM,CACd,MAAM,EACN;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CACxD,EACD,WAAW,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAC7C,OAAO,CAAC,MAAM,CAAC,CAmGjB;AAED,wBAAsB,cAAc,CAClC,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,YAAY,GAAG,WAAW,EACrC,QAAQ,EACJ;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,GACjB;IACE,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;CACtB,EACL,OAAO,GAAE,MAAU,GAClB,OAAO,CAAC;IACT,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,YAAY,GAAG,WAAW,CAAC;CACvC,CAAC,CA0CD"}
package/dist/config/index.d.ts CHANGED
@@ -1,5 +1,6 @@
1
1
  type GlobalConfig = {
2
2
  name: string;
3
+ mode: "fast" | "normal" | "expert";
3
4
  platform: "windows" | "mac" | "linux";
4
5
  maxReactNum: number;
5
6
  maxTokens: number;
@@ -12,6 +13,8 @@ type GlobalConfig = {
12
13
  maxDialogueImgFileNum: number;
13
14
  toolResultMultimodal: boolean;
14
15
  parallelToolCalls: boolean;
16
+ markImageMode: "dom" | "draw";
17
+ /** @deprecated please use mode set to expert */
15
18
  expertMode: boolean;
16
19
  expertModeTodoLoopNum: number;
17
20
  };
package/dist/config/index.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,KAAK,YAAY,GAAG;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,SAAS,GAAG,KAAK,GAAG,OAAO,CAAC;IACtC,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,OAAO,CAAC;IACvB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,uBAAuB,EAAE,MAAM,CAAC;IAChC,eAAe,EAAE,MAAM,CAAC;IACxB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,oBAAoB,EAAE,OAAO,CAAC;IAC9B,iBAAiB,EAAE,OAAO,CAAC;IAC3B,UAAU,EAAE,OAAO,CAAC;IACpB,qBAAqB,EAAE,MAAM,CAAC;CAC/B,CAAA;AAED,QAAA,MAAM,MAAM,EAAE,YAgBb,CAAC;AAEF,eAAe,MAAM,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,KAAK,YAAY,GAAG;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACnC,QAAQ,EAAE,SAAS,GAAG,KAAK,GAAG,OAAO,CAAC;IACtC,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,OAAO,CAAC;IACvB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,uBAAuB,EAAE,MAAM,CAAC;IAChC,eAAe,EAAE,MAAM,CAAC;IACxB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,oBAAoB,EAAE,OAAO,CAAC;IAC9B,iBAAiB,EAAE,OAAO,CAAC;IAC3B,aAAa,EAAE,KAAK,GAAG,MAAM,CAAC;IAC9B,gDAAgD;IAChD,UAAU,EAAE,OAAO,CAAC;IACpB,qBAAqB,EAAE,MAAM,CAAC;CAC/B,CAAA;AAED,QAAA,MAAM,MAAM,EAAE,YAkBb,CAAC;AAEF,eAAe,MAAM,CAAC"}
package/dist/index.cjs.js CHANGED
@@ -6,6 +6,7 @@ var buffer = require('buffer');
6
6
 
7
7
  const config$1 = {
8
8
  name: "Eko",
9
+ mode: "normal",
9
10
  platform: "mac",
10
11
  maxReactNum: 500,
11
12
  maxTokens: 16000,
@@ -18,6 +19,7 @@ const config$1 = {
18
19
  maxDialogueImgFileNum: 1,
19
20
  toolResultMultimodal: true,
20
21
  parallelToolCalls: true,
22
+ markImageMode: "dom",
21
23
  expertMode: false,
22
24
  expertModeTodoLoopNum: 10,
23
25
  };
@@ -31972,7 +31974,7 @@ class Eko {
31972
31974
  results.push(agent_results.join("\n\n"));
31973
31975
  }
31974
31976
  context.conversation.splice(0, context.conversation.length);
31975
- if (config$1.expertMode &&
31977
+ if ((config$1.mode == "expert" || config$1.expertMode) &&
31976
31978
  !workflow.modified &&
31977
31979
  agentTree.nextAgent &&
31978
31980
  lastAgent?.AgentContext &&
@@ -34219,12 +34221,12 @@ class Agent {
34219
34221
  const finalResult = await this.handleCallResult(agentContext, messages, agentTools, results);
34220
34222
  loopNum++;
34221
34223
  if (!finalResult) {
34222
- if (config$1.expertMode && loopNum % config$1.expertModeTodoLoopNum == 0) {
34224
+ if ((config$1.mode == "expert" || config$1.expertMode) && loopNum % config$1.expertModeTodoLoopNum == 0) {
34223
34225
  await doTodoListManager(agentContext, rlm, messages, llm_tools);
34224
34226
  }
34225
34227
  continue;
34226
34228
  }
34227
- if (config$1.expertMode && checkNum == 0) {
34229
+ if ((config$1.mode == "expert" || config$1.expertMode) && checkNum == 0) {
34228
34230
  checkNum++;
34229
34231
  const { completionStatus } = await doTaskResultCheck(agentContext, rlm, messages, llm_tools);
34230
34232
  if (completionStatus == "incomplete") {
@@ -35116,10 +35118,140 @@ function extract_page_content(max_url_length = 200, max_content_length = 50000)
35116
35118
  result = result.replace(/\s*\n/g, "\n").replace(/\n+/g, "\n").trim();
35117
35119
  if (result.length > max_content_length) {
35118
35120
  // result = result.slice(0, max_content_length) + "...";
35119
- result = Array.from(result).slice(0, max_content_length).join('') + "...";
35121
+ result = Array.from(result).slice(0, max_content_length).join("") + "...";
35120
35122
  }
35121
35123
  return result;
35122
35124
  }
35125
+ function mark_screenshot_highlight_elements(screenshot, area_map, client_rect) {
35126
+ return new Promise(async (resolve, reject) => {
35127
+ try {
35128
+ // Convert base64 to Blob
35129
+ const base64Data = screenshot.imageBase64;
35130
+ const binaryString = atob(base64Data);
35131
+ const bytes = new Uint8Array(binaryString.length);
35132
+ for (let i = 0; i < binaryString.length; i++) {
35133
+ bytes[i] = binaryString.charCodeAt(i);
35134
+ }
35135
+ const blob = new Blob([bytes], { type: screenshot.imageType });
35136
+ const imageBitmap = await createImageBitmap(blob, {
35137
+ resizeQuality: "high",
35138
+ resizeWidth: client_rect.width,
35139
+ resizeHeight: client_rect.height,
35140
+ });
35141
+ const canvas = new OffscreenCanvas(imageBitmap.width, imageBitmap.height);
35142
+ const ctx = canvas.getContext("2d");
35143
+ if (!ctx) {
35144
+ reject(new Error("Failed to get canvas context"));
35145
+ return;
35146
+ }
35147
+ ctx.imageSmoothingEnabled = true;
35148
+ ctx.imageSmoothingQuality = "high";
35149
+ ctx.drawImage(imageBitmap, 0, 0);
35150
+ const sortedEntries = Object.entries(area_map)
35151
+ .filter(([id, area]) => area.width > 0 && area.height > 0)
35152
+ .sort((a, b) => {
35153
+ const areaA = a[1].width * a[1].height;
35154
+ const areaB = b[1].width * b[1].height;
35155
+ return areaA - areaB;
35156
+ });
35157
+ const colors = [
35158
+ "#FF0000",
35159
+ "#00FF00",
35160
+ "#0000FF",
35161
+ "#FFA500",
35162
+ "#800080",
35163
+ "#008080",
35164
+ "#FF69B4",
35165
+ "#4B0082",
35166
+ "#FF4500",
35167
+ "#2E8B57",
35168
+ "#DC143C",
35169
+ "#4682B4",
35170
+ ];
35171
+ sortedEntries.forEach(([id, area], index) => {
35172
+ const color = colors[index % colors.length];
35173
+ // Draw a border
35174
+ ctx.strokeStyle = color;
35175
+ ctx.lineWidth = 2;
35176
+ ctx.strokeRect(area.x, area.y, area.width, area.height);
35177
+ // Draw ID tag background
35178
+ const fontSize = Math.min(12, Math.max(8, area.height / 2));
35179
+ ctx.font = `${fontSize}px sans-serif`;
35180
+ const textMetrics = ctx.measureText(id);
35181
+ const padding = 4;
35182
+ const labelWidth = textMetrics.width + padding * 2;
35183
+ const labelHeight = fontSize + padding * 2;
35184
+ // The tag position is in the upper right corner.
35185
+ const labelX = area.x + area.width - labelWidth;
35186
+ const labelY = area.y;
35187
+ // Draw label background
35188
+ ctx.fillStyle = color;
35189
+ ctx.fillRect(labelX, labelY, labelWidth, labelHeight);
35190
+ // Draw ID text
35191
+ ctx.fillStyle = "#FFFFFF";
35192
+ ctx.textBaseline = "top";
35193
+ ctx.fillText(id, labelX + padding, labelY + padding);
35194
+ });
35195
+ // Convert OffscreenCanvas to Blob, then to base64
35196
+ const resultBlob = await canvas.convertToBlob({
35197
+ type: screenshot.imageType,
35198
+ });
35199
+ const reader = new FileReader();
35200
+ reader.onloadend = () => {
35201
+ const resultBase64 = reader.result;
35202
+ resolve(resultBase64);
35203
+ };
35204
+ reader.onerror = () => {
35205
+ reject(new Error("Failed to convert blob to base64"));
35206
+ };
35207
+ reader.readAsDataURL(resultBlob);
35208
+ }
35209
+ catch (error) {
35210
+ reject(error);
35211
+ }
35212
+ });
35213
+ }
35214
+ async function compress_image(imageBase64, imageType, compress, quality = 1) {
35215
+ const base64Data = imageBase64;
35216
+ const binaryString = atob(base64Data);
35217
+ const bytes = new Uint8Array(binaryString.length);
35218
+ for (let i = 0; i < binaryString.length; i++) {
35219
+ bytes[i] = binaryString.charCodeAt(i);
35220
+ }
35221
+ const blob = new Blob([bytes], { type: imageType });
35222
+ const bitmap = await createImageBitmap(blob);
35223
+ const width = compress.scale
35224
+ ? bitmap.width * compress.scale
35225
+ : compress.resizeWidth;
35226
+ const height = compress.scale
35227
+ ? bitmap.height * compress.scale
35228
+ : compress.resizeHeight;
35229
+ if (bitmap.width == width && bitmap.height == height && quality == 1) {
35230
+ return {
35231
+ imageBase64: imageBase64,
35232
+ imageType: imageType,
35233
+ };
35234
+ }
35235
+ const canvas = new OffscreenCanvas(width, height);
35236
+ const ctx = canvas.getContext("2d");
35237
+ ctx.drawImage(bitmap, 0, 0, width, height);
35238
+ const resultBlob = await canvas.convertToBlob({
35239
+ type: "image/jpeg",
35240
+ quality: quality,
35241
+ });
35242
+ return new Promise((resolve) => {
35243
+ const reader = new FileReader();
35244
+ reader.onloadend = () => {
35245
+ let imageDataUrl = reader.result;
35246
+ let imageBase64 = imageDataUrl.substring(imageDataUrl.indexOf("base64,") + 7);
35247
+ resolve({
35248
+ imageBase64: imageBase64,
35249
+ imageType: "image/jpeg",
35250
+ });
35251
+ };
35252
+ reader.readAsDataURL(resultBlob);
35253
+ });
35254
+ }
35123
35255
 
35124
35256
  const AGENT_NAME = "Browser";
35125
35257
  class BaseBrowserAgent extends Agent {
@@ -35301,19 +35433,30 @@ function run_build_dom_tree() {
35301
35433
  /**
35302
35434
  * Get clickable elements on the page
35303
35435
  *
35304
- * @param {*} doHighlightElements Is highlighted
35436
+ * @param {*} markHighlightElements Is mark highlighted
35305
35437
  * @param {*} includeAttributes [attr_names...]
35306
- * @returns { element_str, selector_map }
35438
+ * @returns { element_str, client_rect, selector_map, area_map }
35307
35439
  */
35308
- function get_clickable_elements(doHighlightElements = true, includeAttributes) {
35440
+ function get_clickable_elements(markHighlightElements = true, includeAttributes) {
35309
35441
  window.clickable_elements = {};
35310
35442
  computedStyleCache = new WeakMap();
35311
35443
  document.querySelectorAll("[eko-user-highlight-id]").forEach(ele => ele.removeAttribute("eko-user-highlight-id"));
35312
- let page_tree = build_dom_tree(doHighlightElements);
35444
+ let page_tree = build_dom_tree(markHighlightElements);
35313
35445
  let element_tree = parse_node(page_tree);
35314
- let selector_map = create_selector_map(element_tree);
35315
35446
  let element_str = clickable_elements_to_string(element_tree, includeAttributes);
35316
- return { element_str, selector_map };
35447
+ let client_rect = {
35448
+ width: window.innerWidth || document.documentElement.clientWidth,
35449
+ height: window.innerHeight || document.documentElement.clientHeight,
35450
+ };
35451
+ if (markHighlightElements) {
35452
+ let selector_map = {};
35453
+ // selector_map = create_selector_map(element_tree);
35454
+ return { element_str, client_rect, selector_map };
35455
+ }
35456
+ else {
35457
+ let area_map = create_area_map(element_tree);
35458
+ return { element_str, client_rect, area_map };
35459
+ }
35317
35460
  }
35318
35461
  function get_highlight_element(highlightIndex) {
35319
35462
  let element = document.querySelector(`[eko-user-highlight-id="eko-highlight-${highlightIndex}"]`);
@@ -35412,12 +35555,13 @@ function run_build_dom_tree() {
35412
35555
  process_node(element_tree);
35413
35556
  return formatted_text.join('\n');
35414
35557
  }
35415
- function create_selector_map(element_tree) {
35416
- let selector_map = {};
35558
+ function create_area_map(element_tree) {
35559
+ let area_map = {};
35417
35560
  function process_node(node) {
35418
35561
  if (node.tagName) {
35419
35562
  if (node.highlightIndex != null) {
35420
- selector_map[node.highlightIndex] = node;
35563
+ const element = window.clickable_elements[node.highlightIndex];
35564
+ area_map[node.highlightIndex] = get_element_real_bounding_rect(element);
35421
35565
  }
35422
35566
  for (let i = 0; i < node.children.length; i++) {
35423
35567
  process_node(node.children[i]);
@@ -35425,7 +35569,38 @@ function run_build_dom_tree() {
35425
35569
  }
35426
35570
  }
35427
35571
  process_node(element_tree);
35428
- return selector_map;
35572
+ return area_map;
35573
+ }
35574
+ function get_element_real_bounding_rect(element) {
35575
+ if (!element || !(element instanceof Element)) {
35576
+ return { x: 0, y: 0, width: 0, height: 0 };
35577
+ }
35578
+ let rect = element.getBoundingClientRect();
35579
+ let x = rect.left;
35580
+ let y = rect.top;
35581
+ let width = rect.width;
35582
+ let height = rect.height;
35583
+ let win = element.ownerDocument.defaultView;
35584
+ let maxDepth = 10;
35585
+ let depth = 0;
35586
+ while (win && win !== win.parent && depth < maxDepth) {
35587
+ depth++;
35588
+ const frameElement = win.frameElement;
35589
+ if (!frameElement) {
35590
+ break;
35591
+ }
35592
+ const frameRect = frameElement.getBoundingClientRect();
35593
+ x += frameRect.left;
35594
+ y += frameRect.top;
35595
+ // Consider the border and padding of the iframe.
35596
+ const frameStyle = getCachedComputedStyle(frameElement);
35597
+ x += parseFloat(frameStyle.borderLeftWidth) || 0;
35598
+ y += parseFloat(frameStyle.borderTopWidth) || 0;
35599
+ x += parseFloat(frameStyle.paddingLeft) || 0;
35600
+ y += parseFloat(frameStyle.paddingTop) || 0;
35601
+ win = win.parent;
35602
+ }
35603
+ return { x, y, width, height };
35429
35604
  }
35430
35605
  function parse_node(node_data, parent) {
35431
35606
  if (!node_data) {
@@ -35465,7 +35640,7 @@ function run_build_dom_tree() {
35465
35640
  }
35466
35641
  return element_node;
35467
35642
  }
35468
- function build_dom_tree(doHighlightElements) {
35643
+ function build_dom_tree(markHighlightElements) {
35469
35644
  let highlightIndex = 0; // Reset highlight index
35470
35645
  function highlightElement(element, index, parentIframe = null) {
35471
35646
  // Create or get highlight container
@@ -35856,7 +36031,7 @@ function run_build_dom_tree() {
35856
36031
  if (shouldHighlight) {
35857
36032
  nodeData.highlightIndex = highlightIndex++;
35858
36033
  window.clickable_elements[nodeData.highlightIndex] = node;
35859
- if (doHighlightElements) {
36034
+ if (markHighlightElements) {
35860
36035
  highlightElement(node, nodeData.highlightIndex, parentIframe);
35861
36036
  }
35862
36037
  }
@@ -36017,26 +36192,37 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
36017
36192
  }
36018
36193
  async screenshot_and_html(agentContext) {
36019
36194
  try {
36020
- let element_result = null;
36195
+ let element_result;
36196
+ let double_screenshots;
36021
36197
  for (let i = 0; i < 5; i++) {
36022
36198
  await sleep(200);
36023
36199
  await this.execute_script(agentContext, run_build_dom_tree, []);
36024
36200
  await sleep(50);
36025
- element_result = (await this.execute_script(agentContext, () => {
36026
- return window.get_clickable_elements(true);
36027
- }, []));
36201
+ element_result = (await this.execute_script(agentContext, (markHighlightElements) => {
36202
+ return window.get_clickable_elements(markHighlightElements);
36203
+ }, [config$1.mode != "fast" && config$1.markImageMode == "dom"]));
36028
36204
  if (element_result) {
36029
36205
  break;
36030
36206
  }
36031
36207
  }
36032
36208
  await sleep(100);
36033
- let screenshot = await this.screenshot(agentContext);
36034
- // agentContext.variables.set("selector_map", element_result.selector_map);
36035
- let pseudoHtml = element_result?.element_str || "";
36209
+ const screenshot = config$1.mode == "fast"
36210
+ ? undefined
36211
+ : await this.screenshot_and_compress(agentContext, element_result.client_rect);
36212
+ if (config$1.markImageMode == "draw" &&
36213
+ screenshot?.imageBase64 &&
36214
+ element_result.area_map) {
36215
+ double_screenshots = { ...screenshot };
36216
+ const markImageBase64 = await mark_screenshot_highlight_elements(screenshot, element_result.area_map, element_result.client_rect);
36217
+ screenshot.imageBase64 = markImageBase64;
36218
+ }
36219
+ const pseudoHtml = element_result.element_str || "";
36036
36220
  return {
36037
- imageBase64: screenshot.imageBase64,
36038
- imageType: screenshot.imageType,
36221
+ double_screenshots: double_screenshots,
36222
+ imageBase64: screenshot?.imageBase64,
36223
+ imageType: screenshot?.imageType,
36039
36224
  pseudoHtml: pseudoHtml,
36225
+ client_rect: element_result.client_rect,
36040
36226
  };
36041
36227
  }
36042
36228
  finally {
@@ -36048,6 +36234,20 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
36048
36234
  catch (e) { }
36049
36235
  }
36050
36236
  }
36237
+ async screenshot_and_compress(agentContext, client_rect) {
36238
+ const screenshot = await this.screenshot(agentContext);
36239
+ if (!client_rect || !screenshot) {
36240
+ return screenshot;
36241
+ }
36242
+ const compressedImage = await compress_image(screenshot.imageBase64, screenshot.imageType, {
36243
+ resizeWidth: client_rect.width,
36244
+ resizeHeight: client_rect.height,
36245
+ });
36246
+ return {
36247
+ imageBase64: compressedImage.imageBase64,
36248
+ imageType: compressedImage.imageType,
36249
+ };
36250
+ }
36051
36251
  get_element_script(index) {
36052
36252
  return `window.get_highlight_element(${index});`;
36053
36253
  }
@@ -36307,7 +36507,7 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
36307
36507
  ];
36308
36508
  }
36309
36509
  async double_screenshots(agentContext, messages, tools) {
36310
- return true;
36510
+ return config$1.mode != "fast";
36311
36511
  }
36312
36512
  async handleMessages(agentContext, messages, tools) {
36313
36513
  const pseudoHtmlDescription = "This is the environmental information after the operation, including the latest browser screenshot and page elements. Please perform the next operation based on the environmental information. Do not output the following elements and index information in your response.\n\nIndex and elements:\n";
@@ -36317,23 +36517,27 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
36317
36517
  lastTool.toolName !== "get_all_tabs" &&
36318
36518
  lastTool.toolName !== "variable_storage") {
36319
36519
  await sleep(300);
36320
- let image_contents = [];
36520
+ const image_contents = [];
36521
+ const result = await this.screenshot_and_html(agentContext);
36321
36522
  if (await this.double_screenshots(agentContext, messages, tools)) {
36322
- let imageResult = await this.screenshot(agentContext);
36323
- let image = toImage(imageResult.imageBase64);
36523
+ const imageResult = result.double_screenshots
36524
+ ? result.double_screenshots
36525
+ : await this.screenshot_and_compress(agentContext, result.client_rect);
36526
+ const image = toImage(imageResult.imageBase64);
36324
36527
  image_contents.push({
36325
36528
  type: "file",
36326
36529
  data: image,
36327
36530
  mediaType: imageResult.imageType,
36328
36531
  });
36329
36532
  }
36330
- let result = await this.screenshot_and_html(agentContext);
36331
- let image = toImage(result.imageBase64);
36332
- image_contents.push({
36333
- type: "file",
36334
- data: image,
36335
- mediaType: result.imageType,
36336
- });
36533
+ if (result.imageBase64) {
36534
+ const image = toImage(result.imageBase64);
36535
+ image_contents.push({
36536
+ type: "file",
36537
+ data: image,
36538
+ mediaType: result.imageType || "image/png",
36539
+ });
36540
+ }
36337
36541
  messages.push({
36338
36542
  role: "user",
36339
36543
  content: [
@@ -36482,7 +36686,7 @@ function do_click(params) {
36482
36686
  cancelable: true,
36483
36687
  button, // 0 left; 1 middle; 2 right
36484
36688
  });
36485
- if (eventType === 'click' && element.click) {
36689
+ if (eventType === "click" && element.click) {
36486
36690
  // support shadow dom element
36487
36691
  element.click();
36488
36692
  }