@eko-ai/eko 2.0.2-alpha.1 → 2.0.2-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/browser/browser_base.d.ts +11 -2
- package/dist/agent/browser/browser_base.d.ts.map +1 -1
- package/dist/agent/browser/browser_labels.d.ts.map +1 -1
- package/dist/agent/browser/browser_screen.d.ts.map +1 -1
- package/dist/agent/computer.d.ts +1 -2
- package/dist/agent/computer.d.ts.map +1 -1
- package/dist/agent/shell.d.ts +1 -1
- package/dist/agent/shell.d.ts.map +1 -1
- package/dist/index.cjs.js +98 -21
- package/dist/index.esm.js +98 -21
- package/package.json +1 -1
|
@@ -11,6 +11,16 @@ export default abstract class BaseBrowserAgent extends Agent {
|
|
|
11
11
|
url: string;
|
|
12
12
|
title?: string;
|
|
13
13
|
}>;
|
|
14
|
+
protected abstract get_all_tabs(agentContext: AgentContext): Promise<Array<{
|
|
15
|
+
tabId: number;
|
|
16
|
+
url: string;
|
|
17
|
+
title: string;
|
|
18
|
+
}>>;
|
|
19
|
+
protected abstract switch_tab(agentContext: AgentContext, tabId: number): Promise<{
|
|
20
|
+
tabId: number;
|
|
21
|
+
url: string;
|
|
22
|
+
title: string;
|
|
23
|
+
}>;
|
|
14
24
|
protected go_back(agentContext: AgentContext): Promise<void>;
|
|
15
25
|
protected extract_content(agentContext: AgentContext): Promise<string>;
|
|
16
26
|
protected controlMcpTools(agentContext: AgentContext, messages: LanguageModelV1Prompt, loopNum: number): Promise<{
|
|
@@ -21,8 +31,7 @@ export default abstract class BaseBrowserAgent extends Agent {
|
|
|
21
31
|
protected get_current_page(agentContext: AgentContext): Promise<{
|
|
22
32
|
url: string;
|
|
23
33
|
title?: string;
|
|
24
|
-
tabId?:
|
|
25
|
-
windowId?: string;
|
|
34
|
+
tabId?: number;
|
|
26
35
|
}>;
|
|
27
36
|
protected lastToolResult(messages: LanguageModelV1Prompt): {
|
|
28
37
|
id: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser_base.d.ts","sourceRoot":"","sources":["../../../src/agent/browser/browser_base.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAA+B,MAAM,kBAAkB,CAAC;AACtF,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAEhC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAc,UAAU,EAAE,MAAM,aAAa,CAAC;AAEnE,MAAM,CAAC,OAAO,CAAC,QAAQ,OAAO,gBAAiB,SAAQ,KAAK;IAC1D,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,YAAY,EAAE,YAAY,GAAG,OAAO,CAAC;QACjE,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,YAAY,GAAG,WAAW,CAAC;KACvC,CAAC;IAEF,SAAS,CAAC,QAAQ,CAAC,WAAW,CAC5B,YAAY,EAAE,YAAY,EAC1B,GAAG,EAAE,MAAM,GACV,OAAO,CAAC;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;cAEc,OAAO,CAAC,YAAY,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;cAalD,eAAe,CAAC,YAAY,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC;cAe5D,eAAe,CAC7B,YAAY,EAAE,YAAY,EAC1B,QAAQ,EAAE,qBAAqB,EAC/B,OAAO,EAAE,MAAM,GACd,OAAO,CAAC;QAAE,QAAQ,EAAE,OAAO,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAE,CAAC;IAatE,SAAS,CAAC,YAAY,CAAC,SAAS,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,YAAY;cAmDzD,gBAAgB,CAAC,YAAY,EAAE,YAAY,GAAG,OAAO,CAAC;QACpE,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"browser_base.d.ts","sourceRoot":"","sources":["../../../src/agent/browser/browser_base.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAA+B,MAAM,kBAAkB,CAAC;AACtF,OAAO,EAAE,KAAK,EAAE,MAAM,SAAS,CAAC;AAEhC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAc,UAAU,EAAE,MAAM,aAAa,CAAC;AAEnE,MAAM,CAAC,OAAO,CAAC,QAAQ,OAAO,gBAAiB,SAAQ,KAAK;IAC1D,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,YAAY,EAAE,YAAY,GAAG,OAAO,CAAC;QACjE,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,YAAY,GAAG,WAAW,CAAC;KACvC,CAAC;IAEF,SAAS,CAAC,QAAQ,CAAC,WAAW,CAC5B,YAAY,EAAE,YAAY,EAC1B,GAAG,EAAE,MAAM,GACV,OAAO,CAAC;QACT,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;IAEF,SAAS,CAAC,QAAQ,CAAC,YAAY,CAAC,YAAY,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC;QACzE,KAAK,EAAE,MAAM,CAAC;QACd,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,CAAC,CAAC;IAEH,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC;QAChF,KAAK,EAAE,MAAM,CAAC;QACd,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;cAEc,OAAO,CAAC,YAAY,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;cAalD,eAAe,CAAC,YAAY,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC;cAe5D,eAAe,CAC7B,YAAY,EAAE,YAAY,EAC1B,QAAQ,EAAE,qBAAqB,EAC/B,OAAO,EAAE,MAAM,GACd,OAAO,CAAC;QAAE,QAAQ,EAAE,OAAO,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAE,CAAC;IAatE,SAAS,CAAC,YAAY,CAAC,SAAS,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,YAAY;cAmDzD,gBAAgB,CAAC,YAAY,EAAE,YAAY,GAAG,OAAO,CAAC;QACpE,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,KAAK,CAAC,EAAE,MAAM,CAAC;KAChB,CAAC;IAaF,SAAS,CAAC,cAAc,CAAC,QAAQ,EAAE,qBAAqB,GAAG;QACzD,EAAE,EAAE,MAAM,CAAC;QACX,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,EAAE,OAAO,CAAC;QACd,MAAM,EAAE,OAAO,CAAC;QAChB,OAAO,CAAC,EAAE,OAAO,CAAC;KACnB,GAAG,IAAI;IAyCR,SAAS,CAAC,QAAQ,CAAC,cAAc,CAC/B,YAAY,EAAE,YAAY,EAC1B,IAAI,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,EAC9B,IAAI,EAAE,GAAG,EAAE,GACV,OAAO,CAAC,GAAG,CAAC;cAEC,kBAAkB,CAChC,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS,CAAC;CAG9D"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser_labels.d.ts","sourceRoot":"","sources":["../../../src/agent/browser/browser_labels.ts"],"names":[],"mappings":"AACA,OAAO,gBAAgB,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,IAAI,EAAc,UAAU,EAAE,MAAM,aAAa,CAAC;AAE3D,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAGzD,MAAM,CAAC,OAAO,CAAC,QAAQ,OAAO,sBAAuB,SAAQ,gBAAgB;gBAC/D,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE,SAAS,CAAC,EAAE,UAAU;cAmCvD,UAAU,CACxB,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,OAAO,GACb,OAAO,CAAC,IAAI,CAAC;cAOA,aAAa,CAC3B,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,GAClC,OAAO,CAAC,IAAI,CAAC;cAMA,iBAAiB,CAC/B,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,IAAI,CAAC;cAaA,kBAAkB,CAChC,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC;cAWA,gBAAgB,CAC9B,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,IAAI,CAAC;cAIA,mBAAmB,CAAC,YAAY,EAAE,YAAY,GAAG,OAAO,CAAC;QACvE,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,YAAY,GAAG,WAAW,CAAC;QACtC,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IAqCF,SAAS,CAAC,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAInD,OAAO,CAAC,cAAc;
|
|
1
|
+
{"version":3,"file":"browser_labels.d.ts","sourceRoot":"","sources":["../../../src/agent/browser/browser_labels.ts"],"names":[],"mappings":"AACA,OAAO,gBAAgB,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,IAAI,EAAc,UAAU,EAAE,MAAM,aAAa,CAAC;AAE3D,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAGzD,MAAM,CAAC,OAAO,CAAC,QAAQ,OAAO,sBAAuB,SAAQ,gBAAgB;gBAC/D,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE,SAAS,CAAC,EAAE,UAAU;cAmCvD,UAAU,CACxB,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,OAAO,GACb,OAAO,CAAC,IAAI,CAAC;cAOA,aAAa,CAC3B,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,GAClC,OAAO,CAAC,IAAI,CAAC;cAMA,iBAAiB,CAC/B,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,IAAI,CAAC;cAaA,kBAAkB,CAChC,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC;cAWA,gBAAgB,CAC9B,YAAY,EAAE,YAAY,EAC1B,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,IAAI,CAAC;cAIA,mBAAmB,CAAC,YAAY,EAAE,YAAY,GAAG,OAAO,CAAC;QACvE,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,YAAY,GAAG,WAAW,CAAC;QACtC,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IAqCF,SAAS,CAAC,kBAAkB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAInD,OAAO,CAAC,cAAc;cA4RN,cAAc,CAC5B,YAAY,EAAE,YAAY,EAC1B,QAAQ,EAAE,qBAAqB,GAC9B,OAAO,CAAC,IAAI,CAAC;CAyBjB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"browser_screen.d.ts","sourceRoot":"","sources":["../../../src/agent/browser/browser_screen.ts"],"names":[],"mappings":"AACA,OAAO,gBAAgB,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,IAAI,EAAc,UAAU,EAAE,MAAM,aAAa,CAAC;AAC3D,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAGzD,MAAM,CAAC,OAAO,CAAC,QAAQ,OAAO,sBAAuB,SAAQ,gBAAgB;gBAC/D,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE,SAAS,CAAC,EAAE,UAAU;IA6BvE,SAAS,CAAC,QAAQ,CAAC,MAAM,CACvB,YAAY,EAAE,YAAY,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,IAAI,CAAC;IAEhB,SAAS,CAAC,QAAQ,CAAC,KAAK,CACtB,YAAY,EAAE,YAAY,EAC1B,CAAC,EAAE,MAAM,EACT,CAAC,EAAE,MAAM,EACT,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,GACvC,OAAO,CAAC,IAAI,CAAC;IAEhB,SAAS,CAAC,QAAQ,CAAC,MAAM,CACvB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC;IAEhB,SAAS,CAAC,QAAQ,CAAC,OAAO,CACxB,YAAY,EAAE,YAAY,EAC1B,CAAC,EAAE,MAAM,EACT,CAAC,EAAE,MAAM,GACR,OAAO,CAAC,IAAI,CAAC;IAEhB,SAAS,CAAC,QAAQ,CAAC,KAAK,CACtB,YAAY,EAAE,YAAY,EAC1B,GAAG,EAAE,OAAO,GAAG,KAAK,GAAG,OAAO,GAAG,WAAW,GAAG,QAAQ,GACtD,OAAO,CAAC,IAAI,CAAC;IAEhB,SAAS,CAAC,QAAQ,CAAC,aAAa,CAC9B,YAAY,EAAE,YAAY,EAC1B,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,GACT,OAAO,CAAC,IAAI,CAAC;IAEhB,OAAO,CAAC,cAAc;
|
|
1
|
+
{"version":3,"file":"browser_screen.d.ts","sourceRoot":"","sources":["../../../src/agent/browser/browser_screen.ts"],"names":[],"mappings":"AACA,OAAO,gBAAgB,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,IAAI,EAAc,UAAU,EAAE,MAAM,aAAa,CAAC;AAC3D,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAGzD,MAAM,CAAC,OAAO,CAAC,QAAQ,OAAO,sBAAuB,SAAQ,gBAAgB;gBAC/D,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE,SAAS,CAAC,EAAE,UAAU;IA6BvE,SAAS,CAAC,QAAQ,CAAC,MAAM,CACvB,YAAY,EAAE,YAAY,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,IAAI,CAAC;IAEhB,SAAS,CAAC,QAAQ,CAAC,KAAK,CACtB,YAAY,EAAE,YAAY,EAC1B,CAAC,EAAE,MAAM,EACT,CAAC,EAAE,MAAM,EACT,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,GACvC,OAAO,CAAC,IAAI,CAAC;IAEhB,SAAS,CAAC,QAAQ,CAAC,MAAM,CACvB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC;IAEhB,SAAS,CAAC,QAAQ,CAAC,OAAO,CACxB,YAAY,EAAE,YAAY,EAC1B,CAAC,EAAE,MAAM,EACT,CAAC,EAAE,MAAM,GACR,OAAO,CAAC,IAAI,CAAC;IAEhB,SAAS,CAAC,QAAQ,CAAC,KAAK,CACtB,YAAY,EAAE,YAAY,EAC1B,GAAG,EAAE,OAAO,GAAG,KAAK,GAAG,OAAO,GAAG,WAAW,GAAG,QAAQ,GACtD,OAAO,CAAC,IAAI,CAAC;IAEhB,SAAS,CAAC,QAAQ,CAAC,aAAa,CAC9B,YAAY,EAAE,YAAY,EAC1B,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,GACT,OAAO,CAAC,IAAI,CAAC;IAEhB,OAAO,CAAC,cAAc;cAoUN,cAAc,CAC5B,YAAY,EAAE,YAAY,EAC1B,QAAQ,EAAE,qBAAqB,GAC9B,OAAO,CAAC,IAAI,CAAC;CAuBjB"}
|
package/dist/agent/computer.d.ts
CHANGED
|
@@ -4,8 +4,7 @@ import { Tool, IMcpClient } from "../types";
|
|
|
4
4
|
import { LanguageModelV1Prompt } from "@ai-sdk/provider";
|
|
5
5
|
export declare const AGENT_NAME = "Computer";
|
|
6
6
|
export default abstract class BaseComputerAgent extends Agent {
|
|
7
|
-
|
|
8
|
-
constructor(llms?: string[], ext_tools?: Tool[], mcpClient?: IMcpClient);
|
|
7
|
+
constructor(llms?: string[], ext_tools?: Tool[], mcpClient?: IMcpClient, keyboardKeys?: string[]);
|
|
9
8
|
protected abstract screenshot(agentContext: AgentContext): Promise<{
|
|
10
9
|
imageBase64: string;
|
|
11
10
|
imageType: "image/jpeg" | "image/png";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"computer.d.ts","sourceRoot":"","sources":["../../src/agent/computer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;
|
|
1
|
+
{"version":3,"file":"computer.d.ts","sourceRoot":"","sources":["../../src/agent/computer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAE/B,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAc,UAAU,EAAE,MAAM,UAAU,CAAC;AACxD,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AAGzD,eAAO,MAAM,UAAU,aAAa,CAAC;AAErC,MAAM,CAAC,OAAO,CAAC,QAAQ,OAAO,iBAAkB,SAAQ,KAAK;gBAE/C,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE,SAAS,CAAC,EAAE,UAAU,EAAE,YAAY,CAAC,EAAE,MAAM,EAAE;IAgDhG,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,YAAY,EAAE,YAAY,GAAG,OAAO,CAAC;QACjE,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,YAAY,GAAG,WAAW,CAAC;KACvC,CAAC;IAEF,SAAS,CAAC,QAAQ,CAAC,MAAM,CACvB,YAAY,EAAE,YAAY,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,IAAI,CAAC;IAEhB,SAAS,CAAC,QAAQ,CAAC,KAAK,CACtB,YAAY,EAAE,YAAY,EAC1B,CAAC,EAAE,MAAM,EACT,CAAC,EAAE,MAAM,EACT,UAAU,EAAE,MAAM,EAClB,WAAW,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,GACvC,OAAO,CAAC,IAAI,CAAC;IAEhB,SAAS,CAAC,QAAQ,CAAC,MAAM,CACvB,YAAY,EAAE,YAAY,EAC1B,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC;IAEhB,SAAS,CAAC,QAAQ,CAAC,OAAO,CACxB,YAAY,EAAE,YAAY,EAC1B,CAAC,EAAE,MAAM,EACT,CAAC,EAAE,MAAM,GACR,OAAO,CAAC,IAAI,CAAC;IAEhB,SAAS,CAAC,QAAQ,CAAC,KAAK,CACtB,YAAY,EAAE,YAAY,EAC1B,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,IAAI,CAAC;IAEhB,SAAS,CAAC,QAAQ,CAAC,MAAM,CACvB,YAAY,EAAE,YAAY,EAC1B,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,IAAI,CAAC;IAEhB,SAAS,CAAC,QAAQ,CAAC,aAAa,CAC9B,YAAY,EAAE,YAAY,EAC1B,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,EACV,EAAE,EAAE,MAAM,GACT,OAAO,CAAC,IAAI,CAAC;IAEhB,OAAO,CAAC,cAAc;cAwON,cAAc,CAC5B,YAAY,EAAE,YAAY,EAC1B,QAAQ,EAAE,qBAAqB,GAC9B,OAAO,CAAC,IAAI,CAAC;CAuCjB"}
|
package/dist/agent/shell.d.ts
CHANGED
|
@@ -8,7 +8,7 @@ export default abstract class BaseShellAgent extends Agent {
|
|
|
8
8
|
session_id: string;
|
|
9
9
|
}>;
|
|
10
10
|
protected abstract shell_exec(agentContext: AgentContext, session_id: string, command: string): Promise<string>;
|
|
11
|
-
protected abstract close_session(agentContext: AgentContext, session_id: string): Promise<
|
|
11
|
+
protected abstract close_session(agentContext: AgentContext, session_id: string): Promise<void>;
|
|
12
12
|
private buildInitTools;
|
|
13
13
|
}
|
|
14
14
|
//# sourceMappingURL=shell.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"shell.d.ts","sourceRoot":"","sources":["../../src/agent/shell.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAE/B,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAc,UAAU,EAAE,MAAM,UAAU,CAAC;AAExD,eAAO,MAAM,UAAU,UAAU,CAAC;AAElC,MAAM,CAAC,OAAO,CAAC,QAAQ,OAAO,cAAe,SAAQ,KAAK;gBAC5C,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE,SAAS,CAAC,EAAE,UAAU;IAiBvE,SAAS,CAAC,QAAQ,CAAC,cAAc,CAC/B,YAAY,EAAE,YAAY,EAC1B,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC;QACT,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IAEF,SAAS,CAAC,QAAQ,CAAC,UAAU,CAC3B,YAAY,EAAE,YAAY,EAC1B,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,MAAM,CAAC;IAElB,SAAS,CAAC,QAAQ,CAAC,aAAa,CAC9B,YAAY,EAAE,YAAY,EAC1B,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,
|
|
1
|
+
{"version":3,"file":"shell.d.ts","sourceRoot":"","sources":["../../src/agent/shell.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AAE/B,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAc,UAAU,EAAE,MAAM,UAAU,CAAC;AAExD,eAAO,MAAM,UAAU,UAAU,CAAC;AAElC,MAAM,CAAC,OAAO,CAAC,QAAQ,OAAO,cAAe,SAAQ,KAAK;gBAC5C,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE,SAAS,CAAC,EAAE,UAAU;IAiBvE,SAAS,CAAC,QAAQ,CAAC,cAAc,CAC/B,YAAY,EAAE,YAAY,EAC1B,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC;QACT,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;IAEF,SAAS,CAAC,QAAQ,CAAC,UAAU,CAC3B,YAAY,EAAE,YAAY,EAC1B,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,MAAM,CAAC;IAElB,SAAS,CAAC,QAAQ,CAAC,aAAa,CAC9B,YAAY,EAAE,YAAY,EAC1B,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,IAAI,CAAC;IAEhB,OAAO,CAAC,cAAc;CA+EvB"}
|
package/dist/index.cjs.js
CHANGED
|
@@ -17473,34 +17473,55 @@ class BaseTimerAgent extends Agent {
|
|
|
17473
17473
|
|
|
17474
17474
|
const AGENT_NAME$1 = "Computer";
|
|
17475
17475
|
class BaseComputerAgent extends Agent {
|
|
17476
|
-
constructor(llms, ext_tools, mcpClient) {
|
|
17476
|
+
constructor(llms, ext_tools, mcpClient, keyboardKeys) {
|
|
17477
17477
|
const _tools_ = [];
|
|
17478
17478
|
super({
|
|
17479
17479
|
name: AGENT_NAME$1,
|
|
17480
|
-
description:
|
|
17480
|
+
description: `You are a computer operation agent, who interacts with the computer using mouse and keyboard, completing specified tasks step by step based on the given tasks and screenshots. After each of your operations, you will receive the latest computer screenshot to evaluate the task execution status.
|
|
17481
|
+
This is a computer GUI interface, observe the execution through screenshots, and specify action sequences to complete designated tasks.
|
|
17482
|
+
* COMPUTER OPERATIONS:
|
|
17483
|
+
- You can operate the application using shortcuts.
|
|
17484
|
+
- If stuck, try alternative approaches`,
|
|
17481
17485
|
tools: _tools_,
|
|
17482
17486
|
llms: llms,
|
|
17483
17487
|
mcpClient: mcpClient,
|
|
17484
|
-
planDescription: "Computer operation agent, interact with the computer using the mouse and keyboard."
|
|
17488
|
+
planDescription: "Computer operation agent, interact with the computer using the mouse and keyboard, operation application."
|
|
17485
17489
|
});
|
|
17486
|
-
|
|
17487
|
-
|
|
17488
|
-
|
|
17489
|
-
|
|
17490
|
-
|
|
17491
|
-
|
|
17492
|
-
|
|
17493
|
-
|
|
17494
|
-
|
|
17495
|
-
|
|
17496
|
-
|
|
17497
|
-
|
|
17490
|
+
if (!keyboardKeys) {
|
|
17491
|
+
if (config.platform == "windows") {
|
|
17492
|
+
keyboardKeys = [
|
|
17493
|
+
'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
|
|
17494
|
+
'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
|
|
17495
|
+
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
|
|
17496
|
+
'enter', 'esc', 'backspace', 'tab', 'space', 'delete',
|
|
17497
|
+
'ctrl', 'alt', 'shift', 'win',
|
|
17498
|
+
'up', 'down', 'left', 'right',
|
|
17499
|
+
'f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'f10', 'f11', 'f12',
|
|
17500
|
+
'ctrl+c', 'ctrl+v', 'ctrl+x', 'ctrl+z', 'ctrl+a', 'ctrl+s',
|
|
17501
|
+
'alt+tab', 'alt+f4', 'ctrl+alt+delete'
|
|
17502
|
+
];
|
|
17503
|
+
}
|
|
17504
|
+
else {
|
|
17505
|
+
keyboardKeys = [
|
|
17506
|
+
'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
|
|
17507
|
+
'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
|
|
17508
|
+
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
|
|
17509
|
+
'enter', 'esc', 'backspace', 'tab', 'space', 'delete',
|
|
17510
|
+
'command', 'option', 'shift', 'control',
|
|
17511
|
+
'up', 'down', 'left', 'right',
|
|
17512
|
+
'f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'f10', 'f11', 'f12',
|
|
17513
|
+
'command+c', 'command+v', 'command+x', 'command+z', 'command+a', 'command+s',
|
|
17514
|
+
'command+tab', 'command+q', 'command+escape'
|
|
17515
|
+
];
|
|
17516
|
+
}
|
|
17517
|
+
}
|
|
17518
|
+
let init_tools = this.buildInitTools(keyboardKeys);
|
|
17498
17519
|
if (ext_tools && ext_tools.length > 0) {
|
|
17499
17520
|
init_tools = mergeTools(init_tools, ext_tools);
|
|
17500
17521
|
}
|
|
17501
17522
|
init_tools.forEach((tool) => _tools_.push(tool));
|
|
17502
17523
|
}
|
|
17503
|
-
buildInitTools() {
|
|
17524
|
+
buildInitTools(keyboardKeys) {
|
|
17504
17525
|
return [
|
|
17505
17526
|
{
|
|
17506
17527
|
name: "typing",
|
|
@@ -17601,7 +17622,7 @@ class BaseComputerAgent extends Agent {
|
|
|
17601
17622
|
key: {
|
|
17602
17623
|
type: "string",
|
|
17603
17624
|
description: "Key to press",
|
|
17604
|
-
enum:
|
|
17625
|
+
enum: keyboardKeys,
|
|
17605
17626
|
},
|
|
17606
17627
|
},
|
|
17607
17628
|
required: ["key"],
|
|
@@ -17619,7 +17640,7 @@ class BaseComputerAgent extends Agent {
|
|
|
17619
17640
|
keys: {
|
|
17620
17641
|
type: "string",
|
|
17621
17642
|
description: "Key combination to press",
|
|
17622
|
-
enum:
|
|
17643
|
+
enum: keyboardKeys,
|
|
17623
17644
|
},
|
|
17624
17645
|
},
|
|
17625
17646
|
required: ["keys"],
|
|
@@ -18491,7 +18512,7 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
|
|
|
18491
18512
|
try {
|
|
18492
18513
|
let element_result = null;
|
|
18493
18514
|
for (let i = 0; i < 5; i++) {
|
|
18494
|
-
await sleep(
|
|
18515
|
+
await sleep(200);
|
|
18495
18516
|
await this.execute_script(agentContext, run_build_dom_tree, []);
|
|
18496
18517
|
element_result = (await this.execute_script(agentContext, () => {
|
|
18497
18518
|
return window.get_clickable_elements(true);
|
|
@@ -18684,6 +18705,34 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
|
|
|
18684
18705
|
return await this.callInnerTool(() => this.extract_content(agentContext));
|
|
18685
18706
|
},
|
|
18686
18707
|
},
|
|
18708
|
+
{
|
|
18709
|
+
name: "get_all_tabs",
|
|
18710
|
+
description: "Get all tabs of the current browser",
|
|
18711
|
+
parameters: {
|
|
18712
|
+
type: "object",
|
|
18713
|
+
properties: {},
|
|
18714
|
+
},
|
|
18715
|
+
execute: async (args, agentContext) => {
|
|
18716
|
+
return await this.callInnerTool(() => this.get_all_tabs(agentContext));
|
|
18717
|
+
},
|
|
18718
|
+
},
|
|
18719
|
+
{
|
|
18720
|
+
name: "switch_tab",
|
|
18721
|
+
description: "Switch to the specified tab page",
|
|
18722
|
+
parameters: {
|
|
18723
|
+
type: "object",
|
|
18724
|
+
properties: {
|
|
18725
|
+
tabId: {
|
|
18726
|
+
type: "number",
|
|
18727
|
+
description: "Tab ID, obtained through get_all_tabs",
|
|
18728
|
+
},
|
|
18729
|
+
},
|
|
18730
|
+
required: ["tabId"],
|
|
18731
|
+
},
|
|
18732
|
+
execute: async (args, agentContext) => {
|
|
18733
|
+
return await this.callInnerTool(() => this.switch_tab(agentContext, args.tabId));
|
|
18734
|
+
},
|
|
18735
|
+
},
|
|
18687
18736
|
{
|
|
18688
18737
|
name: "wait",
|
|
18689
18738
|
description: "Wait for specified duration",
|
|
@@ -18708,7 +18757,7 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
|
|
|
18708
18757
|
}
|
|
18709
18758
|
async handleMessages(agentContext, messages) {
|
|
18710
18759
|
let lastTool = this.lastToolResult(messages);
|
|
18711
|
-
if (lastTool && lastTool.toolName !== "extract_content") {
|
|
18760
|
+
if (lastTool && lastTool.toolName !== "extract_content" && lastTool.toolName !== "get_all_tabs") {
|
|
18712
18761
|
await sleep(200);
|
|
18713
18762
|
let result = await this.screenshot_and_html(agentContext);
|
|
18714
18763
|
let image = toImage(result.imageBase64);
|
|
@@ -19054,6 +19103,34 @@ class BaseBrowserScreenAgent extends BaseBrowserAgent {
|
|
|
19054
19103
|
return await this.callInnerTool(() => this.drag_and_drop(agentContext, args.x1, args.y1, args.x2, args.y2));
|
|
19055
19104
|
},
|
|
19056
19105
|
},
|
|
19106
|
+
{
|
|
19107
|
+
name: "get_all_tabs",
|
|
19108
|
+
description: "Get all tabs of the current browser",
|
|
19109
|
+
parameters: {
|
|
19110
|
+
type: "object",
|
|
19111
|
+
properties: {},
|
|
19112
|
+
},
|
|
19113
|
+
execute: async (args, agentContext) => {
|
|
19114
|
+
return await this.callInnerTool(() => this.get_all_tabs(agentContext));
|
|
19115
|
+
},
|
|
19116
|
+
},
|
|
19117
|
+
{
|
|
19118
|
+
name: "switch_tab",
|
|
19119
|
+
description: "Switch to the specified tab page",
|
|
19120
|
+
parameters: {
|
|
19121
|
+
type: "object",
|
|
19122
|
+
properties: {
|
|
19123
|
+
tabId: {
|
|
19124
|
+
type: "number",
|
|
19125
|
+
description: "Tab ID, obtained through get_all_tabs",
|
|
19126
|
+
},
|
|
19127
|
+
},
|
|
19128
|
+
required: ["tabId"],
|
|
19129
|
+
},
|
|
19130
|
+
execute: async (args, agentContext) => {
|
|
19131
|
+
return await this.callInnerTool(() => this.switch_tab(agentContext, args.tabId));
|
|
19132
|
+
},
|
|
19133
|
+
},
|
|
19057
19134
|
{
|
|
19058
19135
|
name: "wait",
|
|
19059
19136
|
description: "Wait for specified duration",
|
|
@@ -19078,7 +19155,7 @@ class BaseBrowserScreenAgent extends BaseBrowserAgent {
|
|
|
19078
19155
|
}
|
|
19079
19156
|
async handleMessages(agentContext, messages) {
|
|
19080
19157
|
let lastTool = this.lastToolResult(messages);
|
|
19081
|
-
if (lastTool && lastTool.toolName !== "extract_content") {
|
|
19158
|
+
if (lastTool && lastTool.toolName !== "extract_content" && lastTool.toolName !== "get_all_tabs") {
|
|
19082
19159
|
await sleep(200);
|
|
19083
19160
|
let result = await this.screenshot(agentContext);
|
|
19084
19161
|
let image = toImage(result.imageBase64);
|
package/dist/index.esm.js
CHANGED
|
@@ -17438,34 +17438,55 @@ class BaseTimerAgent extends Agent {
|
|
|
17438
17438
|
|
|
17439
17439
|
const AGENT_NAME$1 = "Computer";
|
|
17440
17440
|
class BaseComputerAgent extends Agent {
|
|
17441
|
-
constructor(llms, ext_tools, mcpClient) {
|
|
17441
|
+
constructor(llms, ext_tools, mcpClient, keyboardKeys) {
|
|
17442
17442
|
const _tools_ = [];
|
|
17443
17443
|
super({
|
|
17444
17444
|
name: AGENT_NAME$1,
|
|
17445
|
-
description:
|
|
17445
|
+
description: `You are a computer operation agent, who interacts with the computer using mouse and keyboard, completing specified tasks step by step based on the given tasks and screenshots. After each of your operations, you will receive the latest computer screenshot to evaluate the task execution status.
|
|
17446
|
+
This is a computer GUI interface, observe the execution through screenshots, and specify action sequences to complete designated tasks.
|
|
17447
|
+
* COMPUTER OPERATIONS:
|
|
17448
|
+
- You can operate the application using shortcuts.
|
|
17449
|
+
- If stuck, try alternative approaches`,
|
|
17446
17450
|
tools: _tools_,
|
|
17447
17451
|
llms: llms,
|
|
17448
17452
|
mcpClient: mcpClient,
|
|
17449
|
-
planDescription: "Computer operation agent, interact with the computer using the mouse and keyboard."
|
|
17453
|
+
planDescription: "Computer operation agent, interact with the computer using the mouse and keyboard, operation application."
|
|
17450
17454
|
});
|
|
17451
|
-
|
|
17452
|
-
|
|
17453
|
-
|
|
17454
|
-
|
|
17455
|
-
|
|
17456
|
-
|
|
17457
|
-
|
|
17458
|
-
|
|
17459
|
-
|
|
17460
|
-
|
|
17461
|
-
|
|
17462
|
-
|
|
17455
|
+
if (!keyboardKeys) {
|
|
17456
|
+
if (config.platform == "windows") {
|
|
17457
|
+
keyboardKeys = [
|
|
17458
|
+
'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
|
|
17459
|
+
'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
|
|
17460
|
+
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
|
|
17461
|
+
'enter', 'esc', 'backspace', 'tab', 'space', 'delete',
|
|
17462
|
+
'ctrl', 'alt', 'shift', 'win',
|
|
17463
|
+
'up', 'down', 'left', 'right',
|
|
17464
|
+
'f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'f10', 'f11', 'f12',
|
|
17465
|
+
'ctrl+c', 'ctrl+v', 'ctrl+x', 'ctrl+z', 'ctrl+a', 'ctrl+s',
|
|
17466
|
+
'alt+tab', 'alt+f4', 'ctrl+alt+delete'
|
|
17467
|
+
];
|
|
17468
|
+
}
|
|
17469
|
+
else {
|
|
17470
|
+
keyboardKeys = [
|
|
17471
|
+
'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
|
|
17472
|
+
'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
|
|
17473
|
+
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
|
|
17474
|
+
'enter', 'esc', 'backspace', 'tab', 'space', 'delete',
|
|
17475
|
+
'command', 'option', 'shift', 'control',
|
|
17476
|
+
'up', 'down', 'left', 'right',
|
|
17477
|
+
'f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'f10', 'f11', 'f12',
|
|
17478
|
+
'command+c', 'command+v', 'command+x', 'command+z', 'command+a', 'command+s',
|
|
17479
|
+
'command+tab', 'command+q', 'command+escape'
|
|
17480
|
+
];
|
|
17481
|
+
}
|
|
17482
|
+
}
|
|
17483
|
+
let init_tools = this.buildInitTools(keyboardKeys);
|
|
17463
17484
|
if (ext_tools && ext_tools.length > 0) {
|
|
17464
17485
|
init_tools = mergeTools(init_tools, ext_tools);
|
|
17465
17486
|
}
|
|
17466
17487
|
init_tools.forEach((tool) => _tools_.push(tool));
|
|
17467
17488
|
}
|
|
17468
|
-
buildInitTools() {
|
|
17489
|
+
buildInitTools(keyboardKeys) {
|
|
17469
17490
|
return [
|
|
17470
17491
|
{
|
|
17471
17492
|
name: "typing",
|
|
@@ -17566,7 +17587,7 @@ class BaseComputerAgent extends Agent {
|
|
|
17566
17587
|
key: {
|
|
17567
17588
|
type: "string",
|
|
17568
17589
|
description: "Key to press",
|
|
17569
|
-
enum:
|
|
17590
|
+
enum: keyboardKeys,
|
|
17570
17591
|
},
|
|
17571
17592
|
},
|
|
17572
17593
|
required: ["key"],
|
|
@@ -17584,7 +17605,7 @@ class BaseComputerAgent extends Agent {
|
|
|
17584
17605
|
keys: {
|
|
17585
17606
|
type: "string",
|
|
17586
17607
|
description: "Key combination to press",
|
|
17587
|
-
enum:
|
|
17608
|
+
enum: keyboardKeys,
|
|
17588
17609
|
},
|
|
17589
17610
|
},
|
|
17590
17611
|
required: ["keys"],
|
|
@@ -18456,7 +18477,7 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
|
|
|
18456
18477
|
try {
|
|
18457
18478
|
let element_result = null;
|
|
18458
18479
|
for (let i = 0; i < 5; i++) {
|
|
18459
|
-
await sleep(
|
|
18480
|
+
await sleep(200);
|
|
18460
18481
|
await this.execute_script(agentContext, run_build_dom_tree, []);
|
|
18461
18482
|
element_result = (await this.execute_script(agentContext, () => {
|
|
18462
18483
|
return window.get_clickable_elements(true);
|
|
@@ -18649,6 +18670,34 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
|
|
|
18649
18670
|
return await this.callInnerTool(() => this.extract_content(agentContext));
|
|
18650
18671
|
},
|
|
18651
18672
|
},
|
|
18673
|
+
{
|
|
18674
|
+
name: "get_all_tabs",
|
|
18675
|
+
description: "Get all tabs of the current browser",
|
|
18676
|
+
parameters: {
|
|
18677
|
+
type: "object",
|
|
18678
|
+
properties: {},
|
|
18679
|
+
},
|
|
18680
|
+
execute: async (args, agentContext) => {
|
|
18681
|
+
return await this.callInnerTool(() => this.get_all_tabs(agentContext));
|
|
18682
|
+
},
|
|
18683
|
+
},
|
|
18684
|
+
{
|
|
18685
|
+
name: "switch_tab",
|
|
18686
|
+
description: "Switch to the specified tab page",
|
|
18687
|
+
parameters: {
|
|
18688
|
+
type: "object",
|
|
18689
|
+
properties: {
|
|
18690
|
+
tabId: {
|
|
18691
|
+
type: "number",
|
|
18692
|
+
description: "Tab ID, obtained through get_all_tabs",
|
|
18693
|
+
},
|
|
18694
|
+
},
|
|
18695
|
+
required: ["tabId"],
|
|
18696
|
+
},
|
|
18697
|
+
execute: async (args, agentContext) => {
|
|
18698
|
+
return await this.callInnerTool(() => this.switch_tab(agentContext, args.tabId));
|
|
18699
|
+
},
|
|
18700
|
+
},
|
|
18652
18701
|
{
|
|
18653
18702
|
name: "wait",
|
|
18654
18703
|
description: "Wait for specified duration",
|
|
@@ -18673,7 +18722,7 @@ class BaseBrowserLabelsAgent extends BaseBrowserAgent {
|
|
|
18673
18722
|
}
|
|
18674
18723
|
async handleMessages(agentContext, messages) {
|
|
18675
18724
|
let lastTool = this.lastToolResult(messages);
|
|
18676
|
-
if (lastTool && lastTool.toolName !== "extract_content") {
|
|
18725
|
+
if (lastTool && lastTool.toolName !== "extract_content" && lastTool.toolName !== "get_all_tabs") {
|
|
18677
18726
|
await sleep(200);
|
|
18678
18727
|
let result = await this.screenshot_and_html(agentContext);
|
|
18679
18728
|
let image = toImage(result.imageBase64);
|
|
@@ -19019,6 +19068,34 @@ class BaseBrowserScreenAgent extends BaseBrowserAgent {
|
|
|
19019
19068
|
return await this.callInnerTool(() => this.drag_and_drop(agentContext, args.x1, args.y1, args.x2, args.y2));
|
|
19020
19069
|
},
|
|
19021
19070
|
},
|
|
19071
|
+
{
|
|
19072
|
+
name: "get_all_tabs",
|
|
19073
|
+
description: "Get all tabs of the current browser",
|
|
19074
|
+
parameters: {
|
|
19075
|
+
type: "object",
|
|
19076
|
+
properties: {},
|
|
19077
|
+
},
|
|
19078
|
+
execute: async (args, agentContext) => {
|
|
19079
|
+
return await this.callInnerTool(() => this.get_all_tabs(agentContext));
|
|
19080
|
+
},
|
|
19081
|
+
},
|
|
19082
|
+
{
|
|
19083
|
+
name: "switch_tab",
|
|
19084
|
+
description: "Switch to the specified tab page",
|
|
19085
|
+
parameters: {
|
|
19086
|
+
type: "object",
|
|
19087
|
+
properties: {
|
|
19088
|
+
tabId: {
|
|
19089
|
+
type: "number",
|
|
19090
|
+
description: "Tab ID, obtained through get_all_tabs",
|
|
19091
|
+
},
|
|
19092
|
+
},
|
|
19093
|
+
required: ["tabId"],
|
|
19094
|
+
},
|
|
19095
|
+
execute: async (args, agentContext) => {
|
|
19096
|
+
return await this.callInnerTool(() => this.switch_tab(agentContext, args.tabId));
|
|
19097
|
+
},
|
|
19098
|
+
},
|
|
19022
19099
|
{
|
|
19023
19100
|
name: "wait",
|
|
19024
19101
|
description: "Wait for specified duration",
|
|
@@ -19043,7 +19120,7 @@ class BaseBrowserScreenAgent extends BaseBrowserAgent {
|
|
|
19043
19120
|
}
|
|
19044
19121
|
async handleMessages(agentContext, messages) {
|
|
19045
19122
|
let lastTool = this.lastToolResult(messages);
|
|
19046
|
-
if (lastTool && lastTool.toolName !== "extract_content") {
|
|
19123
|
+
if (lastTool && lastTool.toolName !== "extract_content" && lastTool.toolName !== "get_all_tabs") {
|
|
19047
19124
|
await sleep(200);
|
|
19048
19125
|
let result = await this.screenshot(agentContext);
|
|
19049
19126
|
let image = toImage(result.imageBase64);
|