@bini-bar-labs/atomic-web-agent-core 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +15 -0
- package/README.md +153 -0
- package/dist/AWAgent/AWAgent.d.ts +35 -0
- package/dist/AWAgent/AWAgent.d.ts.map +1 -0
- package/dist/AWAgent/AWAgent.js +115 -0
- package/dist/AWAgent/AWAgent.js.map +1 -0
- package/dist/AWAgent/AWAgent.types.d.ts +3 -0
- package/dist/AWAgent/AWAgent.types.d.ts.map +1 -0
- package/dist/AWAgent/AWAgent.types.js +2 -0
- package/dist/AWAgent/AWAgent.types.js.map +1 -0
- package/dist/AWAgent/middlewares/logging-calls.middleware.d.ts +2 -0
- package/dist/AWAgent/middlewares/logging-calls.middleware.d.ts.map +1 -0
- package/dist/AWAgent/middlewares/logging-calls.middleware.js +17 -0
- package/dist/AWAgent/middlewares/logging-calls.middleware.js.map +1 -0
- package/dist/AWAgent/middlewares/trim-messages-history.middleware.d.ts +2 -0
- package/dist/AWAgent/middlewares/trim-messages-history.middleware.d.ts.map +1 -0
- package/dist/AWAgent/middlewares/trim-messages-history.middleware.js +11 -0
- package/dist/AWAgent/middlewares/trim-messages-history.middleware.js.map +1 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +7 -0
- package/dist/index.js.map +1 -0
- package/dist/tools/click-by-element-id.tool.d.ts +14 -0
- package/dist/tools/click-by-element-id.tool.d.ts.map +1 -0
- package/dist/tools/click-by-element-id.tool.js +32 -0
- package/dist/tools/click-by-element-id.tool.js.map +1 -0
- package/dist/tools/click-by-position.tool.d.ts +17 -0
- package/dist/tools/click-by-position.tool.d.ts.map +1 -0
- package/dist/tools/click-by-position.tool.js +18 -0
- package/dist/tools/click-by-position.tool.js.map +1 -0
- package/dist/tools/click-by-selector.tool.d.ts +13 -0
- package/dist/tools/click-by-selector.tool.d.ts.map +1 -0
- package/dist/tools/click-by-selector.tool.js +17 -0
- package/dist/tools/click-by-selector.tool.js.map +1 -0
- package/dist/tools/get-DOM-snapshot.tool.d.ts +4 -0
- package/dist/tools/get-DOM-snapshot.tool.d.ts.map +1 -0
- package/dist/tools/get-DOM-snapshot.tool.js +26 -0
- package/dist/tools/get-DOM-snapshot.tool.js.map +1 -0
- package/dist/tools/get-page-screenshot.tool.d.ts +3 -0
- package/dist/tools/get-page-screenshot.tool.d.ts.map +1 -0
- package/dist/tools/get-page-screenshot.tool.js +13 -0
- package/dist/tools/get-page-screenshot.tool.js.map +1 -0
- package/dist/tools/input-by-element-id.tool.d.ts +18 -0
- package/dist/tools/input-by-element-id.tool.d.ts.map +1 -0
- package/dist/tools/input-by-element-id.tool.js +39 -0
- package/dist/tools/input-by-element-id.tool.js.map +1 -0
- package/dist/tools/input.tool.d.ts +17 -0
- package/dist/tools/input.tool.d.ts.map +1 -0
- package/dist/tools/input.tool.js +21 -0
- package/dist/tools/input.tool.js.map +1 -0
- package/dist/tools/navigate.tool.d.ts +10 -0
- package/dist/tools/navigate.tool.d.ts.map +1 -0
- package/dist/tools/navigate.tool.js +16 -0
- package/dist/tools/navigate.tool.js.map +1 -0
- package/dist/tools/print-to-console.tool.d.ts +12 -0
- package/dist/tools/print-to-console.tool.d.ts.map +1 -0
- package/dist/tools/print-to-console.tool.js +16 -0
- package/dist/tools/print-to-console.tool.js.map +1 -0
- package/dist/tools/utils/accessibility-snapshot.util.d.ts +34 -0
- package/dist/tools/utils/accessibility-snapshot.util.d.ts.map +1 -0
- package/dist/tools/utils/accessibility-snapshot.util.js +244 -0
- package/dist/tools/utils/accessibility-snapshot.util.js.map +1 -0
- package/dist/tools/utils/element-registry.util.d.ts +39 -0
- package/dist/tools/utils/element-registry.util.d.ts.map +1 -0
- package/dist/tools/utils/element-registry.util.js +55 -0
- package/dist/tools/utils/element-registry.util.js.map +1 -0
- package/dist/tools/utils/minimize-snapshot.util.d.ts +2 -0
- package/dist/tools/utils/minimize-snapshot.util.d.ts.map +1 -0
- package/dist/tools/utils/minimize-snapshot.util.js +43 -0
- package/dist/tools/utils/minimize-snapshot.util.js.map +1 -0
- package/dist/tools/wait.tool.d.ts +12 -0
- package/dist/tools/wait.tool.d.ts.map +1 -0
- package/dist/tools/wait.tool.js +16 -0
- package/dist/tools/wait.tool.js.map +1 -0
- package/package.json +64 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
ISC License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025, Bini Barazany <bgt636@gmail.com>
|
|
4
|
+
|
|
5
|
+
Permission to use, copy, modify, and/or distribute this software for any
|
|
6
|
+
purpose with or without fee is hereby granted, provided that the above
|
|
7
|
+
copyright notice and this permission notice appear in all copies.
|
|
8
|
+
|
|
9
|
+
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
|
10
|
+
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
|
11
|
+
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
|
12
|
+
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
|
13
|
+
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
|
14
|
+
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
|
15
|
+
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
# @bini-bar-labs/atomic-web-agent-core
|
|
2
|
+
|
|
3
|
+
The core of the Atomic Web Agent, providing essential functionalities for AI-powered web interaction and automation.
|
|
4
|
+
|
|
5
|
+
## Overview
|
|
6
|
+
|
|
7
|
+
`@bini-bar-labs/atomic-web-agent-core` is a powerful library that combines the capabilities of [Playwright](https://playwright.dev/) for browser automation with [LangChain](https://js.langchain.com/) for AI agent orchestration. It enables you to create intelligent agents that can interact with web applications autonomously.
|
|
8
|
+
|
|
9
|
+
## Features
|
|
10
|
+
|
|
11
|
+
- **AI-Powered Browser Automation**: Control browser interactions using AI models (Anthropic Claude, OpenAI GPT)
|
|
12
|
+
- **Built-in Tools**: Pre-configured tools for common web interactions (clicking, typing, navigation, screenshots)
|
|
13
|
+
- **Extensible**: Easy to add custom tools for specific use cases
|
|
14
|
+
- **Type-Safe**: Full TypeScript support with comprehensive type definitions
|
|
15
|
+
- **Accessibility-First**: Uses accessibility snapshots for robust element interaction
|
|
16
|
+
|
|
17
|
+
## Installation
|
|
18
|
+
|
|
19
|
+
```bash
|
|
20
|
+
npm install @bini-bar-labs/atomic-web-agent-core
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
or with pnpm:
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
pnpm add @bini-bar-labs/atomic-web-agent-core
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
## Quick Start
|
|
30
|
+
|
|
31
|
+
```typescript
|
|
32
|
+
import { AWAgent } from "@bini-bar-labs/atomic-web-agent-core";
|
|
33
|
+
import { ChatAnthropic } from "@langchain/anthropic";
|
|
34
|
+
|
|
35
|
+
// Initialize the model
|
|
36
|
+
const model = new ChatAnthropic({
|
|
37
|
+
apiKey: process.env.ANTHROPIC_API_KEY,
|
|
38
|
+
model: "claude-3-5-sonnet-20241022",
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
// Create the agent
|
|
42
|
+
const agent = new AWAgent(
|
|
43
|
+
model,
|
|
44
|
+
"You are a helpful web automation assistant."
|
|
45
|
+
);
|
|
46
|
+
|
|
47
|
+
// Initialize and run
|
|
48
|
+
await agent.init();
|
|
49
|
+
await agent.run("Navigate to https://example.com and take a screenshot");
|
|
50
|
+
await agent.close();
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## API Reference
|
|
54
|
+
|
|
55
|
+
### AWAgent
|
|
56
|
+
|
|
57
|
+
The main class for creating and controlling web agents.
|
|
58
|
+
|
|
59
|
+
#### Constructor
|
|
60
|
+
|
|
61
|
+
```typescript
|
|
62
|
+
new AWAgent(
|
|
63
|
+
model: ChatAnthropic | ChatOpenAI,
|
|
64
|
+
systemMessage: string,
|
|
65
|
+
options?: {
|
|
66
|
+
overrideTools?: {
|
|
67
|
+
getDOMSnapshotTool?: (page: Page, registry: ElementLocatorRegistry) => AgentTool;
|
|
68
|
+
};
|
|
69
|
+
customTools?: ((page: Page) => AgentTool)[];
|
|
70
|
+
}
|
|
71
|
+
)
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
#### Methods
|
|
75
|
+
|
|
76
|
+
- `init(launchOptions?: LaunchOptions, contextOptions?: BrowserContextOptions): Promise<void>` - Initialize the browser and agent
|
|
77
|
+
- `run(message: string): Promise<void>` - Execute a task with the agent
|
|
78
|
+
- `close(): Promise<void>` - Close the browser and clean up resources
|
|
79
|
+
|
|
80
|
+
### Exports
|
|
81
|
+
|
|
82
|
+
```typescript
|
|
83
|
+
export { AWAgent } from "@bini-bar-labs/atomic-web-agent-core";
|
|
84
|
+
export { type PlaywrightPage } from "@bini-bar-labs/atomic-web-agent-core";
|
|
85
|
+
export { createTool } from "@bini-bar-labs/atomic-web-agent-core";
|
|
86
|
+
export { type AgentTool } from "@bini-bar-labs/atomic-web-agent-core";
|
|
87
|
+
export { ElementLocatorRegistry } from "@bini-bar-labs/atomic-web-agent-core";
|
|
88
|
+
export {
|
|
89
|
+
type ElementSnapshot,
|
|
90
|
+
type PageSnapshot,
|
|
91
|
+
generateAccessibilitySnapshot,
|
|
92
|
+
} from "@bini-bar-labs/atomic-web-agent-core";
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
## Built-in Tools
|
|
96
|
+
|
|
97
|
+
The agent comes with several pre-configured tools:
|
|
98
|
+
|
|
99
|
+
- **Navigate**: Navigate to URLs
|
|
100
|
+
- **Click**: Click elements by ID or position
|
|
101
|
+
- **Input**: Type text into input fields
|
|
102
|
+
- **Screenshot**: Capture page screenshots
|
|
103
|
+
- **DOM Snapshot**: Get accessibility-based page structure
|
|
104
|
+
- **Wait**: Wait for specified durations
|
|
105
|
+
- **Console Print**: Output messages to console
|
|
106
|
+
|
|
107
|
+
## Custom Tools
|
|
108
|
+
|
|
109
|
+
You can extend the agent with custom tools:
|
|
110
|
+
|
|
111
|
+
```typescript
|
|
112
|
+
import { AWAgent, createTool } from "@bini-bar-labs/atomic-web-agent-core";
|
|
113
|
+
|
|
114
|
+
const myCustomTool = (page: Page) =>
|
|
115
|
+
createTool(
|
|
116
|
+
async ({ input }) => {
|
|
117
|
+
// Your custom logic here
|
|
118
|
+
return "Result";
|
|
119
|
+
},
|
|
120
|
+
{
|
|
121
|
+
name: "my_custom_tool",
|
|
122
|
+
description: "Description of what this tool does",
|
|
123
|
+
schema: z.object({
|
|
124
|
+
input: z.string(),
|
|
125
|
+
}),
|
|
126
|
+
}
|
|
127
|
+
);
|
|
128
|
+
|
|
129
|
+
const agent = new AWAgent(model, systemMessage, {
|
|
130
|
+
customTools: [myCustomTool],
|
|
131
|
+
});
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
## Requirements
|
|
135
|
+
|
|
136
|
+
- Node.js >= 18
|
|
137
|
+
- An API key for Anthropic Claude or OpenAI
|
|
138
|
+
|
|
139
|
+
## License
|
|
140
|
+
|
|
141
|
+
ISC
|
|
142
|
+
|
|
143
|
+
## Repository
|
|
144
|
+
|
|
145
|
+
[https://github.com/binikingi/atomic-web-agent](https://github.com/binikingi/atomic-web-agent)
|
|
146
|
+
|
|
147
|
+
## Issues
|
|
148
|
+
|
|
149
|
+
Report issues at [https://github.com/binikingi/atomic-web-agent/issues](https://github.com/binikingi/atomic-web-agent/issues)
|
|
150
|
+
|
|
151
|
+
## Author
|
|
152
|
+
|
|
153
|
+
Bini Barazany <bgt636@gmail.com>
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { ChatAnthropic } from "@langchain/anthropic";
|
|
2
|
+
import { ChatOpenAI } from "@langchain/openai";
|
|
3
|
+
import { Tool } from "langchain";
|
|
4
|
+
import { type BrowserContextOptions, type LaunchOptions, type Page } from "playwright";
|
|
5
|
+
import { ElementLocatorRegistry } from "../tools/utils/element-registry.util.js";
|
|
6
|
+
import type { AgentTool } from "./AWAgent.types.js";
|
|
7
|
+
export declare class AWAgent {
|
|
8
|
+
private browser;
|
|
9
|
+
private context;
|
|
10
|
+
private pages;
|
|
11
|
+
private currentPageContext;
|
|
12
|
+
private model;
|
|
13
|
+
private agent;
|
|
14
|
+
private systemMessage;
|
|
15
|
+
private elementRegistry;
|
|
16
|
+
private overrideTools;
|
|
17
|
+
private customTools;
|
|
18
|
+
constructor(model: ChatAnthropic | ChatOpenAI, systemMessage: string, options?: {
|
|
19
|
+
customTools?: ((page: Page) => Tool)[];
|
|
20
|
+
overrideTools?: {
|
|
21
|
+
getDOMSnapshotTool?: (page: Page, registry: ElementLocatorRegistry) => AgentTool;
|
|
22
|
+
};
|
|
23
|
+
});
|
|
24
|
+
init(type: "chromium", options?: {
|
|
25
|
+
launchOptions?: LaunchOptions;
|
|
26
|
+
contextOptions?: BrowserContextOptions;
|
|
27
|
+
}): Promise<void>;
|
|
28
|
+
newPage(): Promise<Page>;
|
|
29
|
+
getCurrentPage(): Page;
|
|
30
|
+
getAllPages(): Page[];
|
|
31
|
+
do(task: string): Promise<void>;
|
|
32
|
+
close(): Promise<void>;
|
|
33
|
+
private createTools;
|
|
34
|
+
}
|
|
35
|
+
//# sourceMappingURL=AWAgent.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AWAgent.d.ts","sourceRoot":"","sources":["../../src/AWAgent/AWAgent.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAKL,IAAI,EACL,MAAM,WAAW,CAAC;AACnB,OAAO,EAGL,KAAK,qBAAqB,EAE1B,KAAK,aAAa,EAClB,KAAK,IAAI,EACV,MAAM,YAAY,CAAC;AAQpB,OAAO,EAAE,sBAAsB,EAAE,MAAM,yCAAyC,CAAC;AAEjF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAIpD,qBAAa,OAAO;IAClB,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,OAAO,CAA+B;IAC9C,OAAO,CAAC,KAAK,CAAuB;IACpC,OAAO,CAAC,kBAAkB,CAAqB;IAC/C,OAAO,CAAC,KAAK,CAA6B;IAC1C,OAAO,CAAC,KAAK,CAA2B;IACxC,OAAO,CAAC,aAAa,CAAgB;IACrC,OAAO,CAAC,eAAe,CAAyB;IAChD,OAAO,CAAC,aAAa,CAKnB;IACF,OAAO,CAAC,WAAW,CAAgC;gBAGjD,KAAK,EAAE,aAAa,GAAG,UAAU,EACjC,aAAa,EAAE,MAAM,EACrB,OAAO,CAAC,EAAE;QACR,WAAW,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QACvC,aAAa,CAAC,EAAE;YACd,kBAAkB,CAAC,EAAE,CACnB,IAAI,EAAE,IAAI,EACV,QAAQ,EAAE,sBAAsB,KAC7B,SAAS,CAAC;SAChB,CAAC;KACH;IASG,IAAI,CACR,IAAI,EAAE,UAAU,EAChB,OAAO,CAAC,EAAE;QACR,aAAa,CAAC,EAAE,aAAa,CAAC;QAC9B,cAAc,CAAC,EAAE,qBAAqB,CAAC;KACxC;IAWG,OAAO;IASb,cAAc;IAOd,WAAW;IAQL,EAAE,CAAC,IAAI,EAAE,MAAM;IAuBf,KAAK;IAeX,OAAO,CAAC,WAAW;CAiBpB"}
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { ChatAnthropic } from "@langchain/anthropic";
|
|
2
|
+
import { ChatOpenAI } from "@langchain/openai";
|
|
3
|
+
import { createAgent, HumanMessage, SystemMessage, Tool, } from "langchain";
|
|
4
|
+
import { chromium, } from "playwright";
|
|
5
|
+
import { clickByElementIdTool } from "../tools/click-by-element-id.tool.js";
|
|
6
|
+
import { clickByPositionTool } from "../tools/click-by-position.tool.js";
|
|
7
|
+
import { getDOMSnapshotTool } from "../tools/get-DOM-snapshot.tool.js";
|
|
8
|
+
import { getPageScreenShotTool } from "../tools/get-page-screenshot.tool.js";
|
|
9
|
+
import { inputByElementIdTool } from "../tools/input-by-element-id.tool.js";
|
|
10
|
+
import { navigateTool } from "../tools/navigate.tool.js";
|
|
11
|
+
import { printToConsoleTool } from "../tools/print-to-console.tool.js";
|
|
12
|
+
import { ElementLocatorRegistry } from "../tools/utils/element-registry.util.js";
|
|
13
|
+
import { waitTool } from "../tools/wait.tool.js";
|
|
14
|
+
import { loggingMiddleware } from "./middlewares/logging-calls.middleware.js";
|
|
15
|
+
import { trimMessagesHistoryMiddleware } from "./middlewares/trim-messages-history.middleware.js";
|
|
16
|
+
export class AWAgent {
|
|
17
|
+
browser = null;
|
|
18
|
+
context = null;
|
|
19
|
+
pages = null;
|
|
20
|
+
currentPageContext = null;
|
|
21
|
+
model;
|
|
22
|
+
agent = null;
|
|
23
|
+
systemMessage;
|
|
24
|
+
elementRegistry;
|
|
25
|
+
overrideTools;
|
|
26
|
+
customTools;
|
|
27
|
+
constructor(model, systemMessage, options) {
|
|
28
|
+
this.model = model;
|
|
29
|
+
this.systemMessage = new SystemMessage(systemMessage);
|
|
30
|
+
this.elementRegistry = new ElementLocatorRegistry();
|
|
31
|
+
this.overrideTools = options?.overrideTools ?? {};
|
|
32
|
+
this.customTools = options?.customTools ?? [];
|
|
33
|
+
}
|
|
34
|
+
async init(type, options) {
|
|
35
|
+
switch (type) {
|
|
36
|
+
case "chromium": {
|
|
37
|
+
this.browser = await chromium.launch(options?.launchOptions);
|
|
38
|
+
this.context = await this.browser.newContext(options?.contextOptions);
|
|
39
|
+
this.currentPageContext = await this.context.newPage();
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
async newPage() {
|
|
44
|
+
if (!this.context) {
|
|
45
|
+
throw new Error("Browser not initialized");
|
|
46
|
+
}
|
|
47
|
+
const page = await this.context.newPage();
|
|
48
|
+
this.currentPageContext = page;
|
|
49
|
+
return page;
|
|
50
|
+
}
|
|
51
|
+
getCurrentPage() {
|
|
52
|
+
if (!this.currentPageContext) {
|
|
53
|
+
throw new Error("Agent is not initialized");
|
|
54
|
+
}
|
|
55
|
+
return this.currentPageContext;
|
|
56
|
+
}
|
|
57
|
+
getAllPages() {
|
|
58
|
+
if (!this.context) {
|
|
59
|
+
throw new Error("Browser not initialized");
|
|
60
|
+
}
|
|
61
|
+
this.pages = this.context.pages();
|
|
62
|
+
return this.pages;
|
|
63
|
+
}
|
|
64
|
+
async do(task) {
|
|
65
|
+
if (!this.currentPageContext) {
|
|
66
|
+
throw new Error("Agent is not initialized");
|
|
67
|
+
}
|
|
68
|
+
if (!this.agent) {
|
|
69
|
+
const tools = this.createTools(this.currentPageContext);
|
|
70
|
+
const agent = createAgent({
|
|
71
|
+
model: this.model,
|
|
72
|
+
tools: tools,
|
|
73
|
+
systemPrompt: this.systemMessage,
|
|
74
|
+
middleware: [loggingMiddleware, trimMessagesHistoryMiddleware],
|
|
75
|
+
});
|
|
76
|
+
this.agent = agent;
|
|
77
|
+
}
|
|
78
|
+
const response = await this.agent.invoke({
|
|
79
|
+
messages: [new HumanMessage(task)],
|
|
80
|
+
});
|
|
81
|
+
return console.log(response.messages.at(-1).content);
|
|
82
|
+
}
|
|
83
|
+
async close() {
|
|
84
|
+
if (this.context) {
|
|
85
|
+
for (const page of this.context.pages()) {
|
|
86
|
+
await page.close();
|
|
87
|
+
}
|
|
88
|
+
await this.context.close();
|
|
89
|
+
this.context = null;
|
|
90
|
+
this.currentPageContext = null;
|
|
91
|
+
}
|
|
92
|
+
if (this.browser) {
|
|
93
|
+
await this.browser.close();
|
|
94
|
+
this.browser = null;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
createTools(page) {
|
|
98
|
+
return [
|
|
99
|
+
// Element ID-based tools (recommended for use with accessibility snapshots)
|
|
100
|
+
clickByElementIdTool(page, this.elementRegistry),
|
|
101
|
+
inputByElementIdTool(page, this.elementRegistry),
|
|
102
|
+
// Snapshot tool with element registry integration
|
|
103
|
+
this.overrideTools.getDOMSnapshotTool?.(page, this.elementRegistry) ??
|
|
104
|
+
getDOMSnapshotTool(page, this.elementRegistry),
|
|
105
|
+
// Other utility tools
|
|
106
|
+
waitTool(),
|
|
107
|
+
navigateTool(page),
|
|
108
|
+
getPageScreenShotTool(page),
|
|
109
|
+
clickByPositionTool(page),
|
|
110
|
+
printToConsoleTool(),
|
|
111
|
+
...this.customTools.map((buildTool) => buildTool(page)),
|
|
112
|
+
];
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
//# sourceMappingURL=AWAgent.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AWAgent.js","sourceRoot":"","sources":["../../src/AWAgent/AWAgent.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,WAAW,EACX,YAAY,EAEZ,aAAa,EACb,IAAI,GACL,MAAM,WAAW,CAAC;AACnB,OAAO,EAIL,QAAQ,GAGT,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,oBAAoB,EAAE,MAAM,sCAAsC,CAAC;AAC5E,OAAO,EAAE,mBAAmB,EAAE,MAAM,oCAAoC,CAAC;AACzE,OAAO,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AACvE,OAAO,EAAE,qBAAqB,EAAE,MAAM,sCAAsC,CAAC;AAC7E,OAAO,EAAE,oBAAoB,EAAE,MAAM,sCAAsC,CAAC;AAC5E,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,mCAAmC,CAAC;AACvE,OAAO,EAAE,sBAAsB,EAAE,MAAM,yCAAyC,CAAC;AACjF,OAAO,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAEjD,OAAO,EAAE,iBAAiB,EAAE,MAAM,2CAA2C,CAAC;AAC9E,OAAO,EAAE,6BAA6B,EAAE,MAAM,mDAAmD,CAAC;AAElG,MAAM,OAAO,OAAO;IACV,OAAO,GAAmB,IAAI,CAAC;IAC/B,OAAO,GAA0B,IAAI,CAAC;IACtC,KAAK,GAAkB,IAAI,CAAC;IAC5B,kBAAkB,GAAgB,IAAI,CAAC;IACvC,KAAK,CAA6B;IAClC,KAAK,GAAsB,IAAI,CAAC;IAChC,aAAa,CAAgB;IAC7B,eAAe,CAAyB;IACxC,aAAa,CAKnB;IACM,WAAW,CAAgC;IAEnD,YACE,KAAiC,EACjC,aAAqB,EACrB,OAQC;QAED,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,aAAa,GAAG,IAAI,aAAa,CAAC,aAAa,CAAC,CAAC;QACtD,IAAI,CAAC,eAAe,GAAG,IAAI,sBAAsB,EAAE,CAAC;QACpD,IAAI,CAAC,aAAa,GAAG,OAAO,EAAE,aAAa,IAAI,EAAE,CAAC;QAClD,IAAI,CAAC,WAAW,GAAG,OAAO,EAAE,WAAW,IAAI,EAAE,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,IAAI,CACR,IAAgB,EAChB,OAGC;QAED,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,UAAU,CAAC,CAAC,CAAC;gBAChB,IAAI,CAAC,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;gBAC7D,IAAI,CAAC,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;gBACtE,IAAI,CAAC,kBAAkB,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YACzD,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC7C,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAC1C,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;QAC/B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,cAAc;QACZ,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC9C,CAAC;QACD,OAAO,IAAI,CAAC,kBAAkB,CAAC;IACjC,CAAC;IAED,WAAW;QACT,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC7C,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QAClC,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,EAAE,CAAC,IAAY;QACnB,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC9C,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAChB,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YAExD,MAAM,KAAK,GAAG,WAAW,CAAC;gBACxB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,KAAK,EAAE,KAAK;gBACZ,YAAY,EAAE,IAAI,CAAC,aAAa;gBAChC,UAAU,EAAE,CAAC,iBAAiB,EAAE,6BAA6B,CAAC;aAC/D,CAAC,CAAC;YACH,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACrB,CAAC;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;YACvC,QAAQ,EAAE,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC;SACnC,CAAC,CAAC;QAEH,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,OAAO,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;gBACxC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACrB,CAAC;YACD,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YACpB,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;QACjC,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,IAAU;QAC5B,OAAO;YACL,4EAA4E;YAC5E,oBAAoB,CAAC,IAAI,EAAE,IAAI,CAAC,eAAe,CAAC;YAChD,oBAAoB,CAAC,IAAI,EAAE,IAAI,CAAC,eAAe,CAAC;YAChD,kDAAkD;YAClD,IAAI,CAAC,aAAa,CAAC,kBAAkB,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,eAAe,CAAC;gBACjE,kBAAkB,CAAC,IAAI,EAAE,IAAI,CAAC,eAAe,CAAC;YAChD,sBAAsB;YACtB,QAAQ,EAAE;YACV,YAAY,CAAC,IAAI,CAAC;YAClB,qBAAqB,CAAC,IAAI,CAAC;YAC3B,mBAAmB,CAAC,IAAI,CAAC;YACzB,kBAAkB,EAAE;YACpB,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;SACxD,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AWAgent.types.d.ts","sourceRoot":"","sources":["../../src/AWAgent/AWAgent.types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAEpE,MAAM,MAAM,SAAS,GAAG,UAAU,GAAG,UAAU,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"AWAgent.types.js","sourceRoot":"","sources":["../../src/AWAgent/AWAgent.types.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"logging-calls.middleware.d.ts","sourceRoot":"","sources":["../../../src/AWAgent/middlewares/logging-calls.middleware.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,iBAAiB,gEAe5B,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { createMiddleware } from "langchain";
|
|
2
|
+
export const loggingMiddleware = createMiddleware({
|
|
3
|
+
name: "LoggingMiddleware",
|
|
4
|
+
wrapModelCall: (request, handler) => {
|
|
5
|
+
console.log("Model call request:", {
|
|
6
|
+
messages: request.messages.map((msg) => ({
|
|
7
|
+
name: msg.name,
|
|
8
|
+
type: msg.type,
|
|
9
|
+
content: msg.content.length > 1000
|
|
10
|
+
? `${msg.content.slice(0, 1000)}...`
|
|
11
|
+
: msg.content,
|
|
12
|
+
})),
|
|
13
|
+
});
|
|
14
|
+
return handler(request);
|
|
15
|
+
},
|
|
16
|
+
});
|
|
17
|
+
//# sourceMappingURL=logging-calls.middleware.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"logging-calls.middleware.js","sourceRoot":"","sources":["../../../src/AWAgent/middlewares/logging-calls.middleware.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAE7C,MAAM,CAAC,MAAM,iBAAiB,GAAG,gBAAgB,CAAC;IAChD,IAAI,EAAE,mBAAmB;IACzB,aAAa,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE;QAClC,OAAO,CAAC,GAAG,CAAC,qBAAqB,EAAE;YACjC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;gBACvC,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,OAAO,EACL,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI;oBACvB,CAAC,CAAC,GAAI,GAAG,CAAC,OAAkB,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK;oBAChD,CAAC,CAAC,GAAG,CAAC,OAAO;aAClB,CAAC,CAAC;SACJ,CAAC,CAAC;QACH,OAAO,OAAO,CAAC,OAAO,CAAC,CAAC;IAC1B,CAAC;CACF,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"trim-messages-history.middleware.d.ts","sourceRoot":"","sources":["../../../src/AWAgent/middlewares/trim-messages-history.middleware.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,6BAA6B,gEAQxC,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { createMiddleware } from "langchain";
|
|
2
|
+
export const trimMessagesHistoryMiddleware = createMiddleware({
|
|
3
|
+
name: "TrimMessageHistory",
|
|
4
|
+
wrapModelCall: (request, handler) => {
|
|
5
|
+
return handler({
|
|
6
|
+
...request,
|
|
7
|
+
messages: request.messages.slice(-10),
|
|
8
|
+
});
|
|
9
|
+
},
|
|
10
|
+
});
|
|
11
|
+
//# sourceMappingURL=trim-messages-history.middleware.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"trim-messages-history.middleware.js","sourceRoot":"","sources":["../../../src/AWAgent/middlewares/trim-messages-history.middleware.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAE7C,MAAM,CAAC,MAAM,6BAA6B,GAAG,gBAAgB,CAAC;IAC5D,IAAI,EAAE,oBAAoB;IAC1B,aAAa,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE;QAClC,OAAO,OAAO,CAAC;YACb,GAAG,OAAO;YACV,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;SACtC,CAAC,CAAC;IACL,CAAC;CACF,CAAC,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export { AWAgent } from "./AWAgent/AWAgent.js";
|
|
2
|
+
export { type Page as PlaywrightPage } from "playwright";
|
|
3
|
+
export { tool as createTool } from "langchain";
|
|
4
|
+
export { type AgentTool } from "./AWAgent/AWAgent.types.js";
|
|
5
|
+
export { ElementLocatorRegistry } from "./tools/utils/element-registry.util.js";
|
|
6
|
+
export { type ElementSnapshot, type PageSnapshot, generateAccessibilitySnapshot, } from "./tools/utils/accessibility-snapshot.util.js";
|
|
7
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAE,KAAK,IAAI,IAAI,cAAc,EAAE,MAAM,YAAY,CAAC;AACzD,OAAO,EAAE,IAAI,IAAI,UAAU,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,EAAE,KAAK,SAAS,EAAE,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,sBAAsB,EAAE,MAAM,wCAAwC,CAAC;AAChF,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,YAAY,EACjB,6BAA6B,GAC9B,MAAM,8CAA8C,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export { AWAgent } from "./AWAgent/AWAgent.js";
|
|
2
|
+
export {} from "playwright";
|
|
3
|
+
export { tool as createTool } from "langchain";
|
|
4
|
+
export {} from "./AWAgent/AWAgent.types.js";
|
|
5
|
+
export { ElementLocatorRegistry } from "./tools/utils/element-registry.util.js";
|
|
6
|
+
export { generateAccessibilitySnapshot, } from "./tools/utils/accessibility-snapshot.util.js";
|
|
7
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAA+B,MAAM,YAAY,CAAC;AACzD,OAAO,EAAE,IAAI,IAAI,UAAU,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,EAAkB,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAE,sBAAsB,EAAE,MAAM,wCAAwC,CAAC;AAChF,OAAO,EAGL,6BAA6B,GAC9B,MAAM,8CAA8C,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { type Page } from "playwright";
|
|
2
|
+
import z from "zod";
|
|
3
|
+
import { type ElementLocatorRegistry } from "./utils/element-registry.util.js";
|
|
4
|
+
export declare const ClickByElementIdToolSchema: z.ZodObject<{
|
|
5
|
+
elementId: z.ZodString;
|
|
6
|
+
}, z.z.core.$strip>;
|
|
7
|
+
export declare function clickByElementIdTool(_page: Page, elementRegistry: ElementLocatorRegistry): import("langchain").DynamicStructuredTool<z.ZodObject<{
|
|
8
|
+
elementId: z.ZodString;
|
|
9
|
+
}, z.z.core.$strip>, {
|
|
10
|
+
elementId: string;
|
|
11
|
+
}, {
|
|
12
|
+
elementId: string;
|
|
13
|
+
}, string>;
|
|
14
|
+
//# sourceMappingURL=click-by-element-id.tool.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"click-by-element-id.tool.d.ts","sourceRoot":"","sources":["../../src/tools/click-by-element-id.tool.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,IAAI,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,CAAC,MAAM,KAAK,CAAC;AACpB,OAAO,EAAE,KAAK,sBAAsB,EAAE,MAAM,kCAAkC,CAAC;AAE/E,eAAO,MAAM,0BAA0B;;mBAIrC,CAAC;AAEH,wBAAgB,oBAAoB,CAClC,KAAK,EAAE,IAAI,EACX,eAAe,EAAE,sBAAsB;;;;;;WA8BxC"}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { tool } from "langchain";
|
|
2
|
+
import {} from "playwright";
|
|
3
|
+
import z from "zod";
|
|
4
|
+
import {} from "./utils/element-registry.util.js";
|
|
5
|
+
export const ClickByElementIdToolSchema = z.object({
|
|
6
|
+
elementId: z
|
|
7
|
+
.string()
|
|
8
|
+
.describe("The element ID from the accessibility snapshot to click on"),
|
|
9
|
+
});
|
|
10
|
+
export function clickByElementIdTool(_page, elementRegistry) {
|
|
11
|
+
return tool(async ({ elementId }) => {
|
|
12
|
+
console.log(`Clicking on element: ${elementId}`);
|
|
13
|
+
const locator = elementRegistry.get(elementId);
|
|
14
|
+
if (!locator) {
|
|
15
|
+
throw new Error(`Element ID "${elementId}" not found in registry. Please take a fresh DOM snapshot first using GetDOMSnapshot tool.`);
|
|
16
|
+
}
|
|
17
|
+
try {
|
|
18
|
+
await locator.click({ timeout: 5000 });
|
|
19
|
+
return `Successfully clicked on element ${elementId}`;
|
|
20
|
+
}
|
|
21
|
+
catch (error) {
|
|
22
|
+
throw new Error(`Failed to click on element ${elementId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
23
|
+
}
|
|
24
|
+
}, {
|
|
25
|
+
name: "ClickByElementId",
|
|
26
|
+
description: `Click on an element using its ID from the accessibility snapshot.
|
|
27
|
+
The element ID should come from the most recent GetDOMSnapshot result.
|
|
28
|
+
This is the recommended way to interact with page elements instead of using CSS selectors.`,
|
|
29
|
+
schema: ClickByElementIdToolSchema,
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
//# sourceMappingURL=click-by-element-id.tool.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"click-by-element-id.tool.js","sourceRoot":"","sources":["../../src/tools/click-by-element-id.tool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAa,MAAM,YAAY,CAAC;AACvC,OAAO,CAAC,MAAM,KAAK,CAAC;AACpB,OAAO,EAA+B,MAAM,kCAAkC,CAAC;AAE/E,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,MAAM,CAAC;IACjD,SAAS,EAAE,CAAC;SACT,MAAM,EAAE;SACR,QAAQ,CAAC,4DAA4D,CAAC;CAC1E,CAAC,CAAC;AAEH,MAAM,UAAU,oBAAoB,CAClC,KAAW,EACX,eAAuC;IAEvC,OAAO,IAAI,CACT,KAAK,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE;QACtB,OAAO,CAAC,GAAG,CAAC,wBAAwB,SAAS,EAAE,CAAC,CAAC;QAEjD,MAAM,OAAO,GAAG,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC/C,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CACb,eAAe,SAAS,4FAA4F,CACrH,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YACvC,OAAO,mCAAmC,SAAS,EAAE,CAAC;QACxD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,8BAA8B,SAAS,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACrG,CAAC;QACJ,CAAC;IACH,CAAC,EACD;QACE,IAAI,EAAE,kBAAkB;QACxB,WAAW,EAAE;;2FAEwE;QACrF,MAAM,EAAE,0BAA0B;KACnC,CACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { type Page } from "playwright";
|
|
2
|
+
import z from "zod";
|
|
3
|
+
export declare const ClickByPositionToolSchema: z.ZodObject<{
|
|
4
|
+
x: z.ZodNumber;
|
|
5
|
+
y: z.ZodNumber;
|
|
6
|
+
}, z.z.core.$strip>;
|
|
7
|
+
export declare function clickByPositionTool(page: Page): import("langchain").DynamicStructuredTool<z.ZodObject<{
|
|
8
|
+
x: z.ZodNumber;
|
|
9
|
+
y: z.ZodNumber;
|
|
10
|
+
}, z.z.core.$strip>, {
|
|
11
|
+
x: number;
|
|
12
|
+
y: number;
|
|
13
|
+
}, {
|
|
14
|
+
x: number;
|
|
15
|
+
y: number;
|
|
16
|
+
}, void>;
|
|
17
|
+
//# sourceMappingURL=click-by-position.tool.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"click-by-position.tool.d.ts","sourceRoot":"","sources":["../../src/tools/click-by-position.tool.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,IAAI,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,CAAC,MAAM,KAAK,CAAC;AAEpB,eAAO,MAAM,yBAAyB;;;mBAGpC,CAAC;AAEH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,IAAI;;;;;;;;;SAa7C"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { tool } from "langchain";
|
|
2
|
+
import {} from "playwright";
|
|
3
|
+
import z from "zod";
|
|
4
|
+
export const ClickByPositionToolSchema = z.object({
|
|
5
|
+
x: z.number().describe("The x coordinate to click on"),
|
|
6
|
+
y: z.number().describe("The y coordinate to click on"),
|
|
7
|
+
});
|
|
8
|
+
export function clickByPositionTool(page) {
|
|
9
|
+
return tool(async ({ x, y }) => {
|
|
10
|
+
console.log(`Clicking at position: (${x}, ${y})`);
|
|
11
|
+
return await page.mouse.click(x, y);
|
|
12
|
+
}, {
|
|
13
|
+
name: "ClickByPosition",
|
|
14
|
+
description: "Click on a position with the given x and y coordinates. use this tool when you have a screen shot page from the GetPageScreenShot tool and you want to click on a specific position on the page.",
|
|
15
|
+
schema: ClickByPositionToolSchema,
|
|
16
|
+
});
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=click-by-position.tool.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"click-by-position.tool.js","sourceRoot":"","sources":["../../src/tools/click-by-position.tool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAa,MAAM,YAAY,CAAC;AACvC,OAAO,CAAC,MAAM,KAAK,CAAC;AAEpB,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChD,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,8BAA8B,CAAC;IACtD,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,8BAA8B,CAAC;CACvD,CAAC,CAAC;AAEH,MAAM,UAAU,mBAAmB,CAAC,IAAU;IAC5C,OAAO,IAAI,CACT,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE;QACjB,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAClD,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACtC,CAAC,EACD;QACE,IAAI,EAAE,iBAAiB;QACvB,WAAW,EACT,kMAAkM;QACpM,MAAM,EAAE,yBAAyB;KAClC,CACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { type Page } from "playwright";
|
|
2
|
+
import z from "zod";
|
|
3
|
+
export declare const ClickBySelectorToolSchema: z.ZodObject<{
|
|
4
|
+
selector: z.ZodString;
|
|
5
|
+
}, z.z.core.$strip>;
|
|
6
|
+
export declare function clickBySelectorTool(page: Page): import("langchain").DynamicStructuredTool<z.ZodObject<{
|
|
7
|
+
selector: z.ZodString;
|
|
8
|
+
}, z.z.core.$strip>, {
|
|
9
|
+
selector: string;
|
|
10
|
+
}, {
|
|
11
|
+
selector: string;
|
|
12
|
+
}, void>;
|
|
13
|
+
//# sourceMappingURL=click-by-selector.tool.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"click-by-selector.tool.d.ts","sourceRoot":"","sources":["../../src/tools/click-by-selector.tool.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,IAAI,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,CAAC,MAAM,KAAK,CAAC;AAEpB,eAAO,MAAM,yBAAyB;;mBAEpC,CAAC;AAEH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,IAAI;;;;;;SAY7C"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { tool } from "langchain";
|
|
2
|
+
import {} from "playwright";
|
|
3
|
+
import z from "zod";
|
|
4
|
+
export const ClickBySelectorToolSchema = z.object({
|
|
5
|
+
selector: z.string().describe("The CSS selector of the element to click on"),
|
|
6
|
+
});
|
|
7
|
+
export function clickBySelectorTool(page) {
|
|
8
|
+
return tool(async ({ selector }) => {
|
|
9
|
+
console.log(`Clicking on (first) selector: ${selector}`);
|
|
10
|
+
return page.locator(selector).first().click({ timeout: 1000 });
|
|
11
|
+
}, {
|
|
12
|
+
name: "ClickBySelector",
|
|
13
|
+
description: "Click on an element with the given CSS selector",
|
|
14
|
+
schema: ClickBySelectorToolSchema,
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
//# sourceMappingURL=click-by-selector.tool.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"click-by-selector.tool.js","sourceRoot":"","sources":["../../src/tools/click-by-selector.tool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAa,MAAM,YAAY,CAAC;AACvC,OAAO,CAAC,MAAM,KAAK,CAAC;AAEpB,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChD,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,6CAA6C,CAAC;CAC7E,CAAC,CAAC;AAEH,MAAM,UAAU,mBAAmB,CAAC,IAAU;IAC5C,OAAO,IAAI,CACT,KAAK,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE;QACrB,OAAO,CAAC,GAAG,CAAC,iCAAiC,QAAQ,EAAE,CAAC,CAAC;QACzD,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;IACjE,CAAC,EACD;QACE,IAAI,EAAE,iBAAiB;QACvB,WAAW,EAAE,iDAAiD;QAC9D,MAAM,EAAE,yBAAyB;KAClC,CACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { type Page } from "playwright";
|
|
2
|
+
import { type ElementLocatorRegistry } from "./utils/element-registry.util.js";
|
|
3
|
+
export declare function getDOMSnapshotTool(page: Page, elementRegistry: ElementLocatorRegistry): import("langchain").DynamicTool<string>;
|
|
4
|
+
//# sourceMappingURL=get-DOM-snapshot.tool.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"get-DOM-snapshot.tool.d.ts","sourceRoot":"","sources":["../../src/tools/get-DOM-snapshot.tool.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,IAAI,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,EAAE,KAAK,sBAAsB,EAAE,MAAM,kCAAkC,CAAC;AAE/E,wBAAgB,kBAAkB,CAChC,IAAI,EAAE,IAAI,EACV,eAAe,EAAE,sBAAsB,2CA+BxC"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { tool } from "langchain";
|
|
2
|
+
import {} from "playwright";
|
|
3
|
+
import { generateAccessibilitySnapshot } from "./utils/accessibility-snapshot.util.js";
|
|
4
|
+
import {} from "./utils/element-registry.util.js";
|
|
5
|
+
export function getDOMSnapshotTool(page, elementRegistry) {
|
|
6
|
+
return tool(async () => {
|
|
7
|
+
console.log("Taking accessibility-based DOM snapshot...");
|
|
8
|
+
// Generate accessibility snapshot with visual metadata
|
|
9
|
+
const snapshot = await generateAccessibilitySnapshot(page, elementRegistry.getMap());
|
|
10
|
+
// Convert to JSON string for the model
|
|
11
|
+
const snapshotJson = JSON.stringify(snapshot, null, 2);
|
|
12
|
+
const snapshotSize = new Blob([snapshotJson]).size;
|
|
13
|
+
console.log(`Accessibility snapshot generated:`);
|
|
14
|
+
console.log(` - Elements found: ${snapshot.elements.length}`);
|
|
15
|
+
console.log(` - Snapshot size: ${snapshotSize} bytes`);
|
|
16
|
+
console.log(` - URL: ${snapshot.url}`);
|
|
17
|
+
return snapshotJson;
|
|
18
|
+
}, {
|
|
19
|
+
name: "GetDOMSnapshot",
|
|
20
|
+
description: `Get an accessibility-based snapshot of the current webpage.
|
|
21
|
+
Returns a structured JSON representation with interactive elements, their roles, names, and visual properties.
|
|
22
|
+
Each element has a unique 'id' that can be used with action tools (ClickByElementId, InputByElementId).
|
|
23
|
+
The snapshot includes only visible, interactive elements like buttons, links, text inputs, etc.`,
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
//# sourceMappingURL=get-DOM-snapshot.tool.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"get-DOM-snapshot.tool.js","sourceRoot":"","sources":["../../src/tools/get-DOM-snapshot.tool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAa,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,6BAA6B,EAAE,MAAM,wCAAwC,CAAC;AACvF,OAAO,EAA+B,MAAM,kCAAkC,CAAC;AAE/E,MAAM,UAAU,kBAAkB,CAChC,IAAU,EACV,eAAuC;IAEvC,OAAO,IAAI,CACT,KAAK,IAAI,EAAE;QACT,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;QAE1D,uDAAuD;QACvD,MAAM,QAAQ,GAAG,MAAM,6BAA6B,CAClD,IAAI,EACJ,eAAe,CAAC,MAAM,EAAE,CACzB,CAAC;QAEF,uCAAuC;QACvC,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QACvD,MAAM,YAAY,GAAG,IAAI,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC;QAEnD,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;QACjD,OAAO,CAAC,GAAG,CAAC,uBAAuB,QAAQ,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC/D,OAAO,CAAC,GAAG,CAAC,sBAAsB,YAAY,QAAQ,CAAC,CAAC;QACxD,OAAO,CAAC,GAAG,CAAC,YAAY,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC;QAExC,OAAO,YAAY,CAAC;IACtB,CAAC,EACD;QACE,IAAI,EAAE,gBAAgB;QACtB,WAAW,EAAE;;;gGAG6E;KAC3F,CACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"get-page-screenshot.tool.d.ts","sourceRoot":"","sources":["../../src/tools/get-page-screenshot.tool.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,IAAI,EAAE,MAAM,YAAY,CAAC;AAEvC,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,IAAI,4DAa/C"}
|