@mastra/agent-browser 0.0.0-async-hooks-fix-20260405144639
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +74 -0
- package/LICENSE.md +30 -0
- package/dist/index.cjs +1577 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +588 -0
- package/dist/index.d.ts +588 -0
- package/dist/index.js +1557 -0
- package/dist/index.js.map +1 -0
- package/package.json +66 -0
package/dist/index.cjs
ADDED
|
@@ -0,0 +1,1577 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var browser = require('@mastra/core/browser');
|
|
4
|
+
var agentBrowser = require('agent-browser');
|
|
5
|
+
var tools = require('@mastra/core/tools');
|
|
6
|
+
var zod = require('zod');
|
|
7
|
+
|
|
8
|
+
// src/agent-browser.ts
|
|
9
|
+
var AgentBrowserThreadManager = class extends browser.ThreadManager {
|
|
10
|
+
browserConfig;
|
|
11
|
+
resolveCdpUrl;
|
|
12
|
+
onBrowserCreated;
|
|
13
|
+
constructor(config) {
|
|
14
|
+
super(config);
|
|
15
|
+
this.browserConfig = config.browserConfig;
|
|
16
|
+
this.resolveCdpUrl = config.resolveCdpUrl;
|
|
17
|
+
this.onBrowserCreated = config.onBrowserCreated;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Get the page for a specific thread, creating session if needed.
|
|
21
|
+
*/
|
|
22
|
+
async getPageForThread(threadId) {
|
|
23
|
+
const manager = await this.getManagerForThread(threadId);
|
|
24
|
+
return manager.getPage();
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Create a new session for a thread.
|
|
28
|
+
*/
|
|
29
|
+
async createSession(threadId) {
|
|
30
|
+
const savedState = this.getSavedBrowserState(threadId);
|
|
31
|
+
const session = {
|
|
32
|
+
threadId,
|
|
33
|
+
createdAt: Date.now(),
|
|
34
|
+
browserState: savedState
|
|
35
|
+
};
|
|
36
|
+
if (this.scope === "thread") {
|
|
37
|
+
const manager = new agentBrowser.BrowserManager();
|
|
38
|
+
const launchOptions = {
|
|
39
|
+
headless: this.browserConfig.headless ?? true,
|
|
40
|
+
viewport: this.browserConfig.viewport
|
|
41
|
+
};
|
|
42
|
+
if (this.browserConfig.cdpUrl && this.resolveCdpUrl) {
|
|
43
|
+
launchOptions.cdpUrl = await this.resolveCdpUrl(this.browserConfig.cdpUrl);
|
|
44
|
+
}
|
|
45
|
+
try {
|
|
46
|
+
await manager.launch(launchOptions);
|
|
47
|
+
} catch (error) {
|
|
48
|
+
try {
|
|
49
|
+
await manager.close();
|
|
50
|
+
} catch {
|
|
51
|
+
}
|
|
52
|
+
throw error;
|
|
53
|
+
}
|
|
54
|
+
session.manager = manager;
|
|
55
|
+
this.threadManagers.set(threadId, manager);
|
|
56
|
+
try {
|
|
57
|
+
if (savedState && savedState.tabs.length > 0) {
|
|
58
|
+
this.logger?.debug?.(`Restoring browser state for thread ${threadId}: ${savedState.tabs.length} tabs`);
|
|
59
|
+
await this.restoreBrowserState(manager, savedState);
|
|
60
|
+
}
|
|
61
|
+
this.onBrowserCreated?.(manager, threadId);
|
|
62
|
+
} catch (error) {
|
|
63
|
+
this.threadManagers.delete(threadId);
|
|
64
|
+
session.manager = void 0;
|
|
65
|
+
try {
|
|
66
|
+
await manager.close();
|
|
67
|
+
} catch {
|
|
68
|
+
}
|
|
69
|
+
throw error;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return session;
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Restore browser state (multiple tabs) to a browser manager.
|
|
76
|
+
*/
|
|
77
|
+
async restoreBrowserState(manager, state) {
|
|
78
|
+
try {
|
|
79
|
+
const firstTab = state.tabs[0];
|
|
80
|
+
if (firstTab?.url) {
|
|
81
|
+
const page = manager.getPage();
|
|
82
|
+
if (page) {
|
|
83
|
+
await page.goto(firstTab.url, { waitUntil: "domcontentloaded" });
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
for (let i = 1; i < state.tabs.length; i++) {
|
|
87
|
+
const tab = state.tabs[i];
|
|
88
|
+
if (tab?.url) {
|
|
89
|
+
await manager.newTab();
|
|
90
|
+
const page = manager.getPage();
|
|
91
|
+
if (page) {
|
|
92
|
+
await page.goto(tab.url, { waitUntil: "domcontentloaded" });
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
if (state.tabs.length > 1 && state.activeTabIndex >= 0 && state.activeTabIndex < state.tabs.length) {
|
|
97
|
+
await manager.switchTo(state.activeTabIndex);
|
|
98
|
+
}
|
|
99
|
+
} catch (error) {
|
|
100
|
+
this.logger?.warn?.(`Failed to restore browser state: ${error}`);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Get the browser manager for a specific session.
|
|
105
|
+
*/
|
|
106
|
+
getManagerForSession(session) {
|
|
107
|
+
if (this.scope === "thread" && session.manager) {
|
|
108
|
+
return session.manager;
|
|
109
|
+
}
|
|
110
|
+
return this.getSharedManager();
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Destroy a session and clean up resources.
|
|
114
|
+
*/
|
|
115
|
+
async doDestroySession(session) {
|
|
116
|
+
if (this.scope === "thread" && session.manager) {
|
|
117
|
+
await session.manager.close();
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Destroy all sessions (called during browser close).
|
|
122
|
+
* doDestroySession handles closing individual browser managers.
|
|
123
|
+
*/
|
|
124
|
+
async destroyAllSessions() {
|
|
125
|
+
await super.destroyAllSessions();
|
|
126
|
+
}
|
|
127
|
+
};
|
|
128
|
+
var gotoInputSchema = zod.z.object({
|
|
129
|
+
url: zod.z.string().describe("The URL to navigate to"),
|
|
130
|
+
waitUntil: zod.z.enum(["load", "domcontentloaded", "networkidle"]).optional().describe("When to consider navigation complete (default: domcontentloaded)"),
|
|
131
|
+
timeout: zod.z.number().optional().describe("Navigation timeout in milliseconds")
|
|
132
|
+
});
|
|
133
|
+
var snapshotInputSchema = zod.z.object({
|
|
134
|
+
interactiveOnly: zod.z.boolean().optional().describe("Only include interactive elements (default: true)"),
|
|
135
|
+
maxDepth: zod.z.number().optional().describe("Maximum depth of the tree to return")
|
|
136
|
+
});
|
|
137
|
+
var clickInputSchema = zod.z.object({
|
|
138
|
+
ref: zod.z.string().describe("Element ref from snapshot (e.g., @e5)"),
|
|
139
|
+
button: zod.z.enum(["left", "right", "middle"]).optional().describe("Mouse button (default: left)"),
|
|
140
|
+
clickCount: zod.z.number().optional().describe("Number of clicks (default: 1, use 2 for double-click)"),
|
|
141
|
+
modifiers: zod.z.array(zod.z.enum(["Alt", "Control", "Meta", "Shift"])).optional().describe("Modifier keys to hold")
|
|
142
|
+
});
|
|
143
|
+
var typeInputSchema = zod.z.object({
|
|
144
|
+
ref: zod.z.string().describe("Element ref from snapshot"),
|
|
145
|
+
text: zod.z.string().describe("Text to type"),
|
|
146
|
+
clear: zod.z.boolean().optional().describe("Clear existing content before typing (default: false)"),
|
|
147
|
+
delay: zod.z.number().optional().describe("Delay between keystrokes in ms")
|
|
148
|
+
});
|
|
149
|
+
var pressInputSchema = zod.z.object({
|
|
150
|
+
key: zod.z.string().describe("Key to press (e.g., Enter, Tab, Escape, Control+a)"),
|
|
151
|
+
modifiers: zod.z.array(zod.z.enum(["Alt", "Control", "Meta", "Shift"])).optional().describe("Modifier keys to hold")
|
|
152
|
+
});
|
|
153
|
+
var selectInputSchema = zod.z.object({
|
|
154
|
+
ref: zod.z.string().describe("Select element ref from snapshot"),
|
|
155
|
+
value: zod.z.string().optional().describe("Option value to select"),
|
|
156
|
+
label: zod.z.string().optional().describe("Option label to select"),
|
|
157
|
+
index: zod.z.number().int().min(0).optional().describe("Option index to select (0-based)")
|
|
158
|
+
}).superRefine((data, ctx) => {
|
|
159
|
+
if (data.value === void 0 && data.label === void 0 && data.index === void 0) {
|
|
160
|
+
ctx.addIssue({
|
|
161
|
+
code: zod.z.ZodIssueCode.custom,
|
|
162
|
+
message: "At least one of value, label, or index is required"
|
|
163
|
+
});
|
|
164
|
+
}
|
|
165
|
+
});
|
|
166
|
+
var scrollInputSchema = zod.z.object({
|
|
167
|
+
direction: zod.z.enum(["up", "down", "left", "right"]).describe("Scroll direction"),
|
|
168
|
+
amount: zod.z.number().optional().describe("Scroll amount in pixels (default: 300)"),
|
|
169
|
+
ref: zod.z.string().optional().describe("Element ref to scroll (scrolls page if omitted)")
|
|
170
|
+
});
|
|
171
|
+
var closeInputSchema = zod.z.object({});
|
|
172
|
+
var hoverInputSchema = zod.z.object({
|
|
173
|
+
ref: zod.z.string().describe("Element ref from snapshot")
|
|
174
|
+
});
|
|
175
|
+
var backInputSchema = zod.z.object({});
|
|
176
|
+
var dialogInputSchema = zod.z.object({
|
|
177
|
+
triggerRef: zod.z.string().describe("Element ref that triggers the dialog (e.g., @e5)"),
|
|
178
|
+
action: zod.z.enum(["accept", "dismiss"]).describe("Accept or dismiss the dialog"),
|
|
179
|
+
text: zod.z.string().optional().describe("Text to enter for prompt dialogs")
|
|
180
|
+
});
|
|
181
|
+
var waitInputSchema = zod.z.object({
|
|
182
|
+
ref: zod.z.string().optional().describe("Element ref to wait for"),
|
|
183
|
+
state: zod.z.enum(["visible", "hidden", "attached", "detached"]).optional().describe("State to wait for (default: visible)"),
|
|
184
|
+
timeout: zod.z.number().optional().describe("Maximum wait time in ms (default: 30000)")
|
|
185
|
+
});
|
|
186
|
+
var tabsInputSchema = zod.z.object({
|
|
187
|
+
action: zod.z.enum(["list", "new", "switch", "close"]).describe("Tab action"),
|
|
188
|
+
index: zod.z.number().int().min(0).optional().describe("Tab index for switch/close"),
|
|
189
|
+
url: zod.z.string().optional().describe("URL to open in new tab")
|
|
190
|
+
}).superRefine((value, ctx) => {
|
|
191
|
+
if (value.action === "switch" && value.index === void 0) {
|
|
192
|
+
ctx.addIssue({
|
|
193
|
+
code: zod.z.ZodIssueCode.custom,
|
|
194
|
+
path: ["index"],
|
|
195
|
+
message: 'index is required when action is "switch"'
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
});
|
|
199
|
+
var dragInputSchema = zod.z.object({
|
|
200
|
+
sourceRef: zod.z.string().optional().describe("Element ref to drag from (e.g., @e5)"),
|
|
201
|
+
targetRef: zod.z.string().optional().describe("Element ref to drag to (e.g., @e7)"),
|
|
202
|
+
sourceSelector: zod.z.string().optional().describe("CSS selector for source element (use if ref not available)"),
|
|
203
|
+
targetSelector: zod.z.string().optional().describe("CSS selector for target element (use if ref not available)")
|
|
204
|
+
}).superRefine((data, ctx) => {
|
|
205
|
+
if (!data.sourceRef && !data.sourceSelector) {
|
|
206
|
+
ctx.addIssue({
|
|
207
|
+
code: zod.z.ZodIssueCode.custom,
|
|
208
|
+
path: ["sourceRef"],
|
|
209
|
+
message: "Either sourceRef or sourceSelector is required"
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
if (!data.targetRef && !data.targetSelector) {
|
|
213
|
+
ctx.addIssue({
|
|
214
|
+
code: zod.z.ZodIssueCode.custom,
|
|
215
|
+
path: ["targetRef"],
|
|
216
|
+
message: "Either targetRef or targetSelector is required"
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
});
|
|
220
|
+
var evaluateInputSchema = zod.z.object({
|
|
221
|
+
script: zod.z.string().describe("JavaScript code to execute"),
|
|
222
|
+
arg: zod.z.unknown().optional().describe("Argument to pass to the script (JSON-serializable)")
|
|
223
|
+
});
|
|
224
|
+
var browserSchemas = {
|
|
225
|
+
// Core
|
|
226
|
+
goto: gotoInputSchema,
|
|
227
|
+
snapshot: snapshotInputSchema,
|
|
228
|
+
click: clickInputSchema,
|
|
229
|
+
type: typeInputSchema,
|
|
230
|
+
press: pressInputSchema,
|
|
231
|
+
select: selectInputSchema,
|
|
232
|
+
scroll: scrollInputSchema,
|
|
233
|
+
close: closeInputSchema,
|
|
234
|
+
// Extended
|
|
235
|
+
hover: hoverInputSchema,
|
|
236
|
+
back: backInputSchema,
|
|
237
|
+
dialog: dialogInputSchema,
|
|
238
|
+
wait: waitInputSchema,
|
|
239
|
+
tabs: tabsInputSchema,
|
|
240
|
+
drag: dragInputSchema,
|
|
241
|
+
// Escape hatch
|
|
242
|
+
evaluate: evaluateInputSchema
|
|
243
|
+
};
|
|
244
|
+
|
|
245
|
+
// src/tools/constants.ts
|
|
246
|
+
var BROWSER_TOOLS = {
|
|
247
|
+
// Core
|
|
248
|
+
GOTO: "browser_goto",
|
|
249
|
+
SNAPSHOT: "browser_snapshot",
|
|
250
|
+
CLICK: "browser_click",
|
|
251
|
+
TYPE: "browser_type",
|
|
252
|
+
PRESS: "browser_press",
|
|
253
|
+
SELECT: "browser_select",
|
|
254
|
+
SCROLL: "browser_scroll",
|
|
255
|
+
CLOSE: "browser_close",
|
|
256
|
+
// Extended
|
|
257
|
+
HOVER: "browser_hover",
|
|
258
|
+
BACK: "browser_back",
|
|
259
|
+
DIALOG: "browser_dialog",
|
|
260
|
+
WAIT: "browser_wait",
|
|
261
|
+
TABS: "browser_tabs",
|
|
262
|
+
DRAG: "browser_drag",
|
|
263
|
+
// Escape hatch
|
|
264
|
+
EVALUATE: "browser_evaluate"
|
|
265
|
+
};
|
|
266
|
+
|
|
267
|
+
// src/tools/back.ts
|
|
268
|
+
function createBackTool(browser) {
|
|
269
|
+
return tools.createTool({
|
|
270
|
+
id: BROWSER_TOOLS.BACK,
|
|
271
|
+
description: "Go back to the previous page in browser history.",
|
|
272
|
+
inputSchema: backInputSchema,
|
|
273
|
+
execute: async (_input, { agent }) => {
|
|
274
|
+
const threadId = agent?.threadId;
|
|
275
|
+
browser.setCurrentThread(threadId);
|
|
276
|
+
await browser.ensureReady();
|
|
277
|
+
return browser.back(threadId);
|
|
278
|
+
}
|
|
279
|
+
});
|
|
280
|
+
}
|
|
281
|
+
function createClickTool(browser) {
|
|
282
|
+
return tools.createTool({
|
|
283
|
+
id: BROWSER_TOOLS.CLICK,
|
|
284
|
+
description: "Click an element using its ref from a snapshot. Use clickCount: 2 for double-click.",
|
|
285
|
+
inputSchema: clickInputSchema,
|
|
286
|
+
execute: async (input, { agent }) => {
|
|
287
|
+
const threadId = agent?.threadId;
|
|
288
|
+
browser.setCurrentThread(threadId);
|
|
289
|
+
await browser.ensureReady();
|
|
290
|
+
return browser.click(input, threadId);
|
|
291
|
+
}
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
function createCloseTool(browser) {
|
|
295
|
+
return tools.createTool({
|
|
296
|
+
id: BROWSER_TOOLS.CLOSE,
|
|
297
|
+
description: "Close the browser. Only use when done with all browsing.",
|
|
298
|
+
inputSchema: closeInputSchema,
|
|
299
|
+
execute: async (_input, { agent }) => {
|
|
300
|
+
const threadId = agent?.threadId;
|
|
301
|
+
if (browser.getScope() !== "shared") {
|
|
302
|
+
if (!threadId) {
|
|
303
|
+
throw new Error("browser_close requires agent.threadId when browser scope is not shared");
|
|
304
|
+
}
|
|
305
|
+
await browser.closeThreadSession(threadId);
|
|
306
|
+
return { success: true, hint: "Thread's browser session closed. A new session will be created on next use." };
|
|
307
|
+
}
|
|
308
|
+
await browser.close();
|
|
309
|
+
return { success: true, hint: "Browser closed. It will be re-launched automatically on next use." };
|
|
310
|
+
}
|
|
311
|
+
});
|
|
312
|
+
}
|
|
313
|
+
function createDialogTool(browser) {
|
|
314
|
+
return tools.createTool({
|
|
315
|
+
id: BROWSER_TOOLS.DIALOG,
|
|
316
|
+
description: "Click an element that triggers a browser dialog (alert, confirm, prompt) and handle it. Use this instead of browser_click when you expect a dialog to appear.",
|
|
317
|
+
inputSchema: dialogInputSchema,
|
|
318
|
+
execute: async (input, { agent }) => {
|
|
319
|
+
const threadId = agent?.threadId;
|
|
320
|
+
browser.setCurrentThread(threadId);
|
|
321
|
+
await browser.ensureReady();
|
|
322
|
+
return browser.dialog(input, threadId);
|
|
323
|
+
}
|
|
324
|
+
});
|
|
325
|
+
}
|
|
326
|
+
function createDragTool(browser) {
|
|
327
|
+
return tools.createTool({
|
|
328
|
+
id: BROWSER_TOOLS.DRAG,
|
|
329
|
+
description: "Drag an element to another element. Use refs from snapshot when available, or CSS selectors for elements not exposed in the accessibility tree.",
|
|
330
|
+
inputSchema: dragInputSchema,
|
|
331
|
+
execute: async (input, { agent }) => {
|
|
332
|
+
const threadId = agent?.threadId;
|
|
333
|
+
browser.setCurrentThread(threadId);
|
|
334
|
+
await browser.ensureReady();
|
|
335
|
+
return browser.drag(input, threadId);
|
|
336
|
+
}
|
|
337
|
+
});
|
|
338
|
+
}
|
|
339
|
+
function createEvaluateTool(browser) {
|
|
340
|
+
return tools.createTool({
|
|
341
|
+
id: BROWSER_TOOLS.EVALUATE,
|
|
342
|
+
description: "Execute JavaScript in the browser. Use for complex interactions not covered by other tools. Returns the script result.",
|
|
343
|
+
inputSchema: evaluateInputSchema,
|
|
344
|
+
execute: async (input, { agent }) => {
|
|
345
|
+
const threadId = agent?.threadId;
|
|
346
|
+
browser.setCurrentThread(threadId);
|
|
347
|
+
await browser.ensureReady();
|
|
348
|
+
return browser.evaluate(input, threadId);
|
|
349
|
+
}
|
|
350
|
+
});
|
|
351
|
+
}
|
|
352
|
+
function createGotoTool(browser) {
|
|
353
|
+
return tools.createTool({
|
|
354
|
+
id: BROWSER_TOOLS.GOTO,
|
|
355
|
+
description: "Navigate the browser to a URL.",
|
|
356
|
+
inputSchema: gotoInputSchema,
|
|
357
|
+
execute: async (input, { agent }) => {
|
|
358
|
+
const threadId = agent?.threadId;
|
|
359
|
+
browser.setCurrentThread(threadId);
|
|
360
|
+
await browser.ensureReady();
|
|
361
|
+
return browser.goto(input, threadId);
|
|
362
|
+
}
|
|
363
|
+
});
|
|
364
|
+
}
|
|
365
|
+
function createHoverTool(browser) {
|
|
366
|
+
return tools.createTool({
|
|
367
|
+
id: BROWSER_TOOLS.HOVER,
|
|
368
|
+
description: "Hover over an element to trigger hover states (dropdowns, tooltips).",
|
|
369
|
+
inputSchema: hoverInputSchema,
|
|
370
|
+
execute: async (input, { agent }) => {
|
|
371
|
+
const threadId = agent?.threadId;
|
|
372
|
+
browser.setCurrentThread(threadId);
|
|
373
|
+
await browser.ensureReady();
|
|
374
|
+
return browser.hover(input, threadId);
|
|
375
|
+
}
|
|
376
|
+
});
|
|
377
|
+
}
|
|
378
|
+
function createPressTool(browser) {
|
|
379
|
+
return tools.createTool({
|
|
380
|
+
id: BROWSER_TOOLS.PRESS,
|
|
381
|
+
description: "Press a keyboard key (e.g., Enter, Tab, Escape, Control+a).",
|
|
382
|
+
inputSchema: pressInputSchema,
|
|
383
|
+
execute: async (input, { agent }) => {
|
|
384
|
+
const threadId = agent?.threadId;
|
|
385
|
+
browser.setCurrentThread(threadId);
|
|
386
|
+
await browser.ensureReady();
|
|
387
|
+
return browser.press(input, threadId);
|
|
388
|
+
}
|
|
389
|
+
});
|
|
390
|
+
}
|
|
391
|
+
function createScrollTool(browser) {
|
|
392
|
+
return tools.createTool({
|
|
393
|
+
id: BROWSER_TOOLS.SCROLL,
|
|
394
|
+
description: "Scroll the page or a specific element.",
|
|
395
|
+
inputSchema: scrollInputSchema,
|
|
396
|
+
execute: async (input, { agent }) => {
|
|
397
|
+
const threadId = agent?.threadId;
|
|
398
|
+
browser.setCurrentThread(threadId);
|
|
399
|
+
await browser.ensureReady();
|
|
400
|
+
return browser.scroll(input, threadId);
|
|
401
|
+
}
|
|
402
|
+
});
|
|
403
|
+
}
|
|
404
|
+
function createSelectTool(browser) {
|
|
405
|
+
return tools.createTool({
|
|
406
|
+
id: BROWSER_TOOLS.SELECT,
|
|
407
|
+
description: "Select an option from a dropdown by value, label, or index.",
|
|
408
|
+
inputSchema: selectInputSchema,
|
|
409
|
+
execute: async (input, { agent }) => {
|
|
410
|
+
const threadId = agent?.threadId;
|
|
411
|
+
browser.setCurrentThread(threadId);
|
|
412
|
+
await browser.ensureReady();
|
|
413
|
+
return browser.select(input, threadId);
|
|
414
|
+
}
|
|
415
|
+
});
|
|
416
|
+
}
|
|
417
|
+
function createSnapshotTool(browser) {
|
|
418
|
+
return tools.createTool({
|
|
419
|
+
id: BROWSER_TOOLS.SNAPSHOT,
|
|
420
|
+
description: "Get accessibility tree snapshot of the page. Returns text-based representation with element refs like [ref=e1], [ref=e2] for targeting.",
|
|
421
|
+
inputSchema: snapshotInputSchema,
|
|
422
|
+
execute: async (input, { agent }) => {
|
|
423
|
+
const threadId = agent?.threadId;
|
|
424
|
+
browser.setCurrentThread(threadId);
|
|
425
|
+
await browser.ensureReady();
|
|
426
|
+
return browser.snapshot(input, threadId);
|
|
427
|
+
}
|
|
428
|
+
});
|
|
429
|
+
}
|
|
430
|
+
function createTabsTool(browser) {
|
|
431
|
+
return tools.createTool({
|
|
432
|
+
id: BROWSER_TOOLS.TABS,
|
|
433
|
+
description: "Manage browser tabs: list, open new, switch, or close tabs.",
|
|
434
|
+
inputSchema: tabsInputSchema,
|
|
435
|
+
execute: async (input, { agent }) => {
|
|
436
|
+
const threadId = agent?.threadId;
|
|
437
|
+
browser.setCurrentThread(threadId);
|
|
438
|
+
await browser.ensureReady();
|
|
439
|
+
return browser.tabs(input, threadId);
|
|
440
|
+
}
|
|
441
|
+
});
|
|
442
|
+
}
|
|
443
|
+
function createTypeTool(browser) {
|
|
444
|
+
return tools.createTool({
|
|
445
|
+
id: BROWSER_TOOLS.TYPE,
|
|
446
|
+
description: "Type text into an input element. Use clear: true to replace existing content.",
|
|
447
|
+
inputSchema: typeInputSchema,
|
|
448
|
+
execute: async (input, { agent }) => {
|
|
449
|
+
const threadId = agent?.threadId;
|
|
450
|
+
browser.setCurrentThread(threadId);
|
|
451
|
+
await browser.ensureReady();
|
|
452
|
+
return browser.type(input, threadId);
|
|
453
|
+
}
|
|
454
|
+
});
|
|
455
|
+
}
|
|
456
|
+
function createWaitTool(browser) {
|
|
457
|
+
return tools.createTool({
|
|
458
|
+
id: BROWSER_TOOLS.WAIT,
|
|
459
|
+
description: "Wait for an element to appear, disappear, or reach a state.",
|
|
460
|
+
inputSchema: waitInputSchema,
|
|
461
|
+
execute: async (input, { agent }) => {
|
|
462
|
+
const threadId = agent?.threadId;
|
|
463
|
+
browser.setCurrentThread(threadId);
|
|
464
|
+
await browser.ensureReady();
|
|
465
|
+
return browser.wait(input, threadId);
|
|
466
|
+
}
|
|
467
|
+
});
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
// src/tools/index.ts
|
|
471
|
+
function createAgentBrowserTools(browser) {
|
|
472
|
+
return {
|
|
473
|
+
// Core (9)
|
|
474
|
+
[BROWSER_TOOLS.GOTO]: createGotoTool(browser),
|
|
475
|
+
[BROWSER_TOOLS.SNAPSHOT]: createSnapshotTool(browser),
|
|
476
|
+
[BROWSER_TOOLS.CLICK]: createClickTool(browser),
|
|
477
|
+
[BROWSER_TOOLS.TYPE]: createTypeTool(browser),
|
|
478
|
+
[BROWSER_TOOLS.PRESS]: createPressTool(browser),
|
|
479
|
+
[BROWSER_TOOLS.SELECT]: createSelectTool(browser),
|
|
480
|
+
[BROWSER_TOOLS.SCROLL]: createScrollTool(browser),
|
|
481
|
+
[BROWSER_TOOLS.CLOSE]: createCloseTool(browser),
|
|
482
|
+
// Extended
|
|
483
|
+
[BROWSER_TOOLS.HOVER]: createHoverTool(browser),
|
|
484
|
+
[BROWSER_TOOLS.BACK]: createBackTool(browser),
|
|
485
|
+
[BROWSER_TOOLS.DIALOG]: createDialogTool(browser),
|
|
486
|
+
[BROWSER_TOOLS.WAIT]: createWaitTool(browser),
|
|
487
|
+
[BROWSER_TOOLS.TABS]: createTabsTool(browser),
|
|
488
|
+
[BROWSER_TOOLS.DRAG]: createDragTool(browser),
|
|
489
|
+
// Escape hatch (1)
|
|
490
|
+
[BROWSER_TOOLS.EVALUATE]: createEvaluateTool(browser)
|
|
491
|
+
};
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
// src/agent-browser.ts
|
|
495
|
+
var AgentBrowser = class extends browser.MastraBrowser {
|
|
496
|
+
id;
|
|
497
|
+
name = "AgentBrowser";
|
|
498
|
+
provider = "vercel-labs/agent-browser";
|
|
499
|
+
defaultTimeout = 3e4;
|
|
500
|
+
constructor(config = {}) {
|
|
501
|
+
super(config);
|
|
502
|
+
this.id = `agent-browser-${Date.now()}`;
|
|
503
|
+
if (config.timeout) {
|
|
504
|
+
this.defaultTimeout = config.timeout;
|
|
505
|
+
}
|
|
506
|
+
const effectiveScope = config.cdpUrl ? config.scope ?? "shared" : config.scope ?? "thread";
|
|
507
|
+
this.threadManager = new AgentBrowserThreadManager({
|
|
508
|
+
scope: effectiveScope,
|
|
509
|
+
browserConfig: config,
|
|
510
|
+
resolveCdpUrl: this.resolveCdpUrl.bind(this),
|
|
511
|
+
logger: this.logger,
|
|
512
|
+
// When a new thread session is created, notify listeners so screencast can start
|
|
513
|
+
onSessionCreated: (session) => {
|
|
514
|
+
this.notifyBrowserReady(session.threadId);
|
|
515
|
+
},
|
|
516
|
+
// When a new browser is created for a thread, set up close listener
|
|
517
|
+
onBrowserCreated: (manager, threadId) => {
|
|
518
|
+
this.setupCloseListenerForThread(manager, threadId);
|
|
519
|
+
}
|
|
520
|
+
});
|
|
521
|
+
}
|
|
522
|
+
// ---------------------------------------------------------------------------
|
|
523
|
+
// Thread Scope (delegated to ThreadManager)
|
|
524
|
+
// ---------------------------------------------------------------------------
|
|
525
|
+
/**
|
|
526
|
+
* Ensure browser is ready and thread session exists.
|
|
527
|
+
* Creates a new page/context for the current thread if needed.
|
|
528
|
+
*
|
|
529
|
+
* For 'thread' scope, we need to create the thread session BEFORE
|
|
530
|
+
* calling super.ensureReady() because the base class's ensureReady() will
|
|
531
|
+
* call checkBrowserAlive(), which needs at least one thread browser to exist.
|
|
532
|
+
*/
|
|
533
|
+
async ensureReady() {
|
|
534
|
+
const scope = this.threadManager.getScope();
|
|
535
|
+
const threadId = this.getCurrentThread();
|
|
536
|
+
const existingSession = this.threadManager.hasSession(threadId);
|
|
537
|
+
if (scope === "thread" && threadId !== browser.DEFAULT_THREAD_ID && !existingSession) {
|
|
538
|
+
await this.getManagerForThread(threadId);
|
|
539
|
+
}
|
|
540
|
+
await super.ensureReady();
|
|
541
|
+
if (scope === "thread" && threadId !== browser.DEFAULT_THREAD_ID && existingSession) {
|
|
542
|
+
await this.getManagerForThread(threadId);
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
/**
|
|
546
|
+
* Get the browser manager for the current thread.
|
|
547
|
+
* Delegates to ThreadManager for scope handling.
|
|
548
|
+
*/
|
|
549
|
+
async getManagerForThread(threadId) {
|
|
550
|
+
const effectiveThreadId = threadId ?? this.getCurrentThread();
|
|
551
|
+
const scope = this.threadManager.getScope();
|
|
552
|
+
if (scope === "thread" && (!effectiveThreadId || effectiveThreadId === browser.DEFAULT_THREAD_ID)) {
|
|
553
|
+
const existingManager = this.threadManager.getExistingManagerForThread(effectiveThreadId);
|
|
554
|
+
if (existingManager) {
|
|
555
|
+
return existingManager;
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
return this.threadManager.getManagerForThread(effectiveThreadId);
|
|
559
|
+
}
|
|
560
|
+
// ---------------------------------------------------------------------------
|
|
561
|
+
// Lifecycle
|
|
562
|
+
// ---------------------------------------------------------------------------
|
|
563
|
+
async doLaunch() {
|
|
564
|
+
const scope = this.threadManager.getScope();
|
|
565
|
+
if (scope === "thread") {
|
|
566
|
+
this.sharedManager = new agentBrowser.BrowserManager();
|
|
567
|
+
this.threadManager.setSharedManager(this.sharedManager);
|
|
568
|
+
return;
|
|
569
|
+
}
|
|
570
|
+
this.sharedManager = new agentBrowser.BrowserManager();
|
|
571
|
+
const localConfig = this.config;
|
|
572
|
+
const launchOptions = {
|
|
573
|
+
headless: localConfig.headless ?? true,
|
|
574
|
+
viewport: localConfig.viewport
|
|
575
|
+
};
|
|
576
|
+
if (localConfig.cdpUrl) {
|
|
577
|
+
launchOptions.cdpUrl = await this.resolveCdpUrl(localConfig.cdpUrl);
|
|
578
|
+
}
|
|
579
|
+
await this.sharedManager.launch(launchOptions);
|
|
580
|
+
this.threadManager.setSharedManager(this.sharedManager);
|
|
581
|
+
this.setupCloseListenerForSharedScope(this.sharedManager);
|
|
582
|
+
}
|
|
583
|
+
/**
|
|
584
|
+
* Set up close event listeners for 'shared' scope browser.
|
|
585
|
+
* This handles the case where the shared browser is closed externally.
|
|
586
|
+
*/
|
|
587
|
+
setupCloseListenerForSharedScope(manager) {
|
|
588
|
+
try {
|
|
589
|
+
let disconnectHandled = false;
|
|
590
|
+
const handleDisconnect = () => {
|
|
591
|
+
if (disconnectHandled) return;
|
|
592
|
+
disconnectHandled = true;
|
|
593
|
+
this.handleBrowserDisconnected();
|
|
594
|
+
};
|
|
595
|
+
const context = manager.getContext();
|
|
596
|
+
if (context) {
|
|
597
|
+
context.on("close", handleDisconnect);
|
|
598
|
+
}
|
|
599
|
+
const pages = manager.getPages();
|
|
600
|
+
for (const page of pages) {
|
|
601
|
+
page.on("close", () => {
|
|
602
|
+
const remainingPages = manager.getPages();
|
|
603
|
+
if (remainingPages.length === 0) {
|
|
604
|
+
handleDisconnect();
|
|
605
|
+
}
|
|
606
|
+
});
|
|
607
|
+
}
|
|
608
|
+
} catch {
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
async doClose() {
|
|
612
|
+
await this.threadManager.destroyAllSessions();
|
|
613
|
+
this.setCurrentThread(void 0);
|
|
614
|
+
const scope = this.threadManager.getScope();
|
|
615
|
+
if (scope === "shared" && this.sharedManager) {
|
|
616
|
+
await this.sharedManager.close();
|
|
617
|
+
}
|
|
618
|
+
this.sharedManager = null;
|
|
619
|
+
}
|
|
620
|
+
/**
|
|
621
|
+
* Check if the browser is still alive by verifying the page is connected.
|
|
622
|
+
* Called by base class ensureReady() to detect externally closed browsers.
|
|
623
|
+
*/
|
|
624
|
+
async checkBrowserAlive() {
|
|
625
|
+
const scope = this.threadManager.getScope();
|
|
626
|
+
if (scope === "thread") {
|
|
627
|
+
return this.threadManager.hasActiveThreadManagers();
|
|
628
|
+
}
|
|
629
|
+
if (!this.sharedManager) {
|
|
630
|
+
return false;
|
|
631
|
+
}
|
|
632
|
+
try {
|
|
633
|
+
const page = this.sharedManager.getPage();
|
|
634
|
+
const url = page.url();
|
|
635
|
+
if (url && url !== "about:blank") {
|
|
636
|
+
const state = await this.getBrowserState();
|
|
637
|
+
if (state) {
|
|
638
|
+
this.lastBrowserState = state;
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
return true;
|
|
642
|
+
} catch (error) {
|
|
643
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
644
|
+
if (this.isDisconnectionError(msg)) {
|
|
645
|
+
this.logger.debug?.("Browser was externally closed");
|
|
646
|
+
}
|
|
647
|
+
return false;
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
// ---------------------------------------------------------------------------
|
|
651
|
+
// Tools
|
|
652
|
+
// ---------------------------------------------------------------------------
|
|
653
|
+
/**
|
|
654
|
+
* Get the browser tools for this provider.
|
|
655
|
+
* Returns 17 flat tools for browser automation.
|
|
656
|
+
*/
|
|
657
|
+
getTools() {
|
|
658
|
+
return createAgentBrowserTools(this);
|
|
659
|
+
}
|
|
660
|
+
// ---------------------------------------------------------------------------
|
|
661
|
+
// Helpers
|
|
662
|
+
// ---------------------------------------------------------------------------
|
|
663
|
+
/**
|
|
664
|
+
* Get the page for the current thread.
|
|
665
|
+
* Uses thread scope if enabled, otherwise returns the shared page.
|
|
666
|
+
* @param explicitThreadId - Optional thread ID to use instead of getCurrentThread()
|
|
667
|
+
* Use this to avoid race conditions in concurrent tool calls.
|
|
668
|
+
*/
|
|
669
|
+
async getPage(explicitThreadId) {
|
|
670
|
+
const scope = this.getScope();
|
|
671
|
+
const threadId = explicitThreadId ?? this.getCurrentThread();
|
|
672
|
+
if (scope === "thread") {
|
|
673
|
+
return this.threadManager.getPageForThread(threadId);
|
|
674
|
+
}
|
|
675
|
+
if (!this.sharedManager) throw new Error("Browser not launched");
|
|
676
|
+
return this.sharedManager.getPage();
|
|
677
|
+
}
|
|
678
|
+
/**
|
|
679
|
+
* Get the active page for a thread (implements abstract method from base class).
|
|
680
|
+
* Returns null if no page is available, unlike getPage which throws.
|
|
681
|
+
*/
|
|
682
|
+
async getActivePage(threadId) {
|
|
683
|
+
try {
|
|
684
|
+
return await this.getPage(threadId);
|
|
685
|
+
} catch {
|
|
686
|
+
return null;
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
/**
|
|
690
|
+
* Set up close event listener for a thread's browser manager.
|
|
691
|
+
* This handles the case where a thread's browser is closed externally.
|
|
692
|
+
*/
|
|
693
|
+
setupCloseListenerForThread(manager, threadId) {
|
|
694
|
+
try {
|
|
695
|
+
let disconnectHandled = false;
|
|
696
|
+
const handleDisconnect = () => {
|
|
697
|
+
if (disconnectHandled) return;
|
|
698
|
+
disconnectHandled = true;
|
|
699
|
+
this.handleThreadBrowserDisconnected(threadId);
|
|
700
|
+
};
|
|
701
|
+
const context = manager.getContext();
|
|
702
|
+
if (context) {
|
|
703
|
+
context.on("close", handleDisconnect);
|
|
704
|
+
}
|
|
705
|
+
const pages = manager.getPages();
|
|
706
|
+
for (const page of pages) {
|
|
707
|
+
page.on("close", () => {
|
|
708
|
+
const remainingPages = manager.getPages();
|
|
709
|
+
if (remainingPages.length === 0) {
|
|
710
|
+
handleDisconnect();
|
|
711
|
+
}
|
|
712
|
+
});
|
|
713
|
+
}
|
|
714
|
+
} catch {
|
|
715
|
+
}
|
|
716
|
+
}
|
|
717
|
+
/**
|
|
718
|
+
* Create an error response from an exception.
|
|
719
|
+
* Extends base class to add agent-browser specific error handling.
|
|
720
|
+
*/
|
|
721
|
+
createErrorFromException(error, context) {
|
|
722
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
723
|
+
if (msg.includes("stale") || msg.includes("Stale")) {
|
|
724
|
+
return this.createError(
|
|
725
|
+
"stale_ref",
|
|
726
|
+
"Element ref is no longer valid.",
|
|
727
|
+
"Get a fresh snapshot and use updated refs."
|
|
728
|
+
);
|
|
729
|
+
}
|
|
730
|
+
if (msg.includes("not found") || msg.includes("No element")) {
|
|
731
|
+
return this.createError(
|
|
732
|
+
"element_not_found",
|
|
733
|
+
"Element not found.",
|
|
734
|
+
"Check the ref is correct or get a fresh snapshot."
|
|
735
|
+
);
|
|
736
|
+
}
|
|
737
|
+
return super.createErrorFromException(error, context);
|
|
738
|
+
}
|
|
739
|
+
async requireLocator(ref, threadId) {
|
|
740
|
+
const manager = await this.getManagerForThread(threadId);
|
|
741
|
+
return manager.getLocatorFromRef(ref);
|
|
742
|
+
}
|
|
743
|
+
async getScrollInfo(threadId) {
|
|
744
|
+
const page = await this.getPage(threadId);
|
|
745
|
+
const info = await page.evaluate(`({
|
|
746
|
+
scrollY: Math.round(window.scrollY),
|
|
747
|
+
scrollHeight: document.documentElement.scrollHeight,
|
|
748
|
+
viewportHeight: window.innerHeight
|
|
749
|
+
})`);
|
|
750
|
+
if (!info || typeof info.scrollHeight !== "number") {
|
|
751
|
+
return {
|
|
752
|
+
scrollY: 0,
|
|
753
|
+
scrollHeight: 0,
|
|
754
|
+
viewportHeight: 0,
|
|
755
|
+
atTop: true,
|
|
756
|
+
atBottom: true,
|
|
757
|
+
percentDown: 0
|
|
758
|
+
};
|
|
759
|
+
}
|
|
760
|
+
const maxScroll = info.scrollHeight - info.viewportHeight;
|
|
761
|
+
return {
|
|
762
|
+
...info,
|
|
763
|
+
atTop: info.scrollY < 50,
|
|
764
|
+
atBottom: info.scrollY >= maxScroll - 50,
|
|
765
|
+
percentDown: maxScroll > 0 ? Math.round(info.scrollY / maxScroll * 100) : 0
|
|
766
|
+
};
|
|
767
|
+
}
|
|
768
|
+
// ---------------------------------------------------------------------------
|
|
769
|
+
// URL Access
|
|
770
|
+
// ---------------------------------------------------------------------------
|
|
771
|
+
/**
|
|
772
|
+
* Get the current page URL without launching the browser.
|
|
773
|
+
* @param threadId - Optional thread ID for thread-isolated browsers
|
|
774
|
+
* @returns The current URL string, or null if browser is not running
|
|
775
|
+
*/
|
|
776
|
+
async getCurrentUrl(threadId) {
|
|
777
|
+
if (!this.isBrowserRunning()) {
|
|
778
|
+
return null;
|
|
779
|
+
}
|
|
780
|
+
try {
|
|
781
|
+
const effectiveThreadId = threadId ?? this.getCurrentThread();
|
|
782
|
+
const scope = this.threadManager.getScope();
|
|
783
|
+
if (scope === "thread" && effectiveThreadId) {
|
|
784
|
+
const manager2 = this.threadManager.getExistingManagerForThread(effectiveThreadId);
|
|
785
|
+
if (!manager2) {
|
|
786
|
+
return null;
|
|
787
|
+
}
|
|
788
|
+
const url2 = manager2.getPage().url();
|
|
789
|
+
if (url2 && url2 !== "about:blank") {
|
|
790
|
+
const state = this.getBrowserStateForManager(manager2);
|
|
791
|
+
if (state) {
|
|
792
|
+
this.threadManager.updateBrowserState(effectiveThreadId, state);
|
|
793
|
+
}
|
|
794
|
+
}
|
|
795
|
+
return url2;
|
|
796
|
+
}
|
|
797
|
+
const manager = await this.getManagerForThread(threadId);
|
|
798
|
+
const url = manager.getPage().url();
|
|
799
|
+
if (url && url !== "about:blank") {
|
|
800
|
+
const state = this.getBrowserStateForManager(manager);
|
|
801
|
+
if (state) {
|
|
802
|
+
this.lastBrowserState = state;
|
|
803
|
+
}
|
|
804
|
+
}
|
|
805
|
+
return url;
|
|
806
|
+
} catch {
|
|
807
|
+
return null;
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
/**
|
|
811
|
+
* Navigate to a URL (simple form). Used internally for restoring state on relaunch.
|
|
812
|
+
*/
|
|
813
|
+
async navigateTo(url) {
|
|
814
|
+
if (!this.isBrowserRunning()) {
|
|
815
|
+
return;
|
|
816
|
+
}
|
|
817
|
+
try {
|
|
818
|
+
const page = await this.getPage();
|
|
819
|
+
await page.goto(url, {
|
|
820
|
+
timeout: this.defaultTimeout,
|
|
821
|
+
waitUntil: "domcontentloaded"
|
|
822
|
+
});
|
|
823
|
+
} catch {
|
|
824
|
+
}
|
|
825
|
+
}
|
|
826
|
+
/**
|
|
827
|
+
* Get the current browser state (all tabs and active tab index).
|
|
828
|
+
*/
|
|
829
|
+
async getBrowserState(threadId) {
|
|
830
|
+
if (!this.isBrowserRunning()) {
|
|
831
|
+
return null;
|
|
832
|
+
}
|
|
833
|
+
try {
|
|
834
|
+
const manager = await this.getManagerForThread(threadId);
|
|
835
|
+
return this.getBrowserStateForManager(manager);
|
|
836
|
+
} catch {
|
|
837
|
+
return null;
|
|
838
|
+
}
|
|
839
|
+
}
|
|
840
|
+
/**
|
|
841
|
+
* Get browser state for a thread (implements abstract method from base class).
|
|
842
|
+
* Sync version that uses existing manager lookup without creating sessions.
|
|
843
|
+
*/
|
|
844
|
+
getBrowserStateForThread(threadId) {
|
|
845
|
+
const effectiveThreadId = threadId ?? this.getCurrentThread() ?? browser.DEFAULT_THREAD_ID;
|
|
846
|
+
const manager = this.threadManager.getExistingManagerForThread(effectiveThreadId);
|
|
847
|
+
if (!manager) return null;
|
|
848
|
+
return this.getBrowserStateForManager(manager);
|
|
849
|
+
}
|
|
850
|
+
/**
|
|
851
|
+
* Get browser state from a specific manager instance.
|
|
852
|
+
*/
|
|
853
|
+
getBrowserStateForManager(manager) {
|
|
854
|
+
try {
|
|
855
|
+
const pages = manager.getPages();
|
|
856
|
+
const activeIndex = manager.getActiveIndex();
|
|
857
|
+
const tabs = pages.map((page) => ({
|
|
858
|
+
url: page.url()
|
|
859
|
+
}));
|
|
860
|
+
return {
|
|
861
|
+
tabs,
|
|
862
|
+
activeTabIndex: activeIndex
|
|
863
|
+
};
|
|
864
|
+
} catch {
|
|
865
|
+
return null;
|
|
866
|
+
}
|
|
867
|
+
}
|
|
868
|
+
/**
|
|
869
|
+
* Get all open tabs with their URLs and titles.
|
|
870
|
+
*/
|
|
871
|
+
async getTabState(threadId) {
|
|
872
|
+
const state = await this.getBrowserState(threadId);
|
|
873
|
+
return state?.tabs ?? [];
|
|
874
|
+
}
|
|
875
|
+
/**
|
|
876
|
+
* Get the active tab index.
|
|
877
|
+
*/
|
|
878
|
+
async getActiveTabIndex(threadId) {
|
|
879
|
+
if (!this.isBrowserRunning()) {
|
|
880
|
+
return 0;
|
|
881
|
+
}
|
|
882
|
+
try {
|
|
883
|
+
const manager = await this.getManagerForThread(threadId);
|
|
884
|
+
return manager.getActiveIndex();
|
|
885
|
+
} catch {
|
|
886
|
+
return 0;
|
|
887
|
+
}
|
|
888
|
+
}
|
|
889
|
+
// ---------------------------------------------------------------------------
|
|
890
|
+
// 1. browser_goto - Navigate to URL
|
|
891
|
+
// ---------------------------------------------------------------------------
|
|
892
|
+
async goto(input, threadId) {
|
|
893
|
+
try {
|
|
894
|
+
const page = await this.getPage(threadId);
|
|
895
|
+
await page.goto(input.url, {
|
|
896
|
+
timeout: input.timeout ?? this.defaultTimeout,
|
|
897
|
+
waitUntil: input.waitUntil ?? "domcontentloaded"
|
|
898
|
+
});
|
|
899
|
+
return {
|
|
900
|
+
success: true,
|
|
901
|
+
url: page.url(),
|
|
902
|
+
title: await page.title(),
|
|
903
|
+
hint: "Take a snapshot to see interactive elements and get refs."
|
|
904
|
+
};
|
|
905
|
+
} catch (error) {
|
|
906
|
+
return this.createErrorFromException(error, "Goto");
|
|
907
|
+
}
|
|
908
|
+
}
|
|
909
|
+
// ---------------------------------------------------------------------------
|
|
910
|
+
// 2. browser_snapshot - Capture accessibility tree
|
|
911
|
+
// ---------------------------------------------------------------------------
|
|
912
|
+
async snapshot(input, threadId) {
|
|
913
|
+
try {
|
|
914
|
+
const manager = await this.getManagerForThread(threadId);
|
|
915
|
+
const page = await this.getPage(threadId);
|
|
916
|
+
const rawSnapshot = await manager.getSnapshot({
|
|
917
|
+
interactive: input.interactiveOnly ?? true,
|
|
918
|
+
compact: true
|
|
919
|
+
});
|
|
920
|
+
const snapshot = (rawSnapshot.tree ?? "").replace(/\[ref=(\w+)\]/g, "@$1");
|
|
921
|
+
const scrollInfo = await this.getScrollInfo(threadId);
|
|
922
|
+
let scrollText;
|
|
923
|
+
if (scrollInfo.atTop && !scrollInfo.atBottom) {
|
|
924
|
+
scrollText = "TOP - more content below";
|
|
925
|
+
} else if (scrollInfo.atBottom) {
|
|
926
|
+
scrollText = "BOTTOM of page";
|
|
927
|
+
} else {
|
|
928
|
+
scrollText = `${scrollInfo.percentDown}% down`;
|
|
929
|
+
}
|
|
930
|
+
const refs = snapshot.match(/@e\d+/g) || [];
|
|
931
|
+
const elementCount = new Set(refs).size;
|
|
932
|
+
return {
|
|
933
|
+
success: true,
|
|
934
|
+
snapshot,
|
|
935
|
+
url: page.url(),
|
|
936
|
+
title: await page.title(),
|
|
937
|
+
elementCount,
|
|
938
|
+
scroll: scrollText,
|
|
939
|
+
hint: elementCount === 0 ? "No interactive elements found. Try scrolling or setting interactiveOnly:false." : void 0
|
|
940
|
+
};
|
|
941
|
+
} catch (error) {
|
|
942
|
+
return this.createErrorFromException(error, "Snapshot");
|
|
943
|
+
}
|
|
944
|
+
}
|
|
945
|
+
// ---------------------------------------------------------------------------
|
|
946
|
+
// 3. browser_click - Click on element
|
|
947
|
+
// ---------------------------------------------------------------------------
|
|
948
|
+
async click(input, threadId) {
|
|
949
|
+
try {
|
|
950
|
+
const page = await this.getPage(threadId);
|
|
951
|
+
const locator = await this.requireLocator(input.ref, threadId);
|
|
952
|
+
if (!locator) {
|
|
953
|
+
return this.createError(
|
|
954
|
+
"stale_ref",
|
|
955
|
+
`Ref ${input.ref} not found. The page has changed.`,
|
|
956
|
+
"Take a new snapshot to see the current page state and get fresh refs."
|
|
957
|
+
);
|
|
958
|
+
}
|
|
959
|
+
await locator.click({
|
|
960
|
+
button: input.button ?? "left",
|
|
961
|
+
clickCount: input.clickCount ?? 1,
|
|
962
|
+
modifiers: input.modifiers,
|
|
963
|
+
timeout: this.defaultTimeout
|
|
964
|
+
});
|
|
965
|
+
return {
|
|
966
|
+
success: true,
|
|
967
|
+
url: page.url(),
|
|
968
|
+
hint: "Take a new snapshot to see updated page state and get fresh refs."
|
|
969
|
+
};
|
|
970
|
+
} catch (error) {
|
|
971
|
+
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
972
|
+
if (errorMsg.includes("intercepts pointer events")) {
|
|
973
|
+
return this.createError(
|
|
974
|
+
"element_blocked",
|
|
975
|
+
`Element ${input.ref} is blocked by another element.`,
|
|
976
|
+
"Take a new snapshot to see what is blocking. Dismiss any modals or scroll the element into view."
|
|
977
|
+
);
|
|
978
|
+
}
|
|
979
|
+
return this.createErrorFromException(error, "Click");
|
|
980
|
+
}
|
|
981
|
+
}
|
|
982
|
+
// ---------------------------------------------------------------------------
|
|
983
|
+
// 4. browser_type - Type text into element
|
|
984
|
+
// ---------------------------------------------------------------------------
|
|
985
|
+
async type(input, threadId) {
|
|
986
|
+
try {
|
|
987
|
+
const page = await this.getPage(threadId);
|
|
988
|
+
const locator = await this.requireLocator(input.ref, threadId);
|
|
989
|
+
if (!locator) {
|
|
990
|
+
return this.createError(
|
|
991
|
+
"stale_ref",
|
|
992
|
+
`Ref ${input.ref} not found. The page has changed.`,
|
|
993
|
+
"Take a new snapshot to see the current page state and get fresh refs."
|
|
994
|
+
);
|
|
995
|
+
}
|
|
996
|
+
if (input.clear) {
|
|
997
|
+
await locator.fill("", { timeout: this.defaultTimeout });
|
|
998
|
+
}
|
|
999
|
+
if (input.delay) {
|
|
1000
|
+
await locator.focus();
|
|
1001
|
+
for (const char of input.text) {
|
|
1002
|
+
await page.keyboard.press(char);
|
|
1003
|
+
await new Promise((r) => setTimeout(r, input.delay));
|
|
1004
|
+
}
|
|
1005
|
+
} else {
|
|
1006
|
+
await locator.fill(input.text, { timeout: this.defaultTimeout });
|
|
1007
|
+
}
|
|
1008
|
+
const value = await locator.inputValue({ timeout: 1e3 }).catch(() => input.text);
|
|
1009
|
+
return {
|
|
1010
|
+
success: true,
|
|
1011
|
+
value,
|
|
1012
|
+
url: page.url(),
|
|
1013
|
+
hint: "Take a new snapshot if you need to interact with more elements."
|
|
1014
|
+
};
|
|
1015
|
+
} catch (error) {
|
|
1016
|
+
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
1017
|
+
if (errorMsg.includes("is not an <input>") || errorMsg.includes("not an input") || errorMsg.includes("Cannot type") || errorMsg.includes("not focusable")) {
|
|
1018
|
+
return this.createError(
|
|
1019
|
+
"not_focusable",
|
|
1020
|
+
`Element ${input.ref} is not a text input field.`,
|
|
1021
|
+
'Take a new snapshot and look for elements with role "textbox" or "searchbox".'
|
|
1022
|
+
);
|
|
1023
|
+
}
|
|
1024
|
+
return this.createErrorFromException(error, "Type");
|
|
1025
|
+
}
|
|
1026
|
+
}
|
|
1027
|
+
// ---------------------------------------------------------------------------
|
|
1028
|
+
// 5. browser_press - Press keyboard key(s)
|
|
1029
|
+
// ---------------------------------------------------------------------------
|
|
1030
|
+
async press(input, threadId) {
|
|
1031
|
+
try {
|
|
1032
|
+
const page = await this.getPage(threadId);
|
|
1033
|
+
await page.keyboard.press(input.key);
|
|
1034
|
+
return {
|
|
1035
|
+
success: true,
|
|
1036
|
+
url: page.url(),
|
|
1037
|
+
hint: "Take a new snapshot if the page may have changed."
|
|
1038
|
+
};
|
|
1039
|
+
} catch (error) {
|
|
1040
|
+
return this.createErrorFromException(error, "Press");
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
1043
|
+
// ---------------------------------------------------------------------------
|
|
1044
|
+
// 6. browser_select - Select dropdown option
|
|
1045
|
+
// ---------------------------------------------------------------------------
|
|
1046
|
+
async select(input, threadId) {
|
|
1047
|
+
try {
|
|
1048
|
+
const page = await this.getPage(threadId);
|
|
1049
|
+
const locator = await this.requireLocator(input.ref, threadId);
|
|
1050
|
+
if (!locator) {
|
|
1051
|
+
return this.createError(
|
|
1052
|
+
"stale_ref",
|
|
1053
|
+
`Ref ${input.ref} not found. The page has changed.`,
|
|
1054
|
+
"Take a new snapshot to get fresh refs."
|
|
1055
|
+
);
|
|
1056
|
+
}
|
|
1057
|
+
const selectValue = {};
|
|
1058
|
+
if (input.value) selectValue.value = input.value;
|
|
1059
|
+
if (input.label) selectValue.label = input.label;
|
|
1060
|
+
if (input.index !== void 0) selectValue.index = input.index;
|
|
1061
|
+
const selected = await locator.selectOption(selectValue, {
|
|
1062
|
+
timeout: this.defaultTimeout
|
|
1063
|
+
});
|
|
1064
|
+
return {
|
|
1065
|
+
success: true,
|
|
1066
|
+
selected,
|
|
1067
|
+
url: page.url(),
|
|
1068
|
+
hint: "Selection complete. Take a snapshot if you need to continue."
|
|
1069
|
+
};
|
|
1070
|
+
} catch (error) {
|
|
1071
|
+
return this.createErrorFromException(error, "Select");
|
|
1072
|
+
}
|
|
1073
|
+
}
|
|
1074
|
+
// ---------------------------------------------------------------------------
|
|
1075
|
+
// 7. browser_scroll - Scroll page or element
|
|
1076
|
+
// ---------------------------------------------------------------------------
|
|
1077
|
+
async scroll(input, threadId) {
|
|
1078
|
+
try {
|
|
1079
|
+
const page = await this.getPage(threadId);
|
|
1080
|
+
if (input.ref) {
|
|
1081
|
+
const locator = await this.requireLocator(input.ref, threadId);
|
|
1082
|
+
if (locator) {
|
|
1083
|
+
await locator.scrollIntoViewIfNeeded({ timeout: this.defaultTimeout });
|
|
1084
|
+
}
|
|
1085
|
+
} else {
|
|
1086
|
+
const direction = input.direction;
|
|
1087
|
+
const amount = input.amount ?? 300;
|
|
1088
|
+
let deltaX = 0;
|
|
1089
|
+
let deltaY = 0;
|
|
1090
|
+
switch (direction) {
|
|
1091
|
+
case "up":
|
|
1092
|
+
deltaY = -amount;
|
|
1093
|
+
break;
|
|
1094
|
+
case "down":
|
|
1095
|
+
deltaY = amount;
|
|
1096
|
+
break;
|
|
1097
|
+
case "left":
|
|
1098
|
+
deltaX = -amount;
|
|
1099
|
+
break;
|
|
1100
|
+
case "right":
|
|
1101
|
+
deltaX = amount;
|
|
1102
|
+
break;
|
|
1103
|
+
}
|
|
1104
|
+
await page.evaluate(
|
|
1105
|
+
({ x, y }) => {
|
|
1106
|
+
globalThis.scrollBy(x, y);
|
|
1107
|
+
},
|
|
1108
|
+
{ x: deltaX, y: deltaY }
|
|
1109
|
+
);
|
|
1110
|
+
}
|
|
1111
|
+
const scrollInfo = await this.getScrollInfo(threadId);
|
|
1112
|
+
let scrollText;
|
|
1113
|
+
if (scrollInfo.atTop && !scrollInfo.atBottom) {
|
|
1114
|
+
scrollText = "TOP - more content below";
|
|
1115
|
+
} else if (scrollInfo.atBottom) {
|
|
1116
|
+
scrollText = "BOTTOM of page";
|
|
1117
|
+
} else {
|
|
1118
|
+
scrollText = `${scrollInfo.percentDown}% down`;
|
|
1119
|
+
}
|
|
1120
|
+
return {
|
|
1121
|
+
success: true,
|
|
1122
|
+
position: { x: 0, y: scrollInfo.scrollY },
|
|
1123
|
+
scroll: scrollText,
|
|
1124
|
+
hint: "Take a new snapshot to see elements in the new viewport."
|
|
1125
|
+
};
|
|
1126
|
+
} catch (error) {
|
|
1127
|
+
return this.createErrorFromException(error, "Scroll");
|
|
1128
|
+
}
|
|
1129
|
+
}
|
|
1130
|
+
// ---------------------------------------------------------------------------
|
|
1131
|
+
// 8. browser_hover - Hover over element
|
|
1132
|
+
// ---------------------------------------------------------------------------
|
|
1133
|
+
async hover(input, threadId) {
|
|
1134
|
+
try {
|
|
1135
|
+
const page = await this.getPage(threadId);
|
|
1136
|
+
const locator = await this.requireLocator(input.ref, threadId);
|
|
1137
|
+
if (!locator) {
|
|
1138
|
+
return this.createError(
|
|
1139
|
+
"stale_ref",
|
|
1140
|
+
`Ref ${input.ref} not found. The page has changed.`,
|
|
1141
|
+
"Take a new snapshot to get fresh refs."
|
|
1142
|
+
);
|
|
1143
|
+
}
|
|
1144
|
+
await locator.hover({ timeout: this.defaultTimeout });
|
|
1145
|
+
return {
|
|
1146
|
+
success: true,
|
|
1147
|
+
url: page.url(),
|
|
1148
|
+
hint: "Take a new snapshot to see any hover-triggered elements (dropdowns, tooltips)."
|
|
1149
|
+
};
|
|
1150
|
+
} catch (error) {
|
|
1151
|
+
return this.createErrorFromException(error, "Hover");
|
|
1152
|
+
}
|
|
1153
|
+
}
|
|
1154
|
+
// ---------------------------------------------------------------------------
|
|
1155
|
+
// 10. browser_back - Navigate back
|
|
1156
|
+
// ---------------------------------------------------------------------------
|
|
1157
|
+
async back(threadId) {
|
|
1158
|
+
try {
|
|
1159
|
+
const page = await this.getPage(threadId);
|
|
1160
|
+
await page.goBack({ timeout: this.defaultTimeout });
|
|
1161
|
+
return {
|
|
1162
|
+
success: true,
|
|
1163
|
+
url: page.url(),
|
|
1164
|
+
title: await page.title(),
|
|
1165
|
+
hint: "Take a new snapshot to see the previous page."
|
|
1166
|
+
};
|
|
1167
|
+
} catch (error) {
|
|
1168
|
+
return this.createErrorFromException(error, "Back");
|
|
1169
|
+
}
|
|
1170
|
+
}
|
|
1171
|
+
// ---------------------------------------------------------------------------
|
|
1172
|
+
// 11. browser_dialog - Click element that triggers dialog and handle it
|
|
1173
|
+
// ---------------------------------------------------------------------------
|
|
1174
|
+
async dialog(input, threadId) {
|
|
1175
|
+
try {
|
|
1176
|
+
const page = await this.getPage(threadId);
|
|
1177
|
+
const locator = await this.requireLocator(input.triggerRef, threadId);
|
|
1178
|
+
if (!locator) {
|
|
1179
|
+
return this.createError(
|
|
1180
|
+
"stale_ref",
|
|
1181
|
+
`Trigger ref ${input.triggerRef} not found.`,
|
|
1182
|
+
"Take a new snapshot to get fresh refs."
|
|
1183
|
+
);
|
|
1184
|
+
}
|
|
1185
|
+
return new Promise((resolve, reject) => {
|
|
1186
|
+
const timeout = setTimeout(() => {
|
|
1187
|
+
page.off("dialog", dialogHandler);
|
|
1188
|
+
reject(
|
|
1189
|
+
new Error(`No dialog appeared after clicking ${input.triggerRef}. The element may not trigger a dialog.`)
|
|
1190
|
+
);
|
|
1191
|
+
}, this.defaultTimeout);
|
|
1192
|
+
const dialogHandler = async (dialog) => {
|
|
1193
|
+
clearTimeout(timeout);
|
|
1194
|
+
try {
|
|
1195
|
+
const dialogType = dialog.type();
|
|
1196
|
+
const message = dialog.message();
|
|
1197
|
+
if (input.action === "accept") {
|
|
1198
|
+
await dialog.accept(input.text);
|
|
1199
|
+
} else {
|
|
1200
|
+
await dialog.dismiss();
|
|
1201
|
+
}
|
|
1202
|
+
resolve({
|
|
1203
|
+
success: true,
|
|
1204
|
+
action: input.action,
|
|
1205
|
+
dialogType,
|
|
1206
|
+
message,
|
|
1207
|
+
hint: "Dialog handled. Take a snapshot to continue."
|
|
1208
|
+
});
|
|
1209
|
+
} catch (e) {
|
|
1210
|
+
reject(e);
|
|
1211
|
+
}
|
|
1212
|
+
};
|
|
1213
|
+
page.once("dialog", dialogHandler);
|
|
1214
|
+
locator.click({ timeout: this.defaultTimeout }).catch((e) => {
|
|
1215
|
+
clearTimeout(timeout);
|
|
1216
|
+
page.off("dialog", dialogHandler);
|
|
1217
|
+
reject(e);
|
|
1218
|
+
});
|
|
1219
|
+
});
|
|
1220
|
+
} catch (error) {
|
|
1221
|
+
return this.createErrorFromException(error, "Dialog");
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
// ---------------------------------------------------------------------------
|
|
1225
|
+
// 13. browser_wait - Wait for element or condition
|
|
1226
|
+
// ---------------------------------------------------------------------------
|
|
1227
|
+
async wait(input, threadId) {
|
|
1228
|
+
try {
|
|
1229
|
+
const timeout = input.timeout ?? this.defaultTimeout;
|
|
1230
|
+
if (input.ref) {
|
|
1231
|
+
const locator = await this.requireLocator(input.ref, threadId);
|
|
1232
|
+
if (!locator) {
|
|
1233
|
+
return this.createError("stale_ref", `Ref ${input.ref} not found.`, "Take a new snapshot to get fresh refs.");
|
|
1234
|
+
}
|
|
1235
|
+
const state = input.state ?? "visible";
|
|
1236
|
+
await locator.waitFor({ state, timeout });
|
|
1237
|
+
return {
|
|
1238
|
+
success: true,
|
|
1239
|
+
hint: `Element is now ${state}. Take a snapshot to continue.`
|
|
1240
|
+
};
|
|
1241
|
+
} else {
|
|
1242
|
+
const page = await this.getPage(threadId);
|
|
1243
|
+
await page.waitForTimeout(timeout);
|
|
1244
|
+
return {
|
|
1245
|
+
success: true,
|
|
1246
|
+
hint: "Wait complete. Take a snapshot to see current state."
|
|
1247
|
+
};
|
|
1248
|
+
}
|
|
1249
|
+
} catch (error) {
|
|
1250
|
+
return this.createErrorFromException(error, "Wait");
|
|
1251
|
+
}
|
|
1252
|
+
}
|
|
1253
|
+
// ---------------------------------------------------------------------------
|
|
1254
|
+
// 14. browser_tabs - Manage browser tabs
|
|
1255
|
+
// ---------------------------------------------------------------------------
|
|
1256
|
+
async tabs(input, threadId) {
|
|
1257
|
+
try {
|
|
1258
|
+
const browser = await this.getManagerForThread(threadId);
|
|
1259
|
+
if (!browser) {
|
|
1260
|
+
return this.createError(
|
|
1261
|
+
"browser_closed",
|
|
1262
|
+
"Browser not launched",
|
|
1263
|
+
"Call a navigation tool first to launch the browser."
|
|
1264
|
+
);
|
|
1265
|
+
}
|
|
1266
|
+
switch (input.action) {
|
|
1267
|
+
case "list": {
|
|
1268
|
+
if (!browser.listTabs) {
|
|
1269
|
+
return this.createError(
|
|
1270
|
+
"browser_error",
|
|
1271
|
+
"Tab management not supported",
|
|
1272
|
+
"This browser provider does not support tab management."
|
|
1273
|
+
);
|
|
1274
|
+
}
|
|
1275
|
+
const tabsList = await browser.listTabs();
|
|
1276
|
+
return {
|
|
1277
|
+
success: true,
|
|
1278
|
+
tabs: tabsList,
|
|
1279
|
+
hint: 'Use browser_tabs with action:"switch" and index to change tabs.'
|
|
1280
|
+
};
|
|
1281
|
+
}
|
|
1282
|
+
case "new": {
|
|
1283
|
+
if (!browser.newTab) {
|
|
1284
|
+
return this.createError(
|
|
1285
|
+
"browser_error",
|
|
1286
|
+
"Tab management not supported",
|
|
1287
|
+
"This browser provider does not support tab management."
|
|
1288
|
+
);
|
|
1289
|
+
}
|
|
1290
|
+
const result = await browser.newTab();
|
|
1291
|
+
if (input.url) {
|
|
1292
|
+
const page = await this.getPage(threadId);
|
|
1293
|
+
await page.goto(input.url);
|
|
1294
|
+
}
|
|
1295
|
+
this.updateSessionBrowserState(threadId);
|
|
1296
|
+
return {
|
|
1297
|
+
success: true,
|
|
1298
|
+
...result,
|
|
1299
|
+
hint: "New tab opened. Take a snapshot to see its content."
|
|
1300
|
+
};
|
|
1301
|
+
}
|
|
1302
|
+
case "switch": {
|
|
1303
|
+
if (!browser.switchTo) {
|
|
1304
|
+
return this.createError(
|
|
1305
|
+
"browser_error",
|
|
1306
|
+
"Tab management not supported",
|
|
1307
|
+
"This browser provider does not support tab management."
|
|
1308
|
+
);
|
|
1309
|
+
}
|
|
1310
|
+
await browser.switchTo(input.index);
|
|
1311
|
+
await this.reconnectScreencastForThread(threadId, "tab switch");
|
|
1312
|
+
const page = browser.getPage();
|
|
1313
|
+
const pageUrl = page.url();
|
|
1314
|
+
const streamKey = this.getStreamKey(threadId);
|
|
1315
|
+
const stream = this.activeScreencastStreams.get(streamKey);
|
|
1316
|
+
if (pageUrl && stream?.isActive()) {
|
|
1317
|
+
stream.emitUrl(pageUrl);
|
|
1318
|
+
}
|
|
1319
|
+
this.updateSessionBrowserState(threadId);
|
|
1320
|
+
return {
|
|
1321
|
+
success: true,
|
|
1322
|
+
index: input.index,
|
|
1323
|
+
url: pageUrl,
|
|
1324
|
+
title: await page.title(),
|
|
1325
|
+
hint: "Tab switched. Take a snapshot to see its content."
|
|
1326
|
+
};
|
|
1327
|
+
}
|
|
1328
|
+
case "close": {
|
|
1329
|
+
if (!browser.closeTab) {
|
|
1330
|
+
return this.createError(
|
|
1331
|
+
"browser_error",
|
|
1332
|
+
"Tab management not supported",
|
|
1333
|
+
"This browser provider does not support tab management."
|
|
1334
|
+
);
|
|
1335
|
+
}
|
|
1336
|
+
await browser.closeTab(input.index);
|
|
1337
|
+
await this.reconnectScreencastForThread(threadId, "tab close");
|
|
1338
|
+
this.updateSessionBrowserState(threadId);
|
|
1339
|
+
const tabsList = await browser.listTabs?.() ?? [];
|
|
1340
|
+
return {
|
|
1341
|
+
success: true,
|
|
1342
|
+
remaining: tabsList.length,
|
|
1343
|
+
hint: tabsList.length > 0 ? "Tab closed. Take a snapshot to see current tab." : "All tabs closed."
|
|
1344
|
+
};
|
|
1345
|
+
}
|
|
1346
|
+
default:
|
|
1347
|
+
return this.createError(
|
|
1348
|
+
"browser_error",
|
|
1349
|
+
`Unknown tabs action: ${input.action}`,
|
|
1350
|
+
'Use "list", "new", "switch", or "close".'
|
|
1351
|
+
);
|
|
1352
|
+
}
|
|
1353
|
+
} catch (error) {
|
|
1354
|
+
return this.createErrorFromException(error, "Tabs");
|
|
1355
|
+
}
|
|
1356
|
+
}
|
|
1357
|
+
// ---------------------------------------------------------------------------
|
|
1358
|
+
// 15. browser_drag - Drag element to target
|
|
1359
|
+
// ---------------------------------------------------------------------------
|
|
1360
|
+
async drag(input, threadId) {
|
|
1361
|
+
try {
|
|
1362
|
+
const page = await this.getPage(threadId);
|
|
1363
|
+
let sourceLocator = null;
|
|
1364
|
+
if (input.sourceRef) {
|
|
1365
|
+
sourceLocator = await this.requireLocator(input.sourceRef, threadId);
|
|
1366
|
+
} else if (input.sourceSelector) {
|
|
1367
|
+
sourceLocator = page.locator(input.sourceSelector);
|
|
1368
|
+
}
|
|
1369
|
+
if (!sourceLocator) {
|
|
1370
|
+
return this.createError(
|
|
1371
|
+
"stale_ref",
|
|
1372
|
+
input.sourceRef ? `Source ref ${input.sourceRef} not found.` : "No source element specified. Provide sourceRef or sourceSelector.",
|
|
1373
|
+
input.sourceRef ? "Take a new snapshot to get fresh refs, or use sourceSelector for elements not in the accessibility tree." : void 0
|
|
1374
|
+
);
|
|
1375
|
+
}
|
|
1376
|
+
let targetLocator = null;
|
|
1377
|
+
if (input.targetRef) {
|
|
1378
|
+
targetLocator = await this.requireLocator(input.targetRef, threadId);
|
|
1379
|
+
} else if (input.targetSelector) {
|
|
1380
|
+
targetLocator = page.locator(input.targetSelector);
|
|
1381
|
+
}
|
|
1382
|
+
if (!targetLocator) {
|
|
1383
|
+
return this.createError(
|
|
1384
|
+
"stale_ref",
|
|
1385
|
+
input.targetRef ? `Target ref ${input.targetRef} not found.` : "No target element specified. Provide targetRef or targetSelector.",
|
|
1386
|
+
input.targetRef ? "Take a new snapshot to get fresh refs, or use targetSelector for elements not in the accessibility tree." : void 0
|
|
1387
|
+
);
|
|
1388
|
+
}
|
|
1389
|
+
await sourceLocator.dragTo(targetLocator, { timeout: this.defaultTimeout });
|
|
1390
|
+
return {
|
|
1391
|
+
success: true,
|
|
1392
|
+
url: page.url(),
|
|
1393
|
+
hint: "Drag complete. Take a snapshot to see the result."
|
|
1394
|
+
};
|
|
1395
|
+
} catch (error) {
|
|
1396
|
+
return this.createErrorFromException(error, "Drag");
|
|
1397
|
+
}
|
|
1398
|
+
}
|
|
1399
|
+
// ---------------------------------------------------------------------------
|
|
1400
|
+
// 16. browser_evaluate - Execute JavaScript
|
|
1401
|
+
// ---------------------------------------------------------------------------
|
|
1402
|
+
async evaluate(input, threadId) {
|
|
1403
|
+
try {
|
|
1404
|
+
const page = await this.getPage(threadId);
|
|
1405
|
+
const wrappedScript = `(async () => { ${input.script} })()`;
|
|
1406
|
+
const result = await page.evaluate(wrappedScript);
|
|
1407
|
+
return {
|
|
1408
|
+
success: true,
|
|
1409
|
+
result,
|
|
1410
|
+
hint: "JavaScript executed. Take a snapshot if the page may have changed."
|
|
1411
|
+
};
|
|
1412
|
+
} catch (error) {
|
|
1413
|
+
return this.createErrorFromException(error, "Evaluate");
|
|
1414
|
+
}
|
|
1415
|
+
}
|
|
1416
|
+
// ---------------------------------------------------------------------------
|
|
1417
|
+
// 17. browser_close - Close browser
|
|
1418
|
+
// ---------------------------------------------------------------------------
|
|
1419
|
+
async closeBrowser() {
|
|
1420
|
+
try {
|
|
1421
|
+
await this.close();
|
|
1422
|
+
return {
|
|
1423
|
+
success: true,
|
|
1424
|
+
hint: "Browser closed. Call browser_goto to start a new session."
|
|
1425
|
+
};
|
|
1426
|
+
} catch (error) {
|
|
1427
|
+
return this.createErrorFromException(error, "Close");
|
|
1428
|
+
}
|
|
1429
|
+
}
|
|
1430
|
+
// ---------------------------------------------------------------------------
|
|
1431
|
+
// Screencast (for Studio live view)
|
|
1432
|
+
// ---------------------------------------------------------------------------
|
|
1433
|
+
async startScreencast(_options) {
|
|
1434
|
+
const requestedThreadId = _options?.threadId;
|
|
1435
|
+
const effectiveThreadId = this.getScope() === "thread" ? requestedThreadId ?? this.getCurrentThread() ?? browser.DEFAULT_THREAD_ID : requestedThreadId;
|
|
1436
|
+
let browserManager;
|
|
1437
|
+
if (this.getScope() === "thread") {
|
|
1438
|
+
browserManager = await this.getManagerForThread(effectiveThreadId);
|
|
1439
|
+
} else {
|
|
1440
|
+
if (!this.sharedManager) throw new Error("Browser not launched");
|
|
1441
|
+
browserManager = this.sharedManager;
|
|
1442
|
+
}
|
|
1443
|
+
const provider = {
|
|
1444
|
+
getCdpSession: async () => {
|
|
1445
|
+
const currentPage = browserManager.getPage();
|
|
1446
|
+
if (!currentPage) {
|
|
1447
|
+
throw new Error("No active page available");
|
|
1448
|
+
}
|
|
1449
|
+
const cdpSession = await currentPage.context().newCDPSession(currentPage);
|
|
1450
|
+
return cdpSession;
|
|
1451
|
+
},
|
|
1452
|
+
isBrowserRunning: () => browserManager.isLaunched()
|
|
1453
|
+
};
|
|
1454
|
+
const stream = new browser.ScreencastStreamImpl(provider, _options);
|
|
1455
|
+
const streamKey = this.getStreamKey(effectiveThreadId);
|
|
1456
|
+
this.activeScreencastStreams.set(streamKey, stream);
|
|
1457
|
+
const context = browserManager.getContext();
|
|
1458
|
+
if (context) {
|
|
1459
|
+
const onNewPage = (_newPage) => {
|
|
1460
|
+
setTimeout(() => {
|
|
1461
|
+
if (stream.isActive()) {
|
|
1462
|
+
stream.reconnect().catch(() => {
|
|
1463
|
+
});
|
|
1464
|
+
}
|
|
1465
|
+
}, 100);
|
|
1466
|
+
};
|
|
1467
|
+
context.on("page", onNewPage);
|
|
1468
|
+
const pageCloseHandlers = /* @__PURE__ */ new Map();
|
|
1469
|
+
const frameNavigatedHandlers = /* @__PURE__ */ new Map();
|
|
1470
|
+
const setupPageListeners = (page) => {
|
|
1471
|
+
const onFrameNavigated = (frame) => {
|
|
1472
|
+
if (!frame.parentFrame()) {
|
|
1473
|
+
stream.emitUrl(frame.url());
|
|
1474
|
+
this.updateSessionBrowserState(effectiveThreadId);
|
|
1475
|
+
}
|
|
1476
|
+
};
|
|
1477
|
+
page.on("framenavigated", onFrameNavigated);
|
|
1478
|
+
frameNavigatedHandlers.set(page, onFrameNavigated);
|
|
1479
|
+
const onClose = () => {
|
|
1480
|
+
pageCloseHandlers.delete(page);
|
|
1481
|
+
const navHandler = frameNavigatedHandlers.get(page);
|
|
1482
|
+
if (navHandler) {
|
|
1483
|
+
page.off("framenavigated", navHandler);
|
|
1484
|
+
frameNavigatedHandlers.delete(page);
|
|
1485
|
+
}
|
|
1486
|
+
setTimeout(() => {
|
|
1487
|
+
const remainingPages = browserManager.getPages();
|
|
1488
|
+
if (stream.isActive() && remainingPages.length > 0) {
|
|
1489
|
+
stream.reconnect().catch(() => {
|
|
1490
|
+
});
|
|
1491
|
+
const activePage = remainingPages[browserManager.getActiveIndex()] || remainingPages[0];
|
|
1492
|
+
if (activePage) {
|
|
1493
|
+
const url = activePage.url();
|
|
1494
|
+
if (url && url !== "about:blank") {
|
|
1495
|
+
stream.emitUrl(url);
|
|
1496
|
+
}
|
|
1497
|
+
}
|
|
1498
|
+
}
|
|
1499
|
+
}, 100);
|
|
1500
|
+
};
|
|
1501
|
+
page.once("close", onClose);
|
|
1502
|
+
pageCloseHandlers.set(page, onClose);
|
|
1503
|
+
};
|
|
1504
|
+
const setupPageCloseListener = setupPageListeners;
|
|
1505
|
+
for (const page of browserManager.getPages()) {
|
|
1506
|
+
setupPageCloseListener(page);
|
|
1507
|
+
}
|
|
1508
|
+
const onNewPageWithCloseListener = (newPage) => {
|
|
1509
|
+
setupPageCloseListener(newPage);
|
|
1510
|
+
const url = newPage.url();
|
|
1511
|
+
if (url && url !== "about:blank") {
|
|
1512
|
+
stream.emitUrl(url);
|
|
1513
|
+
}
|
|
1514
|
+
onNewPage();
|
|
1515
|
+
};
|
|
1516
|
+
context.off("page", onNewPage);
|
|
1517
|
+
context.on("page", onNewPageWithCloseListener);
|
|
1518
|
+
stream.once("stop", () => {
|
|
1519
|
+
context.off("page", onNewPageWithCloseListener);
|
|
1520
|
+
for (const [page, handler] of pageCloseHandlers) {
|
|
1521
|
+
page.off("close", handler);
|
|
1522
|
+
}
|
|
1523
|
+
pageCloseHandlers.clear();
|
|
1524
|
+
for (const [page, handler] of frameNavigatedHandlers) {
|
|
1525
|
+
page.off("framenavigated", handler);
|
|
1526
|
+
}
|
|
1527
|
+
frameNavigatedHandlers.clear();
|
|
1528
|
+
this.activeScreencastStreams.delete(streamKey);
|
|
1529
|
+
});
|
|
1530
|
+
}
|
|
1531
|
+
await stream.start();
|
|
1532
|
+
return stream;
|
|
1533
|
+
}
|
|
1534
|
+
// ---------------------------------------------------------------------------
|
|
1535
|
+
// Event Injection (for Studio live view interactivity)
|
|
1536
|
+
// ---------------------------------------------------------------------------
|
|
1537
|
+
async injectMouseEvent(event, threadId) {
|
|
1538
|
+
const effectiveThreadId = threadId ?? this.getCurrentThread();
|
|
1539
|
+
const manager = await this.getManagerForThread(effectiveThreadId);
|
|
1540
|
+
await manager.injectMouseEvent(event);
|
|
1541
|
+
}
|
|
1542
|
+
async injectKeyboardEvent(event, threadId) {
|
|
1543
|
+
const effectiveThreadId = threadId ?? this.getCurrentThread();
|
|
1544
|
+
const manager = await this.getManagerForThread(effectiveThreadId);
|
|
1545
|
+
const cdp = await manager.getCDPSession();
|
|
1546
|
+
await cdp.send("Input.dispatchKeyEvent", {
|
|
1547
|
+
type: event.type,
|
|
1548
|
+
key: event.key,
|
|
1549
|
+
code: event.code,
|
|
1550
|
+
text: event.text,
|
|
1551
|
+
modifiers: event.modifiers ?? 0,
|
|
1552
|
+
windowsVirtualKeyCode: event.windowsVirtualKeyCode
|
|
1553
|
+
});
|
|
1554
|
+
}
|
|
1555
|
+
};
|
|
1556
|
+
|
|
1557
|
+
exports.AgentBrowser = AgentBrowser;
|
|
1558
|
+
exports.BROWSER_TOOLS = BROWSER_TOOLS;
|
|
1559
|
+
exports.backInputSchema = backInputSchema;
|
|
1560
|
+
exports.browserSchemas = browserSchemas;
|
|
1561
|
+
exports.clickInputSchema = clickInputSchema;
|
|
1562
|
+
exports.closeInputSchema = closeInputSchema;
|
|
1563
|
+
exports.createAgentBrowserTools = createAgentBrowserTools;
|
|
1564
|
+
exports.dialogInputSchema = dialogInputSchema;
|
|
1565
|
+
exports.dragInputSchema = dragInputSchema;
|
|
1566
|
+
exports.evaluateInputSchema = evaluateInputSchema;
|
|
1567
|
+
exports.gotoInputSchema = gotoInputSchema;
|
|
1568
|
+
exports.hoverInputSchema = hoverInputSchema;
|
|
1569
|
+
exports.pressInputSchema = pressInputSchema;
|
|
1570
|
+
exports.scrollInputSchema = scrollInputSchema;
|
|
1571
|
+
exports.selectInputSchema = selectInputSchema;
|
|
1572
|
+
exports.snapshotInputSchema = snapshotInputSchema;
|
|
1573
|
+
exports.tabsInputSchema = tabsInputSchema;
|
|
1574
|
+
exports.typeInputSchema = typeInputSchema;
|
|
1575
|
+
exports.waitInputSchema = waitInputSchema;
|
|
1576
|
+
//# sourceMappingURL=index.cjs.map
|
|
1577
|
+
//# sourceMappingURL=index.cjs.map
|