@mastra/agent-browser 0.4.1 → 0.5.0-alpha.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,1779 +1,1656 @@
1
- import { ThreadManager, MastraBrowser, DEFAULT_THREAD_ID, createBrowserRecordingTools, ScreencastStreamImpl } from '@mastra/core/browser';
2
- import { BrowserManager } from 'agent-browser';
3
- import { createTool } from '@mastra/core/tools';
4
- import { z } from 'zod';
5
-
6
- // src/agent-browser.ts
1
+ import { DEFAULT_THREAD_ID, MastraBrowser, ScreencastStreamImpl, ThreadManager, createBrowserRecordingTools } from "@mastra/core/browser";
2
+ import { BrowserManager } from "agent-browser";
3
+ import { createTool } from "@mastra/core/tools";
4
+ import { z } from "zod";
5
+ //#region src/thread-manager.ts
6
+ /**
7
+ * AgentBrowserThreadManager - Thread scope management for AgentBrowser
8
+ *
9
+ * Manages thread-scoped browser sessions using agent-browser's
10
+ * BrowserManager capabilities (newWindow, switchTo, closeTab).
11
+ */
12
+ /**
13
+ * Thread manager implementation for AgentBrowser.
14
+ *
15
+ * Supports two scope modes:
16
+ * - 'shared': All threads share the shared browser manager
17
+ * - 'thread': Each thread gets a dedicated browser manager instance
18
+ */
7
19
  var AgentBrowserThreadManager = class extends ThreadManager {
8
- browserConfig;
9
- resolveCdpUrl;
10
- onBrowserCreated;
11
- constructor(config) {
12
- super(config);
13
- this.browserConfig = config.browserConfig;
14
- this.resolveCdpUrl = config.resolveCdpUrl;
15
- this.onBrowserCreated = config.onBrowserCreated;
16
- }
17
- /**
18
- * Get the page for a specific thread, creating session if needed.
19
- */
20
- async getPageForThread(threadId) {
21
- const manager = await this.getManagerForThread(threadId);
22
- return manager.getPage();
23
- }
24
- /**
25
- * Create a new session for a thread.
26
- */
27
- async createSession(threadId) {
28
- const savedState = this.getSavedBrowserState(threadId);
29
- const session = {
30
- threadId,
31
- createdAt: Date.now(),
32
- browserState: savedState
33
- };
34
- if (this.scope === "thread") {
35
- const manager = new BrowserManager();
36
- const launchOptions = {
37
- headless: this.browserConfig.headless,
38
- viewport: this.browserConfig.viewport,
39
- profile: this.browserConfig.profile,
40
- executablePath: this.browserConfig.executablePath,
41
- storageState: this.browserConfig.storageState
42
- };
43
- if (this.browserConfig.cdpUrl && this.resolveCdpUrl) {
44
- launchOptions.cdpUrl = await this.resolveCdpUrl(this.browserConfig.cdpUrl);
45
- }
46
- try {
47
- await manager.launch(launchOptions);
48
- } catch (error) {
49
- try {
50
- await manager.close();
51
- } catch {
52
- }
53
- throw error;
54
- }
55
- session.manager = manager;
56
- this.threadManagers.set(threadId, manager);
57
- try {
58
- if (savedState && savedState.tabs.length > 0) {
59
- this.logger?.debug?.(`Restoring browser state for thread ${threadId}: ${savedState.tabs.length} tabs`);
60
- await this.restoreBrowserState(manager, savedState);
61
- }
62
- this.onBrowserCreated?.(manager, threadId);
63
- } catch (error) {
64
- this.threadManagers.delete(threadId);
65
- session.manager = void 0;
66
- try {
67
- await manager.close();
68
- } catch {
69
- }
70
- throw error;
71
- }
72
- }
73
- return session;
74
- }
75
- /**
76
- * Restore browser state (multiple tabs) to a browser manager.
77
- */
78
- async restoreBrowserState(manager, state) {
79
- try {
80
- const firstTab = state.tabs[0];
81
- if (firstTab?.url) {
82
- const page = manager.getPage();
83
- if (page) {
84
- await page.goto(firstTab.url, { waitUntil: "domcontentloaded" });
85
- }
86
- }
87
- for (let i = 1; i < state.tabs.length; i++) {
88
- const tab = state.tabs[i];
89
- if (tab?.url) {
90
- await manager.newTab();
91
- const page = manager.getPage();
92
- if (page) {
93
- await page.goto(tab.url, { waitUntil: "domcontentloaded" });
94
- }
95
- }
96
- }
97
- if (state.tabs.length > 1 && state.activeTabIndex >= 0 && state.activeTabIndex < state.tabs.length) {
98
- await manager.switchTo(state.activeTabIndex);
99
- }
100
- } catch (error) {
101
- this.logger?.warn?.(`Failed to restore browser state: ${error}`);
102
- }
103
- }
104
- /**
105
- * Get the browser manager for a specific session.
106
- */
107
- getManagerForSession(session) {
108
- if (this.scope === "thread" && session.manager) {
109
- return session.manager;
110
- }
111
- return this.getSharedManager();
112
- }
113
- /**
114
- * Destroy a session and clean up resources.
115
- */
116
- async doDestroySession(session) {
117
- if (this.scope === "thread" && session.manager) {
118
- await session.manager.close();
119
- }
120
- }
121
- /**
122
- * Destroy all sessions (called during browser close).
123
- * doDestroySession handles closing individual browser managers.
124
- */
125
- async destroyAllSessions() {
126
- await super.destroyAllSessions();
127
- }
20
+ browserConfig;
21
+ resolveCdpUrl;
22
+ onBrowserCreated;
23
+ constructor(config) {
24
+ super(config);
25
+ this.browserConfig = config.browserConfig;
26
+ this.resolveCdpUrl = config.resolveCdpUrl;
27
+ this.onBrowserCreated = config.onBrowserCreated;
28
+ }
29
+ /**
30
+ * Get the page for a specific thread, creating session if needed.
31
+ */
32
+ async getPageForThread(threadId) {
33
+ return (await this.getManagerForThread(threadId)).getPage();
34
+ }
35
+ /**
36
+ * Create a new session for a thread.
37
+ */
38
+ async createSession(threadId) {
39
+ const savedState = this.getSavedBrowserState(threadId);
40
+ const session = {
41
+ threadId,
42
+ createdAt: Date.now(),
43
+ browserState: savedState
44
+ };
45
+ if (this.scope === "thread") {
46
+ const manager = new BrowserManager();
47
+ const launchOptions = {
48
+ headless: this.browserConfig.headless,
49
+ viewport: this.browserConfig.viewport,
50
+ profile: this.browserConfig.profile,
51
+ executablePath: this.browserConfig.executablePath,
52
+ storageState: this.browserConfig.storageState
53
+ };
54
+ if (this.browserConfig.cdpUrl && this.resolveCdpUrl) launchOptions.cdpUrl = await this.resolveCdpUrl(this.browserConfig.cdpUrl);
55
+ if (this.browserConfig.cdpHeaders) launchOptions.cdpHeaders = this.browserConfig.cdpHeaders;
56
+ try {
57
+ await manager.launch(launchOptions);
58
+ } catch (error) {
59
+ try {
60
+ await manager.close();
61
+ } catch {}
62
+ throw error;
63
+ }
64
+ session.manager = manager;
65
+ this.threadManagers.set(threadId, manager);
66
+ try {
67
+ if (savedState && savedState.tabs.length > 0) {
68
+ this.logger?.debug?.(`Restoring browser state for thread ${threadId}: ${savedState.tabs.length} tabs`);
69
+ await this.restoreBrowserState(manager, savedState);
70
+ }
71
+ this.onBrowserCreated?.(manager, threadId);
72
+ } catch (error) {
73
+ this.threadManagers.delete(threadId);
74
+ session.manager = void 0;
75
+ try {
76
+ await manager.close();
77
+ } catch {}
78
+ throw error;
79
+ }
80
+ }
81
+ return session;
82
+ }
83
+ /**
84
+ * Restore browser state (multiple tabs) to a browser manager.
85
+ */
86
+ async restoreBrowserState(manager, state) {
87
+ try {
88
+ const firstTab = state.tabs[0];
89
+ if (firstTab?.url) {
90
+ const page = manager.getPage();
91
+ if (page) await page.goto(firstTab.url, { waitUntil: "domcontentloaded" });
92
+ }
93
+ for (let i = 1; i < state.tabs.length; i++) {
94
+ const tab = state.tabs[i];
95
+ if (tab?.url) {
96
+ await manager.newTab();
97
+ const page = manager.getPage();
98
+ if (page) await page.goto(tab.url, { waitUntil: "domcontentloaded" });
99
+ }
100
+ }
101
+ if (state.tabs.length > 1 && state.activeTabIndex >= 0 && state.activeTabIndex < state.tabs.length) await manager.switchTo(state.activeTabIndex);
102
+ } catch (error) {
103
+ this.logger?.warn?.(`Failed to restore browser state: ${error}`);
104
+ }
105
+ }
106
+ /**
107
+ * Get the browser manager for a specific session.
108
+ */
109
+ getManagerForSession(session) {
110
+ if (this.scope === "thread" && session.manager) return session.manager;
111
+ return this.getSharedManager();
112
+ }
113
+ /**
114
+ * Destroy a session and clean up resources.
115
+ */
116
+ async doDestroySession(session) {
117
+ if (this.scope === "thread" && session.manager) await session.manager.close();
118
+ }
119
+ /**
120
+ * Destroy all sessions (called during browser close).
121
+ * doDestroySession handles closing individual browser managers.
122
+ */
123
+ async destroyAllSessions() {
124
+ await super.destroyAllSessions();
125
+ }
128
126
  };
129
- var gotoInputSchema = z.object({
130
- url: z.string().describe("The URL to navigate to"),
131
- waitUntil: z.enum(["load", "domcontentloaded", "networkidle"]).optional().describe("When to consider navigation complete (default: domcontentloaded)"),
132
- timeout: z.number().optional().describe("Navigation timeout in milliseconds")
127
+ //#endregion
128
+ //#region src/schemas.ts
129
+ /**
130
+ * AgentBrowser Tool Schemas
131
+ *
132
+ * Flat schemas for browser tools. Each tool has a single-purpose schema
133
+ * without discriminated unions, making them easier for LLMs to understand.
134
+ *
135
+ * Tools:
136
+ * - Core: goto, snapshot, click, type, press, select, scroll, close
137
+ * - Extended: hover, back, dialog, wait, tabs, drag
138
+ * - Escape Hatch: evaluate
139
+ */
140
+ /**
141
+ * browser_goto - Navigate to a URL
142
+ */
143
+ const gotoInputSchema = z.object({
144
+ url: z.string().describe("The URL to navigate to"),
145
+ waitUntil: z.enum([
146
+ "load",
147
+ "domcontentloaded",
148
+ "networkidle"
149
+ ]).optional().describe("When to consider navigation complete (default: domcontentloaded)"),
150
+ timeout: z.number().optional().describe("Navigation timeout in milliseconds")
133
151
  });
134
- var snapshotInputSchema = z.object({
135
- interactiveOnly: z.boolean().optional().describe("Only include interactive elements (default: true)"),
136
- maxDepth: z.number().optional().describe("Maximum depth of the tree to return")
152
+ /**
153
+ * browser_snapshot - Get accessibility tree snapshot
154
+ */
155
+ const snapshotInputSchema = z.object({
156
+ interactiveOnly: z.boolean().optional().describe("Only include interactive elements (default: true)"),
157
+ maxDepth: z.number().optional().describe("Maximum depth of the tree to return")
137
158
  });
138
- var clickInputSchema = z.object({
139
- ref: z.string().describe("Element ref from snapshot (e.g., @e5)"),
140
- button: z.enum(["left", "right", "middle"]).optional().describe("Mouse button (default: left)"),
141
- clickCount: z.number().optional().describe("Number of clicks (default: 1, use 2 for double-click)"),
142
- modifiers: z.array(z.enum(["Alt", "Control", "Meta", "Shift"])).optional().describe("Modifier keys to hold"),
143
- waitUntil: z.enum(["load", "domcontentloaded", "networkidle"]).optional().describe("If the click triggers a navigation, wait for this page load state before returning"),
144
- timeout: z.number().nonnegative().optional().describe("Timeout in milliseconds for the click and optional waitUntil")
159
+ /**
160
+ * browser_click - Click an element
161
+ */
162
+ const clickInputSchema = z.object({
163
+ ref: z.string().describe("Element ref from snapshot (e.g., @e5)"),
164
+ button: z.enum([
165
+ "left",
166
+ "right",
167
+ "middle"
168
+ ]).optional().describe("Mouse button (default: left)"),
169
+ clickCount: z.number().optional().describe("Number of clicks (default: 1, use 2 for double-click)"),
170
+ modifiers: z.array(z.enum([
171
+ "Alt",
172
+ "Control",
173
+ "Meta",
174
+ "Shift"
175
+ ])).optional().describe("Modifier keys to hold"),
176
+ waitUntil: z.enum([
177
+ "load",
178
+ "domcontentloaded",
179
+ "networkidle"
180
+ ]).optional().describe("If the click triggers a navigation, wait for this page load state before returning"),
181
+ timeout: z.number().nonnegative().optional().describe("Timeout in milliseconds for the click and optional waitUntil")
145
182
  });
146
- var typeInputSchema = z.object({
147
- ref: z.string().describe("Element ref from snapshot"),
148
- text: z.string().describe("Text to type"),
149
- clear: z.boolean().optional().describe("Clear existing content before typing (default: false)"),
150
- delay: z.number().optional().describe("Delay between keystrokes in ms")
183
+ /**
184
+ * browser_type - Type text into an element
185
+ */
186
+ const typeInputSchema = z.object({
187
+ ref: z.string().describe("Element ref from snapshot"),
188
+ text: z.string().describe("Text to type"),
189
+ clear: z.boolean().optional().describe("Clear existing content before typing (default: false)"),
190
+ delay: z.number().optional().describe("Delay between keystrokes in ms")
151
191
  });
152
- var pressInputSchema = z.object({
153
- key: z.string().describe("Key to press (e.g., Enter, Tab, Escape, Control+a)"),
154
- modifiers: z.array(z.enum(["Alt", "Control", "Meta", "Shift"])).optional().describe("Modifier keys to hold"),
155
- waitUntil: z.enum(["load", "domcontentloaded", "networkidle"]).optional().describe("If the key press triggers a navigation, wait for this page load state before returning"),
156
- timeout: z.number().nonnegative().optional().describe("Timeout in milliseconds for the optional waitUntil")
192
+ /**
193
+ * browser_press - Press a keyboard key
194
+ */
195
+ const pressInputSchema = z.object({
196
+ key: z.string().describe("Key to press (e.g., Enter, Tab, Escape, Control+a)"),
197
+ modifiers: z.array(z.enum([
198
+ "Alt",
199
+ "Control",
200
+ "Meta",
201
+ "Shift"
202
+ ])).optional().describe("Modifier keys to hold"),
203
+ waitUntil: z.enum([
204
+ "load",
205
+ "domcontentloaded",
206
+ "networkidle"
207
+ ]).optional().describe("If the key press triggers a navigation, wait for this page load state before returning"),
208
+ timeout: z.number().nonnegative().optional().describe("Timeout in milliseconds for the optional waitUntil")
157
209
  });
158
- var selectInputSchema = z.object({
159
- ref: z.string().describe("Select element ref from snapshot"),
160
- value: z.string().optional().describe("Option value to select"),
161
- label: z.string().optional().describe("Option label to select"),
162
- index: z.number().int().min(0).optional().describe("Option index to select (0-based)"),
163
- waitUntil: z.enum(["load", "domcontentloaded", "networkidle"]).optional().describe("If the selection triggers a navigation, wait for this page load state before returning"),
164
- timeout: z.number().nonnegative().optional().describe("Timeout in milliseconds for the selection and optional waitUntil")
210
+ /**
211
+ * browser_select - Select option from dropdown
212
+ */
213
+ const selectInputSchema = z.object({
214
+ ref: z.string().describe("Select element ref from snapshot"),
215
+ value: z.string().optional().describe("Option value to select"),
216
+ label: z.string().optional().describe("Option label to select"),
217
+ index: z.number().int().min(0).optional().describe("Option index to select (0-based)"),
218
+ waitUntil: z.enum([
219
+ "load",
220
+ "domcontentloaded",
221
+ "networkidle"
222
+ ]).optional().describe("If the selection triggers a navigation, wait for this page load state before returning"),
223
+ timeout: z.number().nonnegative().optional().describe("Timeout in milliseconds for the selection and optional waitUntil")
165
224
  }).superRefine((data, ctx) => {
166
- if (data.value === void 0 && data.label === void 0 && data.index === void 0) {
167
- ctx.addIssue({
168
- code: z.ZodIssueCode.custom,
169
- message: "At least one of value, label, or index is required"
170
- });
171
- }
172
- });
173
- var scrollInputSchema = z.object({
174
- direction: z.enum(["up", "down", "left", "right"]).describe("Scroll direction"),
175
- amount: z.number().optional().describe("Scroll amount in pixels (default: 300)"),
176
- ref: z.string().optional().describe("Element ref to scroll (scrolls page if omitted)")
225
+ if (data.value === void 0 && data.label === void 0 && data.index === void 0) ctx.addIssue({
226
+ code: z.ZodIssueCode.custom,
227
+ message: "At least one of value, label, or index is required"
228
+ });
177
229
  });
178
- var closeInputSchema = z.object({});
179
- var hoverInputSchema = z.object({
180
- ref: z.string().describe("Element ref from snapshot")
230
+ /**
231
+ * browser_scroll - Scroll the page or element
232
+ */
233
+ const scrollInputSchema = z.object({
234
+ direction: z.enum([
235
+ "up",
236
+ "down",
237
+ "left",
238
+ "right"
239
+ ]).describe("Scroll direction"),
240
+ amount: z.number().optional().describe("Scroll amount in pixels (default: 300)"),
241
+ ref: z.string().optional().describe("Element ref to scroll (scrolls page if omitted)")
181
242
  });
182
- var backInputSchema = z.object({});
183
- var dialogInputSchema = z.object({
184
- triggerRef: z.string().describe("Element ref that triggers the dialog (e.g., @e5)"),
185
- action: z.enum(["accept", "dismiss"]).describe("Accept or dismiss the dialog"),
186
- text: z.string().optional().describe("Text to enter for prompt dialogs")
243
+ /**
244
+ * browser_close - Close the browser
245
+ */
246
+ const closeInputSchema = z.object({});
247
+ /**
248
+ * browser_hover - Hover over an element
249
+ */
250
+ const hoverInputSchema = z.object({ ref: z.string().describe("Element ref from snapshot") });
251
+ /**
252
+ * browser_back - Go back in browser history
253
+ */
254
+ const backInputSchema = z.object({});
255
+ /**
256
+ * browser_dialog - Click an element that triggers a dialog and handle it
257
+ */
258
+ const dialogInputSchema = z.object({
259
+ triggerRef: z.string().describe("Element ref that triggers the dialog (e.g., @e5)"),
260
+ action: z.enum(["accept", "dismiss"]).describe("Accept or dismiss the dialog"),
261
+ text: z.string().optional().describe("Text to enter for prompt dialogs")
187
262
  });
188
- var waitInputSchema = z.object({
189
- ref: z.string().optional().describe("Element ref to wait for"),
190
- state: z.enum(["visible", "hidden", "attached", "detached"]).optional().describe("State to wait for (default: visible)"),
191
- timeout: z.number().optional().describe("Maximum wait time in ms (default: 30000)")
263
+ /**
264
+ * browser_wait - Wait for an element or condition
265
+ */
266
+ const waitInputSchema = z.object({
267
+ ref: z.string().optional().describe("Element ref to wait for"),
268
+ state: z.enum([
269
+ "visible",
270
+ "hidden",
271
+ "attached",
272
+ "detached"
273
+ ]).optional().describe("State to wait for (default: visible)"),
274
+ timeout: z.number().optional().describe("Maximum wait time in ms (default: 30000)")
192
275
  });
193
- var tabsInputSchema = z.object({
194
- action: z.enum(["list", "new", "switch", "close"]).describe("Tab action"),
195
- index: z.number().int().min(0).optional().describe("Tab index for switch/close"),
196
- url: z.string().optional().describe("URL to open in new tab")
276
+ /**
277
+ * browser_tabs - Manage browser tabs
278
+ */
279
+ const tabsInputSchema = z.object({
280
+ action: z.enum([
281
+ "list",
282
+ "new",
283
+ "switch",
284
+ "close"
285
+ ]).describe("Tab action"),
286
+ index: z.number().int().min(0).optional().describe("Tab index for switch/close"),
287
+ url: z.string().optional().describe("URL to open in new tab")
197
288
  }).superRefine((value, ctx) => {
198
- if (value.action === "switch" && value.index === void 0) {
199
- ctx.addIssue({
200
- code: z.ZodIssueCode.custom,
201
- path: ["index"],
202
- message: 'index is required when action is "switch"'
203
- });
204
- }
289
+ if (value.action === "switch" && value.index === void 0) ctx.addIssue({
290
+ code: z.ZodIssueCode.custom,
291
+ path: ["index"],
292
+ message: "index is required when action is \"switch\""
293
+ });
205
294
  });
206
- var dragInputSchema = z.object({
207
- sourceRef: z.string().optional().describe("Element ref to drag from (e.g., @e5)"),
208
- targetRef: z.string().optional().describe("Element ref to drag to (e.g., @e7)"),
209
- sourceSelector: z.string().optional().describe("CSS selector for source element (use if ref not available)"),
210
- targetSelector: z.string().optional().describe("CSS selector for target element (use if ref not available)")
295
+ /**
296
+ * browser_drag - Drag an element to another element
297
+ */
298
+ const dragInputSchema = z.object({
299
+ sourceRef: z.string().optional().describe("Element ref to drag from (e.g., @e5)"),
300
+ targetRef: z.string().optional().describe("Element ref to drag to (e.g., @e7)"),
301
+ sourceSelector: z.string().optional().describe("CSS selector for source element (use if ref not available)"),
302
+ targetSelector: z.string().optional().describe("CSS selector for target element (use if ref not available)")
211
303
  }).superRefine((data, ctx) => {
212
- if (!data.sourceRef && !data.sourceSelector) {
213
- ctx.addIssue({
214
- code: z.ZodIssueCode.custom,
215
- path: ["sourceRef"],
216
- message: "Either sourceRef or sourceSelector is required"
217
- });
218
- }
219
- if (!data.targetRef && !data.targetSelector) {
220
- ctx.addIssue({
221
- code: z.ZodIssueCode.custom,
222
- path: ["targetRef"],
223
- message: "Either targetRef or targetSelector is required"
224
- });
225
- }
226
- });
227
- var screenshotInputSchema = z.object({
228
- fullPage: z.boolean().optional().describe("Capture the full scrollable page instead of just the viewport (default: false)")
304
+ if (!data.sourceRef && !data.sourceSelector) ctx.addIssue({
305
+ code: z.ZodIssueCode.custom,
306
+ path: ["sourceRef"],
307
+ message: "Either sourceRef or sourceSelector is required"
308
+ });
309
+ if (!data.targetRef && !data.targetSelector) ctx.addIssue({
310
+ code: z.ZodIssueCode.custom,
311
+ path: ["targetRef"],
312
+ message: "Either targetRef or targetSelector is required"
313
+ });
229
314
  });
230
- var evaluateInputSchema = z.object({
231
- script: z.string().describe(
232
- "JavaScript expression to evaluate in the browser and return the result. Do not use `return` \u2014 write a bare expression like `document.title` or `1 + 1`. For async code, wrap in an async IIFE: `(async () => { ... })()`."
233
- ),
234
- arg: z.unknown().optional().describe("Argument to pass to the script (JSON-serializable)")
315
+ /**
316
+ * browser_screenshot - Capture a screenshot of the current page
317
+ */
318
+ const screenshotInputSchema = z.object({ fullPage: z.boolean().optional().describe("Capture the full scrollable page instead of just the viewport (default: false)") });
319
+ /**
320
+ * browser_evaluate - Execute JavaScript in the browser
321
+ */
322
+ const evaluateInputSchema = z.object({
323
+ script: z.string().describe("JavaScript expression to evaluate in the browser and return the result. Do not use `return` — write a bare expression like `document.title` or `1 + 1`. For async code, wrap in an async IIFE: `(async () => { ... })()`."),
324
+ arg: z.unknown().optional().describe("Argument to pass to the script (JSON-serializable)")
235
325
  });
236
- var browserSchemas = {
237
- // Core
238
- goto: gotoInputSchema,
239
- snapshot: snapshotInputSchema,
240
- click: clickInputSchema,
241
- type: typeInputSchema,
242
- press: pressInputSchema,
243
- select: selectInputSchema,
244
- scroll: scrollInputSchema,
245
- close: closeInputSchema,
246
- // Extended
247
- hover: hoverInputSchema,
248
- back: backInputSchema,
249
- dialog: dialogInputSchema,
250
- wait: waitInputSchema,
251
- tabs: tabsInputSchema,
252
- drag: dragInputSchema,
253
- // Utility
254
- screenshot: screenshotInputSchema,
255
- // Escape hatch
256
- evaluate: evaluateInputSchema
326
+ const browserSchemas = {
327
+ goto: gotoInputSchema,
328
+ snapshot: snapshotInputSchema,
329
+ click: clickInputSchema,
330
+ type: typeInputSchema,
331
+ press: pressInputSchema,
332
+ select: selectInputSchema,
333
+ scroll: scrollInputSchema,
334
+ close: closeInputSchema,
335
+ hover: hoverInputSchema,
336
+ back: backInputSchema,
337
+ dialog: dialogInputSchema,
338
+ wait: waitInputSchema,
339
+ tabs: tabsInputSchema,
340
+ drag: dragInputSchema,
341
+ screenshot: screenshotInputSchema,
342
+ evaluate: evaluateInputSchema
257
343
  };
258
-
259
- // src/tools/constants.ts
260
- var BROWSER_TOOLS = {
261
- // Core
262
- GOTO: "browser_goto",
263
- SNAPSHOT: "browser_snapshot",
264
- CLICK: "browser_click",
265
- TYPE: "browser_type",
266
- PRESS: "browser_press",
267
- SELECT: "browser_select",
268
- SCROLL: "browser_scroll",
269
- CLOSE: "browser_close",
270
- // Extended
271
- HOVER: "browser_hover",
272
- BACK: "browser_back",
273
- DIALOG: "browser_dialog",
274
- WAIT: "browser_wait",
275
- TABS: "browser_tabs",
276
- DRAG: "browser_drag",
277
- // Utility
278
- SCREENSHOT: "browser_screenshot",
279
- // Escape hatch
280
- EVALUATE: "browser_evaluate"
344
+ //#endregion
345
+ //#region src/tools/constants.ts
346
+ /**
347
+ * Browser Tool Constants
348
+ */
349
+ const BROWSER_TOOLS = {
350
+ GOTO: "browser_goto",
351
+ SNAPSHOT: "browser_snapshot",
352
+ CLICK: "browser_click",
353
+ TYPE: "browser_type",
354
+ PRESS: "browser_press",
355
+ SELECT: "browser_select",
356
+ SCROLL: "browser_scroll",
357
+ CLOSE: "browser_close",
358
+ HOVER: "browser_hover",
359
+ BACK: "browser_back",
360
+ DIALOG: "browser_dialog",
361
+ WAIT: "browser_wait",
362
+ TABS: "browser_tabs",
363
+ DRAG: "browser_drag",
364
+ SCREENSHOT: "browser_screenshot",
365
+ EVALUATE: "browser_evaluate"
281
366
  };
282
-
283
- // src/tools/back.ts
367
+ //#endregion
368
+ //#region src/tools/back.ts
369
+ /**
370
+ * browser_back - Go back in browser history
371
+ */
284
372
  function createBackTool(browser) {
285
- return createTool({
286
- id: BROWSER_TOOLS.BACK,
287
- description: "Go back to the previous page in browser history.",
288
- inputSchema: backInputSchema,
289
- execute: async (_input, { agent }) => {
290
- const threadId = agent?.threadId;
291
- browser.setCurrentThread(threadId);
292
- await browser.ensureReady();
293
- return browser.back(threadId);
294
- }
295
- });
373
+ return createTool({
374
+ id: BROWSER_TOOLS.BACK,
375
+ description: "Go back to the previous page in browser history.",
376
+ inputSchema: backInputSchema,
377
+ execute: async (_input, { agent }) => {
378
+ const threadId = agent?.threadId;
379
+ browser.setCurrentThread(threadId);
380
+ await browser.ensureReady();
381
+ return browser.back(threadId);
382
+ }
383
+ });
296
384
  }
385
+ //#endregion
386
+ //#region src/tools/click.ts
387
+ /**
388
+ * browser_click - Click an element
389
+ */
297
390
  function createClickTool(browser) {
298
- return createTool({
299
- id: BROWSER_TOOLS.CLICK,
300
- description: "Click an element using its ref from a snapshot. Use clickCount: 2 for double-click. Pass waitUntil when the click triggers navigation so the page settles before the next snapshot.",
301
- inputSchema: clickInputSchema,
302
- execute: async (input, { agent }) => {
303
- const threadId = agent?.threadId;
304
- browser.setCurrentThread(threadId);
305
- await browser.ensureReady();
306
- return browser.click(input, threadId);
307
- }
308
- });
391
+ return createTool({
392
+ id: BROWSER_TOOLS.CLICK,
393
+ description: "Click an element using its ref from a snapshot. Use clickCount: 2 for double-click. Pass waitUntil when the click triggers navigation so the page settles before the next snapshot.",
394
+ inputSchema: clickInputSchema,
395
+ execute: async (input, { agent }) => {
396
+ const threadId = agent?.threadId;
397
+ browser.setCurrentThread(threadId);
398
+ await browser.ensureReady();
399
+ return browser.click(input, threadId);
400
+ }
401
+ });
309
402
  }
403
+ //#endregion
404
+ //#region src/tools/close.ts
405
+ /**
406
+ * browser_close - Close the browser
407
+ */
310
408
  function createCloseTool(browser) {
311
- return createTool({
312
- id: BROWSER_TOOLS.CLOSE,
313
- description: "Close the browser. Only use when done with all browsing.",
314
- inputSchema: closeInputSchema,
315
- execute: async (_input, { agent }) => {
316
- const threadId = agent?.threadId;
317
- browser.setCurrentThread(threadId);
318
- if (browser.getScope() !== "shared") {
319
- if (!threadId) {
320
- throw new Error("browser_close requires agent.threadId when browser scope is not shared");
321
- }
322
- browser.markBrowserCloseReason("agent", threadId);
323
- await browser.closeThreadSession(threadId);
324
- return { success: true, hint: "Thread's browser session closed. A new session will be created on next use." };
325
- }
326
- browser.markBrowserCloseReason("agent");
327
- await browser.close();
328
- return { success: true, hint: "Browser closed. It will be re-launched automatically on next use." };
329
- }
330
- });
409
+ return createTool({
410
+ id: BROWSER_TOOLS.CLOSE,
411
+ description: "Close the browser. Only use when done with all browsing.",
412
+ inputSchema: closeInputSchema,
413
+ execute: async (_input, { agent }) => {
414
+ const threadId = agent?.threadId;
415
+ browser.setCurrentThread(threadId);
416
+ if (browser.getScope() !== "shared") {
417
+ if (!threadId) throw new Error("browser_close requires agent.threadId when browser scope is not shared");
418
+ browser.markBrowserCloseReason("agent", threadId);
419
+ await browser.closeThreadSession(threadId);
420
+ return {
421
+ success: true,
422
+ hint: "Thread's browser session closed. A new session will be created on next use."
423
+ };
424
+ }
425
+ browser.markBrowserCloseReason("agent");
426
+ await browser.close();
427
+ return {
428
+ success: true,
429
+ hint: "Browser closed. It will be re-launched automatically on next use."
430
+ };
431
+ }
432
+ });
331
433
  }
434
+ //#endregion
435
+ //#region src/tools/dialog.ts
436
+ /**
437
+ * browser_dialog - Click element and handle resulting dialog
438
+ */
332
439
  function createDialogTool(browser) {
333
- return createTool({
334
- id: BROWSER_TOOLS.DIALOG,
335
- description: "Click an element that triggers a browser dialog (alert, confirm, prompt) and handle it. Use this instead of browser_click when you expect a dialog to appear.",
336
- inputSchema: dialogInputSchema,
337
- execute: async (input, { agent }) => {
338
- const threadId = agent?.threadId;
339
- browser.setCurrentThread(threadId);
340
- await browser.ensureReady();
341
- return browser.dialog(input, threadId);
342
- }
343
- });
440
+ return createTool({
441
+ id: BROWSER_TOOLS.DIALOG,
442
+ description: "Click an element that triggers a browser dialog (alert, confirm, prompt) and handle it. Use this instead of browser_click when you expect a dialog to appear.",
443
+ inputSchema: dialogInputSchema,
444
+ execute: async (input, { agent }) => {
445
+ const threadId = agent?.threadId;
446
+ browser.setCurrentThread(threadId);
447
+ await browser.ensureReady();
448
+ return browser.dialog(input, threadId);
449
+ }
450
+ });
344
451
  }
452
+ //#endregion
453
+ //#region src/tools/drag.ts
454
+ /**
455
+ * browser_drag - Drag an element to another element
456
+ */
345
457
  function createDragTool(browser) {
346
- return createTool({
347
- id: BROWSER_TOOLS.DRAG,
348
- description: "Drag an element to another element. Use refs from snapshot when available, or CSS selectors for elements not exposed in the accessibility tree.",
349
- inputSchema: dragInputSchema,
350
- execute: async (input, { agent }) => {
351
- const threadId = agent?.threadId;
352
- browser.setCurrentThread(threadId);
353
- await browser.ensureReady();
354
- return browser.drag(input, threadId);
355
- }
356
- });
458
+ return createTool({
459
+ id: BROWSER_TOOLS.DRAG,
460
+ description: "Drag an element to another element. Use refs from snapshot when available, or CSS selectors for elements not exposed in the accessibility tree.",
461
+ inputSchema: dragInputSchema,
462
+ execute: async (input, { agent }) => {
463
+ const threadId = agent?.threadId;
464
+ browser.setCurrentThread(threadId);
465
+ await browser.ensureReady();
466
+ return browser.drag(input, threadId);
467
+ }
468
+ });
357
469
  }
470
+ //#endregion
471
+ //#region src/tools/evaluate.ts
472
+ /**
473
+ * browser_evaluate - Execute JavaScript in the browser
474
+ */
358
475
  function createEvaluateTool(browser) {
359
- return createTool({
360
- id: BROWSER_TOOLS.EVALUATE,
361
- description: "Execute JavaScript in the browser. Use for complex interactions not covered by other tools. Returns the script result.",
362
- inputSchema: evaluateInputSchema,
363
- execute: async (input, { agent }) => {
364
- const threadId = agent?.threadId;
365
- browser.setCurrentThread(threadId);
366
- await browser.ensureReady();
367
- return browser.evaluate(input, threadId);
368
- }
369
- });
476
+ return createTool({
477
+ id: BROWSER_TOOLS.EVALUATE,
478
+ description: "Execute JavaScript in the browser. Use for complex interactions not covered by other tools. Returns the script result.",
479
+ inputSchema: evaluateInputSchema,
480
+ execute: async (input, { agent }) => {
481
+ const threadId = agent?.threadId;
482
+ browser.setCurrentThread(threadId);
483
+ await browser.ensureReady();
484
+ return browser.evaluate(input, threadId);
485
+ }
486
+ });
370
487
  }
488
+ //#endregion
489
+ //#region src/tools/goto.ts
490
+ /**
491
+ * browser_goto - Navigate to a URL
492
+ */
371
493
  function createGotoTool(browser) {
372
- return createTool({
373
- id: BROWSER_TOOLS.GOTO,
374
- description: "Navigate the browser to a URL.",
375
- inputSchema: gotoInputSchema,
376
- execute: async (input, { agent }) => {
377
- const threadId = agent?.threadId;
378
- browser.setCurrentThread(threadId);
379
- await browser.ensureReady();
380
- return browser.goto(input, threadId);
381
- }
382
- });
494
+ return createTool({
495
+ id: BROWSER_TOOLS.GOTO,
496
+ description: "Navigate the browser to a URL.",
497
+ inputSchema: gotoInputSchema,
498
+ execute: async (input, { agent }) => {
499
+ const threadId = agent?.threadId;
500
+ browser.setCurrentThread(threadId);
501
+ await browser.ensureReady();
502
+ return browser.goto(input, threadId);
503
+ }
504
+ });
383
505
  }
506
+ //#endregion
507
+ //#region src/tools/hover.ts
508
+ /**
509
+ * browser_hover - Hover over an element
510
+ */
384
511
  function createHoverTool(browser) {
385
- return createTool({
386
- id: BROWSER_TOOLS.HOVER,
387
- description: "Hover over an element to trigger hover states (dropdowns, tooltips).",
388
- inputSchema: hoverInputSchema,
389
- execute: async (input, { agent }) => {
390
- const threadId = agent?.threadId;
391
- browser.setCurrentThread(threadId);
392
- await browser.ensureReady();
393
- return browser.hover(input, threadId);
394
- }
395
- });
512
+ return createTool({
513
+ id: BROWSER_TOOLS.HOVER,
514
+ description: "Hover over an element to trigger hover states (dropdowns, tooltips).",
515
+ inputSchema: hoverInputSchema,
516
+ execute: async (input, { agent }) => {
517
+ const threadId = agent?.threadId;
518
+ browser.setCurrentThread(threadId);
519
+ await browser.ensureReady();
520
+ return browser.hover(input, threadId);
521
+ }
522
+ });
396
523
  }
524
+ //#endregion
525
+ //#region src/tools/press.ts
526
+ /**
527
+ * browser_press - Press a keyboard key
528
+ */
397
529
  function createPressTool(browser) {
398
- return createTool({
399
- id: BROWSER_TOOLS.PRESS,
400
- description: "Press a keyboard key (e.g., Enter, Tab, Escape, Control+a). Pass waitUntil when the keypress triggers navigation (e.g., Enter to submit a form) so the page settles before the next snapshot.",
401
- inputSchema: pressInputSchema,
402
- execute: async (input, { agent }) => {
403
- const threadId = agent?.threadId;
404
- browser.setCurrentThread(threadId);
405
- await browser.ensureReady();
406
- return browser.press(input, threadId);
407
- }
408
- });
530
+ return createTool({
531
+ id: BROWSER_TOOLS.PRESS,
532
+ description: "Press a keyboard key (e.g., Enter, Tab, Escape, Control+a). Pass waitUntil when the keypress triggers navigation (e.g., Enter to submit a form) so the page settles before the next snapshot.",
533
+ inputSchema: pressInputSchema,
534
+ execute: async (input, { agent }) => {
535
+ const threadId = agent?.threadId;
536
+ browser.setCurrentThread(threadId);
537
+ await browser.ensureReady();
538
+ return browser.press(input, threadId);
539
+ }
540
+ });
409
541
  }
542
+ //#endregion
543
+ //#region src/tools/screenshot.ts
544
+ /**
545
+ * browser_screenshot - Capture a screenshot of the current page
546
+ */
410
547
  function createScreenshotTool(browser) {
411
- return createTool({
412
- id: BROWSER_TOOLS.SCREENSHOT,
413
- description: "Capture a screenshot of the current viewport as a visible PNG (set fullPage: true for full-page capture). Use snapshot when you only need text or interactive elements \u2014 screenshots are expensive. Use this when you need to visually inspect the page, e.g. evaluating images, product photos, layout, design, or colors.",
414
- inputSchema: screenshotInputSchema,
415
- execute: async (input, { agent }) => {
416
- const threadId = agent?.threadId;
417
- browser.setCurrentThread(threadId);
418
- await browser.ensureReady();
419
- return await browser.screenshot(input, threadId);
420
- },
421
- toModelOutput(output) {
422
- const result = output;
423
- if (typeof result.base64 !== "string") {
424
- return {
425
- type: "content",
426
- value: [{ type: "text", text: result.message ?? "Failed to capture screenshot." }]
427
- };
428
- }
429
- return {
430
- type: "content",
431
- value: [
432
- {
433
- type: "media",
434
- mediaType: "image/png",
435
- data: result.base64
436
- }
437
- ]
438
- };
439
- }
440
- });
548
+ return createTool({
549
+ id: BROWSER_TOOLS.SCREENSHOT,
550
+ description: "Capture a screenshot of the current viewport as a visible PNG (set fullPage: true for full-page capture). Use snapshot when you only need text or interactive elements screenshots are expensive. Use this when you need to visually inspect the page, e.g. evaluating images, product photos, layout, design, or colors.",
551
+ inputSchema: screenshotInputSchema,
552
+ execute: async (input, { agent }) => {
553
+ const threadId = agent?.threadId;
554
+ browser.setCurrentThread(threadId);
555
+ await browser.ensureReady();
556
+ return await browser.screenshot(input, threadId);
557
+ },
558
+ toModelOutput(output) {
559
+ const result = output;
560
+ if (typeof result.base64 !== "string") return {
561
+ type: "content",
562
+ value: [{
563
+ type: "text",
564
+ text: result.message ?? "Failed to capture screenshot."
565
+ }]
566
+ };
567
+ return {
568
+ type: "content",
569
+ value: [{
570
+ type: "media",
571
+ mediaType: "image/png",
572
+ data: result.base64
573
+ }]
574
+ };
575
+ }
576
+ });
441
577
  }
578
+ //#endregion
579
+ //#region src/tools/scroll.ts
580
+ /**
581
+ * browser_scroll - Scroll the page or element
582
+ */
442
583
  function createScrollTool(browser) {
443
- return createTool({
444
- id: BROWSER_TOOLS.SCROLL,
445
- description: "Scroll the page or a specific element.",
446
- inputSchema: scrollInputSchema,
447
- execute: async (input, { agent }) => {
448
- const threadId = agent?.threadId;
449
- browser.setCurrentThread(threadId);
450
- await browser.ensureReady();
451
- return browser.scroll(input, threadId);
452
- }
453
- });
584
+ return createTool({
585
+ id: BROWSER_TOOLS.SCROLL,
586
+ description: "Scroll the page or a specific element.",
587
+ inputSchema: scrollInputSchema,
588
+ execute: async (input, { agent }) => {
589
+ const threadId = agent?.threadId;
590
+ browser.setCurrentThread(threadId);
591
+ await browser.ensureReady();
592
+ return browser.scroll(input, threadId);
593
+ }
594
+ });
454
595
  }
596
+ //#endregion
597
+ //#region src/tools/select.ts
598
+ /**
599
+ * browser_select - Select option from dropdown
600
+ */
455
601
  function createSelectTool(browser) {
456
- return createTool({
457
- id: BROWSER_TOOLS.SELECT,
458
- description: "Select an option from a dropdown by value, label, or index. Pass waitUntil when the selection triggers navigation so the page settles before the next snapshot.",
459
- inputSchema: selectInputSchema,
460
- execute: async (input, { agent }) => {
461
- const threadId = agent?.threadId;
462
- browser.setCurrentThread(threadId);
463
- await browser.ensureReady();
464
- return browser.select(input, threadId);
465
- }
466
- });
602
+ return createTool({
603
+ id: BROWSER_TOOLS.SELECT,
604
+ description: "Select an option from a dropdown by value, label, or index. Pass waitUntil when the selection triggers navigation so the page settles before the next snapshot.",
605
+ inputSchema: selectInputSchema,
606
+ execute: async (input, { agent }) => {
607
+ const threadId = agent?.threadId;
608
+ browser.setCurrentThread(threadId);
609
+ await browser.ensureReady();
610
+ return browser.select(input, threadId);
611
+ }
612
+ });
467
613
  }
614
+ //#endregion
615
+ //#region src/tools/snapshot.ts
616
+ /**
617
+ * browser_snapshot - Get accessibility tree snapshot
618
+ */
468
619
  function createSnapshotTool(browser) {
469
- return createTool({
470
- id: BROWSER_TOOLS.SNAPSHOT,
471
- description: "Get accessibility tree snapshot of the page. Returns text-based representation with element refs like [ref=e1], [ref=e2] for targeting.",
472
- inputSchema: snapshotInputSchema,
473
- execute: async (input, { agent }) => {
474
- const threadId = agent?.threadId;
475
- browser.setCurrentThread(threadId);
476
- await browser.ensureReady();
477
- return browser.snapshot(input, threadId);
478
- }
479
- });
620
+ return createTool({
621
+ id: BROWSER_TOOLS.SNAPSHOT,
622
+ description: "Get accessibility tree snapshot of the page. Returns text-based representation with element refs like [ref=e1], [ref=e2] for targeting.",
623
+ inputSchema: snapshotInputSchema,
624
+ execute: async (input, { agent }) => {
625
+ const threadId = agent?.threadId;
626
+ browser.setCurrentThread(threadId);
627
+ await browser.ensureReady();
628
+ return browser.snapshot(input, threadId);
629
+ }
630
+ });
480
631
  }
632
+ //#endregion
633
+ //#region src/tools/tabs.ts
634
+ /**
635
+ * browser_tabs - Manage browser tabs
636
+ */
481
637
  function createTabsTool(browser) {
482
- return createTool({
483
- id: BROWSER_TOOLS.TABS,
484
- description: "Manage browser tabs: list, open new, switch, or close tabs.",
485
- inputSchema: tabsInputSchema,
486
- execute: async (input, { agent }) => {
487
- const threadId = agent?.threadId;
488
- browser.setCurrentThread(threadId);
489
- await browser.ensureReady();
490
- return browser.tabs(input, threadId);
491
- }
492
- });
638
+ return createTool({
639
+ id: BROWSER_TOOLS.TABS,
640
+ description: "Manage browser tabs: list, open new, switch, or close tabs.",
641
+ inputSchema: tabsInputSchema,
642
+ execute: async (input, { agent }) => {
643
+ const threadId = agent?.threadId;
644
+ browser.setCurrentThread(threadId);
645
+ await browser.ensureReady();
646
+ return browser.tabs(input, threadId);
647
+ }
648
+ });
493
649
  }
650
+ //#endregion
651
+ //#region src/tools/type.ts
652
+ /**
653
+ * browser_type - Type text into an element
654
+ */
494
655
  function createTypeTool(browser) {
495
- return createTool({
496
- id: BROWSER_TOOLS.TYPE,
497
- description: "Type text into an input element. Use clear: true to replace existing content.",
498
- inputSchema: typeInputSchema,
499
- execute: async (input, { agent }) => {
500
- const threadId = agent?.threadId;
501
- browser.setCurrentThread(threadId);
502
- await browser.ensureReady();
503
- return browser.type(input, threadId);
504
- }
505
- });
656
+ return createTool({
657
+ id: BROWSER_TOOLS.TYPE,
658
+ description: "Type text into an input element. Use clear: true to replace existing content.",
659
+ inputSchema: typeInputSchema,
660
+ execute: async (input, { agent }) => {
661
+ const threadId = agent?.threadId;
662
+ browser.setCurrentThread(threadId);
663
+ await browser.ensureReady();
664
+ return browser.type(input, threadId);
665
+ }
666
+ });
506
667
  }
668
+ //#endregion
669
+ //#region src/tools/wait.ts
670
+ /**
671
+ * browser_wait - Wait for an element or condition
672
+ */
507
673
  function createWaitTool(browser) {
508
- return createTool({
509
- id: BROWSER_TOOLS.WAIT,
510
- description: "Wait for an element to appear, disappear, or reach a state.",
511
- inputSchema: waitInputSchema,
512
- execute: async (input, { agent }) => {
513
- const threadId = agent?.threadId;
514
- browser.setCurrentThread(threadId);
515
- await browser.ensureReady();
516
- return browser.wait(input, threadId);
517
- }
518
- });
674
+ return createTool({
675
+ id: BROWSER_TOOLS.WAIT,
676
+ description: "Wait for an element to appear, disappear, or reach a state.",
677
+ inputSchema: waitInputSchema,
678
+ execute: async (input, { agent }) => {
679
+ const threadId = agent?.threadId;
680
+ browser.setCurrentThread(threadId);
681
+ await browser.ensureReady();
682
+ return browser.wait(input, threadId);
683
+ }
684
+ });
519
685
  }
520
-
521
- // src/tools/index.ts
686
+ //#endregion
687
+ //#region src/tools/index.ts
688
+ /**
689
+ * Creates all browser tools bound to an AgentBrowser instance.
690
+ * The browser is lazily initialized on first tool use.
691
+ */
522
692
  function createAgentBrowserTools(browser) {
523
- return {
524
- // Core (9)
525
- [BROWSER_TOOLS.GOTO]: createGotoTool(browser),
526
- [BROWSER_TOOLS.SNAPSHOT]: createSnapshotTool(browser),
527
- [BROWSER_TOOLS.CLICK]: createClickTool(browser),
528
- [BROWSER_TOOLS.TYPE]: createTypeTool(browser),
529
- [BROWSER_TOOLS.PRESS]: createPressTool(browser),
530
- [BROWSER_TOOLS.SELECT]: createSelectTool(browser),
531
- [BROWSER_TOOLS.SCROLL]: createScrollTool(browser),
532
- [BROWSER_TOOLS.CLOSE]: createCloseTool(browser),
533
- // Utility
534
- [BROWSER_TOOLS.SCREENSHOT]: createScreenshotTool(browser),
535
- // Extended
536
- [BROWSER_TOOLS.HOVER]: createHoverTool(browser),
537
- [BROWSER_TOOLS.BACK]: createBackTool(browser),
538
- [BROWSER_TOOLS.DIALOG]: createDialogTool(browser),
539
- [BROWSER_TOOLS.WAIT]: createWaitTool(browser),
540
- [BROWSER_TOOLS.TABS]: createTabsTool(browser),
541
- [BROWSER_TOOLS.DRAG]: createDragTool(browser),
542
- // Escape hatch (1)
543
- [BROWSER_TOOLS.EVALUATE]: createEvaluateTool(browser)
544
- };
693
+ return {
694
+ [BROWSER_TOOLS.GOTO]: createGotoTool(browser),
695
+ [BROWSER_TOOLS.SNAPSHOT]: createSnapshotTool(browser),
696
+ [BROWSER_TOOLS.CLICK]: createClickTool(browser),
697
+ [BROWSER_TOOLS.TYPE]: createTypeTool(browser),
698
+ [BROWSER_TOOLS.PRESS]: createPressTool(browser),
699
+ [BROWSER_TOOLS.SELECT]: createSelectTool(browser),
700
+ [BROWSER_TOOLS.SCROLL]: createScrollTool(browser),
701
+ [BROWSER_TOOLS.CLOSE]: createCloseTool(browser),
702
+ [BROWSER_TOOLS.SCREENSHOT]: createScreenshotTool(browser),
703
+ [BROWSER_TOOLS.HOVER]: createHoverTool(browser),
704
+ [BROWSER_TOOLS.BACK]: createBackTool(browser),
705
+ [BROWSER_TOOLS.DIALOG]: createDialogTool(browser),
706
+ [BROWSER_TOOLS.WAIT]: createWaitTool(browser),
707
+ [BROWSER_TOOLS.TABS]: createTabsTool(browser),
708
+ [BROWSER_TOOLS.DRAG]: createDragTool(browser),
709
+ [BROWSER_TOOLS.EVALUATE]: createEvaluateTool(browser)
710
+ };
545
711
  }
546
-
547
- // src/utils.ts
712
+ //#endregion
713
+ //#region src/utils.ts
714
+ /**
715
+ * Get the browser process PID from a BrowserManager instance via CDP.
716
+ *
717
+ * Playwright doesn't expose the browser process PID directly, so we use CDP's
718
+ * SystemInfo.getProcessInfo to get it. This works for both regular browser
719
+ * launches and persistent contexts (profiles).
720
+ *
721
+ * Returns undefined if the PID can't be retrieved (e.g., browser not running).
722
+ */
548
723
  async function getBrowserPid(manager) {
549
- try {
550
- let browser = manager.getBrowser();
551
- if (!browser) {
552
- const ctx = manager.getContext();
553
- browser = ctx?.browser?.() ?? null;
554
- }
555
- if (!browser) return void 0;
556
- const cdp = await browser.newBrowserCDPSession();
557
- try {
558
- const info = await cdp.send("SystemInfo.getProcessInfo");
559
- const browserProcess = info.processInfo?.find((p) => p.type === "browser");
560
- return browserProcess?.id;
561
- } finally {
562
- await cdp.detach().catch(() => void 0);
563
- }
564
- } catch {
565
- return void 0;
566
- }
724
+ try {
725
+ let browser = manager.getBrowser();
726
+ if (!browser) browser = manager.getContext()?.browser?.() ?? null;
727
+ if (!browser) return void 0;
728
+ const cdp = await browser.newBrowserCDPSession();
729
+ try {
730
+ return ((await cdp.send("SystemInfo.getProcessInfo")).processInfo?.find((p) => p.type === "browser"))?.id;
731
+ } finally {
732
+ await cdp.detach().catch(() => void 0);
733
+ }
734
+ } catch {
735
+ return;
736
+ }
567
737
  }
568
-
569
- // src/agent-browser.ts
738
+ //#endregion
739
+ //#region src/agent-browser.ts
740
+ /**
741
+ * AgentBrowser - Browser automation using agent-browser (vercel-labs/agent-browser)
742
+ *
743
+ * Uses snapshot + refs pattern for LLM-friendly element targeting.
744
+ */
570
745
  var AgentBrowser = class extends MastraBrowser {
571
- id;
572
- name = "AgentBrowser";
573
- provider = "vercel-labs/agent-browser";
574
- defaultTimeout = 3e4;
575
- /** Pending PID lookups — awaited in disconnect handlers to avoid racing. */
576
- pidLookups = /* @__PURE__ */ new Set();
577
- pendingCloseReasons = /* @__PURE__ */ new Map();
578
- activeUrlChangeSources = /* @__PURE__ */ new Map();
579
- browserConfig;
580
- constructor(config = {}) {
581
- super(config);
582
- this.browserConfig = config;
583
- this.id = `agent-browser-${Date.now()}`;
584
- if (config.timeout) {
585
- this.defaultTimeout = config.timeout;
586
- }
587
- const effectiveScope = config.cdpUrl ? config.scope ?? "shared" : config.scope ?? "thread";
588
- const threadManagerConfig = {
589
- scope: effectiveScope,
590
- browserConfig: { ...config, headless: this.headless },
591
- resolveCdpUrl: this.resolveCdpUrl.bind(this),
592
- logger: this.logger,
593
- // When a new thread session is created, notify listeners so screencast can start
594
- onSessionCreated: (session) => {
595
- this.notifyBrowserReady(session.threadId);
596
- },
597
- // When a new browser is created for a thread, set up close listener
598
- onBrowserCreated: (manager, threadId) => {
599
- this.setupCloseListenerForThread(manager, threadId);
600
- }
601
- };
602
- const createTm = config.createThreadManager ?? ((opts) => new AgentBrowserThreadManager(opts));
603
- this.threadManager = createTm(threadManagerConfig);
604
- }
605
- // ---------------------------------------------------------------------------
606
- // Thread Scope (delegated to ThreadManager)
607
- // ---------------------------------------------------------------------------
608
- /**
609
- * Ensure browser is ready and thread session exists.
610
- * Creates a new page/context for the current thread if needed.
611
- *
612
- * For 'thread' scope, we need to create the thread session BEFORE
613
- * calling super.ensureReady() because the base class's ensureReady() will
614
- * call checkBrowserAlive(), which needs at least one thread browser to exist.
615
- */
616
- async ensureReady() {
617
- const scope = this.threadManager.getScope();
618
- const threadId = this.getCurrentThread();
619
- const existingSession = this.threadManager.hasSession(threadId);
620
- if (scope === "thread" && !existingSession) {
621
- await this.getManagerForThread(threadId);
622
- }
623
- await super.ensureReady();
624
- if (scope === "thread" && existingSession) {
625
- await this.getManagerForThread(threadId);
626
- }
627
- }
628
- /**
629
- * Get the browser manager for the current thread.
630
- * Delegates to ThreadManager for scope handling.
631
- */
632
- async getManagerForThread(threadId) {
633
- const effectiveThreadId = threadId ?? this.getCurrentThread();
634
- const scope = this.threadManager.getScope();
635
- if (scope === "thread" && (!effectiveThreadId || effectiveThreadId === DEFAULT_THREAD_ID)) {
636
- const existingManager = this.threadManager.getExistingManagerForThread(effectiveThreadId);
637
- if (existingManager) {
638
- return existingManager;
639
- }
640
- }
641
- return this.threadManager.getManagerForThread(effectiveThreadId);
642
- }
643
- // ---------------------------------------------------------------------------
644
- // Lifecycle
645
- // ---------------------------------------------------------------------------
646
- async doLaunch() {
647
- this.pendingCloseReasons.clear();
648
- this.activeUrlChangeSources.clear();
649
- const scope = this.threadManager.getScope();
650
- if (scope === "thread") {
651
- this.sharedManager = new BrowserManager();
652
- this.threadManager.setSharedManager(this.sharedManager);
653
- return;
654
- }
655
- this.sharedManager = new BrowserManager();
656
- const localConfig = this.config;
657
- const launchOptions = {
658
- headless: this.headless,
659
- viewport: localConfig.viewport,
660
- profile: localConfig.profile,
661
- executablePath: localConfig.executablePath,
662
- storageState: localConfig.storageState
663
- };
664
- if (localConfig.cdpUrl) {
665
- launchOptions.cdpUrl = await this.resolveCdpUrl(localConfig.cdpUrl);
666
- }
667
- await this.sharedManager.launch(launchOptions);
668
- this.threadManager.setSharedManager(this.sharedManager);
669
- this.setupCloseListenerForSharedScope(this.sharedManager);
670
- }
671
- /**
672
- * Set up close event listeners for 'shared' scope browser.
673
- * This handles the case where the shared browser is closed externally.
674
- */
675
- setupCloseListenerForSharedScope(manager) {
676
- try {
677
- const pidLookup = getBrowserPid(manager).then((pid) => {
678
- if (pid && this.sharedManager === manager) this.sharedBrowserPid = pid;
679
- }).finally(() => this.pidLookups.delete(pidLookup));
680
- this.pidLookups.add(pidLookup);
681
- let disconnectHandled = false;
682
- const handleDisconnect = () => {
683
- if (disconnectHandled) return;
684
- disconnectHandled = true;
685
- this.rememberClosedBrowserState(manager, "user");
686
- void pidLookup.catch(() => void 0).then(() => this.handleBrowserDisconnected());
687
- };
688
- const context = manager.getContext();
689
- if (context) {
690
- context.on("close", handleDisconnect);
691
- }
692
- const pages = manager.getPages();
693
- for (const page of pages) {
694
- page.on("close", () => {
695
- const remainingPages = manager.getPages();
696
- if (remainingPages.length === 0) {
697
- handleDisconnect();
698
- }
699
- });
700
- }
701
- } catch {
702
- }
703
- }
704
- async doClose() {
705
- await Promise.allSettled([...this.pidLookups]);
706
- this.pidLookups.clear();
707
- await this.threadManager.destroyAllSessions();
708
- this.setCurrentThread(void 0);
709
- const scope = this.threadManager.getScope();
710
- if (scope === "shared" && this.sharedManager) {
711
- await this.sharedManager.close();
712
- }
713
- this.sharedManager = null;
714
- }
715
- async closeThreadSession(threadId) {
716
- const manager = this.threadManager.getExistingManagerForThread(threadId);
717
- if (manager) {
718
- const state = this.getBrowserStateForManager(manager, threadId);
719
- if (state) this.threadManager.updateBrowserState(threadId, state);
720
- }
721
- await super.closeThreadSession(threadId);
722
- }
723
- /**
724
- * Check if the browser is still alive by verifying the page is connected.
725
- * Called by base class ensureReady() to detect externally closed browsers.
726
- */
727
- async checkBrowserAlive() {
728
- const scope = this.threadManager.getScope();
729
- if (scope === "thread") {
730
- return this.threadManager.hasActiveThreadManagers();
731
- }
732
- if (!this.sharedManager) {
733
- return false;
734
- }
735
- try {
736
- const page = this.sharedManager.getPage();
737
- const url = page.url();
738
- if (url && url !== "about:blank") {
739
- const state = await this.getBrowserState();
740
- if (state) {
741
- this.lastBrowserState = state;
742
- }
743
- }
744
- return true;
745
- } catch (error) {
746
- const msg = error instanceof Error ? error.message : String(error);
747
- if (this.isDisconnectionError(msg)) {
748
- this.logger.debug?.("Browser was externally closed");
749
- }
750
- return false;
751
- }
752
- }
753
- // ---------------------------------------------------------------------------
754
- // Tools
755
- // ---------------------------------------------------------------------------
756
- /**
757
- * Get the browser tools for this provider.
758
- * Returns 16 flat tools for browser automation.
759
- */
760
- getTools() {
761
- const tools = createAgentBrowserTools(this);
762
- if (this.browserConfig.recording) {
763
- Object.assign(tools, createBrowserRecordingTools(this, this.browserConfig.recording));
764
- }
765
- const exclude = this.browserConfig.excludeTools;
766
- if (exclude?.length) {
767
- for (const name of exclude) {
768
- delete tools[name];
769
- }
770
- }
771
- return tools;
772
- }
773
- // ---------------------------------------------------------------------------
774
- // Helpers
775
- // ---------------------------------------------------------------------------
776
- browserStateKey(threadId) {
777
- return threadId ?? this.getCurrentThread() ?? DEFAULT_THREAD_ID;
778
- }
779
- markBrowserCloseReason(reason, threadId) {
780
- this.pendingCloseReasons.set(this.browserStateKey(threadId), reason);
781
- }
782
- markActiveUrlChangeSource(source, url, threadId) {
783
- this.activeUrlChangeSources.set(this.browserStateKey(threadId), { url, source });
784
- }
785
- getCloseReason(threadId) {
786
- return this.pendingCloseReasons.get(this.browserStateKey(threadId)) ?? this.pendingCloseReasons.get(DEFAULT_THREAD_ID);
787
- }
788
- getActiveUrlChangeSource(activeUrl, threadId) {
789
- const entry = this.activeUrlChangeSources.get(this.browserStateKey(threadId));
790
- return entry && entry.url === activeUrl ? entry.source : void 0;
791
- }
792
- rememberClosedBrowserState(manager, reason, threadId) {
793
- const state = this.getBrowserStateForManager(manager, threadId);
794
- if (!state || state.tabs.length === 0) return;
795
- const closedState = { ...state, closeReason: this.getCloseReason(threadId) ?? reason };
796
- if (threadId) {
797
- this.threadManager.updateBrowserState(threadId, closedState);
798
- } else {
799
- this.lastBrowserState = closedState;
800
- }
801
- }
802
- /**
803
- * Get the page for the current thread.
804
- * Uses thread scope if enabled, otherwise returns the shared page.
805
- * @param explicitThreadId - Optional thread ID to use instead of getCurrentThread()
806
- * Use this to avoid race conditions in concurrent tool calls.
807
- */
808
- async getPage(explicitThreadId) {
809
- const scope = this.getScope();
810
- const threadId = explicitThreadId ?? this.getCurrentThread();
811
- if (scope === "thread") {
812
- return this.threadManager.getPageForThread(threadId);
813
- }
814
- if (!this.sharedManager) throw new Error("Browser not launched");
815
- return this.sharedManager.getPage();
816
- }
817
- /**
818
- * Get the active page for a thread (implements abstract method from base class).
819
- * Returns null if no page is available, unlike getPage which throws.
820
- */
821
- async getActivePage(threadId) {
822
- try {
823
- return await this.getPage(threadId);
824
- } catch {
825
- return null;
826
- }
827
- }
828
- /**
829
- * Set up close event listener for a thread's browser manager.
830
- * This handles the case where a thread's browser is closed externally.
831
- */
832
- setupCloseListenerForThread(manager, threadId) {
833
- try {
834
- const pidLookup = getBrowserPid(manager).then((pid) => {
835
- if (pid && this.threadManager?.getExistingManagerForThread(threadId) === manager) {
836
- this.threadBrowserPids.set(threadId, pid);
837
- }
838
- }).finally(() => this.pidLookups.delete(pidLookup));
839
- this.pidLookups.add(pidLookup);
840
- let disconnectHandled = false;
841
- const handleDisconnect = () => {
842
- if (disconnectHandled) return;
843
- disconnectHandled = true;
844
- this.rememberClosedBrowserState(manager, "user", threadId);
845
- void pidLookup.catch(() => void 0).then(() => this.handleThreadBrowserDisconnected(threadId));
846
- };
847
- const context = manager.getContext();
848
- if (context) {
849
- context.on("close", handleDisconnect);
850
- }
851
- const pages = manager.getPages();
852
- for (const page of pages) {
853
- page.on("close", () => {
854
- const remainingPages = manager.getPages();
855
- if (remainingPages.length === 0) {
856
- handleDisconnect();
857
- }
858
- });
859
- }
860
- } catch {
861
- }
862
- }
863
- /**
864
- * Create an error response from an exception.
865
- * Extends base class to add agent-browser specific error handling.
866
- */
867
- createErrorFromException(error, context) {
868
- const msg = error instanceof Error ? error.message : String(error);
869
- if (msg.includes("stale") || msg.includes("Stale")) {
870
- return this.createError(
871
- "stale_ref",
872
- "Element ref is no longer valid.",
873
- "Get a fresh snapshot and use updated refs."
874
- );
875
- }
876
- if (msg.includes("not found") || msg.includes("No element")) {
877
- return this.createError(
878
- "element_not_found",
879
- "Element not found.",
880
- "Check the ref is correct or get a fresh snapshot."
881
- );
882
- }
883
- return super.createErrorFromException(error, context);
884
- }
885
- async requireLocator(ref, threadId) {
886
- const manager = await this.getManagerForThread(threadId);
887
- return manager.getLocatorFromRef(ref);
888
- }
889
- async getScrollInfo(threadId) {
890
- const page = await this.getPage(threadId);
891
- const info = await page.evaluate(`({
746
+ id;
747
+ name = "AgentBrowser";
748
+ provider = "vercel-labs/agent-browser";
749
+ defaultTimeout = 3e4;
750
+ /** Pending PID lookups — awaited in disconnect handlers to avoid racing. */
751
+ pidLookups = /* @__PURE__ */ new Set();
752
+ pendingCloseReasons = /* @__PURE__ */ new Map();
753
+ activeUrlChangeSources = /* @__PURE__ */ new Map();
754
+ browserConfig;
755
+ constructor(config = {}) {
756
+ super(config);
757
+ this.browserConfig = config;
758
+ this.id = `agent-browser-${Date.now()}`;
759
+ if (config.timeout) this.defaultTimeout = config.timeout;
760
+ const threadManagerConfig = {
761
+ scope: config.cdpUrl ? config.scope ?? "shared" : config.scope ?? "thread",
762
+ browserConfig: {
763
+ ...config,
764
+ headless: this.headless
765
+ },
766
+ resolveCdpUrl: this.resolveCdpUrl.bind(this),
767
+ logger: this.logger,
768
+ onSessionCreated: (session) => {
769
+ this.notifyBrowserReady(session.threadId);
770
+ },
771
+ onBrowserCreated: (manager, threadId) => {
772
+ this.setupCloseListenerForThread(manager, threadId);
773
+ }
774
+ };
775
+ const createTm = config.createThreadManager ?? ((opts) => new AgentBrowserThreadManager(opts));
776
+ this.threadManager = createTm(threadManagerConfig);
777
+ }
778
+ /**
779
+ * Ensure browser is ready and thread session exists.
780
+ * Creates a new page/context for the current thread if needed.
781
+ *
782
+ * For 'thread' scope, we need to create the thread session BEFORE
783
+ * calling super.ensureReady() because the base class's ensureReady() will
784
+ * call checkBrowserAlive(), which needs at least one thread browser to exist.
785
+ */
786
+ async ensureReady() {
787
+ const scope = this.threadManager.getScope();
788
+ const threadId = this.getCurrentThread();
789
+ const existingSession = this.threadManager.hasSession(threadId);
790
+ if (scope === "thread" && !existingSession) await this.getManagerForThread(threadId);
791
+ await super.ensureReady();
792
+ if (scope === "thread" && existingSession) await this.getManagerForThread(threadId);
793
+ }
794
+ /**
795
+ * Get the browser manager for the current thread.
796
+ * Delegates to ThreadManager for scope handling.
797
+ */
798
+ async getManagerForThread(threadId) {
799
+ const effectiveThreadId = threadId ?? this.getCurrentThread();
800
+ if (this.threadManager.getScope() === "thread" && (!effectiveThreadId || effectiveThreadId === DEFAULT_THREAD_ID)) {
801
+ const existingManager = this.threadManager.getExistingManagerForThread(effectiveThreadId);
802
+ if (existingManager) return existingManager;
803
+ }
804
+ return this.threadManager.getManagerForThread(effectiveThreadId);
805
+ }
806
+ async doLaunch() {
807
+ this.pendingCloseReasons.clear();
808
+ this.activeUrlChangeSources.clear();
809
+ if (this.threadManager.getScope() === "thread") {
810
+ this.sharedManager = new BrowserManager();
811
+ this.threadManager.setSharedManager(this.sharedManager);
812
+ return;
813
+ }
814
+ this.sharedManager = new BrowserManager();
815
+ const localConfig = this.config;
816
+ const launchOptions = {
817
+ headless: this.headless,
818
+ viewport: localConfig.viewport,
819
+ profile: localConfig.profile,
820
+ executablePath: localConfig.executablePath,
821
+ storageState: localConfig.storageState
822
+ };
823
+ if (localConfig.cdpUrl) launchOptions.cdpUrl = await this.resolveCdpUrl(localConfig.cdpUrl);
824
+ if (localConfig.cdpHeaders) launchOptions.cdpHeaders = localConfig.cdpHeaders;
825
+ await this.sharedManager.launch(launchOptions);
826
+ this.threadManager.setSharedManager(this.sharedManager);
827
+ this.setupCloseListenerForSharedScope(this.sharedManager);
828
+ }
829
+ /**
830
+ * Set up close event listeners for 'shared' scope browser.
831
+ * This handles the case where the shared browser is closed externally.
832
+ */
833
+ setupCloseListenerForSharedScope(manager) {
834
+ try {
835
+ const pidLookup = getBrowserPid(manager).then((pid) => {
836
+ if (pid && this.sharedManager === manager) this.sharedBrowserPid = pid;
837
+ }).finally(() => this.pidLookups.delete(pidLookup));
838
+ this.pidLookups.add(pidLookup);
839
+ let disconnectHandled = false;
840
+ const handleDisconnect = () => {
841
+ if (disconnectHandled) return;
842
+ disconnectHandled = true;
843
+ this.rememberClosedBrowserState(manager, "user");
844
+ pidLookup.catch(() => void 0).then(() => this.handleBrowserDisconnected());
845
+ };
846
+ const context = manager.getContext();
847
+ if (context) context.on("close", handleDisconnect);
848
+ const pages = manager.getPages();
849
+ for (const page of pages) page.on("close", () => {
850
+ if (manager.getPages().length === 0) handleDisconnect();
851
+ });
852
+ } catch {}
853
+ }
854
+ async doClose() {
855
+ await Promise.allSettled([...this.pidLookups]);
856
+ this.pidLookups.clear();
857
+ await this.threadManager.destroyAllSessions();
858
+ this.setCurrentThread(void 0);
859
+ if (this.threadManager.getScope() === "shared" && this.sharedManager) await this.sharedManager.close();
860
+ this.sharedManager = null;
861
+ }
862
+ async closeThreadSession(threadId) {
863
+ const manager = this.threadManager.getExistingManagerForThread(threadId);
864
+ if (manager) {
865
+ const state = this.getBrowserStateForManager(manager, threadId);
866
+ if (state) this.threadManager.updateBrowserState(threadId, state);
867
+ }
868
+ await super.closeThreadSession(threadId);
869
+ }
870
+ /**
871
+ * Check if the browser is still alive by verifying the page is connected.
872
+ * Called by base class ensureReady() to detect externally closed browsers.
873
+ */
874
+ async checkBrowserAlive() {
875
+ if (this.threadManager.getScope() === "thread") return this.threadManager.hasActiveThreadManagers();
876
+ if (!this.sharedManager) return false;
877
+ try {
878
+ const url = this.sharedManager.getPage().url();
879
+ if (url && url !== "about:blank") {
880
+ const state = await this.getBrowserState();
881
+ if (state) this.lastBrowserState = state;
882
+ }
883
+ return true;
884
+ } catch (error) {
885
+ const msg = error instanceof Error ? error.message : String(error);
886
+ if (this.isDisconnectionError(msg)) this.logger.debug?.("Browser was externally closed");
887
+ return false;
888
+ }
889
+ }
890
+ /**
891
+ * Get the browser tools for this provider.
892
+ * Returns 16 flat tools for browser automation.
893
+ */
894
+ getTools() {
895
+ const tools = createAgentBrowserTools(this);
896
+ if (this.browserConfig.recording) Object.assign(tools, createBrowserRecordingTools(this, this.browserConfig.recording));
897
+ const exclude = this.browserConfig.excludeTools;
898
+ if (exclude?.length) for (const name of exclude) delete tools[name];
899
+ return tools;
900
+ }
901
+ browserStateKey(threadId) {
902
+ return threadId ?? this.getCurrentThread() ?? DEFAULT_THREAD_ID;
903
+ }
904
+ markBrowserCloseReason(reason, threadId) {
905
+ this.pendingCloseReasons.set(this.browserStateKey(threadId), reason);
906
+ }
907
+ markActiveUrlChangeSource(source, url, threadId) {
908
+ this.activeUrlChangeSources.set(this.browserStateKey(threadId), {
909
+ url,
910
+ source
911
+ });
912
+ }
913
+ getCloseReason(threadId) {
914
+ return this.pendingCloseReasons.get(this.browserStateKey(threadId)) ?? this.pendingCloseReasons.get(DEFAULT_THREAD_ID);
915
+ }
916
+ getActiveUrlChangeSource(activeUrl, threadId) {
917
+ const entry = this.activeUrlChangeSources.get(this.browserStateKey(threadId));
918
+ return entry && entry.url === activeUrl ? entry.source : void 0;
919
+ }
920
+ rememberClosedBrowserState(manager, reason, threadId) {
921
+ const state = this.getBrowserStateForManager(manager, threadId);
922
+ if (!state || state.tabs.length === 0) return;
923
+ const closedState = {
924
+ ...state,
925
+ closeReason: this.getCloseReason(threadId) ?? reason
926
+ };
927
+ if (threadId) this.threadManager.updateBrowserState(threadId, closedState);
928
+ else this.lastBrowserState = closedState;
929
+ }
930
+ /**
931
+ * Get the page for the current thread.
932
+ * Uses thread scope if enabled, otherwise returns the shared page.
933
+ * @param explicitThreadId - Optional thread ID to use instead of getCurrentThread()
934
+ * Use this to avoid race conditions in concurrent tool calls.
935
+ */
936
+ async getPage(explicitThreadId) {
937
+ const scope = this.getScope();
938
+ const threadId = explicitThreadId ?? this.getCurrentThread();
939
+ if (scope === "thread") return this.threadManager.getPageForThread(threadId);
940
+ if (!this.sharedManager) throw new Error("Browser not launched");
941
+ return this.sharedManager.getPage();
942
+ }
943
+ /**
944
+ * Get the active page for a thread (implements abstract method from base class).
945
+ * Returns null if no page is available, unlike getPage which throws.
946
+ */
947
+ async getActivePage(threadId) {
948
+ try {
949
+ return await this.getPage(threadId);
950
+ } catch {
951
+ return null;
952
+ }
953
+ }
954
+ /**
955
+ * Set up close event listener for a thread's browser manager.
956
+ * This handles the case where a thread's browser is closed externally.
957
+ */
958
+ setupCloseListenerForThread(manager, threadId) {
959
+ try {
960
+ const pidLookup = getBrowserPid(manager).then((pid) => {
961
+ if (pid && this.threadManager?.getExistingManagerForThread(threadId) === manager) this.threadBrowserPids.set(threadId, pid);
962
+ }).finally(() => this.pidLookups.delete(pidLookup));
963
+ this.pidLookups.add(pidLookup);
964
+ let disconnectHandled = false;
965
+ const handleDisconnect = () => {
966
+ if (disconnectHandled) return;
967
+ disconnectHandled = true;
968
+ this.rememberClosedBrowserState(manager, "user", threadId);
969
+ pidLookup.catch(() => void 0).then(() => this.handleThreadBrowserDisconnected(threadId));
970
+ };
971
+ const context = manager.getContext();
972
+ if (context) context.on("close", handleDisconnect);
973
+ const pages = manager.getPages();
974
+ for (const page of pages) page.on("close", () => {
975
+ if (manager.getPages().length === 0) handleDisconnect();
976
+ });
977
+ } catch {}
978
+ }
979
+ /**
980
+ * Create an error response from an exception.
981
+ * Extends base class to add agent-browser specific error handling.
982
+ */
983
+ createErrorFromException(error, context) {
984
+ const msg = error instanceof Error ? error.message : String(error);
985
+ if (msg.includes("stale") || msg.includes("Stale")) return this.createError("stale_ref", "Element ref is no longer valid.", "Get a fresh snapshot and use updated refs.");
986
+ if (msg.includes("not found") || msg.includes("No element")) return this.createError("element_not_found", "Element not found.", "Check the ref is correct or get a fresh snapshot.");
987
+ return super.createErrorFromException(error, context);
988
+ }
989
+ async requireLocator(ref, threadId) {
990
+ return (await this.getManagerForThread(threadId)).getLocatorFromRef(ref);
991
+ }
992
+ async getScrollInfo(threadId) {
993
+ const info = await (await this.getPage(threadId)).evaluate(`({
892
994
  scrollY: Math.round(window.scrollY),
893
995
  scrollHeight: document.documentElement.scrollHeight,
894
996
  viewportHeight: window.innerHeight
895
997
  })`);
896
- if (!info || typeof info.scrollHeight !== "number") {
897
- return {
898
- scrollY: 0,
899
- scrollHeight: 0,
900
- viewportHeight: 0,
901
- atTop: true,
902
- atBottom: true,
903
- percentDown: 0
904
- };
905
- }
906
- const maxScroll = info.scrollHeight - info.viewportHeight;
907
- return {
908
- ...info,
909
- atTop: info.scrollY < 50,
910
- atBottom: info.scrollY >= maxScroll - 50,
911
- percentDown: maxScroll > 0 ? Math.round(info.scrollY / maxScroll * 100) : 0
912
- };
913
- }
914
- // ---------------------------------------------------------------------------
915
- // URL Access
916
- // ---------------------------------------------------------------------------
917
- /**
918
- * Get the current page URL without launching the browser.
919
- * @param threadId - Optional thread ID for thread-isolated browsers
920
- * @returns The current URL string, or null if browser is not running
921
- */
922
- async getCurrentUrl(threadId) {
923
- if (!this.isBrowserRunning()) {
924
- return null;
925
- }
926
- try {
927
- const effectiveThreadId = threadId ?? this.getCurrentThread();
928
- const scope = this.threadManager.getScope();
929
- if (scope === "thread" && effectiveThreadId) {
930
- const manager2 = this.threadManager.getExistingManagerForThread(effectiveThreadId);
931
- if (!manager2) {
932
- return null;
933
- }
934
- const url2 = manager2.getPage().url();
935
- if (url2 && url2 !== "about:blank") {
936
- const state = this.getBrowserStateForManager(manager2);
937
- if (state) {
938
- this.threadManager.updateBrowserState(effectiveThreadId, state);
939
- }
940
- }
941
- return url2;
942
- }
943
- const manager = await this.getManagerForThread(threadId);
944
- const url = manager.getPage().url();
945
- if (url && url !== "about:blank") {
946
- const state = this.getBrowserStateForManager(manager);
947
- if (state) {
948
- this.lastBrowserState = state;
949
- }
950
- }
951
- return url;
952
- } catch {
953
- return null;
954
- }
955
- }
956
- /**
957
- * Navigate to a URL (simple form). Used internally for restoring state on relaunch.
958
- */
959
- async navigateTo(url) {
960
- if (!this.isBrowserRunning()) {
961
- return;
962
- }
963
- try {
964
- const page = await this.getPage();
965
- await page.goto(url, {
966
- timeout: this.defaultTimeout,
967
- waitUntil: "domcontentloaded"
968
- });
969
- } catch {
970
- }
971
- }
972
- /**
973
- * Get the current browser state (all tabs and active tab index).
974
- */
975
- async getBrowserState(threadId) {
976
- if (!this.isBrowserRunning(threadId)) {
977
- return null;
978
- }
979
- try {
980
- const manager = await this.getManagerForThread(threadId);
981
- return this.getBrowserStateForManager(manager, threadId);
982
- } catch {
983
- return null;
984
- }
985
- }
986
- /**
987
- * Get browser state for a thread (implements abstract method from base class).
988
- * Sync version that uses existing manager lookup without creating sessions.
989
- */
990
- getBrowserStateForThread(threadId) {
991
- const effectiveThreadId = threadId ?? this.getCurrentThread() ?? DEFAULT_THREAD_ID;
992
- const manager = this.threadManager.getExistingManagerForThread(effectiveThreadId);
993
- if (!manager) return null;
994
- return this.getBrowserStateForManager(manager, effectiveThreadId);
995
- }
996
- /**
997
- * Get browser state from a specific manager instance.
998
- */
999
- getBrowserStateForManager(manager, threadId) {
1000
- try {
1001
- const stateKey = this.browserStateKey(threadId);
1002
- const pages = manager.getPages();
1003
- const activeIndex = manager.getActiveIndex();
1004
- const tabs = pages.map((page) => ({
1005
- url: page.url()
1006
- }));
1007
- const activeUrl = tabs[activeIndex]?.url;
1008
- const previousState = this.threadManager.getSavedBrowserState(stateKey) ?? this.lastBrowserState;
1009
- const previousUrl = previousState?.tabs[previousState.activeTabIndex]?.url;
1010
- const activeUrlChangeSource = this.getActiveUrlChangeSource(activeUrl, stateKey) ?? (previousUrl && activeUrl !== previousUrl ? "user" : void 0);
1011
- const state = {
1012
- tabs,
1013
- activeTabIndex: activeIndex,
1014
- ...this.getCloseReason(stateKey) ? { closeReason: this.getCloseReason(stateKey) } : {},
1015
- ...activeUrlChangeSource ? { activeUrlChangeSource } : {}
1016
- };
1017
- this.threadManager.updateBrowserState(stateKey, state);
1018
- this.lastBrowserState = state;
1019
- return state;
1020
- } catch {
1021
- return null;
1022
- }
1023
- }
1024
- /**
1025
- * Get all open tabs with their URLs and titles.
1026
- */
1027
- async getTabState(threadId) {
1028
- const state = await this.getBrowserState(threadId);
1029
- return state?.tabs ?? [];
1030
- }
1031
- /**
1032
- * Get the active tab index.
1033
- */
1034
- async getActiveTabIndex(threadId) {
1035
- if (!this.isBrowserRunning()) {
1036
- return 0;
1037
- }
1038
- try {
1039
- const manager = await this.getManagerForThread(threadId);
1040
- return manager.getActiveIndex();
1041
- } catch {
1042
- return 0;
1043
- }
1044
- }
1045
- /**
1046
- * Export the current browser session's storage state (cookies, localStorage) to a JSON file.
1047
- * This can later be loaded via the `storageState` config option to restore the session.
1048
- *
1049
- * @param path - File path to save the storage state JSON
1050
- * @param threadId - Optional thread ID (defaults to current thread)
1051
- */
1052
- async exportStorageState(path, threadId) {
1053
- const effectiveThreadId = threadId ?? this.getCurrentThread();
1054
- const manager = this.threadManager.getExistingManagerForThread(effectiveThreadId);
1055
- if (!manager) {
1056
- throw new Error("No browser is running. Launch a browser first before exporting storage state.");
1057
- }
1058
- const context = manager.getContext();
1059
- if (!context) {
1060
- throw new Error("Browser context not available");
1061
- }
1062
- await context.storageState({ path });
1063
- }
1064
- // ---------------------------------------------------------------------------
1065
- // 1. browser_goto - Navigate to URL
1066
- // ---------------------------------------------------------------------------
1067
- async goto(input, threadId) {
1068
- try {
1069
- const page = await this.getPage(threadId);
1070
- await page.goto(input.url, {
1071
- timeout: input.timeout ?? this.defaultTimeout,
1072
- waitUntil: input.waitUntil ?? "domcontentloaded"
1073
- });
1074
- const url = page.url();
1075
- this.markActiveUrlChangeSource("agent", url, threadId);
1076
- return {
1077
- success: true,
1078
- url,
1079
- title: await page.title(),
1080
- hint: "Take a snapshot to see interactive elements and get refs."
1081
- };
1082
- } catch (error) {
1083
- return this.createErrorFromException(error, "Goto");
1084
- }
1085
- }
1086
- // ---------------------------------------------------------------------------
1087
- // 2. browser_snapshot - Capture accessibility tree
1088
- // ---------------------------------------------------------------------------
1089
- async snapshot(input, threadId) {
1090
- try {
1091
- const manager = await this.getManagerForThread(threadId);
1092
- const page = await this.getPage(threadId);
1093
- const rawSnapshot = await manager.getSnapshot({
1094
- interactive: input.interactiveOnly ?? true,
1095
- compact: true
1096
- });
1097
- const snapshot = (rawSnapshot.tree ?? "").replace(/\[ref=(\w+)\]/g, "@$1");
1098
- const scrollInfo = await this.getScrollInfo(threadId);
1099
- let scrollText;
1100
- if (scrollInfo.atTop && !scrollInfo.atBottom) {
1101
- scrollText = "TOP - more content below";
1102
- } else if (scrollInfo.atBottom) {
1103
- scrollText = "BOTTOM of page";
1104
- } else {
1105
- scrollText = `${scrollInfo.percentDown}% down`;
1106
- }
1107
- const refs = snapshot.match(/@e\d+/g) || [];
1108
- const elementCount = new Set(refs).size;
1109
- return {
1110
- success: true,
1111
- snapshot,
1112
- url: page.url(),
1113
- title: await page.title(),
1114
- elementCount,
1115
- scroll: scrollText,
1116
- hint: elementCount === 0 ? "No interactive elements found. Try scrolling or setting interactiveOnly:false." : void 0
1117
- };
1118
- } catch (error) {
1119
- return this.createErrorFromException(error, "Snapshot");
1120
- }
1121
- }
1122
- // ---------------------------------------------------------------------------
1123
- // browser_screenshot - Capture a screenshot of the current page
1124
- // ---------------------------------------------------------------------------
1125
- async screenshot(input, threadId) {
1126
- try {
1127
- const page = await this.getPage(threadId);
1128
- const buffer = await page.screenshot({
1129
- fullPage: input.fullPage ?? false,
1130
- type: "png"
1131
- });
1132
- const base64 = Buffer.from(buffer).toString("base64");
1133
- return {
1134
- base64,
1135
- url: page.url(),
1136
- title: await page.title()
1137
- };
1138
- } catch (error) {
1139
- return this.createErrorFromException(error, "Screenshot");
1140
- }
1141
- }
1142
- /**
1143
- * Start a `waitForNavigation` wait (when `waitUntil` is set) and immediately
1144
- * attach a noop catch handler so a navigation timeout/rejection can't become
1145
- * an unhandled rejection (crashing the process) while the caller's action is
1146
- * still pending. Callers should still `await` the returned promise to observe
1147
- * the original error.
1148
- */
1149
- startNavigationWait(page, waitUntil, timeout) {
1150
- const navigation = waitUntil ? page.waitForNavigation({ waitUntil, timeout }) : void 0;
1151
- navigation?.catch(() => {
1152
- });
1153
- return navigation;
1154
- }
1155
- // ---------------------------------------------------------------------------
1156
- // 3. browser_click - Click on element
1157
- // ---------------------------------------------------------------------------
1158
- async click(input, threadId) {
1159
- try {
1160
- const page = await this.getPage(threadId);
1161
- const locator = await this.requireLocator(input.ref, threadId);
1162
- if (!locator) {
1163
- return this.createError(
1164
- "stale_ref",
1165
- `Ref ${input.ref} not found. The page has changed.`,
1166
- "Take a new snapshot to see the current page state and get fresh refs."
1167
- );
1168
- }
1169
- const timeout = input.timeout ?? this.defaultTimeout;
1170
- const navigation = this.startNavigationWait(page, input.waitUntil, timeout);
1171
- await locator.click({
1172
- button: input.button ?? "left",
1173
- clickCount: input.clickCount ?? 1,
1174
- modifiers: input.modifiers,
1175
- timeout
1176
- });
1177
- await navigation;
1178
- return {
1179
- success: true,
1180
- url: page.url(),
1181
- hint: "Take a new snapshot to see updated page state and get fresh refs."
1182
- };
1183
- } catch (error) {
1184
- const errorMsg = error instanceof Error ? error.message : String(error);
1185
- if (errorMsg.includes("intercepts pointer events")) {
1186
- return this.createError(
1187
- "element_blocked",
1188
- `Element ${input.ref} is blocked by another element.`,
1189
- "Take a new snapshot to see what is blocking. Dismiss any modals or scroll the element into view."
1190
- );
1191
- }
1192
- return this.createErrorFromException(error, "Click");
1193
- }
1194
- }
1195
- // ---------------------------------------------------------------------------
1196
- // 4. browser_type - Type text into element
1197
- // ---------------------------------------------------------------------------
1198
- async type(input, threadId) {
1199
- try {
1200
- const page = await this.getPage(threadId);
1201
- const locator = await this.requireLocator(input.ref, threadId);
1202
- if (!locator) {
1203
- return this.createError(
1204
- "stale_ref",
1205
- `Ref ${input.ref} not found. The page has changed.`,
1206
- "Take a new snapshot to see the current page state and get fresh refs."
1207
- );
1208
- }
1209
- if (input.clear) {
1210
- await locator.fill("", { timeout: this.defaultTimeout });
1211
- }
1212
- if (input.delay) {
1213
- await locator.focus();
1214
- for (const char of input.text) {
1215
- await page.keyboard.press(char);
1216
- await new Promise((r) => setTimeout(r, input.delay));
1217
- }
1218
- } else {
1219
- await locator.fill(input.text, { timeout: this.defaultTimeout });
1220
- }
1221
- const value = await locator.inputValue({ timeout: 1e3 }).catch(() => input.text);
1222
- return {
1223
- success: true,
1224
- value,
1225
- url: page.url(),
1226
- hint: "Take a new snapshot if you need to interact with more elements."
1227
- };
1228
- } catch (error) {
1229
- const errorMsg = error instanceof Error ? error.message : String(error);
1230
- if (errorMsg.includes("is not an <input>") || errorMsg.includes("not an input") || errorMsg.includes("Cannot type") || errorMsg.includes("not focusable")) {
1231
- return this.createError(
1232
- "not_focusable",
1233
- `Element ${input.ref} is not a text input field.`,
1234
- 'Take a new snapshot and look for elements with role "textbox" or "searchbox".'
1235
- );
1236
- }
1237
- return this.createErrorFromException(error, "Type");
1238
- }
1239
- }
1240
- // ---------------------------------------------------------------------------
1241
- // 5. browser_press - Press keyboard key(s)
1242
- // ---------------------------------------------------------------------------
1243
- async press(input, threadId) {
1244
- try {
1245
- const page = await this.getPage(threadId);
1246
- const timeout = input.timeout ?? this.defaultTimeout;
1247
- const navigation = this.startNavigationWait(page, input.waitUntil, timeout);
1248
- await page.keyboard.press(input.key);
1249
- await navigation;
1250
- return {
1251
- success: true,
1252
- url: page.url(),
1253
- hint: "Take a new snapshot if the page may have changed."
1254
- };
1255
- } catch (error) {
1256
- return this.createErrorFromException(error, "Press");
1257
- }
1258
- }
1259
- // ---------------------------------------------------------------------------
1260
- // 6. browser_select - Select dropdown option
1261
- // ---------------------------------------------------------------------------
1262
- async select(input, threadId) {
1263
- try {
1264
- const page = await this.getPage(threadId);
1265
- const locator = await this.requireLocator(input.ref, threadId);
1266
- if (!locator) {
1267
- return this.createError(
1268
- "stale_ref",
1269
- `Ref ${input.ref} not found. The page has changed.`,
1270
- "Take a new snapshot to get fresh refs."
1271
- );
1272
- }
1273
- const selectValue = {};
1274
- if (input.value) selectValue.value = input.value;
1275
- if (input.label) selectValue.label = input.label;
1276
- if (input.index !== void 0) selectValue.index = input.index;
1277
- const timeout = input.timeout ?? this.defaultTimeout;
1278
- const navigation = this.startNavigationWait(page, input.waitUntil, timeout);
1279
- const selected = await locator.selectOption(selectValue, { timeout });
1280
- await navigation;
1281
- return {
1282
- success: true,
1283
- selected,
1284
- url: page.url(),
1285
- hint: "Selection complete. Take a snapshot if you need to continue."
1286
- };
1287
- } catch (error) {
1288
- return this.createErrorFromException(error, "Select");
1289
- }
1290
- }
1291
- // ---------------------------------------------------------------------------
1292
- // 7. browser_scroll - Scroll page or element
1293
- // ---------------------------------------------------------------------------
1294
- async scroll(input, threadId) {
1295
- try {
1296
- const page = await this.getPage(threadId);
1297
- if (input.ref) {
1298
- const locator = await this.requireLocator(input.ref, threadId);
1299
- if (locator) {
1300
- await locator.scrollIntoViewIfNeeded({ timeout: this.defaultTimeout });
1301
- }
1302
- } else {
1303
- const direction = input.direction;
1304
- const amount = input.amount ?? 300;
1305
- let deltaX = 0;
1306
- let deltaY = 0;
1307
- switch (direction) {
1308
- case "up":
1309
- deltaY = -amount;
1310
- break;
1311
- case "down":
1312
- deltaY = amount;
1313
- break;
1314
- case "left":
1315
- deltaX = -amount;
1316
- break;
1317
- case "right":
1318
- deltaX = amount;
1319
- break;
1320
- }
1321
- await page.evaluate(
1322
- ({ x, y }) => {
1323
- globalThis.scrollBy(x, y);
1324
- },
1325
- { x: deltaX, y: deltaY }
1326
- );
1327
- }
1328
- const scrollInfo = await this.getScrollInfo(threadId);
1329
- let scrollText;
1330
- if (scrollInfo.atTop && !scrollInfo.atBottom) {
1331
- scrollText = "TOP - more content below";
1332
- } else if (scrollInfo.atBottom) {
1333
- scrollText = "BOTTOM of page";
1334
- } else {
1335
- scrollText = `${scrollInfo.percentDown}% down`;
1336
- }
1337
- return {
1338
- success: true,
1339
- position: { x: 0, y: scrollInfo.scrollY },
1340
- scroll: scrollText,
1341
- hint: "Take a new snapshot to see elements in the new viewport."
1342
- };
1343
- } catch (error) {
1344
- return this.createErrorFromException(error, "Scroll");
1345
- }
1346
- }
1347
- // ---------------------------------------------------------------------------
1348
- // 8. browser_hover - Hover over element
1349
- // ---------------------------------------------------------------------------
1350
- async hover(input, threadId) {
1351
- try {
1352
- const page = await this.getPage(threadId);
1353
- const locator = await this.requireLocator(input.ref, threadId);
1354
- if (!locator) {
1355
- return this.createError(
1356
- "stale_ref",
1357
- `Ref ${input.ref} not found. The page has changed.`,
1358
- "Take a new snapshot to get fresh refs."
1359
- );
1360
- }
1361
- await locator.hover({ timeout: this.defaultTimeout });
1362
- return {
1363
- success: true,
1364
- url: page.url(),
1365
- hint: "Take a new snapshot to see any hover-triggered elements (dropdowns, tooltips)."
1366
- };
1367
- } catch (error) {
1368
- return this.createErrorFromException(error, "Hover");
1369
- }
1370
- }
1371
- // ---------------------------------------------------------------------------
1372
- // 10. browser_back - Navigate back
1373
- // ---------------------------------------------------------------------------
1374
- async back(threadId) {
1375
- try {
1376
- const page = await this.getPage(threadId);
1377
- await page.goBack({ timeout: this.defaultTimeout });
1378
- const url = page.url();
1379
- this.markActiveUrlChangeSource("agent", url, threadId);
1380
- return {
1381
- success: true,
1382
- url,
1383
- title: await page.title(),
1384
- hint: "Take a new snapshot to see the previous page."
1385
- };
1386
- } catch (error) {
1387
- return this.createErrorFromException(error, "Back");
1388
- }
1389
- }
1390
- // ---------------------------------------------------------------------------
1391
- // 11. browser_dialog - Click element that triggers dialog and handle it
1392
- // ---------------------------------------------------------------------------
1393
- async dialog(input, threadId) {
1394
- try {
1395
- const page = await this.getPage(threadId);
1396
- const locator = await this.requireLocator(input.triggerRef, threadId);
1397
- if (!locator) {
1398
- return this.createError(
1399
- "stale_ref",
1400
- `Trigger ref ${input.triggerRef} not found.`,
1401
- "Take a new snapshot to get fresh refs."
1402
- );
1403
- }
1404
- return new Promise((resolve, reject) => {
1405
- const timeout = setTimeout(() => {
1406
- page.off("dialog", dialogHandler);
1407
- reject(
1408
- new Error(`No dialog appeared after clicking ${input.triggerRef}. The element may not trigger a dialog.`)
1409
- );
1410
- }, this.defaultTimeout);
1411
- const dialogHandler = async (dialog) => {
1412
- clearTimeout(timeout);
1413
- try {
1414
- const dialogType = dialog.type();
1415
- const message = dialog.message();
1416
- if (input.action === "accept") {
1417
- await dialog.accept(input.text);
1418
- } else {
1419
- await dialog.dismiss();
1420
- }
1421
- resolve({
1422
- success: true,
1423
- action: input.action,
1424
- dialogType,
1425
- message,
1426
- hint: "Dialog handled. Take a snapshot to continue."
1427
- });
1428
- } catch (e) {
1429
- reject(e);
1430
- }
1431
- };
1432
- page.once("dialog", dialogHandler);
1433
- locator.click({ timeout: this.defaultTimeout }).catch((e) => {
1434
- clearTimeout(timeout);
1435
- page.off("dialog", dialogHandler);
1436
- reject(e);
1437
- });
1438
- });
1439
- } catch (error) {
1440
- return this.createErrorFromException(error, "Dialog");
1441
- }
1442
- }
1443
- // ---------------------------------------------------------------------------
1444
- // 13. browser_wait - Wait for element or condition
1445
- // ---------------------------------------------------------------------------
1446
- async wait(input, threadId) {
1447
- try {
1448
- const timeout = input.timeout ?? this.defaultTimeout;
1449
- if (input.ref) {
1450
- const locator = await this.requireLocator(input.ref, threadId);
1451
- if (!locator) {
1452
- return this.createError("stale_ref", `Ref ${input.ref} not found.`, "Take a new snapshot to get fresh refs.");
1453
- }
1454
- const state = input.state ?? "visible";
1455
- await locator.waitFor({ state, timeout });
1456
- return {
1457
- success: true,
1458
- hint: `Element is now ${state}. Take a snapshot to continue.`
1459
- };
1460
- } else {
1461
- const page = await this.getPage(threadId);
1462
- await page.waitForTimeout(timeout);
1463
- return {
1464
- success: true,
1465
- hint: "Wait complete. Take a snapshot to see current state."
1466
- };
1467
- }
1468
- } catch (error) {
1469
- return this.createErrorFromException(error, "Wait");
1470
- }
1471
- }
1472
- // ---------------------------------------------------------------------------
1473
- // 14. browser_tabs - Manage browser tabs
1474
- // ---------------------------------------------------------------------------
1475
- async tabs(input, threadId) {
1476
- try {
1477
- const browser = await this.getManagerForThread(threadId);
1478
- if (!browser) {
1479
- return this.createError(
1480
- "browser_closed",
1481
- "Browser not launched",
1482
- "Call a navigation tool first to launch the browser."
1483
- );
1484
- }
1485
- switch (input.action) {
1486
- case "list": {
1487
- if (!browser.listTabs) {
1488
- return this.createError(
1489
- "browser_error",
1490
- "Tab management not supported",
1491
- "This browser provider does not support tab management."
1492
- );
1493
- }
1494
- const tabsList = await browser.listTabs();
1495
- return {
1496
- success: true,
1497
- tabs: tabsList,
1498
- hint: 'Use browser_tabs with action:"switch" and index to change tabs.'
1499
- };
1500
- }
1501
- case "new": {
1502
- if (!browser.newTab) {
1503
- return this.createError(
1504
- "browser_error",
1505
- "Tab management not supported",
1506
- "This browser provider does not support tab management."
1507
- );
1508
- }
1509
- const result = await browser.newTab();
1510
- if (input.url) {
1511
- const page = await this.getPage(threadId);
1512
- await page.goto(input.url);
1513
- this.markActiveUrlChangeSource("agent", page.url(), threadId);
1514
- }
1515
- this.updateSessionBrowserState(threadId);
1516
- return {
1517
- success: true,
1518
- ...result,
1519
- hint: "New tab opened. Take a snapshot to see its content."
1520
- };
1521
- }
1522
- case "switch": {
1523
- if (!browser.switchTo) {
1524
- return this.createError(
1525
- "browser_error",
1526
- "Tab management not supported",
1527
- "This browser provider does not support tab management."
1528
- );
1529
- }
1530
- await browser.switchTo(input.index);
1531
- await this.reconnectScreencastForThread(threadId, "tab switch");
1532
- const page = browser.getPage();
1533
- const pageUrl = page.url();
1534
- this.markActiveUrlChangeSource("agent", pageUrl, threadId);
1535
- const streamKey = this.getStreamKey(threadId);
1536
- const stream = this.activeScreencastStreams.get(streamKey);
1537
- if (pageUrl && stream?.isActive()) {
1538
- stream.emitUrl(pageUrl);
1539
- }
1540
- this.updateSessionBrowserState(threadId);
1541
- return {
1542
- success: true,
1543
- index: input.index,
1544
- url: pageUrl,
1545
- title: await page.title(),
1546
- hint: "Tab switched. Take a snapshot to see its content."
1547
- };
1548
- }
1549
- case "close": {
1550
- if (!browser.closeTab) {
1551
- return this.createError(
1552
- "browser_error",
1553
- "Tab management not supported",
1554
- "This browser provider does not support tab management."
1555
- );
1556
- }
1557
- await browser.closeTab(input.index);
1558
- await this.reconnectScreencastForThread(threadId, "tab close");
1559
- this.updateSessionBrowserState(threadId);
1560
- const tabsList = await browser.listTabs?.() ?? [];
1561
- return {
1562
- success: true,
1563
- remaining: tabsList.length,
1564
- hint: tabsList.length > 0 ? "Tab closed. Take a snapshot to see current tab." : "All tabs closed."
1565
- };
1566
- }
1567
- default:
1568
- return this.createError(
1569
- "browser_error",
1570
- `Unknown tabs action: ${input.action}`,
1571
- 'Use "list", "new", "switch", or "close".'
1572
- );
1573
- }
1574
- } catch (error) {
1575
- return this.createErrorFromException(error, "Tabs");
1576
- }
1577
- }
1578
- // ---------------------------------------------------------------------------
1579
- // 15. browser_drag - Drag element to target
1580
- // ---------------------------------------------------------------------------
1581
- async drag(input, threadId) {
1582
- try {
1583
- const page = await this.getPage(threadId);
1584
- let sourceLocator = null;
1585
- if (input.sourceRef) {
1586
- sourceLocator = await this.requireLocator(input.sourceRef, threadId);
1587
- } else if (input.sourceSelector) {
1588
- sourceLocator = page.locator(input.sourceSelector);
1589
- }
1590
- if (!sourceLocator) {
1591
- return this.createError(
1592
- "stale_ref",
1593
- input.sourceRef ? `Source ref ${input.sourceRef} not found.` : "No source element specified. Provide sourceRef or sourceSelector.",
1594
- input.sourceRef ? "Take a new snapshot to get fresh refs, or use sourceSelector for elements not in the accessibility tree." : void 0
1595
- );
1596
- }
1597
- let targetLocator = null;
1598
- if (input.targetRef) {
1599
- targetLocator = await this.requireLocator(input.targetRef, threadId);
1600
- } else if (input.targetSelector) {
1601
- targetLocator = page.locator(input.targetSelector);
1602
- }
1603
- if (!targetLocator) {
1604
- return this.createError(
1605
- "stale_ref",
1606
- input.targetRef ? `Target ref ${input.targetRef} not found.` : "No target element specified. Provide targetRef or targetSelector.",
1607
- input.targetRef ? "Take a new snapshot to get fresh refs, or use targetSelector for elements not in the accessibility tree." : void 0
1608
- );
1609
- }
1610
- await sourceLocator.dragTo(targetLocator, { timeout: this.defaultTimeout });
1611
- return {
1612
- success: true,
1613
- url: page.url(),
1614
- hint: "Drag complete. Take a snapshot to see the result."
1615
- };
1616
- } catch (error) {
1617
- return this.createErrorFromException(error, "Drag");
1618
- }
1619
- }
1620
- // ---------------------------------------------------------------------------
1621
- // 16. browser_evaluate - Execute JavaScript
1622
- // ---------------------------------------------------------------------------
1623
- async evaluate(input, threadId) {
1624
- try {
1625
- const page = await this.getPage(threadId);
1626
- const result = await page.evaluate(input.script);
1627
- return {
1628
- success: true,
1629
- result,
1630
- hint: "JavaScript executed. Take a snapshot if the page may have changed."
1631
- };
1632
- } catch (error) {
1633
- return this.createErrorFromException(error, "Evaluate");
1634
- }
1635
- }
1636
- // ---------------------------------------------------------------------------
1637
- // 17. browser_close - Close browser
1638
- // ---------------------------------------------------------------------------
1639
- async closeBrowser() {
1640
- try {
1641
- await this.close();
1642
- return {
1643
- success: true,
1644
- hint: "Browser closed. Call browser_goto to start a new session."
1645
- };
1646
- } catch (error) {
1647
- return this.createErrorFromException(error, "Close");
1648
- }
1649
- }
1650
- // ---------------------------------------------------------------------------
1651
- // Screencast (for Studio live view)
1652
- // ---------------------------------------------------------------------------
1653
- async startScreencast(_options) {
1654
- const requestedThreadId = _options?.threadId;
1655
- const effectiveThreadId = this.getScope() === "thread" ? requestedThreadId ?? this.getCurrentThread() ?? DEFAULT_THREAD_ID : requestedThreadId;
1656
- let browserManager;
1657
- if (this.getScope() === "thread") {
1658
- browserManager = await this.getManagerForThread(effectiveThreadId);
1659
- } else {
1660
- if (!this.sharedManager) throw new Error("Browser not launched");
1661
- browserManager = this.sharedManager;
1662
- }
1663
- const provider = {
1664
- getCdpSession: async () => {
1665
- const currentPage = browserManager.getPage();
1666
- if (!currentPage) {
1667
- throw new Error("No active page available");
1668
- }
1669
- const cdpSession = await currentPage.context().newCDPSession(currentPage);
1670
- return cdpSession;
1671
- },
1672
- isBrowserRunning: () => browserManager.isLaunched()
1673
- };
1674
- const stream = new ScreencastStreamImpl(provider, _options);
1675
- const streamKey = this.getStreamKey(effectiveThreadId);
1676
- this.activeScreencastStreams.set(streamKey, stream);
1677
- const context = browserManager.getContext();
1678
- if (context) {
1679
- const onNewPage = (_newPage) => {
1680
- setTimeout(() => {
1681
- if (stream.isActive()) {
1682
- stream.reconnect().catch(() => {
1683
- });
1684
- }
1685
- }, 100);
1686
- };
1687
- context.on("page", onNewPage);
1688
- const pageCloseHandlers = /* @__PURE__ */ new Map();
1689
- const frameNavigatedHandlers = /* @__PURE__ */ new Map();
1690
- const setupPageListeners = (page) => {
1691
- const onFrameNavigated = (frame) => {
1692
- if (!frame.parentFrame()) {
1693
- stream.emitUrl(frame.url());
1694
- this.updateSessionBrowserState(effectiveThreadId);
1695
- }
1696
- };
1697
- page.on("framenavigated", onFrameNavigated);
1698
- frameNavigatedHandlers.set(page, onFrameNavigated);
1699
- const onClose = () => {
1700
- pageCloseHandlers.delete(page);
1701
- const navHandler = frameNavigatedHandlers.get(page);
1702
- if (navHandler) {
1703
- page.off("framenavigated", navHandler);
1704
- frameNavigatedHandlers.delete(page);
1705
- }
1706
- setTimeout(() => {
1707
- const remainingPages = browserManager.getPages();
1708
- if (stream.isActive() && remainingPages.length > 0) {
1709
- stream.reconnect().catch(() => {
1710
- });
1711
- const activePage = remainingPages[browserManager.getActiveIndex()] || remainingPages[0];
1712
- if (activePage) {
1713
- const url = activePage.url();
1714
- if (url && url !== "about:blank") {
1715
- stream.emitUrl(url);
1716
- }
1717
- }
1718
- }
1719
- }, 100);
1720
- };
1721
- page.once("close", onClose);
1722
- pageCloseHandlers.set(page, onClose);
1723
- };
1724
- const setupPageCloseListener = setupPageListeners;
1725
- for (const page of browserManager.getPages()) {
1726
- setupPageCloseListener(page);
1727
- }
1728
- const onNewPageWithCloseListener = (newPage) => {
1729
- setupPageCloseListener(newPage);
1730
- const url = newPage.url();
1731
- if (url && url !== "about:blank") {
1732
- stream.emitUrl(url);
1733
- }
1734
- onNewPage();
1735
- };
1736
- context.off("page", onNewPage);
1737
- context.on("page", onNewPageWithCloseListener);
1738
- stream.once("stop", () => {
1739
- context.off("page", onNewPageWithCloseListener);
1740
- for (const [page, handler] of pageCloseHandlers) {
1741
- page.off("close", handler);
1742
- }
1743
- pageCloseHandlers.clear();
1744
- for (const [page, handler] of frameNavigatedHandlers) {
1745
- page.off("framenavigated", handler);
1746
- }
1747
- frameNavigatedHandlers.clear();
1748
- this.activeScreencastStreams.delete(streamKey);
1749
- });
1750
- }
1751
- await stream.start();
1752
- return stream;
1753
- }
1754
- // ---------------------------------------------------------------------------
1755
- // Event Injection (for Studio live view interactivity)
1756
- // ---------------------------------------------------------------------------
1757
- async injectMouseEvent(event, threadId) {
1758
- const effectiveThreadId = threadId ?? this.getCurrentThread();
1759
- const manager = await this.getManagerForThread(effectiveThreadId);
1760
- await manager.injectMouseEvent(event);
1761
- }
1762
- async injectKeyboardEvent(event, threadId) {
1763
- const effectiveThreadId = threadId ?? this.getCurrentThread();
1764
- const manager = await this.getManagerForThread(effectiveThreadId);
1765
- const cdp = await manager.getCDPSession();
1766
- await cdp.send("Input.dispatchKeyEvent", {
1767
- type: event.type,
1768
- key: event.key,
1769
- code: event.code,
1770
- text: event.text,
1771
- modifiers: event.modifiers ?? 0,
1772
- windowsVirtualKeyCode: event.windowsVirtualKeyCode
1773
- });
1774
- }
998
+ if (!info || typeof info.scrollHeight !== "number") return {
999
+ scrollY: 0,
1000
+ scrollHeight: 0,
1001
+ viewportHeight: 0,
1002
+ atTop: true,
1003
+ atBottom: true,
1004
+ percentDown: 0
1005
+ };
1006
+ const maxScroll = info.scrollHeight - info.viewportHeight;
1007
+ return {
1008
+ ...info,
1009
+ atTop: info.scrollY < 50,
1010
+ atBottom: info.scrollY >= maxScroll - 50,
1011
+ percentDown: maxScroll > 0 ? Math.round(info.scrollY / maxScroll * 100) : 0
1012
+ };
1013
+ }
1014
+ /**
1015
+ * Get the current page URL without launching the browser.
1016
+ * @param threadId - Optional thread ID for thread-isolated browsers
1017
+ * @returns The current URL string, or null if browser is not running
1018
+ */
1019
+ async getCurrentUrl(threadId) {
1020
+ if (!this.isBrowserRunning()) return null;
1021
+ try {
1022
+ const effectiveThreadId = threadId ?? this.getCurrentThread();
1023
+ if (this.threadManager.getScope() === "thread" && effectiveThreadId) {
1024
+ const manager = this.threadManager.getExistingManagerForThread(effectiveThreadId);
1025
+ if (!manager) return null;
1026
+ const url = manager.getPage().url();
1027
+ if (url && url !== "about:blank") {
1028
+ const state = this.getBrowserStateForManager(manager);
1029
+ if (state) this.threadManager.updateBrowserState(effectiveThreadId, state);
1030
+ }
1031
+ return url;
1032
+ }
1033
+ const manager = await this.getManagerForThread(threadId);
1034
+ const url = manager.getPage().url();
1035
+ if (url && url !== "about:blank") {
1036
+ const state = this.getBrowserStateForManager(manager);
1037
+ if (state) this.lastBrowserState = state;
1038
+ }
1039
+ return url;
1040
+ } catch {
1041
+ return null;
1042
+ }
1043
+ }
1044
+ /**
1045
+ * Navigate to a URL (simple form). Used internally for restoring state on relaunch.
1046
+ */
1047
+ async navigateTo(url) {
1048
+ if (!this.isBrowserRunning()) return;
1049
+ try {
1050
+ await (await this.getPage()).goto(url, {
1051
+ timeout: this.defaultTimeout,
1052
+ waitUntil: "domcontentloaded"
1053
+ });
1054
+ } catch {}
1055
+ }
1056
+ /**
1057
+ * Get the current browser state (all tabs and active tab index).
1058
+ */
1059
+ async getBrowserState(threadId) {
1060
+ if (!this.isBrowserRunning(threadId)) return null;
1061
+ try {
1062
+ const manager = await this.getManagerForThread(threadId);
1063
+ return this.getBrowserStateForManager(manager, threadId);
1064
+ } catch {
1065
+ return null;
1066
+ }
1067
+ }
1068
+ /**
1069
+ * Get browser state for a thread (implements abstract method from base class).
1070
+ * Sync version that uses existing manager lookup without creating sessions.
1071
+ */
1072
+ getBrowserStateForThread(threadId) {
1073
+ const effectiveThreadId = threadId ?? this.getCurrentThread() ?? DEFAULT_THREAD_ID;
1074
+ const manager = this.threadManager.getExistingManagerForThread(effectiveThreadId);
1075
+ if (!manager) return null;
1076
+ return this.getBrowserStateForManager(manager, effectiveThreadId);
1077
+ }
1078
+ /**
1079
+ * Get browser state from a specific manager instance.
1080
+ */
1081
+ getBrowserStateForManager(manager, threadId) {
1082
+ try {
1083
+ const stateKey = this.browserStateKey(threadId);
1084
+ const pages = manager.getPages();
1085
+ const activeIndex = manager.getActiveIndex();
1086
+ const tabs = pages.map((page) => ({ url: page.url() }));
1087
+ const activeUrl = tabs[activeIndex]?.url;
1088
+ const previousState = this.threadManager.getSavedBrowserState(stateKey) ?? this.lastBrowserState;
1089
+ const previousUrl = previousState?.tabs[previousState.activeTabIndex]?.url;
1090
+ const activeUrlChangeSource = this.getActiveUrlChangeSource(activeUrl, stateKey) ?? (previousUrl && activeUrl !== previousUrl ? "user" : void 0);
1091
+ const state = {
1092
+ tabs,
1093
+ activeTabIndex: activeIndex,
1094
+ ...this.getCloseReason(stateKey) ? { closeReason: this.getCloseReason(stateKey) } : {},
1095
+ ...activeUrlChangeSource ? { activeUrlChangeSource } : {}
1096
+ };
1097
+ this.threadManager.updateBrowserState(stateKey, state);
1098
+ this.lastBrowserState = state;
1099
+ return state;
1100
+ } catch {
1101
+ return null;
1102
+ }
1103
+ }
1104
+ /**
1105
+ * Get all open tabs with their URLs and titles.
1106
+ */
1107
+ async getTabState(threadId) {
1108
+ return (await this.getBrowserState(threadId))?.tabs ?? [];
1109
+ }
1110
+ /**
1111
+ * Get the active tab index.
1112
+ */
1113
+ async getActiveTabIndex(threadId) {
1114
+ if (!this.isBrowserRunning()) return 0;
1115
+ try {
1116
+ return (await this.getManagerForThread(threadId)).getActiveIndex();
1117
+ } catch {
1118
+ return 0;
1119
+ }
1120
+ }
1121
+ /**
1122
+ * Export the current browser session's storage state (cookies, localStorage) to a JSON file.
1123
+ * This can later be loaded via the `storageState` config option to restore the session.
1124
+ *
1125
+ * @param path - File path to save the storage state JSON
1126
+ * @param threadId - Optional thread ID (defaults to current thread)
1127
+ */
1128
+ async exportStorageState(path, threadId) {
1129
+ const effectiveThreadId = threadId ?? this.getCurrentThread();
1130
+ const manager = this.threadManager.getExistingManagerForThread(effectiveThreadId);
1131
+ if (!manager) throw new Error("No browser is running. Launch a browser first before exporting storage state.");
1132
+ const context = manager.getContext();
1133
+ if (!context) throw new Error("Browser context not available");
1134
+ await context.storageState({ path });
1135
+ }
1136
+ async goto(input, threadId) {
1137
+ try {
1138
+ const page = await this.getPage(threadId);
1139
+ await page.goto(input.url, {
1140
+ timeout: input.timeout ?? this.defaultTimeout,
1141
+ waitUntil: input.waitUntil ?? "domcontentloaded"
1142
+ });
1143
+ const url = page.url();
1144
+ this.markActiveUrlChangeSource("agent", url, threadId);
1145
+ return {
1146
+ success: true,
1147
+ url,
1148
+ title: await page.title(),
1149
+ hint: "Take a snapshot to see interactive elements and get refs."
1150
+ };
1151
+ } catch (error) {
1152
+ return this.createErrorFromException(error, "Goto");
1153
+ }
1154
+ }
1155
+ async snapshot(input, threadId) {
1156
+ try {
1157
+ const manager = await this.getManagerForThread(threadId);
1158
+ const page = await this.getPage(threadId);
1159
+ const snapshot = ((await manager.getSnapshot({
1160
+ interactive: input.interactiveOnly ?? true,
1161
+ compact: true
1162
+ })).tree ?? "").replace(/\[ref=(\w+)\]/g, "@$1");
1163
+ const scrollInfo = await this.getScrollInfo(threadId);
1164
+ let scrollText;
1165
+ if (scrollInfo.atTop && !scrollInfo.atBottom) scrollText = "TOP - more content below";
1166
+ else if (scrollInfo.atBottom) scrollText = "BOTTOM of page";
1167
+ else scrollText = `${scrollInfo.percentDown}% down`;
1168
+ const refs = snapshot.match(/@e\d+/g) || [];
1169
+ const elementCount = new Set(refs).size;
1170
+ return {
1171
+ success: true,
1172
+ snapshot,
1173
+ url: page.url(),
1174
+ title: await page.title(),
1175
+ elementCount,
1176
+ scroll: scrollText,
1177
+ hint: elementCount === 0 ? "No interactive elements found. Try scrolling or setting interactiveOnly:false." : void 0
1178
+ };
1179
+ } catch (error) {
1180
+ return this.createErrorFromException(error, "Snapshot");
1181
+ }
1182
+ }
1183
+ async screenshot(input, threadId) {
1184
+ try {
1185
+ const page = await this.getPage(threadId);
1186
+ const buffer = await page.screenshot({
1187
+ fullPage: input.fullPage ?? false,
1188
+ type: "png"
1189
+ });
1190
+ return {
1191
+ base64: Buffer.from(buffer).toString("base64"),
1192
+ url: page.url(),
1193
+ title: await page.title()
1194
+ };
1195
+ } catch (error) {
1196
+ return this.createErrorFromException(error, "Screenshot");
1197
+ }
1198
+ }
1199
+ /**
1200
+ * Start a `waitForNavigation` wait (when `waitUntil` is set) and immediately
1201
+ * attach a noop catch handler so a navigation timeout/rejection can't become
1202
+ * an unhandled rejection (crashing the process) while the caller's action is
1203
+ * still pending. Callers should still `await` the returned promise to observe
1204
+ * the original error.
1205
+ */
1206
+ startNavigationWait(page, waitUntil, timeout) {
1207
+ const navigation = waitUntil ? page.waitForNavigation({
1208
+ waitUntil,
1209
+ timeout
1210
+ }) : void 0;
1211
+ navigation?.catch(() => {});
1212
+ return navigation;
1213
+ }
1214
+ async click(input, threadId) {
1215
+ try {
1216
+ const page = await this.getPage(threadId);
1217
+ const locator = await this.requireLocator(input.ref, threadId);
1218
+ if (!locator) return this.createError("stale_ref", `Ref ${input.ref} not found. The page has changed.`, "Take a new snapshot to see the current page state and get fresh refs.");
1219
+ const timeout = input.timeout ?? this.defaultTimeout;
1220
+ const navigation = this.startNavigationWait(page, input.waitUntil, timeout);
1221
+ await locator.click({
1222
+ button: input.button ?? "left",
1223
+ clickCount: input.clickCount ?? 1,
1224
+ modifiers: input.modifiers,
1225
+ timeout
1226
+ });
1227
+ await navigation;
1228
+ return {
1229
+ success: true,
1230
+ url: page.url(),
1231
+ hint: "Take a new snapshot to see updated page state and get fresh refs."
1232
+ };
1233
+ } catch (error) {
1234
+ if ((error instanceof Error ? error.message : String(error)).includes("intercepts pointer events")) return this.createError("element_blocked", `Element ${input.ref} is blocked by another element.`, "Take a new snapshot to see what is blocking. Dismiss any modals or scroll the element into view.");
1235
+ return this.createErrorFromException(error, "Click");
1236
+ }
1237
+ }
1238
+ async type(input, threadId) {
1239
+ try {
1240
+ const page = await this.getPage(threadId);
1241
+ const locator = await this.requireLocator(input.ref, threadId);
1242
+ if (!locator) return this.createError("stale_ref", `Ref ${input.ref} not found. The page has changed.`, "Take a new snapshot to see the current page state and get fresh refs.");
1243
+ if (input.clear) await locator.fill("", { timeout: this.defaultTimeout });
1244
+ if (input.delay) {
1245
+ await locator.focus();
1246
+ for (const char of input.text) {
1247
+ await page.keyboard.press(char);
1248
+ await new Promise((r) => setTimeout(r, input.delay));
1249
+ }
1250
+ } else await locator.fill(input.text, { timeout: this.defaultTimeout });
1251
+ return {
1252
+ success: true,
1253
+ value: await locator.inputValue({ timeout: 1e3 }).catch(() => input.text),
1254
+ url: page.url(),
1255
+ hint: "Take a new snapshot if you need to interact with more elements."
1256
+ };
1257
+ } catch (error) {
1258
+ const errorMsg = error instanceof Error ? error.message : String(error);
1259
+ if (errorMsg.includes("is not an <input>") || errorMsg.includes("not an input") || errorMsg.includes("Cannot type") || errorMsg.includes("not focusable")) return this.createError("not_focusable", `Element ${input.ref} is not a text input field.`, "Take a new snapshot and look for elements with role \"textbox\" or \"searchbox\".");
1260
+ return this.createErrorFromException(error, "Type");
1261
+ }
1262
+ }
1263
+ async press(input, threadId) {
1264
+ try {
1265
+ const page = await this.getPage(threadId);
1266
+ const timeout = input.timeout ?? this.defaultTimeout;
1267
+ const navigation = this.startNavigationWait(page, input.waitUntil, timeout);
1268
+ await page.keyboard.press(input.key);
1269
+ await navigation;
1270
+ return {
1271
+ success: true,
1272
+ url: page.url(),
1273
+ hint: "Take a new snapshot if the page may have changed."
1274
+ };
1275
+ } catch (error) {
1276
+ return this.createErrorFromException(error, "Press");
1277
+ }
1278
+ }
1279
+ async select(input, threadId) {
1280
+ try {
1281
+ const page = await this.getPage(threadId);
1282
+ const locator = await this.requireLocator(input.ref, threadId);
1283
+ if (!locator) return this.createError("stale_ref", `Ref ${input.ref} not found. The page has changed.`, "Take a new snapshot to get fresh refs.");
1284
+ const selectValue = {};
1285
+ if (input.value) selectValue.value = input.value;
1286
+ if (input.label) selectValue.label = input.label;
1287
+ if (input.index !== void 0) selectValue.index = input.index;
1288
+ const timeout = input.timeout ?? this.defaultTimeout;
1289
+ const navigation = this.startNavigationWait(page, input.waitUntil, timeout);
1290
+ const selected = await locator.selectOption(selectValue, { timeout });
1291
+ await navigation;
1292
+ return {
1293
+ success: true,
1294
+ selected,
1295
+ url: page.url(),
1296
+ hint: "Selection complete. Take a snapshot if you need to continue."
1297
+ };
1298
+ } catch (error) {
1299
+ return this.createErrorFromException(error, "Select");
1300
+ }
1301
+ }
1302
+ async scroll(input, threadId) {
1303
+ try {
1304
+ const page = await this.getPage(threadId);
1305
+ if (input.ref) {
1306
+ const locator = await this.requireLocator(input.ref, threadId);
1307
+ if (locator) await locator.scrollIntoViewIfNeeded({ timeout: this.defaultTimeout });
1308
+ } else {
1309
+ const direction = input.direction;
1310
+ const amount = input.amount ?? 300;
1311
+ let deltaX = 0;
1312
+ let deltaY = 0;
1313
+ switch (direction) {
1314
+ case "up":
1315
+ deltaY = -amount;
1316
+ break;
1317
+ case "down":
1318
+ deltaY = amount;
1319
+ break;
1320
+ case "left":
1321
+ deltaX = -amount;
1322
+ break;
1323
+ case "right":
1324
+ deltaX = amount;
1325
+ break;
1326
+ }
1327
+ await page.evaluate(({ x, y }) => {
1328
+ globalThis.scrollBy(x, y);
1329
+ }, {
1330
+ x: deltaX,
1331
+ y: deltaY
1332
+ });
1333
+ }
1334
+ const scrollInfo = await this.getScrollInfo(threadId);
1335
+ let scrollText;
1336
+ if (scrollInfo.atTop && !scrollInfo.atBottom) scrollText = "TOP - more content below";
1337
+ else if (scrollInfo.atBottom) scrollText = "BOTTOM of page";
1338
+ else scrollText = `${scrollInfo.percentDown}% down`;
1339
+ return {
1340
+ success: true,
1341
+ position: {
1342
+ x: 0,
1343
+ y: scrollInfo.scrollY
1344
+ },
1345
+ scroll: scrollText,
1346
+ hint: "Take a new snapshot to see elements in the new viewport."
1347
+ };
1348
+ } catch (error) {
1349
+ return this.createErrorFromException(error, "Scroll");
1350
+ }
1351
+ }
1352
+ async hover(input, threadId) {
1353
+ try {
1354
+ const page = await this.getPage(threadId);
1355
+ const locator = await this.requireLocator(input.ref, threadId);
1356
+ if (!locator) return this.createError("stale_ref", `Ref ${input.ref} not found. The page has changed.`, "Take a new snapshot to get fresh refs.");
1357
+ await locator.hover({ timeout: this.defaultTimeout });
1358
+ return {
1359
+ success: true,
1360
+ url: page.url(),
1361
+ hint: "Take a new snapshot to see any hover-triggered elements (dropdowns, tooltips)."
1362
+ };
1363
+ } catch (error) {
1364
+ return this.createErrorFromException(error, "Hover");
1365
+ }
1366
+ }
1367
+ async back(threadId) {
1368
+ try {
1369
+ const page = await this.getPage(threadId);
1370
+ await page.goBack({ timeout: this.defaultTimeout });
1371
+ const url = page.url();
1372
+ this.markActiveUrlChangeSource("agent", url, threadId);
1373
+ return {
1374
+ success: true,
1375
+ url,
1376
+ title: await page.title(),
1377
+ hint: "Take a new snapshot to see the previous page."
1378
+ };
1379
+ } catch (error) {
1380
+ return this.createErrorFromException(error, "Back");
1381
+ }
1382
+ }
1383
+ async dialog(input, threadId) {
1384
+ try {
1385
+ const page = await this.getPage(threadId);
1386
+ const locator = await this.requireLocator(input.triggerRef, threadId);
1387
+ if (!locator) return this.createError("stale_ref", `Trigger ref ${input.triggerRef} not found.`, "Take a new snapshot to get fresh refs.");
1388
+ return new Promise((resolve, reject) => {
1389
+ const timeout = setTimeout(() => {
1390
+ page.off("dialog", dialogHandler);
1391
+ reject(/* @__PURE__ */ new Error(`No dialog appeared after clicking ${input.triggerRef}. The element may not trigger a dialog.`));
1392
+ }, this.defaultTimeout);
1393
+ const dialogHandler = async (dialog) => {
1394
+ clearTimeout(timeout);
1395
+ try {
1396
+ const dialogType = dialog.type();
1397
+ const message = dialog.message();
1398
+ if (input.action === "accept") await dialog.accept(input.text);
1399
+ else await dialog.dismiss();
1400
+ resolve({
1401
+ success: true,
1402
+ action: input.action,
1403
+ dialogType,
1404
+ message,
1405
+ hint: "Dialog handled. Take a snapshot to continue."
1406
+ });
1407
+ } catch (e) {
1408
+ reject(e);
1409
+ }
1410
+ };
1411
+ page.once("dialog", dialogHandler);
1412
+ locator.click({ timeout: this.defaultTimeout }).catch((e) => {
1413
+ clearTimeout(timeout);
1414
+ page.off("dialog", dialogHandler);
1415
+ reject(e);
1416
+ });
1417
+ });
1418
+ } catch (error) {
1419
+ return this.createErrorFromException(error, "Dialog");
1420
+ }
1421
+ }
1422
+ async wait(input, threadId) {
1423
+ try {
1424
+ const timeout = input.timeout ?? this.defaultTimeout;
1425
+ if (input.ref) {
1426
+ const locator = await this.requireLocator(input.ref, threadId);
1427
+ if (!locator) return this.createError("stale_ref", `Ref ${input.ref} not found.`, "Take a new snapshot to get fresh refs.");
1428
+ const state = input.state ?? "visible";
1429
+ await locator.waitFor({
1430
+ state,
1431
+ timeout
1432
+ });
1433
+ return {
1434
+ success: true,
1435
+ hint: `Element is now ${state}. Take a snapshot to continue.`
1436
+ };
1437
+ } else {
1438
+ await (await this.getPage(threadId)).waitForTimeout(timeout);
1439
+ return {
1440
+ success: true,
1441
+ hint: "Wait complete. Take a snapshot to see current state."
1442
+ };
1443
+ }
1444
+ } catch (error) {
1445
+ return this.createErrorFromException(error, "Wait");
1446
+ }
1447
+ }
1448
+ async tabs(input, threadId) {
1449
+ try {
1450
+ const browser = await this.getManagerForThread(threadId);
1451
+ if (!browser) return this.createError("browser_closed", "Browser not launched", "Call a navigation tool first to launch the browser.");
1452
+ switch (input.action) {
1453
+ case "list":
1454
+ if (!browser.listTabs) return this.createError("browser_error", "Tab management not supported", "This browser provider does not support tab management.");
1455
+ return {
1456
+ success: true,
1457
+ tabs: await browser.listTabs(),
1458
+ hint: "Use browser_tabs with action:\"switch\" and index to change tabs."
1459
+ };
1460
+ case "new": {
1461
+ if (!browser.newTab) return this.createError("browser_error", "Tab management not supported", "This browser provider does not support tab management.");
1462
+ const result = await browser.newTab();
1463
+ if (input.url) {
1464
+ const page = await this.getPage(threadId);
1465
+ await page.goto(input.url);
1466
+ this.markActiveUrlChangeSource("agent", page.url(), threadId);
1467
+ }
1468
+ this.updateSessionBrowserState(threadId);
1469
+ return {
1470
+ success: true,
1471
+ ...result,
1472
+ hint: "New tab opened. Take a snapshot to see its content."
1473
+ };
1474
+ }
1475
+ case "switch": {
1476
+ if (!browser.switchTo) return this.createError("browser_error", "Tab management not supported", "This browser provider does not support tab management.");
1477
+ await browser.switchTo(input.index);
1478
+ await this.reconnectScreencastForThread(threadId, "tab switch");
1479
+ const page = browser.getPage();
1480
+ const pageUrl = page.url();
1481
+ this.markActiveUrlChangeSource("agent", pageUrl, threadId);
1482
+ const streamKey = this.getStreamKey(threadId);
1483
+ const stream = this.activeScreencastStreams.get(streamKey);
1484
+ if (pageUrl && stream?.isActive()) stream.emitUrl(pageUrl);
1485
+ this.updateSessionBrowserState(threadId);
1486
+ return {
1487
+ success: true,
1488
+ index: input.index,
1489
+ url: pageUrl,
1490
+ title: await page.title(),
1491
+ hint: "Tab switched. Take a snapshot to see its content."
1492
+ };
1493
+ }
1494
+ case "close": {
1495
+ if (!browser.closeTab) return this.createError("browser_error", "Tab management not supported", "This browser provider does not support tab management.");
1496
+ await browser.closeTab(input.index);
1497
+ await this.reconnectScreencastForThread(threadId, "tab close");
1498
+ this.updateSessionBrowserState(threadId);
1499
+ const tabsList = await browser.listTabs?.() ?? [];
1500
+ return {
1501
+ success: true,
1502
+ remaining: tabsList.length,
1503
+ hint: tabsList.length > 0 ? "Tab closed. Take a snapshot to see current tab." : "All tabs closed."
1504
+ };
1505
+ }
1506
+ default: return this.createError("browser_error", `Unknown tabs action: ${input.action}`, "Use \"list\", \"new\", \"switch\", or \"close\".");
1507
+ }
1508
+ } catch (error) {
1509
+ return this.createErrorFromException(error, "Tabs");
1510
+ }
1511
+ }
1512
+ async drag(input, threadId) {
1513
+ try {
1514
+ const page = await this.getPage(threadId);
1515
+ let sourceLocator = null;
1516
+ if (input.sourceRef) sourceLocator = await this.requireLocator(input.sourceRef, threadId);
1517
+ else if (input.sourceSelector) sourceLocator = page.locator(input.sourceSelector);
1518
+ if (!sourceLocator) return this.createError("stale_ref", input.sourceRef ? `Source ref ${input.sourceRef} not found.` : "No source element specified. Provide sourceRef or sourceSelector.", input.sourceRef ? "Take a new snapshot to get fresh refs, or use sourceSelector for elements not in the accessibility tree." : void 0);
1519
+ let targetLocator = null;
1520
+ if (input.targetRef) targetLocator = await this.requireLocator(input.targetRef, threadId);
1521
+ else if (input.targetSelector) targetLocator = page.locator(input.targetSelector);
1522
+ if (!targetLocator) return this.createError("stale_ref", input.targetRef ? `Target ref ${input.targetRef} not found.` : "No target element specified. Provide targetRef or targetSelector.", input.targetRef ? "Take a new snapshot to get fresh refs, or use targetSelector for elements not in the accessibility tree." : void 0);
1523
+ await sourceLocator.dragTo(targetLocator, { timeout: this.defaultTimeout });
1524
+ return {
1525
+ success: true,
1526
+ url: page.url(),
1527
+ hint: "Drag complete. Take a snapshot to see the result."
1528
+ };
1529
+ } catch (error) {
1530
+ return this.createErrorFromException(error, "Drag");
1531
+ }
1532
+ }
1533
+ async evaluate(input, threadId) {
1534
+ try {
1535
+ return {
1536
+ success: true,
1537
+ result: await (await this.getPage(threadId)).evaluate(input.script),
1538
+ hint: "JavaScript executed. Take a snapshot if the page may have changed."
1539
+ };
1540
+ } catch (error) {
1541
+ return this.createErrorFromException(error, "Evaluate");
1542
+ }
1543
+ }
1544
+ async closeBrowser() {
1545
+ try {
1546
+ await this.close();
1547
+ return {
1548
+ success: true,
1549
+ hint: "Browser closed. Call browser_goto to start a new session."
1550
+ };
1551
+ } catch (error) {
1552
+ return this.createErrorFromException(error, "Close");
1553
+ }
1554
+ }
1555
+ async startScreencast(_options) {
1556
+ const requestedThreadId = _options?.threadId;
1557
+ const effectiveThreadId = this.getScope() === "thread" ? requestedThreadId ?? this.getCurrentThread() ?? DEFAULT_THREAD_ID : requestedThreadId;
1558
+ let browserManager;
1559
+ if (this.getScope() === "thread") browserManager = await this.getManagerForThread(effectiveThreadId);
1560
+ else {
1561
+ if (!this.sharedManager) throw new Error("Browser not launched");
1562
+ browserManager = this.sharedManager;
1563
+ }
1564
+ const stream = new ScreencastStreamImpl({
1565
+ getCdpSession: async () => {
1566
+ const currentPage = browserManager.getPage();
1567
+ if (!currentPage) throw new Error("No active page available");
1568
+ return await currentPage.context().newCDPSession(currentPage);
1569
+ },
1570
+ isBrowserRunning: () => browserManager.isLaunched()
1571
+ }, _options);
1572
+ const streamKey = this.getStreamKey(effectiveThreadId);
1573
+ this.activeScreencastStreams.set(streamKey, stream);
1574
+ const context = browserManager.getContext();
1575
+ if (context) {
1576
+ const onNewPage = (_newPage) => {
1577
+ setTimeout(() => {
1578
+ if (stream.isActive()) stream.reconnect().catch(() => {});
1579
+ }, 100);
1580
+ };
1581
+ context.on("page", onNewPage);
1582
+ const pageCloseHandlers = /* @__PURE__ */ new Map();
1583
+ const frameNavigatedHandlers = /* @__PURE__ */ new Map();
1584
+ const setupPageListeners = (page) => {
1585
+ const onFrameNavigated = (frame) => {
1586
+ if (!frame.parentFrame()) {
1587
+ stream.emitUrl(frame.url());
1588
+ this.updateSessionBrowserState(effectiveThreadId);
1589
+ }
1590
+ };
1591
+ page.on("framenavigated", onFrameNavigated);
1592
+ frameNavigatedHandlers.set(page, onFrameNavigated);
1593
+ const onClose = () => {
1594
+ pageCloseHandlers.delete(page);
1595
+ const navHandler = frameNavigatedHandlers.get(page);
1596
+ if (navHandler) {
1597
+ page.off("framenavigated", navHandler);
1598
+ frameNavigatedHandlers.delete(page);
1599
+ }
1600
+ setTimeout(() => {
1601
+ const remainingPages = browserManager.getPages();
1602
+ if (stream.isActive() && remainingPages.length > 0) {
1603
+ stream.reconnect().catch(() => {});
1604
+ const activePage = remainingPages[browserManager.getActiveIndex()] || remainingPages[0];
1605
+ if (activePage) {
1606
+ const url = activePage.url();
1607
+ if (url && url !== "about:blank") stream.emitUrl(url);
1608
+ }
1609
+ }
1610
+ }, 100);
1611
+ };
1612
+ page.once("close", onClose);
1613
+ pageCloseHandlers.set(page, onClose);
1614
+ };
1615
+ const setupPageCloseListener = setupPageListeners;
1616
+ for (const page of browserManager.getPages()) setupPageCloseListener(page);
1617
+ const onNewPageWithCloseListener = (newPage) => {
1618
+ setupPageCloseListener(newPage);
1619
+ const url = newPage.url();
1620
+ if (url && url !== "about:blank") stream.emitUrl(url);
1621
+ onNewPage(newPage);
1622
+ };
1623
+ context.off("page", onNewPage);
1624
+ context.on("page", onNewPageWithCloseListener);
1625
+ stream.once("stop", () => {
1626
+ context.off("page", onNewPageWithCloseListener);
1627
+ for (const [page, handler] of pageCloseHandlers) page.off("close", handler);
1628
+ pageCloseHandlers.clear();
1629
+ for (const [page, handler] of frameNavigatedHandlers) page.off("framenavigated", handler);
1630
+ frameNavigatedHandlers.clear();
1631
+ this.activeScreencastStreams.delete(streamKey);
1632
+ });
1633
+ }
1634
+ await stream.start();
1635
+ return stream;
1636
+ }
1637
+ async injectMouseEvent(event, threadId) {
1638
+ const effectiveThreadId = threadId ?? this.getCurrentThread();
1639
+ await (await this.getManagerForThread(effectiveThreadId)).injectMouseEvent(event);
1640
+ }
1641
+ async injectKeyboardEvent(event, threadId) {
1642
+ const effectiveThreadId = threadId ?? this.getCurrentThread();
1643
+ await (await (await this.getManagerForThread(effectiveThreadId)).getCDPSession()).send("Input.dispatchKeyEvent", {
1644
+ type: event.type,
1645
+ key: event.key,
1646
+ code: event.code,
1647
+ text: event.text,
1648
+ modifiers: event.modifiers ?? 0,
1649
+ windowsVirtualKeyCode: event.windowsVirtualKeyCode
1650
+ });
1651
+ }
1775
1652
  };
1776
-
1653
+ //#endregion
1777
1654
  export { AgentBrowser, AgentBrowserThreadManager, BROWSER_TOOLS, backInputSchema, browserSchemas, clickInputSchema, closeInputSchema, createAgentBrowserTools, dialogInputSchema, dragInputSchema, evaluateInputSchema, getBrowserPid, gotoInputSchema, hoverInputSchema, pressInputSchema, scrollInputSchema, selectInputSchema, snapshotInputSchema, tabsInputSchema, typeInputSchema, waitInputSchema };
1778
- //# sourceMappingURL=index.js.map
1655
+
1779
1656
  //# sourceMappingURL=index.js.map