@brightdata/brightdata-plugin 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1575 @@
1
+ import { Type } from "@sinclair/typebox";
2
+ import {
3
+ ToolInputError,
4
+ readNumberParam,
5
+ readStringParam,
6
+ } from "openclaw/plugin-sdk/agent-runtime";
7
+ import type {
8
+ OpenClawPluginApi,
9
+ OpenClawPluginToolContext,
10
+ } from "openclaw/plugin-sdk/plugin-runtime";
11
+ import {
12
+ readResponseText,
13
+ withTrustedWebToolsEndpoint,
14
+ } from "openclaw/plugin-sdk/provider-web-search";
15
+ import { wrapExternalContent } from "openclaw/plugin-sdk/security-runtime";
16
+ import { ensureBrightDataBrowserZoneExists } from "./brightdata-client.js";
17
+ import {
18
+ type BrightDataPluginConfig,
19
+ DEFAULT_BRIGHTDATA_BASE_URL,
20
+ resolveBrightDataApiToken,
21
+ resolveBrightDataBaseUrl,
22
+ resolveBrightDataBrowserZone,
23
+ resolveBrightDataBrowserTimeoutSeconds,
24
+ } from "./config.js";
25
+
26
+ function stringEnum<const T extends readonly string[]>(
27
+ values: T,
28
+ options: { description?: string } = {},
29
+ ) {
30
+ return Type.Unsafe<T[number]>({
31
+ type: "string",
32
+ enum: [...values],
33
+ ...options,
34
+ });
35
+ }
36
+
37
+ const DEFAULT_NAVIGATION_TIMEOUT_MS = 120_000;
38
+ const DEFAULT_WAIT_FOR_TIMEOUT_MS = 30_000;
39
+ const PAGE_METADATA_RETRY_DELAY_MS = 250;
40
+ const PAGE_METADATA_RETRY_ATTEMPTS = 3;
41
+ const PAGE_METADATA_WAIT_TIMEOUT_MS = 5_000;
42
+ const BROWSER_SESSION_IDLE_TTL_MS = 10 * 60_000;
43
+ const BROWSER_SESSION_SWEEP_INTERVAL_MS = 60_000;
44
+
45
+ type PlaywrightModuleLike = {
46
+ chromium: {
47
+ connectOverCDP(endpoint: string): Promise<BrowserLike>;
48
+ };
49
+ };
50
+
51
+ type BrowserLike = {
52
+ contexts(): BrowserContextLike[];
53
+ newContext(): Promise<BrowserContextLike>;
54
+ close(): Promise<void>;
55
+ on?(event: "disconnected", handler: () => void): void;
56
+ };
57
+
58
+ type BrowserContextLike = {
59
+ pages(): PageLike[];
60
+ newPage(): Promise<PageLike>;
61
+ };
62
+
63
+ type BrowserRequestLike = {
64
+ method(): string;
65
+ url(): string;
66
+ };
67
+
68
+ type BrowserResponseLike = {
69
+ status(): number;
70
+ statusText(): string;
71
+ request(): BrowserRequestLike;
72
+ };
73
+
74
+ type LocatorLike = {
75
+ click(opts?: { timeout?: number }): Promise<void>;
76
+ fill(value: string): Promise<void>;
77
+ press(key: string): Promise<void>;
78
+ scrollIntoViewIfNeeded(): Promise<void>;
79
+ waitFor(opts?: { timeout?: number }): Promise<void>;
80
+ setChecked(value: boolean): Promise<void>;
81
+ selectOption(value: { label: string }): Promise<void>;
82
+ first?(): LocatorLike;
83
+ };
84
+
85
+ type SnapshotForAIPage = PageLike & {
86
+ _snapshotForAI?: (options?: { timeout?: number; track?: string }) => Promise<{ full?: string }>;
87
+ };
88
+
89
+ type PageLike = {
90
+ goto(
91
+ url: string,
92
+ opts?: { timeout?: number; waitUntil?: "domcontentloaded" | "load" | "networkidle" },
93
+ ): Promise<unknown>;
94
+ title(): Promise<string>;
95
+ url(): string;
96
+ goBack(): Promise<unknown>;
97
+ goForward(): Promise<unknown>;
98
+ locator(selector: string): LocatorLike;
99
+ screenshot(opts?: { fullPage?: boolean }): Promise<Buffer>;
100
+ content(): Promise<string>;
101
+ $eval(selector: string, fn: (element: Element) => unknown): Promise<unknown>;
102
+ evaluate<T>(fn: () => T | Promise<T>): Promise<T>;
103
+ waitForLoadState?(
104
+ state?: "domcontentloaded" | "load" | "networkidle",
105
+ opts?: { timeout?: number },
106
+ ): Promise<unknown>;
107
+ waitForTimeout?(timeout: number): Promise<unknown>;
108
+ on?(event: "request", handler: (request: BrowserRequestLike) => void): void;
109
+ on?(event: "response", handler: (response: BrowserResponseLike) => void): void;
110
+ on?(event: "close", handler: () => void): void;
111
+ };
112
+
113
+ type BrowserFieldType = "textbox" | "checkbox" | "radio" | "combobox" | "slider";
114
+
115
+ type BrowserFormField = {
116
+ name: string;
117
+ ref: string;
118
+ type: BrowserFieldType;
119
+ value: string;
120
+ };
121
+
122
+ type BrowserDomElement = {
123
+ ref: string;
124
+ role?: string;
125
+ name?: string;
126
+ url?: string;
127
+ };
128
+
129
+ const INTERACTIVE_ARIA_ROLES = new Set([
130
+ "button",
131
+ "link",
132
+ "textbox",
133
+ "searchbox",
134
+ "combobox",
135
+ "checkbox",
136
+ "radio",
137
+ "switch",
138
+ "slider",
139
+ "tab",
140
+ "menuitem",
141
+ "option",
142
+ ]);
143
+
144
+ const BrowserNavigateSchema = Type.Object(
145
+ {
146
+ url: Type.String({ description: "The URL to navigate to." }),
147
+ country: Type.Optional(
148
+ Type.String({
149
+ description:
150
+ 'Optional 2-letter ISO country code to route the browser session, for example "US" or "GB".',
151
+ minLength: 2,
152
+ maxLength: 2,
153
+ }),
154
+ ),
155
+ },
156
+ { additionalProperties: false },
157
+ );
158
+
159
+ const BrowserSnapshotSchema = Type.Object(
160
+ {
161
+ filtered: Type.Optional(
162
+ Type.Boolean({
163
+ description:
164
+ "Whether to apply filtering/compaction. Set to true to get a compacted interactive-element snapshot.",
165
+ }),
166
+ ),
167
+ },
168
+ { additionalProperties: false },
169
+ );
170
+
171
+ const BrowserRefActionSchema = Type.Object(
172
+ {
173
+ ref: Type.String({
174
+ description: 'The ref attribute from the page snapshot, for example "23".',
175
+ }),
176
+ element: Type.String({ description: "Description of the element for context." }),
177
+ },
178
+ { additionalProperties: false },
179
+ );
180
+
181
+ const BrowserTypeSchema = Type.Object(
182
+ {
183
+ ref: Type.String({
184
+ description: 'The ref attribute from the page snapshot, for example "23".',
185
+ }),
186
+ element: Type.String({ description: "Description of the element being typed into." }),
187
+ text: Type.String({ description: "Text to type." }),
188
+ submit: Type.Optional(
189
+ Type.Boolean({
190
+ description: "Whether to submit the form after typing by pressing Enter.",
191
+ }),
192
+ ),
193
+ },
194
+ { additionalProperties: false },
195
+ );
196
+
197
+ const BrowserScreenshotSchema = Type.Object(
198
+ {
199
+ full_page: Type.Optional(
200
+ Type.Boolean({
201
+ description:
202
+ "Whether to capture the full page. Avoid this unless the extra height is needed.",
203
+ }),
204
+ ),
205
+ },
206
+ { additionalProperties: false },
207
+ );
208
+
209
+ const BrowserGetHtmlSchema = Type.Object(
210
+ {
211
+ full_page: Type.Optional(
212
+ Type.Boolean({
213
+ description:
214
+ "Whether to return the full page HTML including head and script tags. Default returns only body HTML.",
215
+ }),
216
+ ),
217
+ },
218
+ { additionalProperties: false },
219
+ );
220
+
221
+ const BrowserWaitForSchema = Type.Object(
222
+ {
223
+ ref: Type.String({
224
+ description: 'The ref attribute from the page snapshot, for example "23".',
225
+ }),
226
+ element: Type.String({ description: "Description of the element being waited for." }),
227
+ timeout: Type.Optional(
228
+ Type.Number({
229
+ description: "Maximum time to wait in milliseconds.",
230
+ minimum: 1,
231
+ }),
232
+ ),
233
+ },
234
+ { additionalProperties: false },
235
+ );
236
+
237
+ const BrowserFieldSchema = Type.Object(
238
+ {
239
+ name: Type.String({ description: "Human-readable field name." }),
240
+ ref: Type.String({ description: "Exact target field reference from the page snapshot." }),
241
+ type: stringEnum(["textbox", "checkbox", "radio", "combobox", "slider"] as const, {
242
+ description: "Type of the field.",
243
+ }),
244
+ value: Type.String({
245
+ description:
246
+ 'Value to fill in the field. For checkbox use "true" or "false". For combobox use the visible option label.',
247
+ }),
248
+ },
249
+ { additionalProperties: false },
250
+ );
251
+
252
+ const BrowserFillFormSchema = Type.Object(
253
+ {
254
+ fields: Type.Array(BrowserFieldSchema, {
255
+ description: "Fields to fill in the form.",
256
+ minItems: 1,
257
+ }),
258
+ },
259
+ { additionalProperties: false },
260
+ );
261
+
262
+ type BrowserExternalContentKind = "html" | "snapshot" | "text";
263
+
264
+ function textResult(text: string, details?: Record<string, unknown>) {
265
+ return {
266
+ content: [{ type: "text" as const, text }],
267
+ ...(details ? { details } : {}),
268
+ };
269
+ }
270
+
271
+ function browserExternalTextResult(params: {
272
+ kind: BrowserExternalContentKind;
273
+ text: string;
274
+ details?: Record<string, unknown>;
275
+ }) {
276
+ const wrappedText = wrapExternalContent(params.text, {
277
+ source: "browser",
278
+ includeWarning: true,
279
+ });
280
+ return textResult(wrappedText, {
281
+ ok: true,
282
+ ...params.details,
283
+ externalContent: {
284
+ untrusted: true,
285
+ source: "browser",
286
+ kind: params.kind,
287
+ wrapped: true,
288
+ },
289
+ });
290
+ }
291
+
292
+ function imageResult(text: string, data: Buffer, details?: Record<string, unknown>) {
293
+ return {
294
+ content: [
295
+ { type: "text" as const, text },
296
+ {
297
+ type: "image" as const,
298
+ data: data.toString("base64"),
299
+ mimeType: "image/png",
300
+ },
301
+ ],
302
+ details: {
303
+ mimeType: "image/png",
304
+ ...details,
305
+ },
306
+ };
307
+ }
308
+
309
+ function toSnakeCaseKey(key: string): string {
310
+ return key
311
+ .replace(/([A-Z]+)([A-Z][a-z])/g, "$1_$2")
312
+ .replace(/([a-z0-9])([A-Z])/g, "$1_$2")
313
+ .toLowerCase();
314
+ }
315
+
316
+ function readRawParam(params: Record<string, unknown>, key: string): unknown {
317
+ if (Object.hasOwn(params, key)) {
318
+ return params[key];
319
+ }
320
+ const snakeKey = toSnakeCaseKey(key);
321
+ if (snakeKey !== key && Object.hasOwn(params, snakeKey)) {
322
+ return params[snakeKey];
323
+ }
324
+ return undefined;
325
+ }
326
+
327
+ function readBooleanParam(params: Record<string, unknown>, key: string): boolean | undefined {
328
+ const raw = readRawParam(params, key);
329
+ if (typeof raw === "boolean") {
330
+ return raw;
331
+ }
332
+ if (typeof raw === "string") {
333
+ const normalized = raw.trim().toLowerCase();
334
+ if (normalized === "true") {
335
+ return true;
336
+ }
337
+ if (normalized === "false") {
338
+ return false;
339
+ }
340
+ }
341
+ return undefined;
342
+ }
343
+
344
+ function readArrayParam(
345
+ params: Record<string, unknown>,
346
+ key: string,
347
+ options: { required?: boolean; label?: string } = {},
348
+ ): unknown[] | undefined {
349
+ const raw = readRawParam(params, key);
350
+ if (Array.isArray(raw)) {
351
+ return raw;
352
+ }
353
+ if (options.required) {
354
+ throw new ToolInputError(`${options.label ?? key} required`);
355
+ }
356
+ return undefined;
357
+ }
358
+
359
+ function readFormFields(rawParams: Record<string, unknown>): BrowserFormField[] {
360
+ const rawFields = readArrayParam(rawParams, "fields", { required: true });
361
+ if (!rawFields || rawFields.length === 0) {
362
+ throw new ToolInputError("fields required");
363
+ }
364
+ return rawFields.map((field, index) => {
365
+ if (!field || typeof field !== "object" || Array.isArray(field)) {
366
+ throw new ToolInputError(`fields[${index}] must be an object`);
367
+ }
368
+ const params = field as Record<string, unknown>;
369
+ const name = readStringParam(params, "name", {
370
+ required: true,
371
+ label: `fields[${index}].name`,
372
+ });
373
+ const ref = readStringParam(params, "ref", {
374
+ required: true,
375
+ label: `fields[${index}].ref`,
376
+ });
377
+ const type = readStringParam(params, "type", {
378
+ required: true,
379
+ label: `fields[${index}].type`,
380
+ });
381
+ if (
382
+ type !== "textbox" &&
383
+ type !== "checkbox" &&
384
+ type !== "radio" &&
385
+ type !== "combobox" &&
386
+ type !== "slider"
387
+ ) {
388
+ throw new ToolInputError(`fields[${index}].type invalid`);
389
+ }
390
+ const value = readStringParam(params, "value", {
391
+ required: true,
392
+ label: `fields[${index}].value`,
393
+ allowEmpty: true,
394
+ });
395
+ return { name, ref, type, value };
396
+ });
397
+ }
398
+
399
+ function normalizeCountry(country: string | undefined): string | undefined {
400
+ const trimmed = country?.trim();
401
+ if (!trimmed) {
402
+ return undefined;
403
+ }
404
+ if (!/^[A-Za-z]{2}$/.test(trimmed)) {
405
+ throw new ToolInputError("country must be a 2-letter ISO country code");
406
+ }
407
+ return trimmed.toLowerCase();
408
+ }
409
+
410
+ function resolveEndpoint(baseUrl: string, pathname: string): string {
411
+ const trimmed = baseUrl.trim();
412
+ try {
413
+ const url = new URL(trimmed || DEFAULT_BRIGHTDATA_BASE_URL);
414
+ url.pathname = pathname;
415
+ url.search = "";
416
+ url.hash = "";
417
+ return url.toString();
418
+ } catch {
419
+ return new URL(pathname, DEFAULT_BRIGHTDATA_BASE_URL).toString();
420
+ }
421
+ }
422
+
423
+ function appendQueryParams(
424
+ urlRaw: string,
425
+ params?: Record<string, string | number | boolean | undefined>,
426
+ ): string {
427
+ const url = new URL(urlRaw);
428
+ for (const [key, value] of Object.entries(params ?? {})) {
429
+ if (value === undefined) {
430
+ continue;
431
+ }
432
+ url.searchParams.set(key, String(value));
433
+ }
434
+ return url.toString();
435
+ }
436
+
437
+ async function requestBrowserApiJson(params: {
438
+ pluginConfig?: Record<string, unknown> | BrightDataPluginConfig;
439
+ pathname: string;
440
+ queryParams?: Record<string, string | number | boolean | undefined>;
441
+ errorLabel: string;
442
+ timeoutSeconds: number;
443
+ }): Promise<unknown> {
444
+ const apiToken = resolveBrightDataApiToken(params.pluginConfig);
445
+ if (!apiToken) {
446
+ throw new Error(
447
+ "Bright Data browser tools need a Bright Data API token. Set BRIGHTDATA_API_TOKEN in the Gateway environment, or configure plugins.entries.brightdata.config.webSearch.apiKey.",
448
+ );
449
+ }
450
+ const baseUrl = resolveBrightDataBaseUrl(params.pluginConfig);
451
+ const endpoint = appendQueryParams(resolveEndpoint(baseUrl, params.pathname), params.queryParams);
452
+ return await withTrustedWebToolsEndpoint(
453
+ {
454
+ url: endpoint,
455
+ timeoutSeconds: params.timeoutSeconds,
456
+ init: {
457
+ method: "GET",
458
+ headers: {
459
+ Authorization: `Bearer ${apiToken}`,
460
+ Accept: "application/json",
461
+ },
462
+ },
463
+ },
464
+ async ({ response }) => {
465
+ const textResult = await readResponseText(response, { maxBytes: 64_000 });
466
+ const text = typeof textResult === "string" ? textResult : textResult.text;
467
+ let payload: unknown = null;
468
+ if (text) {
469
+ try {
470
+ payload = JSON.parse(text);
471
+ } catch {
472
+ if (!response.ok) {
473
+ throw new Error(`${params.errorLabel} failed (${response.status}): ${text}`);
474
+ }
475
+ throw new Error(`${params.errorLabel} returned invalid JSON.`);
476
+ }
477
+ }
478
+ if (!response.ok) {
479
+ const detail =
480
+ payload && typeof payload === "object" && !Array.isArray(payload)
481
+ ? ((payload as Record<string, unknown>).error ??
482
+ (payload as Record<string, unknown>).message ??
483
+ text)
484
+ : text;
485
+ const detailText =
486
+ typeof detail === "string" && detail.trim()
487
+ ? detail
488
+ : (() => {
489
+ try {
490
+ const serialized = JSON.stringify(detail);
491
+ return serialized && serialized !== "null" ? serialized : text;
492
+ } catch {
493
+ return text;
494
+ }
495
+ })();
496
+ throw new Error(
497
+ `${params.errorLabel} failed (${response.status}${response.statusText ? ` ${response.statusText}` : ""}): ${detailText || "request failed"}`,
498
+ );
499
+ }
500
+ return payload;
501
+ },
502
+ );
503
+ }
504
+
505
+ function buildBrightDataBrowserCdpEndpoint(params: {
506
+ customer: string;
507
+ zone: string;
508
+ password: string;
509
+ country?: string;
510
+ }): string {
511
+ const countrySuffix = params.country ? `-country-${params.country}` : "";
512
+ return `wss://brd-customer-${params.customer}-zone-${params.zone}${countrySuffix}:${params.password}@brd.superproxy.io:9222`;
513
+ }
514
+
515
+ async function resolveBrightDataBrowserCdpEndpoint(params: {
516
+ pluginConfig?: Record<string, unknown> | BrightDataPluginConfig;
517
+ country?: string;
518
+ }): Promise<string> {
519
+ const country = normalizeCountry(params.country);
520
+ const zone = resolveBrightDataBrowserZone(params.pluginConfig);
521
+ const timeoutSeconds = resolveBrightDataBrowserTimeoutSeconds(params.pluginConfig);
522
+
523
+ const statusPayload = await requestBrowserApiJson({
524
+ pluginConfig: params.pluginConfig,
525
+ pathname: "/status",
526
+ errorLabel: "Bright Data status",
527
+ timeoutSeconds,
528
+ });
529
+ const customer =
530
+ statusPayload &&
531
+ typeof statusPayload === "object" &&
532
+ !Array.isArray(statusPayload) &&
533
+ typeof (statusPayload as Record<string, unknown>).customer === "string"
534
+ ? ((statusPayload as Record<string, unknown>).customer as string)
535
+ : statusPayload &&
536
+ typeof statusPayload === "object" &&
537
+ !Array.isArray(statusPayload) &&
538
+ typeof (statusPayload as Record<string, unknown>).customer === "number"
539
+ ? String((statusPayload as Record<string, unknown>).customer)
540
+ : "";
541
+ if (!customer) {
542
+ throw new Error("Bright Data status returned no customer identifier.");
543
+ }
544
+
545
+ await ensureBrightDataBrowserZoneExists(params.pluginConfig, timeoutSeconds);
546
+
547
+ const passwordsPayload = await requestBrowserApiJson({
548
+ pluginConfig: params.pluginConfig,
549
+ pathname: "/zone/passwords",
550
+ queryParams: { zone },
551
+ errorLabel: `Bright Data browser zone password (${zone})`,
552
+ timeoutSeconds,
553
+ });
554
+ const passwords =
555
+ passwordsPayload &&
556
+ typeof passwordsPayload === "object" &&
557
+ !Array.isArray(passwordsPayload) &&
558
+ Array.isArray((passwordsPayload as Record<string, unknown>).passwords)
559
+ ? ((passwordsPayload as Record<string, unknown>).passwords as unknown[])
560
+ : [];
561
+ const password = passwords.find((entry) => typeof entry === "string" && entry.trim()) as
562
+ | string
563
+ | undefined;
564
+ if (!password) {
565
+ throw new Error(`Bright Data browser zone "${zone}" returned no passwords.`);
566
+ }
567
+
568
+ return buildBrightDataBrowserCdpEndpoint({
569
+ customer,
570
+ zone,
571
+ password,
572
+ ...(country ? { country } : {}),
573
+ });
574
+ }
575
+
576
+ function isModuleNotFoundError(error: unknown): boolean {
577
+ const message = error instanceof Error ? error.message : String(error);
578
+ return (
579
+ message.includes("Cannot find module") ||
580
+ message.includes("Cannot find package") ||
581
+ message.includes("ERR_MODULE_NOT_FOUND") ||
582
+ message.includes("Failed to resolve import")
583
+ );
584
+ }
585
+
586
+ let playwrightPromise: Promise<PlaywrightModuleLike | null> | null = null;
587
+
588
+ async function getPlaywright(): Promise<PlaywrightModuleLike | null> {
589
+ if (!playwrightPromise) {
590
+ playwrightPromise = import("playwright")
591
+ .then((mod) => mod as unknown as PlaywrightModuleLike)
592
+ .catch((error) => {
593
+ if (isModuleNotFoundError(error)) {
594
+ return null;
595
+ }
596
+ throw error;
597
+ });
598
+ }
599
+ return await playwrightPromise;
600
+ }
601
+
602
+ async function requirePlaywright(): Promise<PlaywrightModuleLike> {
603
+ const playwright = await getPlaywright();
604
+ if (playwright) {
605
+ return playwright;
606
+ }
607
+ throw new Error(
608
+ "Playwright is not installed for the Bright Data extension. Add playwright to extensions/brightdata/package.json dependencies and reinstall the extension.",
609
+ );
610
+ }
611
+
612
+ function filterAriaSnapshot(snapshotText: string): string {
613
+ try {
614
+ const lines = snapshotText.split("\n");
615
+ const elements: BrowserDomElement[] = [];
616
+ for (const [index, line] of lines.entries()) {
617
+ const trimmed = line.trim();
618
+ if (!trimmed || !trimmed.startsWith("-")) {
619
+ continue;
620
+ }
621
+ const refMatch = trimmed.match(/\[ref=([^\]]+)\]/);
622
+ if (!refMatch) {
623
+ continue;
624
+ }
625
+ const roleMatch = trimmed.match(/^-\s+([a-zA-Z]+)/);
626
+ if (!roleMatch) {
627
+ continue;
628
+ }
629
+ const role = roleMatch[1] ?? "";
630
+ if (!INTERACTIVE_ARIA_ROLES.has(role)) {
631
+ continue;
632
+ }
633
+ const nameMatch = trimmed.match(/"([^"]*)"/);
634
+ const urlMatch = lines[index + 1]?.match(/\/url:\s*(.+)/);
635
+ elements.push({
636
+ ref: refMatch[1] ?? "",
637
+ role,
638
+ name: nameMatch?.[1] ?? "",
639
+ url: urlMatch?.[1]?.trim().replace(/^["']|["']$/g, "") ?? "",
640
+ });
641
+ }
642
+ if (elements.length === 0) {
643
+ return "No interactive elements found";
644
+ }
645
+ return formatDomElements(elements) ?? "No interactive elements found";
646
+ } catch (error) {
647
+ return `Error filtering snapshot: ${error instanceof Error ? error.message : String(error)}`;
648
+ }
649
+ }
650
+
651
+ function formatDomElements(elements: BrowserDomElement[]): string | null {
652
+ if (elements.length === 0) {
653
+ return null;
654
+ }
655
+ return elements
656
+ .map((entry) => {
657
+ const parts = [`[${entry.ref}]`, entry.role?.trim() || "unknown"];
658
+ const rawName = entry.name?.trim() || "";
659
+ if (rawName) {
660
+ const name = rawName.length > 60 ? `${rawName.slice(0, 57)}...` : rawName;
661
+ parts.push(`"${name}"`);
662
+ }
663
+ const rawUrl = entry.url?.trim() || "";
664
+ if (rawUrl && !rawUrl.startsWith("#")) {
665
+ const url = rawUrl.length > 50 ? `${rawUrl.slice(0, 47)}...` : rawUrl;
666
+ parts.push(`-> ${url}`);
667
+ }
668
+ return parts.join(" ");
669
+ })
670
+ .join("\n");
671
+ }
672
+
673
+ function escapeAttributeValue(value: string): string {
674
+ return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
675
+ }
676
+
677
+ function sleep(ms: number): Promise<void> {
678
+ return new Promise((resolve) => setTimeout(resolve, ms));
679
+ }
680
+
681
+ function isExecutionContextDestroyedError(error: unknown): boolean {
682
+ const message = error instanceof Error ? error.message : String(error);
683
+ return (
684
+ message.includes("Execution context was destroyed") ||
685
+ message.includes("Cannot find context with specified id")
686
+ );
687
+ }
688
+
689
+ async function waitForPageLoadState(
690
+ page: PageLike,
691
+ state: "domcontentloaded" | "load" | "networkidle",
692
+ timeout = PAGE_METADATA_WAIT_TIMEOUT_MS,
693
+ ): Promise<void> {
694
+ await page.waitForLoadState?.(state, { timeout }).catch(() => {});
695
+ }
696
+
697
+ async function waitForPageTimeout(page: PageLike, ms: number): Promise<void> {
698
+ if (typeof page.waitForTimeout === "function") {
699
+ await page.waitForTimeout(ms);
700
+ return;
701
+ }
702
+ await sleep(ms);
703
+ }
704
+
705
+ async function readPageMetadata(page: PageLike): Promise<{ title: string; url: string }> {
706
+ for (let attempt = 0; attempt < PAGE_METADATA_RETRY_ATTEMPTS; attempt += 1) {
707
+ try {
708
+ await waitForPageLoadState(page, "domcontentloaded");
709
+ return {
710
+ title: await page.title(),
711
+ url: page.url(),
712
+ };
713
+ } catch (error) {
714
+ if (!isExecutionContextDestroyedError(error) || attempt >= PAGE_METADATA_RETRY_ATTEMPTS - 1) {
715
+ throw error;
716
+ }
717
+ await waitForPageLoadState(page, "load");
718
+ await waitForPageTimeout(page, PAGE_METADATA_RETRY_DELAY_MS);
719
+ }
720
+ }
721
+ throw new Error("Failed to read page metadata.");
722
+ }
723
+
724
+ class BrightDataBrowserSession {
725
+ private readonly cdpEndpoint: string;
726
+ private browser: BrowserLike | null = null;
727
+ private page: PageLike | null = null;
728
+ private requests = new Map<BrowserRequestLike, BrowserResponseLike | null>();
729
+ private domRefs = new Set<string>();
730
+
731
+ constructor(cdpEndpoint: string) {
732
+ this.cdpEndpoint = cdpEndpoint;
733
+ }
734
+
735
+ async getPage(): Promise<PageLike> {
736
+ if (this.page) {
737
+ return this.page;
738
+ }
739
+ const browser = await this.getBrowser();
740
+ const existingContext = browser.contexts()[0];
741
+ const context = existingContext ?? (await browser.newContext());
742
+ const existingPage = context.pages()[0];
743
+ const page = existingPage ?? (await context.newPage());
744
+
745
+ page.on?.("request", (request) => {
746
+ this.requests.set(request, null);
747
+ });
748
+ page.on?.("response", (response) => {
749
+ this.requests.set(response.request(), response);
750
+ });
751
+ page.on?.("close", () => {
752
+ if (this.page === page) {
753
+ this.page = null;
754
+ }
755
+ });
756
+ this.page = page;
757
+ return page;
758
+ }
759
+
760
+ async getBrowser(): Promise<BrowserLike> {
761
+ if (this.browser) {
762
+ try {
763
+ void this.browser.contexts();
764
+ return this.browser;
765
+ } catch {
766
+ this.browser = null;
767
+ this.page = null;
768
+ }
769
+ }
770
+ const playwright = await requirePlaywright();
771
+ const browser = await playwright.chromium.connectOverCDP(this.cdpEndpoint);
772
+ browser.on?.("disconnected", () => {
773
+ if (this.browser === browser) {
774
+ this.browser = null;
775
+ this.page = null;
776
+ }
777
+ });
778
+ this.browser = browser;
779
+ return browser;
780
+ }
781
+
782
+ async captureSnapshot(filtered: boolean): Promise<{
783
+ url: string;
784
+ title: string;
785
+ ariaSnapshot: string;
786
+ domSnapshot?: string;
787
+ }> {
788
+ const page = await this.getPage();
789
+ const snapshotPage = page as SnapshotForAIPage;
790
+ if (!snapshotPage._snapshotForAI) {
791
+ throw new Error("Playwright _snapshotForAI is not available.");
792
+ }
793
+ const snapshot = await snapshotPage._snapshotForAI({
794
+ timeout: 5_000,
795
+ track: "response",
796
+ });
797
+ const fullSnapshot = String(snapshot?.full ?? "");
798
+ if (!filtered) {
799
+ this.domRefs.clear();
800
+ const metadata = await readPageMetadata(page);
801
+ return {
802
+ url: metadata.url,
803
+ title: metadata.title,
804
+ ariaSnapshot: fullSnapshot,
805
+ };
806
+ }
807
+ const domElements = await page.evaluate<BrowserDomElement[]>(() => {
808
+ const selectors = [
809
+ "a[href]",
810
+ "button",
811
+ "input",
812
+ "select",
813
+ "textarea",
814
+ "option",
815
+ ".radio-item",
816
+ "[role]",
817
+ "[tabindex]",
818
+ "[onclick]",
819
+ "[data-spm-click]",
820
+ "[data-click]",
821
+ "[data-action]",
822
+ "[data-spm-anchor-id]",
823
+ "[aria-pressed]",
824
+ "[aria-label]",
825
+ "[aria-haspopup]",
826
+ ];
827
+ const nodes = Array.from(document.querySelectorAll(selectors.join(",")));
828
+ const elements: BrowserDomElement[] = [];
829
+ let counter = 0;
830
+
831
+ const collapse = (text: string | null | undefined) =>
832
+ (text || "").replace(/\s+/g, " ").trim();
833
+
834
+ const readElementText = (element: Element | null) =>
835
+ collapse((element instanceof HTMLElement ? element.innerText : element?.textContent) || "");
836
+
837
+ const getLabelledBy = (element: Element) => {
838
+ const ids = (element.getAttribute("aria-labelledby") || "").split(/\s+/);
839
+ return ids
840
+ .map((id) => {
841
+ const ref = document.getElementById(id);
842
+ return readElementText(ref);
843
+ })
844
+ .filter(Boolean)
845
+ .join(" ");
846
+ };
847
+
848
+ const getLabelFor = (element: Element) => {
849
+ const id = element.id?.trim();
850
+ if (!id) {
851
+ return "";
852
+ }
853
+ const label = document.querySelector(`label[for="${CSS.escape(id)}"]`);
854
+ return readElementText(label);
855
+ };
856
+
857
+ const isIntrinsic = (element: Element) => {
858
+ const tag = element.tagName.toLowerCase();
859
+ if (["a", "input", "button", "select", "textarea", "option"].includes(tag)) {
860
+ return true;
861
+ }
862
+ const role = (element.getAttribute("role") || "").toLowerCase();
863
+ if (["button", "link", "radio", "option", "tab", "checkbox", "menuitem"].includes(role)) {
864
+ return true;
865
+ }
866
+ if (element.classList.contains("radio-item")) {
867
+ return true;
868
+ }
869
+ return (
870
+ element.hasAttribute("onclick") ||
871
+ element.hasAttribute("data-click") ||
872
+ element.hasAttribute("data-action") ||
873
+ element.hasAttribute("data-spm-click") ||
874
+ element.hasAttribute("data-spm-anchor-id")
875
+ );
876
+ };
877
+
878
+ const isClickable = (element: Element) => {
879
+ const style = window.getComputedStyle(element);
880
+ if (
881
+ style.display === "none" ||
882
+ style.visibility === "hidden" ||
883
+ style.pointerEvents === "none"
884
+ ) {
885
+ return false;
886
+ }
887
+ const rect = element.getBoundingClientRect();
888
+ if (!rect || rect.width === 0 || rect.height === 0) {
889
+ return false;
890
+ }
891
+ const centerX = rect.left + rect.width / 2;
892
+ const centerY = rect.top + rect.height / 2;
893
+ if (
894
+ centerX < 0 ||
895
+ centerX > window.innerWidth ||
896
+ centerY < 0 ||
897
+ centerY > window.innerHeight
898
+ ) {
899
+ return false;
900
+ }
901
+ const topElement = document.elementFromPoint(centerX, centerY);
902
+ if (
903
+ topElement &&
904
+ (topElement === element ||
905
+ topElement.contains(element) ||
906
+ (element instanceof HTMLElement && element.contains(topElement)))
907
+ ) {
908
+ return true;
909
+ }
910
+ return isIntrinsic(element);
911
+ };
912
+
913
+ for (const element of nodes) {
914
+ if (!isClickable(element)) {
915
+ continue;
916
+ }
917
+
918
+ let name =
919
+ collapse(element.getAttribute("aria-label")) ||
920
+ collapse(getLabelledBy(element)) ||
921
+ collapse(element.getAttribute("title")) ||
922
+ collapse(element.getAttribute("alt")) ||
923
+ collapse(element.getAttribute("placeholder")) ||
924
+ collapse(getLabelFor(element));
925
+
926
+ if (!name) {
927
+ name = collapse((element as HTMLElement).innerText || element.textContent);
928
+ }
929
+ if (name.length > 80) {
930
+ name = `${name.slice(0, 77)}...`;
931
+ }
932
+
933
+ const url = (
934
+ (element as HTMLAnchorElement).href ||
935
+ element.getAttribute("data-url") ||
936
+ ""
937
+ ).toString();
938
+ if (!name && !url) {
939
+ continue;
940
+ }
941
+ const htmlElement = element as HTMLElement & { dataset: DOMStringMap };
942
+ if (!htmlElement.dataset.fastmcpRef) {
943
+ htmlElement.dataset.fastmcpRef = `dom-${++counter}`;
944
+ }
945
+ elements.push({
946
+ ref: htmlElement.dataset.fastmcpRef,
947
+ role: element.getAttribute("role") || element.tagName.toLowerCase(),
948
+ name,
949
+ url,
950
+ });
951
+ }
952
+ return elements;
953
+ });
954
+
955
+ this.domRefs = new Set(domElements.map((entry) => entry.ref));
956
+ const domSnapshot = formatDomElements(domElements);
957
+ const metadata = await readPageMetadata(page);
958
+ return {
959
+ url: metadata.url,
960
+ title: metadata.title,
961
+ ariaSnapshot: filterAriaSnapshot(fullSnapshot),
962
+ ...(domSnapshot ? { domSnapshot } : {}),
963
+ };
964
+ }
965
+
966
+ async refLocator(params: { element: string; ref: string }): Promise<LocatorLike> {
967
+ const page = await this.getPage();
968
+ if (this.domRefs.has(params.ref)) {
969
+ const locator = page.locator(`[data-fastmcp-ref="${escapeAttributeValue(params.ref)}"]`);
970
+ return typeof locator.first === "function" ? locator.first() : locator;
971
+ }
972
+
973
+ const snapshotPage = page as SnapshotForAIPage;
974
+ if (!snapshotPage._snapshotForAI) {
975
+ throw new Error("Playwright _snapshotForAI is not available.");
976
+ }
977
+ const snapshot = await snapshotPage._snapshotForAI({
978
+ timeout: 5_000,
979
+ track: "response",
980
+ });
981
+ const fullSnapshot = String(snapshot?.full ?? "");
982
+ if (!fullSnapshot.includes(`[ref=${params.ref}]`)) {
983
+ throw new Error(
984
+ `Ref ${params.ref} not found in the current page snapshot. Capture a new snapshot first.`,
985
+ );
986
+ }
987
+ return page.locator(`aria-ref=${params.ref}`);
988
+ }
989
+
990
+ getRequests() {
991
+ return this.requests;
992
+ }
993
+
994
+ clearRequests() {
995
+ this.requests.clear();
996
+ }
997
+
998
+ clearSnapshotState() {
999
+ this.domRefs.clear();
1000
+ }
1001
+
1002
+ async close() {
1003
+ const browser = this.browser;
1004
+ this.browser = null;
1005
+ this.page = null;
1006
+ this.requests.clear();
1007
+ this.domRefs.clear();
1008
+ if (browser) {
1009
+ await browser.close().catch(() => {});
1010
+ }
1011
+ }
1012
+ }
1013
+
1014
+ type ScopedBrowserSession = {
1015
+ country: string | null;
1016
+ session: BrightDataBrowserSession;
1017
+ lastUsedAtMs: number;
1018
+ activeUseCount: number;
1019
+ };
1020
+
1021
+ const browserSessionsByScope = new Map<string, ScopedBrowserSession>();
1022
+ let browserSessionSweeper: ReturnType<typeof setInterval> | null = null;
1023
+
1024
+ type BrowserSessionParams = {
1025
+ pluginConfig?: Record<string, unknown> | BrightDataPluginConfig;
1026
+ country?: string;
1027
+ context?: OpenClawPluginToolContext;
1028
+ createSession?: (cdpEndpoint: string) => BrightDataBrowserSession;
1029
+ resolveCdpEndpoint?: (params: {
1030
+ pluginConfig?: Record<string, unknown> | BrightDataPluginConfig;
1031
+ country?: string;
1032
+ }) => Promise<string>;
1033
+ };
1034
+
1035
+ function resolveBrowserSessionScopeKey(context?: OpenClawPluginToolContext): string {
1036
+ const sessionId = context?.sessionId?.trim();
1037
+ if (sessionId) {
1038
+ return `session-id:${sessionId}`;
1039
+ }
1040
+ const sessionKey = context?.sessionKey?.trim();
1041
+ if (sessionKey) {
1042
+ return `session-key:${sessionKey}`;
1043
+ }
1044
+ const agentId = context?.agentId?.trim();
1045
+ if (agentId) {
1046
+ return `agent:${agentId}`;
1047
+ }
1048
+ return "global";
1049
+ }
1050
+
1051
+ function touchBrowserSession(entry: ScopedBrowserSession, nowMs = Date.now()): void {
1052
+ entry.lastUsedAtMs = nowMs;
1053
+ }
1054
+
1055
+ function stopBrowserSessionSweeper(): void {
1056
+ if (!browserSessionSweeper) {
1057
+ return;
1058
+ }
1059
+ clearInterval(browserSessionSweeper);
1060
+ browserSessionSweeper = null;
1061
+ }
1062
+
1063
+ function ensureBrowserSessionSweeper(): void {
1064
+ if (browserSessionSweeper || browserSessionsByScope.size === 0) {
1065
+ return;
1066
+ }
1067
+ const sweeper = setInterval(() => {
1068
+ void pruneIdleBrowserSessions();
1069
+ }, BROWSER_SESSION_SWEEP_INTERVAL_MS);
1070
+ const unref = (sweeper as { unref?: () => void }).unref;
1071
+ if (typeof unref === "function") {
1072
+ unref.call(sweeper);
1073
+ }
1074
+ browserSessionSweeper = sweeper;
1075
+ }
1076
+
1077
+ async function closeTrackedBrowserSession(entry: ScopedBrowserSession): Promise<void> {
1078
+ await entry.session.close().catch(() => {});
1079
+ }
1080
+
1081
+ async function pruneIdleBrowserSessions(nowMs = Date.now()): Promise<void> {
1082
+ const staleEntries = Array.from(browserSessionsByScope.entries()).filter(
1083
+ ([, entry]) =>
1084
+ entry.activeUseCount === 0 && nowMs - entry.lastUsedAtMs >= BROWSER_SESSION_IDLE_TTL_MS,
1085
+ );
1086
+ if (staleEntries.length === 0) {
1087
+ if (browserSessionsByScope.size === 0) {
1088
+ stopBrowserSessionSweeper();
1089
+ }
1090
+ return;
1091
+ }
1092
+
1093
+ for (const [scopeKey] of staleEntries) {
1094
+ browserSessionsByScope.delete(scopeKey);
1095
+ }
1096
+
1097
+ if (browserSessionsByScope.size === 0) {
1098
+ stopBrowserSessionSweeper();
1099
+ }
1100
+
1101
+ await Promise.all(staleEntries.map(([, entry]) => closeTrackedBrowserSession(entry)));
1102
+ }
1103
+
1104
+ async function resetBrowserSessions(): Promise<void> {
1105
+ const entries = Array.from(browserSessionsByScope.values());
1106
+ browserSessionsByScope.clear();
1107
+ stopBrowserSessionSweeper();
1108
+ await Promise.all(entries.map((entry) => closeTrackedBrowserSession(entry)));
1109
+ }
1110
+
1111
+ async function requireBrowserSessionEntry(
1112
+ params: BrowserSessionParams,
1113
+ ): Promise<{ scopeKey: string; entry: ScopedBrowserSession }> {
1114
+ const scopeKey = resolveBrowserSessionScopeKey(params.context);
1115
+ const existing = browserSessionsByScope.get(scopeKey);
1116
+ const normalizedCountry =
1117
+ params.country !== undefined
1118
+ ? normalizeCountry(params.country)
1119
+ : (existing?.country ?? undefined);
1120
+ const resolvedCountry = normalizedCountry ?? null;
1121
+ const needsNewSession = !existing || resolvedCountry !== existing.country;
1122
+ if (needsNewSession) {
1123
+ if (existing) {
1124
+ browserSessionsByScope.delete(scopeKey);
1125
+ await closeTrackedBrowserSession(existing);
1126
+ }
1127
+ const resolveCdpEndpoint = params.resolveCdpEndpoint ?? resolveBrightDataBrowserCdpEndpoint;
1128
+ const createSession =
1129
+ params.createSession ?? ((cdpEndpoint: string) => new BrightDataBrowserSession(cdpEndpoint));
1130
+ const entry: ScopedBrowserSession = {
1131
+ country: resolvedCountry,
1132
+ session: createSession(
1133
+ await resolveCdpEndpoint({
1134
+ pluginConfig: params.pluginConfig,
1135
+ ...(resolvedCountry ? { country: resolvedCountry } : {}),
1136
+ }),
1137
+ ),
1138
+ lastUsedAtMs: Date.now(),
1139
+ activeUseCount: 0,
1140
+ };
1141
+ browserSessionsByScope.set(scopeKey, entry);
1142
+ ensureBrowserSessionSweeper();
1143
+ return { scopeKey, entry };
1144
+ }
1145
+ touchBrowserSession(existing);
1146
+ ensureBrowserSessionSweeper();
1147
+ return { scopeKey, entry: existing };
1148
+ }
1149
+
1150
+ async function requireBrowserSession(
1151
+ params: BrowserSessionParams,
1152
+ ): Promise<BrightDataBrowserSession> {
1153
+ return (await requireBrowserSessionEntry(params)).entry.session;
1154
+ }
1155
+
1156
+ async function withBrowserSession<T>(
1157
+ params: BrowserSessionParams,
1158
+ run: (session: BrightDataBrowserSession) => Promise<T>,
1159
+ ): Promise<T> {
1160
+ const { entry } = await requireBrowserSessionEntry(params);
1161
+ entry.activeUseCount += 1;
1162
+ touchBrowserSession(entry);
1163
+ try {
1164
+ return await run(entry.session);
1165
+ } finally {
1166
+ entry.activeUseCount = Math.max(0, entry.activeUseCount - 1);
1167
+ touchBrowserSession(entry);
1168
+ }
1169
+ }
1170
+
1171
+ export const BRIGHTDATA_BROWSER_TOOL_NAMES = [
1172
+ "brightdata_browser_navigate",
1173
+ "brightdata_browser_go_back",
1174
+ "brightdata_browser_go_forward",
1175
+ "brightdata_browser_snapshot",
1176
+ "brightdata_browser_click",
1177
+ "brightdata_browser_type",
1178
+ "brightdata_browser_screenshot",
1179
+ "brightdata_browser_get_html",
1180
+ "brightdata_browser_get_text",
1181
+ "brightdata_browser_scroll",
1182
+ "brightdata_browser_scroll_to",
1183
+ "brightdata_browser_wait_for",
1184
+ "brightdata_browser_network_requests",
1185
+ "brightdata_browser_fill_form",
1186
+ ] as const;
1187
+
1188
+ export function createBrightDataBrowserTools(
1189
+ api: OpenClawPluginApi,
1190
+ context?: OpenClawPluginToolContext,
1191
+ ) {
1192
+ const withToolBrowserSession = <T>(
1193
+ run: (session: BrightDataBrowserSession) => Promise<T>,
1194
+ options?: { country?: string },
1195
+ ) =>
1196
+ withBrowserSession(
1197
+ {
1198
+ pluginConfig: api.pluginConfig,
1199
+ context,
1200
+ ...(options?.country ? { country: options.country } : {}),
1201
+ },
1202
+ run,
1203
+ );
1204
+
1205
+ return [
1206
+ {
1207
+ name: "brightdata_browser_navigate",
1208
+ label: "Bright Data Browser Navigate",
1209
+ description: "Navigate a Bright Data scraping browser session to a new URL.",
1210
+ parameters: BrowserNavigateSchema,
1211
+ execute: async (_toolCallId: string, rawParams: Record<string, unknown>) => {
1212
+ const url = readStringParam(rawParams, "url", { required: true });
1213
+ const country = readStringParam(rawParams, "country");
1214
+ return await withToolBrowserSession(
1215
+ async (session) => {
1216
+ const page = await session.getPage();
1217
+ session.clearRequests();
1218
+ session.clearSnapshotState();
1219
+ await page.goto(url, {
1220
+ timeout: DEFAULT_NAVIGATION_TIMEOUT_MS,
1221
+ waitUntil: "domcontentloaded",
1222
+ });
1223
+ const metadata = await readPageMetadata(page);
1224
+ return textResult(
1225
+ [
1226
+ `Successfully navigated to ${url}`,
1227
+ `Title: ${metadata.title}`,
1228
+ `URL: ${metadata.url}`,
1229
+ ].join("\n"),
1230
+ {
1231
+ url: metadata.url,
1232
+ title: metadata.title,
1233
+ ...(country ? { country: normalizeCountry(country) } : {}),
1234
+ },
1235
+ );
1236
+ },
1237
+ country ? { country } : undefined,
1238
+ );
1239
+ },
1240
+ },
1241
+ {
1242
+ name: "brightdata_browser_go_back",
1243
+ label: "Bright Data Browser Go Back",
1244
+ description: "Go back to the previous page in the Bright Data browser session.",
1245
+ parameters: Type.Object({}, { additionalProperties: false }),
1246
+ execute: async () =>
1247
+ await withToolBrowserSession(async (session) => {
1248
+ const page = await session.getPage();
1249
+ session.clearRequests();
1250
+ session.clearSnapshotState();
1251
+ await page.goBack();
1252
+ const metadata = await readPageMetadata(page);
1253
+ return textResult(
1254
+ [
1255
+ "Successfully navigated back",
1256
+ `Title: ${metadata.title}`,
1257
+ `URL: ${metadata.url}`,
1258
+ ].join("\n"),
1259
+ { url: metadata.url, title: metadata.title },
1260
+ );
1261
+ }),
1262
+ },
1263
+ {
1264
+ name: "brightdata_browser_go_forward",
1265
+ label: "Bright Data Browser Go Forward",
1266
+ description: "Go forward to the next page in the Bright Data browser session.",
1267
+ parameters: Type.Object({}, { additionalProperties: false }),
1268
+ execute: async () =>
1269
+ await withToolBrowserSession(async (session) => {
1270
+ const page = await session.getPage();
1271
+ session.clearRequests();
1272
+ session.clearSnapshotState();
1273
+ await page.goForward();
1274
+ const metadata = await readPageMetadata(page);
1275
+ return textResult(
1276
+ [
1277
+ "Successfully navigated forward",
1278
+ `Title: ${metadata.title}`,
1279
+ `URL: ${metadata.url}`,
1280
+ ].join("\n"),
1281
+ { url: metadata.url, title: metadata.title },
1282
+ );
1283
+ }),
1284
+ },
1285
+ {
1286
+ name: "brightdata_browser_snapshot",
1287
+ label: "Bright Data Browser Snapshot",
1288
+ description:
1289
+ "Capture an ARIA snapshot of the current page showing interactive elements and refs.",
1290
+ parameters: BrowserSnapshotSchema,
1291
+ execute: async (_toolCallId: string, rawParams: Record<string, unknown>) => {
1292
+ const filtered = readBooleanParam(rawParams, "filtered") ?? false;
1293
+ return await withToolBrowserSession(async (session) => {
1294
+ const snapshot = await session.captureSnapshot(filtered);
1295
+ const lines = [
1296
+ `Page: ${snapshot.url}`,
1297
+ `Title: ${snapshot.title}`,
1298
+ "",
1299
+ "Interactive Elements:",
1300
+ snapshot.ariaSnapshot,
1301
+ ];
1302
+ if (snapshot.domSnapshot) {
1303
+ lines.push("", "DOM Interactive Elements:", snapshot.domSnapshot);
1304
+ }
1305
+ return browserExternalTextResult({
1306
+ kind: "snapshot",
1307
+ text: lines.join("\n"),
1308
+ details: {
1309
+ url: snapshot.url,
1310
+ title: snapshot.title,
1311
+ filtered,
1312
+ },
1313
+ });
1314
+ });
1315
+ },
1316
+ },
1317
+ {
1318
+ name: "brightdata_browser_click",
1319
+ label: "Bright Data Browser Click",
1320
+ description: "Click an element using its ref from the Bright Data browser snapshot.",
1321
+ parameters: BrowserRefActionSchema,
1322
+ execute: async (_toolCallId: string, rawParams: Record<string, unknown>) => {
1323
+ const ref = readStringParam(rawParams, "ref", { required: true });
1324
+ const element = readStringParam(rawParams, "element", { required: true });
1325
+ return await withToolBrowserSession(async (session) => {
1326
+ const locator = await session.refLocator({
1327
+ ref,
1328
+ element,
1329
+ });
1330
+ await locator.click({ timeout: 5_000 });
1331
+ return textResult(`Successfully clicked element: ${element} (ref=${ref})`, {
1332
+ ref,
1333
+ element,
1334
+ });
1335
+ });
1336
+ },
1337
+ },
1338
+ {
1339
+ name: "brightdata_browser_type",
1340
+ label: "Bright Data Browser Type",
1341
+ description: "Type text into an element using its ref from the Bright Data browser snapshot.",
1342
+ parameters: BrowserTypeSchema,
1343
+ execute: async (_toolCallId: string, rawParams: Record<string, unknown>) => {
1344
+ const ref = readStringParam(rawParams, "ref", { required: true });
1345
+ const element = readStringParam(rawParams, "element", { required: true });
1346
+ const text = readStringParam(rawParams, "text", {
1347
+ required: true,
1348
+ allowEmpty: true,
1349
+ });
1350
+ const submit = readBooleanParam(rawParams, "submit") ?? false;
1351
+ return await withToolBrowserSession(async (session) => {
1352
+ const locator = await session.refLocator({
1353
+ ref,
1354
+ element,
1355
+ });
1356
+ await locator.fill(text);
1357
+ if (submit) {
1358
+ await locator.press("Enter");
1359
+ }
1360
+ return textResult(
1361
+ `Successfully typed "${text}" into element: ${element} (ref=${ref})${submit ? " and submitted the form" : ""}`,
1362
+ { ref, element, text, submit },
1363
+ );
1364
+ });
1365
+ },
1366
+ },
1367
+ {
1368
+ name: "brightdata_browser_screenshot",
1369
+ label: "Bright Data Browser Screenshot",
1370
+ description: "Take a screenshot of the current page in the Bright Data browser session.",
1371
+ parameters: BrowserScreenshotSchema,
1372
+ execute: async (_toolCallId: string, rawParams: Record<string, unknown>) => {
1373
+ const fullPage = readBooleanParam(rawParams, "full_page") ?? false;
1374
+ return await withToolBrowserSession(async (session) => {
1375
+ const page = await session.getPage();
1376
+ const buffer = await page.screenshot({ fullPage });
1377
+ return imageResult(
1378
+ `Browser screenshot (${fullPage ? "full page" : "viewport"})`,
1379
+ buffer,
1380
+ {
1381
+ fullPage,
1382
+ url: page.url(),
1383
+ },
1384
+ );
1385
+ });
1386
+ },
1387
+ },
1388
+ {
1389
+ name: "brightdata_browser_get_html",
1390
+ label: "Bright Data Browser Get HTML",
1391
+ description: "Get the HTML content of the current page.",
1392
+ parameters: BrowserGetHtmlSchema,
1393
+ execute: async (_toolCallId: string, rawParams: Record<string, unknown>) => {
1394
+ const fullPage = readBooleanParam(rawParams, "full_page") ?? false;
1395
+ return await withToolBrowserSession(async (session) => {
1396
+ const page = await session.getPage();
1397
+ const html = fullPage
1398
+ ? await page.content()
1399
+ : String((await page.$eval("body", (body) => (body as HTMLElement).innerHTML)) ?? "");
1400
+ return browserExternalTextResult({
1401
+ kind: "html",
1402
+ text: html,
1403
+ details: {
1404
+ url: page.url(),
1405
+ fullPage,
1406
+ },
1407
+ });
1408
+ });
1409
+ },
1410
+ },
1411
+ {
1412
+ name: "brightdata_browser_get_text",
1413
+ label: "Bright Data Browser Get Text",
1414
+ description: "Get the text content of the current page.",
1415
+ parameters: Type.Object({}, { additionalProperties: false }),
1416
+ execute: async () =>
1417
+ await withToolBrowserSession(async (session) => {
1418
+ const page = await session.getPage();
1419
+ const text = String(
1420
+ (await page.$eval("body", (body) => (body as HTMLElement).innerText)) ?? "",
1421
+ );
1422
+ return browserExternalTextResult({
1423
+ kind: "text",
1424
+ text,
1425
+ details: { url: page.url() },
1426
+ });
1427
+ }),
1428
+ },
1429
+ {
1430
+ name: "brightdata_browser_scroll",
1431
+ label: "Bright Data Browser Scroll",
1432
+ description: "Scroll to the bottom of the current page.",
1433
+ parameters: Type.Object({}, { additionalProperties: false }),
1434
+ execute: async () =>
1435
+ await withToolBrowserSession(async (session) => {
1436
+ const page = await session.getPage();
1437
+ await page.evaluate(() => {
1438
+ window.scrollTo(0, document.body.scrollHeight);
1439
+ });
1440
+ return textResult("Successfully scrolled to the bottom of the page", {
1441
+ url: page.url(),
1442
+ });
1443
+ }),
1444
+ },
1445
+ {
1446
+ name: "brightdata_browser_scroll_to",
1447
+ label: "Bright Data Browser Scroll To",
1448
+ description:
1449
+ "Scroll to a specific element using its ref from the Bright Data browser snapshot.",
1450
+ parameters: BrowserRefActionSchema,
1451
+ execute: async (_toolCallId: string, rawParams: Record<string, unknown>) => {
1452
+ const ref = readStringParam(rawParams, "ref", { required: true });
1453
+ const element = readStringParam(rawParams, "element", { required: true });
1454
+ return await withToolBrowserSession(async (session) => {
1455
+ const locator = await session.refLocator({
1456
+ ref,
1457
+ element,
1458
+ });
1459
+ await locator.scrollIntoViewIfNeeded();
1460
+ return textResult(`Successfully scrolled to element: ${element} (ref=${ref})`, {
1461
+ ref,
1462
+ element,
1463
+ });
1464
+ });
1465
+ },
1466
+ },
1467
+ {
1468
+ name: "brightdata_browser_wait_for",
1469
+ label: "Bright Data Browser Wait For",
1470
+ description:
1471
+ "Wait for an element to be visible using its ref from the Bright Data browser snapshot.",
1472
+ parameters: BrowserWaitForSchema,
1473
+ execute: async (_toolCallId: string, rawParams: Record<string, unknown>) => {
1474
+ const ref = readStringParam(rawParams, "ref", { required: true });
1475
+ const element = readStringParam(rawParams, "element", { required: true });
1476
+ const timeout =
1477
+ readNumberParam(rawParams, "timeout", {
1478
+ integer: true,
1479
+ }) ?? DEFAULT_WAIT_FOR_TIMEOUT_MS;
1480
+ return await withToolBrowserSession(async (session) => {
1481
+ const locator = await session.refLocator({
1482
+ ref,
1483
+ element,
1484
+ });
1485
+ await locator.waitFor({ timeout });
1486
+ return textResult(`Successfully waited for element: ${element} (ref=${ref})`, {
1487
+ ref,
1488
+ element,
1489
+ timeout,
1490
+ });
1491
+ });
1492
+ },
1493
+ },
1494
+ {
1495
+ name: "brightdata_browser_network_requests",
1496
+ label: "Bright Data Browser Network Requests",
1497
+ description: "Get network requests recorded since the current page was loaded.",
1498
+ parameters: Type.Object({}, { additionalProperties: false }),
1499
+ execute: async () =>
1500
+ await withToolBrowserSession(async (session) => {
1501
+ const requests = Array.from(session.getRequests().entries()).map(
1502
+ ([request, response]) => {
1503
+ const parts = [`[${request.method().toUpperCase()}] ${request.url()}`];
1504
+ if (response) {
1505
+ parts.push(`=> [${response.status()}] ${response.statusText()}`);
1506
+ }
1507
+ return parts.join(" ");
1508
+ },
1509
+ );
1510
+ if (requests.length === 0) {
1511
+ return textResult("No network requests recorded for the current page.", { count: 0 });
1512
+ }
1513
+ return textResult(
1514
+ [`Network Requests (${requests.length} total):`, "", ...requests].join("\n"),
1515
+ { count: requests.length },
1516
+ );
1517
+ }),
1518
+ },
1519
+ {
1520
+ name: "brightdata_browser_fill_form",
1521
+ label: "Bright Data Browser Fill Form",
1522
+ description: "Fill multiple form fields in one operation using refs from the page snapshot.",
1523
+ parameters: BrowserFillFormSchema,
1524
+ execute: async (_toolCallId: string, rawParams: Record<string, unknown>) => {
1525
+ const fields = readFormFields(rawParams);
1526
+ return await withToolBrowserSession(async (session) => {
1527
+ const results: string[] = [];
1528
+
1529
+ for (const field of fields) {
1530
+ const locator = await session.refLocator({
1531
+ element: field.name,
1532
+ ref: field.ref,
1533
+ });
1534
+ if (field.type === "textbox" || field.type === "slider") {
1535
+ await locator.fill(field.value);
1536
+ results.push(`Filled ${field.name} with "${field.value}"`);
1537
+ continue;
1538
+ }
1539
+ if (field.type === "checkbox" || field.type === "radio") {
1540
+ const checked = field.value.trim().toLowerCase() === "true";
1541
+ await locator.setChecked(checked);
1542
+ results.push(`Set ${field.name} to ${checked ? "checked" : "unchecked"}`);
1543
+ continue;
1544
+ }
1545
+ await locator.selectOption({ label: field.value });
1546
+ results.push(`Selected "${field.value}" in ${field.name}`);
1547
+ }
1548
+
1549
+ return textResult(`Successfully filled form:\n${results.join("\n")}`, {
1550
+ filled: fields.length,
1551
+ });
1552
+ });
1553
+ },
1554
+ },
1555
+ ];
1556
+ }
1557
+
1558
+ export const __testing = {
1559
+ BROWSER_SESSION_IDLE_TTL_MS,
1560
+ BROWSER_SESSION_SWEEP_INTERVAL_MS,
1561
+ BRIGHTDATA_BROWSER_TOOL_NAMES,
1562
+ browserExternalTextResult,
1563
+ buildBrightDataBrowserCdpEndpoint,
1564
+ filterAriaSnapshot,
1565
+ formatDomElements,
1566
+ getBrowserSessionCount() {
1567
+ return browserSessionsByScope.size;
1568
+ },
1569
+ pruneIdleBrowserSessions,
1570
+ readPageMetadata,
1571
+ requireBrowserSession,
1572
+ resolveBrowserSessionScopeKey,
1573
+ resolveBrightDataBrowserCdpEndpoint,
1574
+ resetBrowserSessions,
1575
+ };