@brightdata/brightdata-plugin 1.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Bright Data
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,234 @@
+# Bright Data Plugin for OpenClaw
+
+Web search, scraping, browser automation, and 50+ structured data tools — powered by [Bright Data](https://brightdata.com).
+
+## Install
+
+```bash
+openclaw plugins install @brightdata/brightdata-plugin
+```
+
+## Configure
+
+Set your Bright Data API token (get one at [brightdata.com](https://brightdata.com)):
+
+```bash
+# Option A: environment variable
+export BRIGHTDATA_API_TOKEN=your_token
+
+# Option B: OpenClaw config
+openclaw config set plugins.entries.brightdata.config.webSearch.apiKey your_token
+```
+
+The plugin auto-creates two proxy zones on first use: `mcp_unlocker` (Web Unlocker) and `mcp_browser` (Browser API). To use existing zones:
+
+```bash
+export BRIGHTDATA_UNLOCKER_ZONE=my_unlocker_zone
+export BRIGHTDATA_BROWSER_ZONE=my_browser_zone
+```
+
+## What's Included
+
+**66 tools** across five categories:
+
+| Category | Tools | Description |
+|----------|-------|-------------|
+| [Search](#search) | 2 | Web search via Google, Bing, Yandex with geo-targeting |
+| [Scrape](#scrape) | 1 | Page extraction as markdown, text, or HTML — handles bot protection |
+| [Batch](#batch) | 2 | Parallel search and scrape (up to 5 items) |
+| [Browser](#browser-automation) | 14 | Full browser automation via Playwright CDP |
+| [Web Data](#web-data) | 47 | Structured data from Amazon, LinkedIn, Instagram, TikTok, and more |
+
+---
+
+### Search
+
+The plugin registers as an OpenClaw **web search provider** — it appears in provider selection automatically.
+
+| Tool | Description |
+|------|-------------|
+| `brightdata_search` | Search Google, Bing, or Yandex with pagination, geo-targeting, and result count control |
+
+```
+Parameters: query (required), engine ("google"|"bing"|"yandex"), count (1-10),
+            cursor, geo_location (2-letter ISO), timeoutSeconds
+```
+
+### Scrape
+
+| Tool | Description |
+|------|-------------|
+| `brightdata_scrape` | Fetch and extract a page through Bright Data Web Unlocker, including bot-protected pages |
+
+```
+Parameters: url (required), extractMode ("markdown"|"text"|"html"), maxChars (≥100), timeoutSeconds
+```
+
+### Batch
+
+| Tool | Description |
+|------|-------------|
+| `brightdata_search_batch` | Run up to 5 search queries in parallel |
+| `brightdata_scrape_batch` | Scrape up to 5 URLs in parallel |
+
+### Browser Automation
+
+Full browser control via Bright Data's residential proxy network. Sessions are scoped per user context and idle-timeout after 10 minutes.
+
+| Tool | Description |
+|------|-------------|
+| `brightdata_browser_navigate` | Navigate to a URL (optional country routing) |
+| `brightdata_browser_go_back` | Go back |
+| `brightdata_browser_go_forward` | Go forward |
+| `brightdata_browser_snapshot` | Capture ARIA snapshot with interactive element refs |
+| `brightdata_browser_click` | Click an element by ref |
+| `brightdata_browser_type` | Type into an element by ref (optional submit) |
+| `brightdata_browser_screenshot` | Take a screenshot (viewport or full page) |
+| `brightdata_browser_get_html` | Get page HTML |
+| `brightdata_browser_get_text` | Get page text content |
+| `brightdata_browser_scroll` | Scroll to bottom |
+| `brightdata_browser_scroll_to` | Scroll to a specific element by ref |
+| `brightdata_browser_wait_for` | Wait for an element to be visible |
+| `brightdata_browser_network_requests` | List network requests since page load |
+| `brightdata_browser_fill_form` | Fill multiple form fields in one operation |
+
+### Web Data
+
+Structured data extraction from 47 platforms via Bright Data datasets. Each tool accepts a `url` or `keyword` input and returns structured JSON.
+
+<details>
+<summary>All 47 dataset tools</summary>
+
+**E-commerce**
+| Tool | Platform |
+|------|----------|
+| `brightdata_amazon_product` | Amazon product details |
+| `brightdata_amazon_product_reviews` | Amazon reviews |
+| `brightdata_amazon_product_search` | Amazon search results |
+| `brightdata_walmart_product` | Walmart product details |
+| `brightdata_walmart_seller` | Walmart seller info |
+| `brightdata_ebay_product` | eBay product details |
+| `brightdata_homedepot_products` | Home Depot products |
+| `brightdata_zara_products` | Zara products |
+| `brightdata_etsy_products` | Etsy products |
+| `brightdata_bestbuy_products` | Best Buy products |
+
+**Professional Networks**
+| Tool | Platform |
+|------|----------|
+| `brightdata_linkedin_person_profile` | LinkedIn person profile |
+| `brightdata_linkedin_company_profile` | LinkedIn company profile |
+| `brightdata_linkedin_job_listings` | LinkedIn jobs |
+| `brightdata_linkedin_posts` | LinkedIn posts |
+| `brightdata_linkedin_people_search` | LinkedIn people search |
+| `brightdata_crunchbase_company` | Crunchbase company data |
+| `brightdata_zoominfo_company_profile` | ZoomInfo company profile |
+
+**Social Media — Instagram**
+| Tool | Platform |
+|------|----------|
+| `brightdata_instagram_profiles` | Instagram profiles |
+| `brightdata_instagram_posts` | Instagram posts |
+| `brightdata_instagram_reels` | Instagram reels |
+| `brightdata_instagram_comments` | Instagram comments |
+
+**Social Media — Facebook**
+| Tool | Platform |
+|------|----------|
+| `brightdata_facebook_posts` | Facebook posts |
+| `brightdata_facebook_marketplace_listings` | Facebook Marketplace |
+| `brightdata_facebook_company_reviews` | Facebook company reviews |
+| `brightdata_facebook_events` | Facebook events |
+
+**Social Media — TikTok**
+| Tool | Platform |
+|------|----------|
+| `brightdata_tiktok_profiles` | TikTok profiles |
+| `brightdata_tiktok_posts` | TikTok posts |
+| `brightdata_tiktok_shop` | TikTok Shop |
+| `brightdata_tiktok_comments` | TikTok comments |
+
+**Social Media — X (Twitter)**
+| Tool | Platform |
+|------|----------|
+| `brightdata_x_posts` | X posts |
+| `brightdata_x_profile_posts` | X profile posts |
+
+**Social Media — YouTube & Reddit**
+| Tool | Platform |
+|------|----------|
+| `brightdata_youtube_profiles` | YouTube profiles |
+| `brightdata_youtube_videos` | YouTube videos |
+| `brightdata_youtube_comments` | YouTube comments |
+| `brightdata_reddit_posts` | Reddit posts |
+
+**Maps, Shopping & Apps**
+| Tool | Platform |
+|------|----------|
+| `brightdata_google_maps_reviews` | Google Maps reviews |
+| `brightdata_google_shopping` | Google Shopping |
+| `brightdata_google_play_store` | Google Play Store |
+| `brightdata_apple_app_store` | Apple App Store |
+
+**Finance, News & Code**
+| Tool | Platform |
+|------|----------|
+| `brightdata_reuter_news` | Reuters news |
+| `brightdata_yahoo_finance_business` | Yahoo Finance |
+| `brightdata_github_repository_file` | GitHub repository files |
+
+**Real Estate & Travel**
+| Tool | Platform |
+|------|----------|
+| `brightdata_zillow_properties_listing` | Zillow listings |
+| `brightdata_booking_hotel_listings` | Booking.com hotels |
+
+**AI Insights**
+| Tool | Platform |
+|------|----------|
+| `brightdata_chatgpt_ai_insights` | ChatGPT responses |
+| `brightdata_grok_ai_insights` | Grok responses |
+| `brightdata_perplexity_ai_insights` | Perplexity responses |
+
+</details>
+
+---
+
+## Configuration Reference
+
+| Setting | Env Var | Config Path | Default |
+|---------|---------|-------------|---------|
+| API Token | `BRIGHTDATA_API_TOKEN` | `...webSearch.apiKey` | *required* |
+| Base URL | `BRIGHTDATA_BASE_URL` | `...webSearch.baseUrl` | `https://api.brightdata.com` |
+| Unlocker Zone | `BRIGHTDATA_UNLOCKER_ZONE` | `...webSearch.unlockerZone` | `mcp_unlocker` |
+| Browser Zone | `BRIGHTDATA_BROWSER_ZONE` | `...webSearch.browserZone` | `mcp_browser` |
+| Timeout | — | `...webSearch.timeoutSeconds` | `30` (search) / `60` (scrape) |
+| Polling Timeout | — | `...webSearch.pollingTimeoutSeconds` | `600` |
+
+Config paths are prefixed with `plugins.entries.brightdata.config`.
+
+Environment variables take priority over config file values.
+
+## Plugin Management
+
+```bash
+# Install
+openclaw plugins install @brightdata/brightdata-plugin
+
+# Verify
+openclaw plugins inspect brightdata
+
+# Disable / re-enable
+openclaw plugins disable brightdata
+openclaw plugins enable brightdata
+
+# Update
+openclaw plugins update brightdata
+
+# Uninstall
+openclaw plugins uninstall brightdata
+```
+
+## License
+
+MIT

package/index.ts

@@ -0,0 +1,30 @@
+import { definePluginEntry, type AnyAgentTool } from "openclaw/plugin-sdk/plugin-entry";
+import { createBrightDataBatchTools } from "./src/brightdata-batch-tools.js";
+import {
+  BRIGHTDATA_BROWSER_TOOL_NAMES,
+  createBrightDataBrowserTools,
+} from "./src/brightdata-browser-tools.js";
+import { createBrightDataScrapeTool } from "./src/brightdata-scrape-tool.js";
+import { createBrightDataWebSearchProvider } from "./src/brightdata-search-provider.js";
+import { createBrightDataSearchTool } from "./src/brightdata-search-tool.js";
+import { createBrightDataWebDataTools } from "./src/brightdata-web-data-tools.js";
+
+export default definePluginEntry({
+  id: "brightdata",
+  name: "Bright Data Plugin",
+  description: "Standalone Bright Data plugin for search, direct scraping, structured data, and browser automation",
+  register(api) {
+    api.registerWebSearchProvider(createBrightDataWebSearchProvider());
+    api.registerTool(createBrightDataSearchTool(api) as AnyAgentTool);
+    api.registerTool(createBrightDataScrapeTool(api) as AnyAgentTool);
+    for (const tool of createBrightDataBatchTools(api)) {
+      api.registerTool(tool as AnyAgentTool);
+    }
+    api.registerTool((ctx) => createBrightDataBrowserTools(api, ctx) as AnyAgentTool[], {
+      names: [...BRIGHTDATA_BROWSER_TOOL_NAMES],
+    });
+    for (const tool of createBrightDataWebDataTools(api)) {
+      api.registerTool(tool as AnyAgentTool);
+    }
+  },
+});

package/openclaw.plugin.json

@@ -0,0 +1,63 @@
+{
+  "id": "brightdata",
+  "uiHints": {
+    "webSearch.apiKey": {
+      "label": "Bright Data API Token",
+      "help": "Bright Data API token for search and scraping (fallback: BRIGHTDATA_API_TOKEN env var).",
+      "sensitive": true,
+      "placeholder": "..."
+    },
+    "webSearch.baseUrl": {
+      "label": "Bright Data API Base URL",
+      "help": "Bright Data API base URL override."
+    },
+    "webSearch.unlockerZone": {
+      "label": "Bright Data Unlocker Zone",
+      "help": "Bright Data Web Unlocker zone name (default: mcp_unlocker)."
+    },
+    "webSearch.browserZone": {
+      "label": "Bright Data Browser Zone",
+      "help": "Bright Data Browser API zone name (default: mcp_browser)."
+    },
+    "webSearch.timeoutSeconds": {
+      "label": "Bright Data Timeout (sec)",
+      "help": "Default timeout in seconds for Bright Data search, scrape, and browser credential requests."
+    },
+    "webSearch.pollingTimeoutSeconds": {
+      "label": "Bright Data Polling Timeout (sec)",
+      "help": "Default polling timeout in seconds for dataset-backed Bright Data tools."
+    }
+  },
+  "configSchema": {
+    "type": "object",
+    "additionalProperties": false,
+    "properties": {
+      "webSearch": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "apiKey": {
+            "type": ["string", "object"]
+          },
+          "baseUrl": {
+            "type": "string"
+          },
+          "unlockerZone": {
+            "type": "string"
+          },
+          "browserZone": {
+            "type": "string"
+          },
+          "timeoutSeconds": {
+            "type": "integer",
+            "minimum": 1
+          },
+          "pollingTimeoutSeconds": {
+            "type": "integer",
+            "minimum": 1
+          }
+        }
+      }
+    }
+  }
+}

package/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "@brightdata/brightdata-plugin",
+  "version": "1.0.0",
+  "description": "OpenClaw Bright Data plugin",
+  "type": "module",
+  "main": "index.ts",
+  "openclaw": {
+    "extensions": [
+      "./index.ts"
+    ],
+    "bundle": {
+      "stageRuntimeDependencies": true
+    },
+    "compat": {
+      "pluginApi": "^1.0.0"
+    }
+  },
+  "keywords": [
+    "openclaw",
+    "openclaw-plugin",
+    "brightdata",
+    "web-scraping",
+    "browser-automation",
+    "web-search"
+  ],
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/brightdata/openclaw-plugin.git"
+  },
+  "dependencies": {
+    "@sinclair/typebox": "0.34.48",
+    "playwright": "1.58.2"
+  },
+  "devDependencies": {
+    "openclaw": "latest",
+    "typescript": "^5.8.0",
+    "vitest": "^3.0.0"
+  },
+  "scripts": {
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit"
+  }
+}

package/src/brightdata-batch-tools.ts

@@ -0,0 +1,274 @@
+import { Type } from "@sinclair/typebox";
+import {
+  ToolInputError,
+  jsonResult,
+  readNumberParam,
+  readStringArrayParam,
+  readStringParam,
+} from "openclaw/plugin-sdk/agent-runtime";
+import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-runtime";
+import { runBrightDataScrape, runBrightDataSearch } from "./brightdata-client.js";
+
+function optionalStringEnum<const T extends readonly string[]>(
+  values: T,
+  options: { description?: string } = {},
+) {
+  return Type.Optional(
+    Type.Unsafe<T[number]>({
+      type: "string",
+      enum: [...values],
+      ...options,
+    }),
+  );
+}
+
+const MAX_BATCH_ITEMS = 5;
+
+const BrightDataBatchSearchQuerySchema = Type.Object(
+  {
+    query: Type.String({ description: "Search query string." }),
+    engine: optionalStringEnum(["google", "bing", "yandex"] as const, {
+      description: 'Search engine ("google", "bing", or "yandex"). Default: google.',
+    }),
+    count: Type.Optional(
+      Type.Number({
+        description: "Number of results to return (1-10).",
+        minimum: 1,
+        maximum: 10,
+      }),
+    ),
+    cursor: Type.Optional(
+      Type.String({
+        description: "Pagination cursor for the next page.",
+      }),
+    ),
+    geo_location: Type.Optional(
+      Type.String({
+        description: '2-letter country code for geo-targeted results, for example "us" or "uk".',
+        minLength: 2,
+        maxLength: 2,
+      }),
+    ),
+  },
+  { additionalProperties: false },
+);
+
+const BrightDataSearchBatchToolSchema = Type.Object(
+  {
+    queries: Type.Array(BrightDataBatchSearchQuerySchema, {
+      description: "Array of search requests to run in parallel (1-5).",
+      minItems: 1,
+      maxItems: MAX_BATCH_ITEMS,
+    }),
+    timeoutSeconds: Type.Optional(
+      Type.Number({
+        description: "Timeout in seconds for each Bright Data search request.",
+        minimum: 1,
+      }),
+    ),
+  },
+  { additionalProperties: false },
+);
+
+const BrightDataScrapeBatchToolSchema = Type.Object(
+  {
+    urls: Type.Array(Type.String({ description: "HTTP or HTTPS URL to scrape via Bright Data." }), {
+      description: "Array of URLs to scrape in parallel (1-5).",
+      minItems: 1,
+      maxItems: MAX_BATCH_ITEMS,
+    }),
+    extractMode: optionalStringEnum(["markdown", "text", "html"] as const, {
+      description: 'Extraction mode ("markdown", "text", or "html"). Default: markdown.',
+    }),
+    maxChars: Type.Optional(
+      Type.Number({
+        description: "Maximum characters to return for each result.",
+        minimum: 100,
+      }),
+    ),
+    timeoutSeconds: Type.Optional(
+      Type.Number({
+        description: "Timeout in seconds for each Bright Data scrape request.",
+        minimum: 1,
+      }),
+    ),
+  },
+  { additionalProperties: false },
+);
+
+type SearchBatchQuery = {
+  query: string;
+  engine?: "google" | "bing" | "yandex";
+  count?: number;
+  cursor?: string;
+  geoLocation?: string;
+};
+
+function readSearchBatchQueries(rawParams: Record<string, unknown>): SearchBatchQuery[] {
+  const rawQueries = rawParams.queries;
+  if (!Array.isArray(rawQueries) || rawQueries.length === 0) {
+    throw new ToolInputError("queries required");
+  }
+  if (rawQueries.length > MAX_BATCH_ITEMS) {
+    throw new ToolInputError(`queries must contain at most ${MAX_BATCH_ITEMS} items`);
+  }
+  return rawQueries.map((entry, index) => {
+    if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
+      throw new ToolInputError(`queries[${index}] must be an object`);
+    }
+    const params = entry as Record<string, unknown>;
+    const query = readStringParam(params, "query", {
+      required: true,
+      label: `queries[${index}].query`,
+    });
+    const engineRaw = readStringParam(params, "engine", {
+      label: `queries[${index}].engine`,
+    });
+    const engine =
+      engineRaw === "google" || engineRaw === "bing" || engineRaw === "yandex"
+        ? engineRaw
+        : undefined;
+    const count = readNumberParam(params, "count", {
+      integer: true,
+      label: `queries[${index}].count`,
+    });
+    const cursor = readStringParam(params, "cursor", {
+      label: `queries[${index}].cursor`,
+    });
+    const geoLocation = readStringParam(params, "geo_location", {
+      label: `queries[${index}].geo_location`,
+    });
+
+    return {
+      query,
+      engine,
+      count,
+      cursor,
+      geoLocation,
+    };
+  });
+}
+
+function readScrapeBatchUrls(rawParams: Record<string, unknown>): string[] {
+  const urls = readStringArrayParam(rawParams, "urls", { required: true });
+  if (urls.length > MAX_BATCH_ITEMS) {
+    throw new ToolInputError(`urls must contain at most ${MAX_BATCH_ITEMS} items`);
+  }
+  return urls;
+}
+
+function readErrorMessage(error: unknown): string {
+  if (error instanceof Error && error.message) {
+    return error.message;
+  }
+  return String(error);
+}
+
+export function createBrightDataBatchTools(api: OpenClawPluginApi) {
+  return [
+    {
+      name: "brightdata_search_batch",
+      label: "Bright Data Search Batch",
+      description:
+        "Run up to 5 Bright Data search requests in parallel. Returns per-query results and preserves item-level failures.",
+      parameters: BrightDataSearchBatchToolSchema,
+      execute: async (_toolCallId: string, rawParams: Record<string, unknown>) => {
+        const queries = readSearchBatchQueries(rawParams);
+        const timeoutSeconds = readNumberParam(rawParams, "timeoutSeconds", {
+          integer: true,
+        });
+
+        const settled = await Promise.allSettled(
+          queries.map((query) =>
+            runBrightDataSearch({
+              pluginConfig: api.pluginConfig,
+              query: query.query,
+              engine: query.engine,
+              count: query.count,
+              cursor: query.cursor,
+              geoLocation: query.geoLocation,
+              timeoutSeconds,
+            }),
+          ),
+        );
+
+        const results = settled.map((entry, index) =>
+          entry.status === "fulfilled"
+            ? {
+                index,
+                query: queries[index]?.query ?? "",
+                engine: queries[index]?.engine ?? "google",
+                ok: true,
+                result: entry.value,
+              }
+            : {
+                index,
+                query: queries[index]?.query ?? "",
+                engine: queries[index]?.engine ?? "google",
+                ok: false,
+                error: readErrorMessage(entry.reason),
+              },
+        );
+
+        return jsonResult({
+          total: queries.length,
+          succeeded: results.filter((entry) => entry.ok).length,
+          failed: results.filter((entry) => !entry.ok).length,
+          results,
+        });
+      },
+    },
+    {
+      name: "brightdata_scrape_batch",
+      label: "Bright Data Scrape Batch",
+      description:
+        "Fetch and extract up to 5 URLs directly through Bright Data in parallel. Returns per-URL results and preserves item-level failures.",
+      parameters: BrightDataScrapeBatchToolSchema,
+      execute: async (_toolCallId: string, rawParams: Record<string, unknown>) => {
+        const urls = readScrapeBatchUrls(rawParams);
+        const extractModeRaw = readStringParam(rawParams, "extractMode");
+        const extractMode =
+          extractModeRaw === "text" || extractModeRaw === "html" ? extractModeRaw : "markdown";
+        const maxChars = readNumberParam(rawParams, "maxChars", { integer: true });
+        const timeoutSeconds = readNumberParam(rawParams, "timeoutSeconds", {
+          integer: true,
+        });
+
+        const settled = await Promise.allSettled(
+          urls.map((url) =>
+            runBrightDataScrape({
+              pluginConfig: api.pluginConfig,
+              url,
+              extractMode,
+              maxChars,
+              timeoutSeconds,
+            }),
+          ),
+        );
+
+        const results = settled.map((entry, index) =>
+          entry.status === "fulfilled"
+            ? {
+                index,
+                url: urls[index] ?? "",
+                ok: true,
+                result: entry.value,
+              }
+            : {
+                index,
+                url: urls[index] ?? "",
+                ok: false,
+                error: readErrorMessage(entry.reason),
+              },
+        );
+
+        return jsonResult({
+          total: urls.length,
+          succeeded: results.filter((entry) => entry.ok).length,
+          failed: results.filter((entry) => !entry.ok).length,
+          results,
+        });
+      },
+    },
+  ];
+}