@brightdata/brightdata-plugin 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +234 -0
- package/index.ts +30 -0
- package/openclaw.plugin.json +63 -0
- package/package.json +45 -0
- package/src/brightdata-batch-tools.ts +274 -0
- package/src/brightdata-browser-tools.ts +1575 -0
- package/src/brightdata-client.ts +907 -0
- package/src/brightdata-scrape-tool.ts +69 -0
- package/src/brightdata-search-provider.ts +76 -0
- package/src/brightdata-search-tool.ts +88 -0
- package/src/brightdata-web-data-tools.ts +501 -0
- package/src/brightdata-zone-bootstrap.ts +177 -0
- package/src/config.ts +155 -0
package/src/config.ts
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import { normalizeSecretInput } from "openclaw/plugin-sdk/provider-auth";
|
|
2
|
+
import { normalizeResolvedSecretInputString } from "openclaw/plugin-sdk/secret-input";
|
|
3
|
+
|
|
4
|
+
export const DEFAULT_BRIGHTDATA_BASE_URL = "https://api.brightdata.com";
|
|
5
|
+
export const DEFAULT_BRIGHTDATA_UNLOCKER_ZONE = "mcp_unlocker";
|
|
6
|
+
export const DEFAULT_BRIGHTDATA_BROWSER_ZONE = "mcp_browser";
|
|
7
|
+
export const DEFAULT_BRIGHTDATA_SEARCH_TIMEOUT_SECONDS = 30;
|
|
8
|
+
export const DEFAULT_BRIGHTDATA_SCRAPE_TIMEOUT_SECONDS = 60;
|
|
9
|
+
export const DEFAULT_BRIGHTDATA_POLLING_TIMEOUT_SECONDS = 600;
|
|
10
|
+
|
|
11
|
+
export type BrightDataPluginConfig = {
|
|
12
|
+
webSearch?: {
|
|
13
|
+
apiKey?: unknown;
|
|
14
|
+
baseUrl?: string;
|
|
15
|
+
unlockerZone?: string;
|
|
16
|
+
browserZone?: string;
|
|
17
|
+
timeoutSeconds?: number;
|
|
18
|
+
pollingTimeoutSeconds?: number;
|
|
19
|
+
};
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
type BrightDataWebSearchConfig = BrightDataPluginConfig["webSearch"];
|
|
23
|
+
|
|
24
|
+
function resolvePluginConfig(
|
|
25
|
+
pluginConfig?: Record<string, unknown> | BrightDataPluginConfig,
|
|
26
|
+
): BrightDataPluginConfig | undefined {
|
|
27
|
+
if (!pluginConfig || typeof pluginConfig !== "object" || Array.isArray(pluginConfig)) {
|
|
28
|
+
return undefined;
|
|
29
|
+
}
|
|
30
|
+
return pluginConfig as BrightDataPluginConfig;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function resolveBrightDataSearchConfig(
|
|
34
|
+
pluginConfig?: Record<string, unknown> | BrightDataPluginConfig,
|
|
35
|
+
): BrightDataWebSearchConfig {
|
|
36
|
+
const webSearch = resolvePluginConfig(pluginConfig)?.webSearch;
|
|
37
|
+
if (!webSearch || typeof webSearch !== "object" || Array.isArray(webSearch)) {
|
|
38
|
+
return undefined;
|
|
39
|
+
}
|
|
40
|
+
return webSearch;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function normalizeConfiguredSecret(value: unknown, path: string): string | undefined {
|
|
44
|
+
return normalizeSecretInput(
|
|
45
|
+
normalizeResolvedSecretInputString({
|
|
46
|
+
value,
|
|
47
|
+
path,
|
|
48
|
+
}),
|
|
49
|
+
);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function readConfiguredString(value: string | undefined): string {
|
|
53
|
+
return typeof value === "string" ? value.trim() : "";
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function resolveBrightDataApiToken(
|
|
57
|
+
pluginConfig?: Record<string, unknown> | BrightDataPluginConfig,
|
|
58
|
+
): string | undefined {
|
|
59
|
+
const search = resolveBrightDataSearchConfig(pluginConfig);
|
|
60
|
+
return (
|
|
61
|
+
normalizeConfiguredSecret(search?.apiKey, "plugins.entries.brightdata.config.webSearch.apiKey") ||
|
|
62
|
+
normalizeSecretInput(process.env.BRIGHTDATA_API_TOKEN) ||
|
|
63
|
+
undefined
|
|
64
|
+
);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export function resolveBrightDataBaseUrl(
|
|
68
|
+
pluginConfig?: Record<string, unknown> | BrightDataPluginConfig,
|
|
69
|
+
): string {
|
|
70
|
+
const search = resolveBrightDataSearchConfig(pluginConfig);
|
|
71
|
+
const configured =
|
|
72
|
+
readConfiguredString(search?.baseUrl) ||
|
|
73
|
+
normalizeSecretInput(process.env.BRIGHTDATA_BASE_URL) ||
|
|
74
|
+
"";
|
|
75
|
+
return configured || DEFAULT_BRIGHTDATA_BASE_URL;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export function resolveBrightDataUnlockerZone(
|
|
79
|
+
pluginConfig?: Record<string, unknown> | BrightDataPluginConfig,
|
|
80
|
+
): string {
|
|
81
|
+
const search = resolveBrightDataSearchConfig(pluginConfig);
|
|
82
|
+
const configured =
|
|
83
|
+
readConfiguredString(search?.unlockerZone) ||
|
|
84
|
+
normalizeSecretInput(process.env.BRIGHTDATA_UNLOCKER_ZONE) ||
|
|
85
|
+
"";
|
|
86
|
+
return configured || DEFAULT_BRIGHTDATA_UNLOCKER_ZONE;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export function resolveBrightDataBrowserZone(
|
|
90
|
+
pluginConfig?: Record<string, unknown> | BrightDataPluginConfig,
|
|
91
|
+
): string {
|
|
92
|
+
const search = resolveBrightDataSearchConfig(pluginConfig);
|
|
93
|
+
const configured =
|
|
94
|
+
readConfiguredString(search?.browserZone) ||
|
|
95
|
+
normalizeSecretInput(process.env.BRIGHTDATA_BROWSER_ZONE) ||
|
|
96
|
+
"";
|
|
97
|
+
return configured || DEFAULT_BRIGHTDATA_BROWSER_ZONE;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export function resolveBrightDataSearchTimeoutSeconds(override?: number): number {
|
|
101
|
+
if (typeof override === "number" && Number.isFinite(override) && override > 0) {
|
|
102
|
+
return Math.floor(override);
|
|
103
|
+
}
|
|
104
|
+
return DEFAULT_BRIGHTDATA_SEARCH_TIMEOUT_SECONDS;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
export function resolveBrightDataBrowserTimeoutSeconds(
|
|
108
|
+
pluginConfig?: Record<string, unknown> | BrightDataPluginConfig,
|
|
109
|
+
override?: number,
|
|
110
|
+
): number {
|
|
111
|
+
if (typeof override === "number" && Number.isFinite(override) && override > 0) {
|
|
112
|
+
return Math.floor(override);
|
|
113
|
+
}
|
|
114
|
+
const search = resolveBrightDataSearchConfig(pluginConfig);
|
|
115
|
+
if (
|
|
116
|
+
typeof search?.timeoutSeconds === "number" &&
|
|
117
|
+
Number.isFinite(search.timeoutSeconds) &&
|
|
118
|
+
search.timeoutSeconds > 0
|
|
119
|
+
) {
|
|
120
|
+
return Math.floor(search.timeoutSeconds);
|
|
121
|
+
}
|
|
122
|
+
return DEFAULT_BRIGHTDATA_SEARCH_TIMEOUT_SECONDS;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
export function resolveBrightDataScrapeTimeoutSeconds(
|
|
126
|
+
pluginConfig?: Record<string, unknown> | BrightDataPluginConfig,
|
|
127
|
+
override?: number,
|
|
128
|
+
): number {
|
|
129
|
+
if (typeof override === "number" && Number.isFinite(override) && override > 0) {
|
|
130
|
+
return Math.floor(override);
|
|
131
|
+
}
|
|
132
|
+
const search = resolveBrightDataSearchConfig(pluginConfig);
|
|
133
|
+
if (
|
|
134
|
+
typeof search?.timeoutSeconds === "number" &&
|
|
135
|
+
Number.isFinite(search.timeoutSeconds) &&
|
|
136
|
+
search.timeoutSeconds > 0
|
|
137
|
+
) {
|
|
138
|
+
return Math.floor(search.timeoutSeconds);
|
|
139
|
+
}
|
|
140
|
+
return DEFAULT_BRIGHTDATA_SCRAPE_TIMEOUT_SECONDS;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export function resolveBrightDataPollingTimeoutSeconds(
|
|
144
|
+
pluginConfig?: Record<string, unknown> | BrightDataPluginConfig,
|
|
145
|
+
): number {
|
|
146
|
+
const search = resolveBrightDataSearchConfig(pluginConfig);
|
|
147
|
+
if (
|
|
148
|
+
typeof search?.pollingTimeoutSeconds === "number" &&
|
|
149
|
+
Number.isFinite(search.pollingTimeoutSeconds) &&
|
|
150
|
+
search.pollingTimeoutSeconds > 0
|
|
151
|
+
) {
|
|
152
|
+
return Math.floor(search.pollingTimeoutSeconds);
|
|
153
|
+
}
|
|
154
|
+
return DEFAULT_BRIGHTDATA_POLLING_TIMEOUT_SECONDS;
|
|
155
|
+
}
|