@mrscraper/mcp 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +230 -353
- package/dist/api.d.ts +26 -75
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +72 -171
- package/dist/api.js.map +1 -1
- package/dist/app.js +1 -1
- package/dist/app.js.map +1 -1
- package/dist/config.d.ts +5 -12
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +6 -25
- package/dist/config.js.map +1 -1
- package/dist/http.d.ts.map +1 -1
- package/dist/http.js +5 -1
- package/dist/http.js.map +1 -1
- package/dist/index.d.ts +0 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +0 -2
- package/dist/index.js.map +1 -1
- package/dist/scopes.d.ts.map +1 -1
- package/dist/scopes.js +2 -6
- package/dist/scopes.js.map +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +3 -16
- package/dist/server.js.map +1 -1
- package/dist/status.d.ts +0 -2
- package/dist/status.d.ts.map +1 -1
- package/dist/status.js +2 -30
- package/dist/status.js.map +1 -1
- package/dist/tools.d.ts +30 -80
- package/dist/tools.d.ts.map +1 -1
- package/dist/tools.js +184 -569
- package/dist/tools.js.map +1 -1
- package/dist/widgets/bundles.generated.d.ts.map +1 -1
- package/dist/widgets/bundles.generated.js +1 -2
- package/dist/widgets/bundles.generated.js.map +1 -1
- package/dist/widgets/index.d.ts +1 -7
- package/dist/widgets/index.d.ts.map +1 -1
- package/dist/widgets/index.js +1 -7
- package/dist/widgets/index.js.map +1 -1
- package/package.json +4 -4
- package/dist/compliance.d.ts +0 -4
- package/dist/compliance.d.ts.map +0 -1
- package/dist/compliance.js +0 -10
- package/dist/compliance.js.map +0 -1
package/dist/tools.js
CHANGED
|
@@ -1,677 +1,292 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
-
import {
|
|
2
|
+
import { getSubscriptionAccountApi, scraperApi, webSearchApi } from "./api.js";
|
|
3
|
+
import { summarizeSubscriptionAccount } from "./status.js";
|
|
3
4
|
import { widgetMeta } from "./widgets/index.js";
|
|
4
|
-
|
|
5
|
-
const jsonValueSchema = z.json();
|
|
6
|
-
const httpUrlSchema = z.url({ protocol: /^https?$/ });
|
|
7
|
-
const headersSchema = z
|
|
8
|
-
.record(z.string(), z.string())
|
|
9
|
-
.describe("Safe response headers; credentials and cookies are removed.");
|
|
10
|
-
const apiResponseSchema = z
|
|
11
|
-
.object({
|
|
5
|
+
const apiResponseSchema = z.object({
|
|
12
6
|
status_code: z
|
|
13
7
|
.number()
|
|
14
8
|
.int()
|
|
15
9
|
.nullable()
|
|
16
|
-
.describe("HTTP status
|
|
17
|
-
data:
|
|
18
|
-
headers:
|
|
10
|
+
.describe("Upstream HTTP status, or null for a network failure."),
|
|
11
|
+
data: z.json().describe("Response body as parsed JSON or text."),
|
|
12
|
+
headers: z
|
|
13
|
+
.record(z.string(), z.string())
|
|
14
|
+
.describe("Response headers with credentials and cookies removed."),
|
|
19
15
|
error: z.string().optional().describe("Request failure message, if any."),
|
|
20
|
-
})
|
|
21
|
-
.meta({
|
|
22
|
-
title: "MrScraper API response",
|
|
23
|
-
description: "A credential-safe response envelope from MrScraper.",
|
|
24
16
|
});
|
|
25
|
-
export const
|
|
26
|
-
title: "
|
|
27
|
-
description: "
|
|
17
|
+
export const scraperAPIOutputSchema = apiResponseSchema.meta({
|
|
18
|
+
title: "Scraper API response",
|
|
19
|
+
description: "Page content as HTML, Markdown, a screenshot, or a combination.",
|
|
28
20
|
});
|
|
29
|
-
export const
|
|
30
|
-
title: "
|
|
31
|
-
description: "
|
|
21
|
+
export const webSearchOutputSchema = apiResponseSchema.meta({
|
|
22
|
+
title: "Web search response",
|
|
23
|
+
description: "Parsed Google search results or result-page HTML.",
|
|
32
24
|
});
|
|
33
25
|
export const statusOutputSchema = z
|
|
34
26
|
.object({
|
|
35
27
|
kind: z.literal("mrscraper-cli-status-summary").optional(),
|
|
36
28
|
source_endpoints: z.array(z.string()).optional(),
|
|
37
29
|
status_code: z.number().int().nullable(),
|
|
38
|
-
data:
|
|
39
|
-
headers:
|
|
30
|
+
data: z.json(),
|
|
31
|
+
headers: z.record(z.string(), z.string()).optional(),
|
|
40
32
|
error: z.string().optional(),
|
|
41
33
|
})
|
|
42
34
|
.meta({
|
|
43
35
|
title: "Status response",
|
|
44
|
-
description: "Account identity, subscription usage, and
|
|
45
|
-
});
|
|
46
|
-
const serpOutputSchema = apiResponseSchema.meta({
|
|
47
|
-
title: "SERP response",
|
|
48
|
-
description: "Parsed Google results or raw result-page HTML.",
|
|
49
|
-
});
|
|
50
|
-
const rerunOutputSchema = apiResponseSchema.meta({
|
|
51
|
-
title: "Rerun response",
|
|
52
|
-
description: "The MrScraper response for a saved scraper rerun.",
|
|
53
|
-
});
|
|
54
|
-
const resultsOutputSchema = apiResponseSchema.meta({
|
|
55
|
-
title: "Results response",
|
|
56
|
-
description: "Paginated stored MrScraper results.",
|
|
57
|
-
});
|
|
58
|
-
const resultOutputSchema = apiResponseSchema.meta({
|
|
59
|
-
title: "Result response",
|
|
60
|
-
description: "One stored MrScraper result.",
|
|
36
|
+
description: "Account identity, subscription, token limit, token usage, and remaining tokens.",
|
|
61
37
|
});
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
(
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
|
|
69
|
-
structuredContent: result,
|
|
70
|
-
...(isApiFailure(result) ? { isError: true } : {}),
|
|
71
|
-
};
|
|
72
|
-
}
|
|
73
|
-
function buildExtractionMessage(prompt, schemaPrompt) {
|
|
74
|
-
const instruction = prompt.trim();
|
|
75
|
-
if (!instruction) {
|
|
76
|
-
throw new Error("prompt is required for general and listing agents");
|
|
77
|
-
}
|
|
78
|
-
if (schemaPrompt === null || schemaPrompt === undefined)
|
|
79
|
-
return instruction;
|
|
80
|
-
return `${instruction}\n\nBest-effort output guidance: return JSON matching this JSON Schema. The MrScraper API does not validate this schema:\n${JSON.stringify(schemaPrompt, null, 2)}`;
|
|
81
|
-
}
|
|
82
|
-
function unwrapApiData(response) {
|
|
83
|
-
const body = response.data;
|
|
84
|
-
if (body &&
|
|
85
|
-
typeof body === "object" &&
|
|
86
|
-
!Array.isArray(body) &&
|
|
87
|
-
"data" in body) {
|
|
88
|
-
return body.data;
|
|
89
|
-
}
|
|
90
|
-
return body;
|
|
91
|
-
}
|
|
92
|
-
function normalizeDomain(value) {
|
|
93
|
-
const candidate = value.trim();
|
|
94
|
-
if (!candidate)
|
|
95
|
-
throw new Error("domain must not be empty");
|
|
96
|
-
try {
|
|
97
|
-
return new URL(/^https?:\/\//i.test(candidate) ? candidate : `https://${candidate}`).hostname;
|
|
98
|
-
}
|
|
99
|
-
catch {
|
|
100
|
-
throw new Error(`Invalid domain: ${value}`);
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
export const fetchInputSchema = z.object({
|
|
104
|
-
url: httpUrlSchema.describe("Required absolute HTTP or HTTPS URL of the known page to retrieve through Web Unblocker."),
|
|
105
|
-
browser_rendering: z
|
|
38
|
+
export const scraperAPIInputSchema = z.object({
|
|
39
|
+
url: z
|
|
40
|
+
.url({ protocol: /^https?$/ })
|
|
41
|
+
.describe("Required absolute HTTP or HTTPS page URL."),
|
|
42
|
+
html: z.boolean().default(true).describe("Request HTML page content."),
|
|
43
|
+
markdown: z
|
|
106
44
|
.boolean()
|
|
107
45
|
.default(false)
|
|
108
|
-
.describe("
|
|
46
|
+
.describe("Request Markdown page content."),
|
|
47
|
+
timeout: z
|
|
48
|
+
.number()
|
|
49
|
+
.int()
|
|
50
|
+
.min(1)
|
|
51
|
+
.max(300)
|
|
52
|
+
.default(120)
|
|
53
|
+
.describe("Page timeout in seconds (1–300); transport allows 30 additional seconds."),
|
|
109
54
|
geo_code: z
|
|
110
55
|
.string()
|
|
111
|
-
.
|
|
112
|
-
.
|
|
113
|
-
|
|
114
|
-
wait_for_selector: z
|
|
56
|
+
.default("US")
|
|
57
|
+
.describe("Proxy geolocation country code, such as US, ID, or SG."),
|
|
58
|
+
proxy_country: z
|
|
115
59
|
.string()
|
|
116
60
|
.nullable()
|
|
117
61
|
.optional()
|
|
118
|
-
.describe("
|
|
119
|
-
home_page: z
|
|
120
|
-
.boolean()
|
|
121
|
-
.default(false)
|
|
122
|
-
.describe("Visit the site's root page before the target URL, which can help establish cookies or session state."),
|
|
62
|
+
.describe("Compatibility country option sent as proxyCountry."),
|
|
123
63
|
block_resources: z
|
|
124
64
|
.boolean()
|
|
65
|
+
.nullable()
|
|
66
|
+
.optional()
|
|
125
67
|
.default(false)
|
|
126
|
-
.describe("
|
|
127
|
-
|
|
128
|
-
.
|
|
129
|
-
.int()
|
|
130
|
-
.min(0)
|
|
131
|
-
.default(3)
|
|
132
|
-
.describe("Maximum Web Unblocker retry count. Use 0 to disable retries; the default is 3."),
|
|
133
|
-
token_cap: z
|
|
134
|
-
.number()
|
|
135
|
-
.int()
|
|
136
|
-
.positive()
|
|
68
|
+
.describe("Block images, CSS, and fonts during loading."),
|
|
69
|
+
browser_rendering: z
|
|
70
|
+
.boolean()
|
|
137
71
|
.nullable()
|
|
138
72
|
.optional()
|
|
139
|
-
.describe("
|
|
140
|
-
|
|
141
|
-
.
|
|
142
|
-
.
|
|
143
|
-
.
|
|
144
|
-
|
|
145
|
-
.
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
throw new Error("wait_for_selector requires browser_rendering");
|
|
150
|
-
}
|
|
151
|
-
return fetchContentApi({
|
|
152
|
-
token,
|
|
153
|
-
url: input.url,
|
|
154
|
-
browserRendering: input.browser_rendering,
|
|
155
|
-
timeout: input.timeout,
|
|
156
|
-
geoCode: input.geo_code ?? null,
|
|
157
|
-
waitForSelector: input.wait_for_selector ?? null,
|
|
158
|
-
homePage: input.home_page,
|
|
159
|
-
blockResources: input.block_resources,
|
|
160
|
-
maxRetries: input.max_retries,
|
|
161
|
-
tokenCap: input.token_cap ?? null,
|
|
162
|
-
fetchFn: dependencies.fetchFn,
|
|
163
|
-
});
|
|
164
|
-
}
|
|
165
|
-
export const scrapeInputSchema = z.object({
|
|
166
|
-
url: httpUrlSchema.describe("Required absolute HTTP or HTTPS starting URL for the AI scraper."),
|
|
167
|
-
prompt: z
|
|
73
|
+
.describe("True forces browser rendering, false disables it; omit or use null for automatic fallback."),
|
|
74
|
+
super: z
|
|
75
|
+
.boolean()
|
|
76
|
+
.default(false)
|
|
77
|
+
.describe("Use real-device routing independently of browser rendering."),
|
|
78
|
+
use_proxy: z
|
|
79
|
+
.boolean()
|
|
80
|
+
.default(true)
|
|
81
|
+
.describe("Allow the service to use a configured proxy."),
|
|
82
|
+
proxy: z
|
|
168
83
|
.string()
|
|
169
84
|
.nullable()
|
|
170
85
|
.optional()
|
|
171
|
-
.describe("
|
|
172
|
-
|
|
173
|
-
.
|
|
86
|
+
.describe("Custom proxy in username:password@host:port form."),
|
|
87
|
+
wait_for_selector: z
|
|
88
|
+
.string()
|
|
174
89
|
.nullable()
|
|
175
90
|
.optional()
|
|
176
|
-
.describe("
|
|
177
|
-
|
|
178
|
-
.
|
|
179
|
-
.default(
|
|
180
|
-
.describe("
|
|
181
|
-
|
|
91
|
+
.describe("CSS selector to wait for before returning page content."),
|
|
92
|
+
home_page: z
|
|
93
|
+
.boolean()
|
|
94
|
+
.default(false)
|
|
95
|
+
.describe("Visit the site's root before the target URL; sent as warmStart=true."),
|
|
96
|
+
screenshot: z
|
|
97
|
+
.boolean()
|
|
98
|
+
.default(false)
|
|
99
|
+
.describe("Capture a screenshot; always enables browser rendering."),
|
|
100
|
+
bypass_proxy: z
|
|
101
|
+
.boolean()
|
|
102
|
+
.default(false)
|
|
103
|
+
.describe("Enable the API's proxy-bypass/resource-blocking behavior."),
|
|
104
|
+
cookie_jar: z
|
|
182
105
|
.string()
|
|
183
106
|
.nullable()
|
|
184
107
|
.optional()
|
|
185
|
-
.describe("
|
|
186
|
-
|
|
187
|
-
.
|
|
188
|
-
.int()
|
|
189
|
-
.positive()
|
|
108
|
+
.describe("Cookie configuration string, such as domain=example.com&country=us&max_used=100&max_age=50m."),
|
|
109
|
+
headers: z
|
|
110
|
+
.union([z.record(z.string(), z.string()), z.string()])
|
|
190
111
|
.nullable()
|
|
191
112
|
.optional()
|
|
192
|
-
.describe("
|
|
193
|
-
|
|
113
|
+
.describe("Target page request headers as an object or JSON string."),
|
|
114
|
+
max_retries: z
|
|
194
115
|
.number()
|
|
195
116
|
.int()
|
|
196
|
-
.
|
|
197
|
-
.
|
|
198
|
-
.
|
|
199
|
-
|
|
200
|
-
limit: z
|
|
117
|
+
.min(0)
|
|
118
|
+
.default(3)
|
|
119
|
+
.describe("Maximum Web Unblocker retries; 0 disables retries."),
|
|
120
|
+
token_cap: z
|
|
201
121
|
.number()
|
|
202
122
|
.int()
|
|
203
|
-
.
|
|
123
|
+
.min(0)
|
|
204
124
|
.nullable()
|
|
205
125
|
.optional()
|
|
206
|
-
.describe("
|
|
207
|
-
|
|
208
|
-
.
|
|
126
|
+
.describe("Optional nonnegative retry-token budget."),
|
|
127
|
+
stream: z
|
|
128
|
+
.boolean()
|
|
209
129
|
.nullable()
|
|
210
130
|
.optional()
|
|
211
|
-
.describe("
|
|
212
|
-
|
|
131
|
+
.describe("Ask the API to stream; the MCP server buffers the response into the normal envelope."),
|
|
132
|
+
discard_token_secret: z
|
|
213
133
|
.string()
|
|
214
|
-
.
|
|
215
|
-
.
|
|
216
|
-
.describe("Regular expression removing matching URLs from map-agent results; not accepted by general or listing."),
|
|
134
|
+
.default("")
|
|
135
|
+
.describe("Optional value for the x-discard-token-secret upstream header."),
|
|
217
136
|
});
|
|
218
|
-
export
|
|
219
|
-
|
|
220
|
-
const agent = input.agent;
|
|
221
|
-
const mapOnlyOptions = [
|
|
222
|
-
["max_depth", input.max_depth],
|
|
223
|
-
["limit", input.limit],
|
|
224
|
-
["include_patterns", input.include_patterns],
|
|
225
|
-
["exclude_patterns", input.exclude_patterns],
|
|
226
|
-
];
|
|
227
|
-
if (agent === "map") {
|
|
228
|
-
if (supplied(input.prompt)) {
|
|
229
|
-
throw new Error("prompt is not accepted by the map agent");
|
|
230
|
-
}
|
|
231
|
-
if (supplied(input.schema_prompt)) {
|
|
232
|
-
throw new Error("schema_prompt is not accepted by the map agent");
|
|
233
|
-
}
|
|
234
|
-
if (supplied(input.proxy_country)) {
|
|
235
|
-
throw new Error("proxy_country is not accepted by the map agent");
|
|
236
|
-
}
|
|
237
|
-
}
|
|
238
|
-
else {
|
|
239
|
-
if (!input.prompt?.trim()) {
|
|
240
|
-
throw new Error("prompt is required for general and listing agents");
|
|
241
|
-
}
|
|
242
|
-
const invalidMapOptions = mapOnlyOptions
|
|
243
|
-
.filter(([, value]) => supplied(value))
|
|
244
|
-
.map(([name]) => name);
|
|
245
|
-
if (invalidMapOptions.length) {
|
|
246
|
-
throw new Error(`${invalidMapOptions.join(", ")} ${invalidMapOptions.length === 1 ? "is" : "are"} only accepted by the map agent`);
|
|
247
|
-
}
|
|
248
|
-
if (agent === "general" && supplied(input.max_pages)) {
|
|
249
|
-
throw new Error("max_pages is only accepted by listing and map agents");
|
|
250
|
-
}
|
|
251
|
-
}
|
|
252
|
-
return createAiScraperApi({
|
|
253
|
-
token,
|
|
254
|
-
url: input.url,
|
|
255
|
-
message: agent === "map"
|
|
256
|
-
? undefined
|
|
257
|
-
: buildExtractionMessage(input.prompt, input.schema_prompt),
|
|
258
|
-
agent,
|
|
259
|
-
proxyCountry: input.proxy_country ?? null,
|
|
260
|
-
maxPages: input.max_pages ?? undefined,
|
|
261
|
-
maxDepth: input.max_depth ?? undefined,
|
|
262
|
-
limit: input.limit ?? undefined,
|
|
263
|
-
includePatterns: input.include_patterns ?? undefined,
|
|
264
|
-
excludePatterns: input.exclude_patterns ?? undefined,
|
|
265
|
-
fetchFn: dependencies.fetchFn,
|
|
266
|
-
});
|
|
267
|
-
}
|
|
268
|
-
export const serpInputSchema = z.object({
|
|
269
|
-
query_or_url: z
|
|
137
|
+
export const webSearchInputSchema = z.object({
|
|
138
|
+
url: z
|
|
270
139
|
.string()
|
|
140
|
+
.trim()
|
|
271
141
|
.min(1)
|
|
272
|
-
.describe("Required
|
|
142
|
+
.describe("Required search query text or complete Google search URL."),
|
|
143
|
+
raw: z
|
|
144
|
+
.boolean()
|
|
145
|
+
.default(true)
|
|
146
|
+
.describe("SDK compatibility default: true requests HTML and overrides format. Set false for parsed JSON results."),
|
|
273
147
|
region: z
|
|
274
148
|
.string()
|
|
275
149
|
.nullable()
|
|
276
150
|
.optional()
|
|
277
|
-
.describe("
|
|
151
|
+
.describe("Country code; overrides gl from a Google search URL."),
|
|
278
152
|
language: z
|
|
279
153
|
.string()
|
|
280
154
|
.nullable()
|
|
281
155
|
.optional()
|
|
282
|
-
.describe("
|
|
156
|
+
.describe("Language code; overrides hl from a Google search URL."),
|
|
283
157
|
page: z
|
|
284
158
|
.number()
|
|
285
159
|
.int()
|
|
286
160
|
.positive()
|
|
287
161
|
.nullable()
|
|
288
162
|
.optional()
|
|
289
|
-
.describe("
|
|
163
|
+
.describe("One-based result page; overrides the page derived from start in a Google URL."),
|
|
290
164
|
format: z
|
|
291
165
|
.enum(["json", "html"])
|
|
292
166
|
.default("json")
|
|
293
|
-
.describe("Response representation
|
|
167
|
+
.describe("Response representation when raw=false."),
|
|
294
168
|
render_js: z
|
|
295
169
|
.boolean()
|
|
296
170
|
.default(false)
|
|
297
|
-
.describe("Render
|
|
298
|
-
|
|
299
|
-
.boolean()
|
|
300
|
-
.default(false)
|
|
301
|
-
.describe("Deprecated alias for format=html; takes precedence over format."),
|
|
302
|
-
client_timeout: z
|
|
171
|
+
.describe("Render JavaScript-dependent search features."),
|
|
172
|
+
timeout: z
|
|
303
173
|
.number()
|
|
304
|
-
.int()
|
|
305
174
|
.positive()
|
|
306
|
-
.default(
|
|
307
|
-
.describe("Local upstream HTTP
|
|
175
|
+
.default(600)
|
|
176
|
+
.describe("Local upstream HTTP deadline in seconds; omitted from the JSON body."),
|
|
308
177
|
});
|
|
309
|
-
export async function
|
|
310
|
-
return
|
|
178
|
+
export async function scraperAPITool(token, input, dependencies = {}) {
|
|
179
|
+
return scraperApi({
|
|
311
180
|
token,
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
181
|
+
url: input.url,
|
|
182
|
+
html: input.html,
|
|
183
|
+
markdown: input.markdown,
|
|
184
|
+
timeout: input.timeout,
|
|
185
|
+
geoCode: input.geo_code,
|
|
186
|
+
proxyCountry: input.proxy_country,
|
|
187
|
+
blockResources: input.block_resources,
|
|
188
|
+
browserRendering: input.browser_rendering,
|
|
189
|
+
super: input.super,
|
|
190
|
+
useProxy: input.use_proxy,
|
|
191
|
+
proxy: input.proxy,
|
|
192
|
+
waitForSelector: input.wait_for_selector,
|
|
193
|
+
homePage: input.home_page,
|
|
194
|
+
screenshot: input.screenshot,
|
|
195
|
+
bypassProxy: input.bypass_proxy,
|
|
196
|
+
cookieJar: input.cookie_jar,
|
|
197
|
+
headers: input.headers,
|
|
198
|
+
maxRetries: input.max_retries,
|
|
199
|
+
tokenCap: input.token_cap,
|
|
200
|
+
stream: input.stream,
|
|
201
|
+
discardTokenSecret: input.discard_token_secret,
|
|
202
|
+
fetchFn: dependencies.fetchFn,
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
export async function webSearchTool(token, input, dependencies = {}) {
|
|
206
|
+
return webSearchApi({
|
|
207
|
+
token,
|
|
208
|
+
url: input.url,
|
|
209
|
+
raw: input.raw,
|
|
210
|
+
region: input.region,
|
|
211
|
+
language: input.language,
|
|
212
|
+
page: input.page,
|
|
316
213
|
format: input.format,
|
|
317
214
|
renderJs: input.render_js,
|
|
318
|
-
|
|
319
|
-
timeout: input.client_timeout,
|
|
215
|
+
timeout: input.timeout,
|
|
320
216
|
fetchFn: dependencies.fetchFn,
|
|
321
217
|
});
|
|
322
218
|
}
|
|
323
|
-
export const statusInputSchema = z.object({
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
.
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
.
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
.nullable()
|
|
340
|
-
.optional()
|
|
341
|
-
.describe("Optional exact MrScraper action filter applied to domain analytics; used only when domain is supplied."),
|
|
342
|
-
api_token_name: z
|
|
343
|
-
.string()
|
|
344
|
-
.nullable()
|
|
345
|
-
.optional()
|
|
346
|
-
.describe("Optional API token name used to filter domain analytics; used only when domain is supplied."),
|
|
347
|
-
});
|
|
348
|
-
export async function statusTool(token, input, dependencies = {}) {
|
|
349
|
-
const accountResponse = await getSubscriptionAccountApi(token, dependencies);
|
|
350
|
-
if (isApiFailure(accountResponse))
|
|
351
|
-
return accountResponse;
|
|
352
|
-
const account = unwrapApiData(accountResponse);
|
|
353
|
-
const output = {
|
|
219
|
+
export const statusInputSchema = z.object({});
|
|
220
|
+
function isApiFailure(result) {
|
|
221
|
+
return Boolean(result.error ||
|
|
222
|
+
(typeof result.status_code === "number" && result.status_code >= 400));
|
|
223
|
+
}
|
|
224
|
+
export async function statusTool(token, _input = {}, dependencies = {}) {
|
|
225
|
+
const response = await getSubscriptionAccountApi(token, {
|
|
226
|
+
fetchFn: dependencies.fetchFn,
|
|
227
|
+
});
|
|
228
|
+
if (isApiFailure(response))
|
|
229
|
+
return response;
|
|
230
|
+
const body = response.data;
|
|
231
|
+
const account = body && typeof body === "object" && !Array.isArray(body) && "data" in body
|
|
232
|
+
? body.data
|
|
233
|
+
: body;
|
|
234
|
+
return {
|
|
354
235
|
kind: "mrscraper-cli-status-summary",
|
|
355
236
|
source_endpoints: ["/subscription-accounts"],
|
|
356
|
-
status_code:
|
|
237
|
+
status_code: response.status_code,
|
|
357
238
|
data: {
|
|
358
239
|
account: summarizeSubscriptionAccount(account && typeof account === "object" && !Array.isArray(account)
|
|
359
240
|
? account
|
|
360
241
|
: {}),
|
|
361
242
|
},
|
|
362
243
|
};
|
|
363
|
-
if (!input.domain)
|
|
364
|
-
return output;
|
|
365
|
-
output.source_endpoints.push("/analytic/statuses");
|
|
366
|
-
const domain = normalizeDomain(input.domain);
|
|
367
|
-
const now = dependencies.now?.() || new Date();
|
|
368
|
-
const end = parseStatusDate(input.to, now, "now");
|
|
369
|
-
const start = parseStatusDate(input.from, end, "24h");
|
|
370
|
-
if (start >= end)
|
|
371
|
-
throw new Error("from must be earlier than to");
|
|
372
|
-
const startDate = formatApiDate(start);
|
|
373
|
-
const endDate = formatApiDate(end);
|
|
374
|
-
const analyticsResponse = await getAnalyticStatusesApi({
|
|
375
|
-
token,
|
|
376
|
-
domain,
|
|
377
|
-
startDate,
|
|
378
|
-
endDate,
|
|
379
|
-
action: input.action || "",
|
|
380
|
-
apiTokenName: input.api_token_name || "",
|
|
381
|
-
fetchFn: dependencies.fetchFn,
|
|
382
|
-
});
|
|
383
|
-
const data = output.data;
|
|
384
|
-
if (analyticsResponse.error) {
|
|
385
|
-
output.error = "Account loaded, but analytics could not be loaded";
|
|
386
|
-
data.analytics = analyticsResponse;
|
|
387
|
-
}
|
|
388
|
-
else {
|
|
389
|
-
const analytics = unwrapApiData(analyticsResponse);
|
|
390
|
-
data.analytics = {
|
|
391
|
-
domain,
|
|
392
|
-
from: `${startDate} UTC`,
|
|
393
|
-
to: `${endDate} UTC`,
|
|
394
|
-
...(analytics &&
|
|
395
|
-
typeof analytics === "object" &&
|
|
396
|
-
!Array.isArray(analytics)
|
|
397
|
-
? analytics
|
|
398
|
-
: { data: analytics }),
|
|
399
|
-
};
|
|
400
|
-
}
|
|
401
|
-
return output;
|
|
402
|
-
}
|
|
403
|
-
export const rerunInputSchema = z.object({
|
|
404
|
-
target: z
|
|
405
|
-
.string()
|
|
406
|
-
.describe("Target URL for a single rerun, or comma/newline-separated target URLs when bulk=true."),
|
|
407
|
-
type: z
|
|
408
|
-
.enum(["ai", "manual"])
|
|
409
|
-
.describe("How the saved scraper was created: ai for an AI scraper or manual for a dashboard-built step workflow."),
|
|
410
|
-
bulk: z
|
|
411
|
-
.boolean()
|
|
412
|
-
.default(false)
|
|
413
|
-
.describe("Independently select target count: false for one URL or true to submit all parsed targets as one asynchronous bulk job."),
|
|
414
|
-
scraper_id: z
|
|
415
|
-
.string()
|
|
416
|
-
.nullable()
|
|
417
|
-
.optional()
|
|
418
|
-
.describe("Required saved scraper UUID when bulk=false. Use the scraperId returned by scrape for an AI scraper, or a dashboard workflow's UUID for manual."),
|
|
419
|
-
id: z
|
|
420
|
-
.string()
|
|
421
|
-
.nullable()
|
|
422
|
-
.optional()
|
|
423
|
-
.describe("Required saved scraper UUID when bulk=true. This identifies the configuration to run and is not a result ID."),
|
|
424
|
-
max_depth: z
|
|
425
|
-
.number()
|
|
426
|
-
.int()
|
|
427
|
-
.positive()
|
|
428
|
-
.nullable()
|
|
429
|
-
.optional()
|
|
430
|
-
.describe("Maximum crawl depth for a single AI rerun; defaults to 2 when omitted."),
|
|
431
|
-
max_pages: z
|
|
432
|
-
.number()
|
|
433
|
-
.int()
|
|
434
|
-
.positive()
|
|
435
|
-
.nullable()
|
|
436
|
-
.optional()
|
|
437
|
-
.describe("Maximum pages for a single AI rerun; defaults to 50 when omitted."),
|
|
438
|
-
limit: z
|
|
439
|
-
.number()
|
|
440
|
-
.int()
|
|
441
|
-
.positive()
|
|
442
|
-
.nullable()
|
|
443
|
-
.optional()
|
|
444
|
-
.describe("Maximum results for a single AI rerun; defaults to 1000 when omitted."),
|
|
445
|
-
include_patterns: z
|
|
446
|
-
.string()
|
|
447
|
-
.nullable()
|
|
448
|
-
.optional()
|
|
449
|
-
.describe("URL include regular expression for a single AI rerun; defaults to an empty string."),
|
|
450
|
-
exclude_patterns: z
|
|
451
|
-
.string()
|
|
452
|
-
.nullable()
|
|
453
|
-
.optional()
|
|
454
|
-
.describe("URL exclude regular expression for a single AI rerun; defaults to an empty string."),
|
|
455
|
-
});
|
|
456
|
-
export async function rerunTool(token, input, dependencies = {}) {
|
|
457
|
-
const aiOptionEntries = [
|
|
458
|
-
["max_depth", input.max_depth],
|
|
459
|
-
["max_pages", input.max_pages],
|
|
460
|
-
["limit", input.limit],
|
|
461
|
-
["include_patterns", input.include_patterns],
|
|
462
|
-
["exclude_patterns", input.exclude_patterns],
|
|
463
|
-
];
|
|
464
|
-
const explicitAiOptions = aiOptionEntries
|
|
465
|
-
.filter(([, value]) => value !== undefined && value !== null)
|
|
466
|
-
.map(([name]) => name);
|
|
467
|
-
let response;
|
|
468
|
-
if (input.bulk) {
|
|
469
|
-
if (!input.id)
|
|
470
|
-
throw new Error("id is required when bulk is true");
|
|
471
|
-
if (input.scraper_id !== undefined && input.scraper_id !== null) {
|
|
472
|
-
throw new Error("scraper_id is only accepted for single reruns; use id when bulk is true");
|
|
473
|
-
}
|
|
474
|
-
if (explicitAiOptions.length) {
|
|
475
|
-
throw new Error(`${explicitAiOptions.join(", ")} ${explicitAiOptions.length === 1 ? "is" : "are"} not accepted by bulk rerun endpoints`);
|
|
476
|
-
}
|
|
477
|
-
const urls = parseBulkUrls(input.target);
|
|
478
|
-
if (!urls.length)
|
|
479
|
-
throw new Error("No URLs found in the bulk target");
|
|
480
|
-
response =
|
|
481
|
-
input.type === "ai"
|
|
482
|
-
? await bulkRerunAiScraperApi({
|
|
483
|
-
token,
|
|
484
|
-
scraperId: input.id,
|
|
485
|
-
urls,
|
|
486
|
-
fetchFn: dependencies.fetchFn,
|
|
487
|
-
})
|
|
488
|
-
: await bulkRerunManualScraperApi({
|
|
489
|
-
token,
|
|
490
|
-
scraperId: input.id,
|
|
491
|
-
urls,
|
|
492
|
-
fetchFn: dependencies.fetchFn,
|
|
493
|
-
});
|
|
494
|
-
}
|
|
495
|
-
else {
|
|
496
|
-
if (!input.scraper_id) {
|
|
497
|
-
throw new Error("scraper_id is required unless bulk is true");
|
|
498
|
-
}
|
|
499
|
-
if (input.id !== undefined && input.id !== null) {
|
|
500
|
-
throw new Error("id is only accepted when bulk is true; use scraper_id for a single rerun");
|
|
501
|
-
}
|
|
502
|
-
if (input.type === "manual" && explicitAiOptions.length) {
|
|
503
|
-
throw new Error(`${explicitAiOptions.join(", ")} ${explicitAiOptions.length === 1 ? "is" : "are"} only accepted by single AI reruns`);
|
|
504
|
-
}
|
|
505
|
-
const url = input.target.trim();
|
|
506
|
-
if (!url)
|
|
507
|
-
throw new Error("target URL must not be empty");
|
|
508
|
-
response =
|
|
509
|
-
input.type === "manual"
|
|
510
|
-
? await rerunManualScraperApi({
|
|
511
|
-
token,
|
|
512
|
-
scraperId: input.scraper_id,
|
|
513
|
-
url,
|
|
514
|
-
fetchFn: dependencies.fetchFn,
|
|
515
|
-
})
|
|
516
|
-
: await rerunAiScraperApi({
|
|
517
|
-
token,
|
|
518
|
-
scraperId: input.scraper_id,
|
|
519
|
-
url,
|
|
520
|
-
maxDepth: input.max_depth ?? 2,
|
|
521
|
-
maxPages: input.max_pages ?? 50,
|
|
522
|
-
limit: input.limit ?? 1000,
|
|
523
|
-
includePatterns: input.include_patterns ?? "",
|
|
524
|
-
excludePatterns: input.exclude_patterns ?? "",
|
|
525
|
-
fetchFn: dependencies.fetchFn,
|
|
526
|
-
});
|
|
527
|
-
}
|
|
528
|
-
return response;
|
|
529
|
-
}
|
|
530
|
-
export const resultsInputSchema = z.object({
|
|
531
|
-
sort_field: z
|
|
532
|
-
.string()
|
|
533
|
-
.min(1)
|
|
534
|
-
.default("updatedAt")
|
|
535
|
-
.describe("Stored-result field used as the sort key; defaults to updatedAt."),
|
|
536
|
-
sort_order: z
|
|
537
|
-
.string()
|
|
538
|
-
.trim()
|
|
539
|
-
.toLowerCase()
|
|
540
|
-
.pipe(z.enum(["asc", "desc"]))
|
|
541
|
-
.default("desc")
|
|
542
|
-
.describe("Sort direction for sort_field: asc or desc, accepted case-insensitively; defaults to desc."),
|
|
543
|
-
page_size: z
|
|
544
|
-
.number()
|
|
545
|
-
.int()
|
|
546
|
-
.positive()
|
|
547
|
-
.default(10)
|
|
548
|
-
.describe("Positive number of stored result records requested per page; defaults to 10."),
|
|
549
|
-
page: z
|
|
550
|
-
.number()
|
|
551
|
-
.int()
|
|
552
|
-
.positive()
|
|
553
|
-
.default(1)
|
|
554
|
-
.describe("One-based page number used to move through the stored result list; defaults to 1."),
|
|
555
|
-
search: z
|
|
556
|
-
.string()
|
|
557
|
-
.nullable()
|
|
558
|
-
.optional()
|
|
559
|
-
.describe("Optional free-text filter used to narrow the stored result list."),
|
|
560
|
-
date_range_column: z
|
|
561
|
-
.string()
|
|
562
|
-
.nullable()
|
|
563
|
-
.optional()
|
|
564
|
-
.describe("Stored-result date column to which start_at and end_at are applied."),
|
|
565
|
-
start_at: z
|
|
566
|
-
.string()
|
|
567
|
-
.nullable()
|
|
568
|
-
.optional()
|
|
569
|
-
.describe("Optional inclusive ISO 8601 start bound applied to date_range_column."),
|
|
570
|
-
end_at: z
|
|
571
|
-
.string()
|
|
572
|
-
.nullable()
|
|
573
|
-
.optional()
|
|
574
|
-
.describe("Optional inclusive ISO 8601 end bound applied to date_range_column."),
|
|
575
|
-
});
|
|
576
|
-
export async function resultsTool(token, input, dependencies = {}) {
|
|
577
|
-
return getAllResultsApi({
|
|
578
|
-
token,
|
|
579
|
-
sortField: input.sort_field,
|
|
580
|
-
sortOrder: input.sort_order.toUpperCase(),
|
|
581
|
-
pageSize: input.page_size,
|
|
582
|
-
page: input.page,
|
|
583
|
-
search: input.search ?? null,
|
|
584
|
-
dateRangeColumn: input.date_range_column ?? null,
|
|
585
|
-
startAt: input.start_at ?? null,
|
|
586
|
-
endAt: input.end_at ?? null,
|
|
587
|
-
fetchFn: dependencies.fetchFn,
|
|
588
|
-
});
|
|
589
244
|
}
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
.
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
}
|
|
596
|
-
export async function resultTool(token, input, dependencies = {}) {
|
|
597
|
-
const resultId = input.result_id.trim();
|
|
598
|
-
if (!resultId)
|
|
599
|
-
throw new Error("result_id must not be empty");
|
|
600
|
-
return getResultByIdApi(token, resultId, dependencies);
|
|
245
|
+
function asStructured(result) {
|
|
246
|
+
return {
|
|
247
|
+
content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
|
|
248
|
+
structuredContent: result,
|
|
249
|
+
...(isApiFailure(result) ? { isError: true } : {}),
|
|
250
|
+
};
|
|
601
251
|
}
|
|
602
|
-
const
|
|
252
|
+
const annotations = {
|
|
603
253
|
readOnlyHint: true,
|
|
604
|
-
openWorldHint:
|
|
605
|
-
destructiveHint: false,
|
|
606
|
-
};
|
|
607
|
-
const writeAnnotations = {
|
|
608
|
-
readOnlyHint: false,
|
|
609
|
-
openWorldHint: false,
|
|
254
|
+
openWorldHint: true,
|
|
610
255
|
destructiveHint: false,
|
|
611
256
|
};
|
|
612
257
|
export const TOOL_DESCRIPTIONS = {
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
status: "Use this to check the current MrScraper account's identity, subscription, and usage before or after web-data work. With no arguments it returns an account summary including the user's name, email, and verification state. Supply domain only when request-outcome analytics are also needed; from and to select that analytics window, while action and api_token_name narrow those domain outcomes. This tool reports account and request health, not scrape-job progress; use result for a known asynchronous result ID.",
|
|
617
|
-
rerun: "Use this only when you already have a saved scraper UUID and want to apply that configuration to the same or new target URLs. Set type=ai for a scraper created by scrape; type=manual selects a dashboard-built step workflow and requires the conversation's compliance acknowledgment. For one URL, leave bulk=false and pass target with scraper_id. A single AI rerun also accepts max_depth, max_pages, limit, include_patterns, and exclude_patterns; manual and bulk reruns reject those controls. For multiple URLs, set bulk=true, pass comma- or newline-separated target URLs and id; bulk jobs are asynchronous, so retain bulkResultId and inspect it with result until completion. Use scrape to create a new AI configuration.",
|
|
618
|
-
results: "Use this to browse or locate stored scrape results when you do not yet know the exact result UUID. sort_field and sort_order control ordering; page_size and page control pagination; search narrows the list; and date_range_column with start_at or end_at applies a date window. This returns a result list rather than one complete record. Once an ID is known, use result for that record.",
|
|
619
|
-
result: "Use this when you already know the exact stored result UUID and need the complete record or current job state. Pass that UUID as result_id; this may also be the bulkResultId returned by an asynchronous bulk rerun. Call result again as needed until an asynchronous job completes. Use results first when the UUID is unknown.",
|
|
258
|
+
scraperAPI: "Retrieves a known page through MrScraper's Scraper API as HTML, Markdown, a screenshot, or a combination. Matches the Python SDK's scraper_api options. Omit browser_rendering for automatic browser fallback; screenshot=true always enables it. super enables real-device routing independently. home_page maps to warmStart. Proxy, cookie, target-header, retry, and timeout options control page loading. Returns status_code, data, and sanitized headers.",
|
|
259
|
+
webSearch: "Searches Google from query text or a Google search URL passed as url. Matches the Python SDK's web_search options. Set raw=false and format=json for parsed results; the SDK-compatible default raw=true returns HTML. Google URLs supply q, gl, hl, and start; explicit region, language, and page override URL-derived values. render_js enables dynamic features; timeout controls the local request deadline. Returns status_code, data, and sanitized headers.",
|
|
260
|
+
status: "Returns the connected MrScraper account's identity (name, email, and verification state), subscription status, token limit, token usage, remaining tokens, and rate limit. Use it to check the account and remaining quota, for example before a large batch of requests. It takes no input and reports account usage, not the progress of a request.",
|
|
620
261
|
};
|
|
621
262
|
export function registerTools(server, getToken, dependencies = {}) {
|
|
622
|
-
server.registerTool("
|
|
623
|
-
title: "
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
}, async (input) => asStructured(await
|
|
629
|
-
server.registerTool("
|
|
630
|
-
title: "
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
}, async (input) => asStructured(await
|
|
637
|
-
server.registerTool("serp", {
|
|
638
|
-
title: "Google Search",
|
|
639
|
-
description: TOOL_DESCRIPTIONS.serp,
|
|
640
|
-
inputSchema: serpInputSchema,
|
|
641
|
-
outputSchema: serpOutputSchema,
|
|
642
|
-
_meta: widgetMeta("serp", "Searching Google…", "Search complete."),
|
|
643
|
-
annotations: readAnnotations,
|
|
644
|
-
}, async (input) => asStructured(await serpTool(getToken(), input, dependencies)));
|
|
263
|
+
server.registerTool("scraperAPI", {
|
|
264
|
+
title: "Scraper API",
|
|
265
|
+
annotations: { title: "Scraper API", ...annotations },
|
|
266
|
+
description: TOOL_DESCRIPTIONS.scraperAPI,
|
|
267
|
+
inputSchema: scraperAPIInputSchema,
|
|
268
|
+
outputSchema: scraperAPIOutputSchema,
|
|
269
|
+
}, async (input) => asStructured(await scraperAPITool(getToken(), input, dependencies)));
|
|
270
|
+
server.registerTool("webSearch", {
|
|
271
|
+
title: "Web Search",
|
|
272
|
+
annotations: { title: "Web Search", ...annotations },
|
|
273
|
+
description: TOOL_DESCRIPTIONS.webSearch,
|
|
274
|
+
inputSchema: webSearchInputSchema,
|
|
275
|
+
outputSchema: webSearchOutputSchema,
|
|
276
|
+
_meta: widgetMeta("webSearch", "Searching Google…", "Search complete."),
|
|
277
|
+
}, async (input) => asStructured(await webSearchTool(getToken(), input, dependencies)));
|
|
645
278
|
server.registerTool("status", {
|
|
646
279
|
title: "Account Status",
|
|
280
|
+
annotations: {
|
|
281
|
+
title: "Account Status",
|
|
282
|
+
readOnlyHint: true,
|
|
283
|
+
openWorldHint: false,
|
|
284
|
+
destructiveHint: false,
|
|
285
|
+
},
|
|
647
286
|
description: TOOL_DESCRIPTIONS.status,
|
|
648
287
|
inputSchema: statusInputSchema,
|
|
649
288
|
outputSchema: statusOutputSchema,
|
|
650
289
|
_meta: widgetMeta("status", "Checking your account…", "Account status loaded."),
|
|
651
|
-
annotations: readAnnotations,
|
|
652
290
|
}, async (input) => asStructured(await statusTool(getToken(), input, dependencies)));
|
|
653
|
-
server.registerTool("rerun", {
|
|
654
|
-
title: "Rerun Saved Scraper",
|
|
655
|
-
description: TOOL_DESCRIPTIONS.rerun,
|
|
656
|
-
inputSchema: rerunInputSchema,
|
|
657
|
-
outputSchema: rerunOutputSchema,
|
|
658
|
-
annotations: writeAnnotations,
|
|
659
|
-
}, async (input) => asStructured(await rerunTool(getToken(), input, dependencies)));
|
|
660
|
-
server.registerTool("results", {
|
|
661
|
-
title: "List Results",
|
|
662
|
-
description: TOOL_DESCRIPTIONS.results,
|
|
663
|
-
inputSchema: resultsInputSchema,
|
|
664
|
-
outputSchema: resultsOutputSchema,
|
|
665
|
-
_meta: widgetMeta("records", "Loading results…", "Results loaded."),
|
|
666
|
-
annotations: readAnnotations,
|
|
667
|
-
}, async (input) => asStructured(await resultsTool(getToken(), input, dependencies)));
|
|
668
|
-
server.registerTool("result", {
|
|
669
|
-
title: "Get Result",
|
|
670
|
-
description: TOOL_DESCRIPTIONS.result,
|
|
671
|
-
inputSchema: resultInputSchema,
|
|
672
|
-
outputSchema: resultOutputSchema,
|
|
673
|
-
_meta: widgetMeta("records", "Loading result…", "Result loaded."),
|
|
674
|
-
annotations: readAnnotations,
|
|
675
|
-
}, async (input) => asStructured(await resultTool(getToken(), input, dependencies)));
|
|
676
291
|
}
|
|
677
292
|
//# sourceMappingURL=tools.js.map
|