@mrscraper/mcp 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +642 -0
- package/dist/api.d.ts +102 -0
- package/dist/api.d.ts.map +1 -0
- package/dist/api.js +203 -0
- package/dist/api.js.map +1 -0
- package/dist/app.d.ts +17 -0
- package/dist/app.d.ts.map +1 -0
- package/dist/app.js +123 -0
- package/dist/app.js.map +1 -0
- package/dist/auth.d.ts +7 -0
- package/dist/auth.d.ts.map +1 -0
- package/dist/auth.js +64 -0
- package/dist/auth.js.map +1 -0
- package/dist/bin.d.ts +3 -0
- package/dist/bin.d.ts.map +1 -0
- package/dist/bin.js +8 -0
- package/dist/bin.js.map +1 -0
- package/dist/compliance.d.ts +4 -0
- package/dist/compliance.d.ts.map +1 -0
- package/dist/compliance.js +10 -0
- package/dist/compliance.js.map +1 -0
- package/dist/config.d.ts +28 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +63 -0
- package/dist/config.js.map +1 -0
- package/dist/http.d.ts +19 -0
- package/dist/http.d.ts.map +1 -0
- package/dist/http.js +156 -0
- package/dist/http.js.map +1 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11 -0
- package/dist/index.js.map +1 -0
- package/dist/runtime.d.ts +8 -0
- package/dist/runtime.d.ts.map +1 -0
- package/dist/runtime.js +46 -0
- package/dist/runtime.js.map +1 -0
- package/dist/server.d.ts +9 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +24 -0
- package/dist/server.js.map +1 -0
- package/dist/status.d.ts +21 -0
- package/dist/status.d.ts.map +1 -0
- package/dist/status.js +62 -0
- package/dist/status.js.map +1 -0
- package/dist/tools.d.ts +122 -0
- package/dist/tools.d.ts.map +1 -0
- package/dist/tools.js +664 -0
- package/dist/tools.js.map +1 -0
- package/package.json +77 -0
package/dist/tools.js
ADDED
|
@@ -0,0 +1,664 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { bulkRerunAiScraperApi, bulkRerunManualScraperApi, createAiScraperApi, fetchContentApi, getAllResultsApi, getAnalyticStatusesApi, getResultByIdApi, getSubscriptionAccountApi, googleSerpSyncApi, parseBulkUrls, rerunAiScraperApi, rerunManualScraperApi, } from "./api.js";
|
|
3
|
+
import { formatApiDate, parseStatusDate, summarizeSubscriptionAccount, } from "./status.js";
|
|
4
|
+
const jsonValueSchema = z.json();
|
|
5
|
+
const httpUrlSchema = z.url({ protocol: /^https?$/ });
|
|
6
|
+
const headersSchema = z
|
|
7
|
+
.record(z.string(), z.string())
|
|
8
|
+
.describe("Safe response headers; credentials and cookies are removed.");
|
|
9
|
+
const apiResponseSchema = z
|
|
10
|
+
.object({
|
|
11
|
+
status_code: z
|
|
12
|
+
.number()
|
|
13
|
+
.int()
|
|
14
|
+
.nullable()
|
|
15
|
+
.describe("HTTP status returned by the MrScraper service."),
|
|
16
|
+
data: jsonValueSchema.describe("Sanitized response payload returned by MrScraper."),
|
|
17
|
+
headers: headersSchema,
|
|
18
|
+
error: z.string().optional().describe("Request failure message, if any."),
|
|
19
|
+
})
|
|
20
|
+
.meta({
|
|
21
|
+
title: "MrScraper API response",
|
|
22
|
+
description: "A credential-safe response envelope from MrScraper.",
|
|
23
|
+
});
|
|
24
|
+
export const fetchOutputSchema = apiResponseSchema.meta({
|
|
25
|
+
title: "Fetch response",
|
|
26
|
+
description: "The response envelope returned by the Web Unblocker API.",
|
|
27
|
+
});
|
|
28
|
+
export const scrapeOutputSchema = apiResponseSchema.meta({
|
|
29
|
+
title: "Scrape response",
|
|
30
|
+
description: "The AI scraper response envelope. A successful run contains scraperId for reproducing the saved configuration with rerun.",
|
|
31
|
+
});
|
|
32
|
+
export const statusOutputSchema = z
|
|
33
|
+
.object({
|
|
34
|
+
kind: z.literal("mrscraper-cli-status-summary").optional(),
|
|
35
|
+
source_endpoints: z.array(z.string()).optional(),
|
|
36
|
+
status_code: z.number().int().nullable(),
|
|
37
|
+
data: jsonValueSchema,
|
|
38
|
+
headers: headersSchema.optional(),
|
|
39
|
+
error: z.string().optional(),
|
|
40
|
+
})
|
|
41
|
+
.meta({
|
|
42
|
+
title: "Status response",
|
|
43
|
+
description: "Account usage and optional domain request-outcome analytics.",
|
|
44
|
+
});
|
|
45
|
+
const serpOutputSchema = apiResponseSchema.meta({
|
|
46
|
+
title: "SERP response",
|
|
47
|
+
description: "Parsed Google results or raw result-page HTML.",
|
|
48
|
+
});
|
|
49
|
+
const rerunOutputSchema = apiResponseSchema.meta({
|
|
50
|
+
title: "Rerun response",
|
|
51
|
+
description: "The MrScraper response for a saved scraper rerun.",
|
|
52
|
+
});
|
|
53
|
+
const resultsOutputSchema = apiResponseSchema.meta({
|
|
54
|
+
title: "Results response",
|
|
55
|
+
description: "Paginated stored MrScraper results.",
|
|
56
|
+
});
|
|
57
|
+
const resultOutputSchema = apiResponseSchema.meta({
|
|
58
|
+
title: "Result response",
|
|
59
|
+
description: "One stored MrScraper result.",
|
|
60
|
+
});
|
|
61
|
+
function isApiFailure(result) {
|
|
62
|
+
return Boolean(result.error ||
|
|
63
|
+
(typeof result.status_code === "number" && result.status_code >= 400));
|
|
64
|
+
}
|
|
65
|
+
function asStructured(result) {
|
|
66
|
+
return {
|
|
67
|
+
content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
|
|
68
|
+
structuredContent: result,
|
|
69
|
+
...(isApiFailure(result) ? { isError: true } : {}),
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
function buildExtractionMessage(prompt, schemaPrompt) {
|
|
73
|
+
const instruction = prompt.trim();
|
|
74
|
+
if (!instruction) {
|
|
75
|
+
throw new Error("prompt is required for general and listing agents");
|
|
76
|
+
}
|
|
77
|
+
if (schemaPrompt === null || schemaPrompt === undefined)
|
|
78
|
+
return instruction;
|
|
79
|
+
return `${instruction}\n\nBest-effort output guidance: return JSON matching this JSON Schema. The MrScraper API does not validate this schema:\n${JSON.stringify(schemaPrompt, null, 2)}`;
|
|
80
|
+
}
|
|
81
|
+
function unwrapApiData(response) {
|
|
82
|
+
const body = response.data;
|
|
83
|
+
if (body &&
|
|
84
|
+
typeof body === "object" &&
|
|
85
|
+
!Array.isArray(body) &&
|
|
86
|
+
"data" in body) {
|
|
87
|
+
return body.data;
|
|
88
|
+
}
|
|
89
|
+
return body;
|
|
90
|
+
}
|
|
91
|
+
function normalizeDomain(value) {
|
|
92
|
+
const candidate = value.trim();
|
|
93
|
+
if (!candidate)
|
|
94
|
+
throw new Error("domain must not be empty");
|
|
95
|
+
try {
|
|
96
|
+
return new URL(/^https?:\/\//i.test(candidate) ? candidate : `https://${candidate}`).hostname;
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
throw new Error(`Invalid domain: ${value}`);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
export const fetchInputSchema = z.object({
|
|
103
|
+
url: httpUrlSchema.describe("Required absolute HTTP or HTTPS URL of the known page to retrieve through Web Unblocker."),
|
|
104
|
+
browser_rendering: z
|
|
105
|
+
.boolean()
|
|
106
|
+
.default(false)
|
|
107
|
+
.describe("Execute page JavaScript in a browser. Enable this for client-rendered content or before using wait_for_selector."),
|
|
108
|
+
geo_code: z
|
|
109
|
+
.string()
|
|
110
|
+
.nullable()
|
|
111
|
+
.optional()
|
|
112
|
+
.describe("Optional proxy-routing country code sent as geoCode when the page must be viewed from a specific location."),
|
|
113
|
+
wait_for_selector: z
|
|
114
|
+
.string()
|
|
115
|
+
.nullable()
|
|
116
|
+
.optional()
|
|
117
|
+
.describe("CSS selector to wait for before returning the page. Requires browser_rendering=true and is useful for content that appears after JavaScript runs."),
|
|
118
|
+
home_page: z
|
|
119
|
+
.boolean()
|
|
120
|
+
.default(false)
|
|
121
|
+
.describe("Visit the site's root page before the target URL, which can help establish cookies or session state."),
|
|
122
|
+
block_resources: z
|
|
123
|
+
.boolean()
|
|
124
|
+
.default(false)
|
|
125
|
+
.describe("Ask Web Unblocker to block nonessential page resources during loading when only page content is needed."),
|
|
126
|
+
max_retries: z
|
|
127
|
+
.number()
|
|
128
|
+
.int()
|
|
129
|
+
.min(0)
|
|
130
|
+
.default(3)
|
|
131
|
+
.describe("Maximum Web Unblocker retry count. Use 0 to disable retries; the default is 3."),
|
|
132
|
+
token_cap: z
|
|
133
|
+
.number()
|
|
134
|
+
.int()
|
|
135
|
+
.positive()
|
|
136
|
+
.nullable()
|
|
137
|
+
.optional()
|
|
138
|
+
.describe("Optional maximum retry-token budget consumed across this fetch request's attempts."),
|
|
139
|
+
timeout: z
|
|
140
|
+
.number()
|
|
141
|
+
.int()
|
|
142
|
+
.positive()
|
|
143
|
+
.default(30)
|
|
144
|
+
.describe("API page-load timeout in seconds; the MCP server allows 30 seconds more for transport."),
|
|
145
|
+
});
|
|
146
|
+
export async function fetchTool(token, input, dependencies = {}) {
|
|
147
|
+
if (input.wait_for_selector && !input.browser_rendering) {
|
|
148
|
+
throw new Error("wait_for_selector requires browser_rendering");
|
|
149
|
+
}
|
|
150
|
+
return fetchContentApi({
|
|
151
|
+
token,
|
|
152
|
+
url: input.url,
|
|
153
|
+
browserRendering: input.browser_rendering,
|
|
154
|
+
timeout: input.timeout,
|
|
155
|
+
geoCode: input.geo_code ?? null,
|
|
156
|
+
waitForSelector: input.wait_for_selector ?? null,
|
|
157
|
+
homePage: input.home_page,
|
|
158
|
+
blockResources: input.block_resources,
|
|
159
|
+
maxRetries: input.max_retries,
|
|
160
|
+
tokenCap: input.token_cap ?? null,
|
|
161
|
+
fetchFn: dependencies.fetchFn,
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
export const scrapeInputSchema = z.object({
|
|
165
|
+
url: httpUrlSchema.describe("Required absolute HTTP or HTTPS starting URL for the AI scraper."),
|
|
166
|
+
prompt: z
|
|
167
|
+
.string()
|
|
168
|
+
.nullable()
|
|
169
|
+
.optional()
|
|
170
|
+
.describe("Natural-language description of the fields or repeated records to extract. Required for general and listing; not accepted by map."),
|
|
171
|
+
schema_prompt: z
|
|
172
|
+
.record(z.string(), jsonValueSchema)
|
|
173
|
+
.nullable()
|
|
174
|
+
.optional()
|
|
175
|
+
.describe("Optional JSON Schema appended to prompt as best-effort output-shape guidance for general or listing extraction."),
|
|
176
|
+
agent: z
|
|
177
|
+
.enum(["general", "listing", "map"])
|
|
178
|
+
.default("general")
|
|
179
|
+
.describe("Extraction mode: general for defined page fields, listing for repeated records across pages, or map for discovering site URLs."),
|
|
180
|
+
proxy_country: z
|
|
181
|
+
.string()
|
|
182
|
+
.nullable()
|
|
183
|
+
.optional()
|
|
184
|
+
.describe("Optional proxy country for general or listing extraction when content varies by location; not accepted by map."),
|
|
185
|
+
max_pages: z
|
|
186
|
+
.number()
|
|
187
|
+
.int()
|
|
188
|
+
.positive()
|
|
189
|
+
.nullable()
|
|
190
|
+
.optional()
|
|
191
|
+
.describe("Maximum pages processed by listing or map. Not accepted by general; omit it to use the service default."),
|
|
192
|
+
max_depth: z
|
|
193
|
+
.number()
|
|
194
|
+
.int()
|
|
195
|
+
.positive()
|
|
196
|
+
.nullable()
|
|
197
|
+
.optional()
|
|
198
|
+
.describe("Maximum link depth followed by the map agent; not accepted by general or listing."),
|
|
199
|
+
limit: z
|
|
200
|
+
.number()
|
|
201
|
+
.int()
|
|
202
|
+
.positive()
|
|
203
|
+
.nullable()
|
|
204
|
+
.optional()
|
|
205
|
+
.describe("Maximum number of discovered URLs returned by the map agent; not accepted by general or listing."),
|
|
206
|
+
include_patterns: z
|
|
207
|
+
.string()
|
|
208
|
+
.nullable()
|
|
209
|
+
.optional()
|
|
210
|
+
.describe("Regular expression selecting URLs the map agent may include; not accepted by general or listing."),
|
|
211
|
+
exclude_patterns: z
|
|
212
|
+
.string()
|
|
213
|
+
.nullable()
|
|
214
|
+
.optional()
|
|
215
|
+
.describe("Regular expression removing matching URLs from map-agent results; not accepted by general or listing."),
|
|
216
|
+
});
|
|
217
|
+
export async function scrapeTool(token, input, dependencies = {}) {
|
|
218
|
+
const supplied = (value) => value !== undefined && value !== null;
|
|
219
|
+
const agent = input.agent;
|
|
220
|
+
const mapOnlyOptions = [
|
|
221
|
+
["max_depth", input.max_depth],
|
|
222
|
+
["limit", input.limit],
|
|
223
|
+
["include_patterns", input.include_patterns],
|
|
224
|
+
["exclude_patterns", input.exclude_patterns],
|
|
225
|
+
];
|
|
226
|
+
if (agent === "map") {
|
|
227
|
+
if (supplied(input.prompt)) {
|
|
228
|
+
throw new Error("prompt is not accepted by the map agent");
|
|
229
|
+
}
|
|
230
|
+
if (supplied(input.schema_prompt)) {
|
|
231
|
+
throw new Error("schema_prompt is not accepted by the map agent");
|
|
232
|
+
}
|
|
233
|
+
if (supplied(input.proxy_country)) {
|
|
234
|
+
throw new Error("proxy_country is not accepted by the map agent");
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
else {
|
|
238
|
+
if (!input.prompt?.trim()) {
|
|
239
|
+
throw new Error("prompt is required for general and listing agents");
|
|
240
|
+
}
|
|
241
|
+
const invalidMapOptions = mapOnlyOptions
|
|
242
|
+
.filter(([, value]) => supplied(value))
|
|
243
|
+
.map(([name]) => name);
|
|
244
|
+
if (invalidMapOptions.length) {
|
|
245
|
+
throw new Error(`${invalidMapOptions.join(", ")} ${invalidMapOptions.length === 1 ? "is" : "are"} only accepted by the map agent`);
|
|
246
|
+
}
|
|
247
|
+
if (agent === "general" && supplied(input.max_pages)) {
|
|
248
|
+
throw new Error("max_pages is only accepted by listing and map agents");
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
return createAiScraperApi({
|
|
252
|
+
token,
|
|
253
|
+
url: input.url,
|
|
254
|
+
message: agent === "map"
|
|
255
|
+
? undefined
|
|
256
|
+
: buildExtractionMessage(input.prompt, input.schema_prompt),
|
|
257
|
+
agent,
|
|
258
|
+
proxyCountry: input.proxy_country ?? null,
|
|
259
|
+
maxPages: input.max_pages ?? undefined,
|
|
260
|
+
maxDepth: input.max_depth ?? undefined,
|
|
261
|
+
limit: input.limit ?? undefined,
|
|
262
|
+
includePatterns: input.include_patterns ?? undefined,
|
|
263
|
+
excludePatterns: input.exclude_patterns ?? undefined,
|
|
264
|
+
fetchFn: dependencies.fetchFn,
|
|
265
|
+
});
|
|
266
|
+
}
|
|
267
|
+
export const serpInputSchema = z.object({
|
|
268
|
+
query_or_url: z
|
|
269
|
+
.string()
|
|
270
|
+
.min(1)
|
|
271
|
+
.describe("Required Google search query text or complete Google search URL from which to retrieve results."),
|
|
272
|
+
region: z
|
|
273
|
+
.string()
|
|
274
|
+
.nullable()
|
|
275
|
+
.optional()
|
|
276
|
+
.describe("Optional country code used to localize the Google results returned by the service."),
|
|
277
|
+
language: z
|
|
278
|
+
.string()
|
|
279
|
+
.nullable()
|
|
280
|
+
.optional()
|
|
281
|
+
.describe("Optional language code used to localize the Google results returned by the service."),
|
|
282
|
+
page: z
|
|
283
|
+
.number()
|
|
284
|
+
.int()
|
|
285
|
+
.positive()
|
|
286
|
+
.nullable()
|
|
287
|
+
.optional()
|
|
288
|
+
.describe("Optional one-based Google results page number for retrieving later result pages."),
|
|
289
|
+
format: z
|
|
290
|
+
.enum(["json", "html"])
|
|
291
|
+
.default("json")
|
|
292
|
+
.describe("Response representation: json returns parsed search results; html returns the result page's raw HTML."),
|
|
293
|
+
render_js: z
|
|
294
|
+
.boolean()
|
|
295
|
+
.default(false)
|
|
296
|
+
.describe("Render the Google results page with JavaScript when dynamic features such as AI Overview are needed."),
|
|
297
|
+
raw: z
|
|
298
|
+
.boolean()
|
|
299
|
+
.default(false)
|
|
300
|
+
.describe("Deprecated alias for format=html; takes precedence over format."),
|
|
301
|
+
client_timeout: z
|
|
302
|
+
.number()
|
|
303
|
+
.int()
|
|
304
|
+
.positive()
|
|
305
|
+
.default(120)
|
|
306
|
+
.describe("Local upstream HTTP timeout in seconds; it is not included in the SERP request body."),
|
|
307
|
+
});
|
|
308
|
+
export async function serpTool(token, input, dependencies = {}) {
|
|
309
|
+
return googleSerpSyncApi({
|
|
310
|
+
token,
|
|
311
|
+
queryOrUrl: input.query_or_url,
|
|
312
|
+
region: input.region ?? null,
|
|
313
|
+
language: input.language ?? null,
|
|
314
|
+
page: input.page ?? null,
|
|
315
|
+
format: input.format,
|
|
316
|
+
renderJs: input.render_js,
|
|
317
|
+
raw: input.raw,
|
|
318
|
+
timeout: input.client_timeout,
|
|
319
|
+
fetchFn: dependencies.fetchFn,
|
|
320
|
+
});
|
|
321
|
+
}
|
|
322
|
+
export const statusInputSchema = z.object({
|
|
323
|
+
domain: z
|
|
324
|
+
.string()
|
|
325
|
+
.nullable()
|
|
326
|
+
.optional()
|
|
327
|
+
.describe("Optional hostname or URL. When supplied, status includes request-outcome analytics for its normalized hostname; omit it for account usage only."),
|
|
328
|
+
from: z
|
|
329
|
+
.string()
|
|
330
|
+
.default("24h")
|
|
331
|
+
.describe("Analytics range start as ISO 8601, now, or a relative duration such as 24h or 7d. Used only when domain is supplied."),
|
|
332
|
+
to: z
|
|
333
|
+
.string()
|
|
334
|
+
.default("now")
|
|
335
|
+
.describe("Analytics range end as ISO 8601, now, or a relative duration. Used only when domain is supplied."),
|
|
336
|
+
action: z
|
|
337
|
+
.string()
|
|
338
|
+
.nullable()
|
|
339
|
+
.optional()
|
|
340
|
+
.describe("Optional exact MrScraper action filter applied to domain analytics; used only when domain is supplied."),
|
|
341
|
+
api_token_name: z
|
|
342
|
+
.string()
|
|
343
|
+
.nullable()
|
|
344
|
+
.optional()
|
|
345
|
+
.describe("Optional API token name used to filter domain analytics; used only when domain is supplied."),
|
|
346
|
+
});
|
|
347
|
+
export async function statusTool(token, input, dependencies = {}) {
|
|
348
|
+
const accountResponse = await getSubscriptionAccountApi(token, dependencies);
|
|
349
|
+
if (isApiFailure(accountResponse))
|
|
350
|
+
return accountResponse;
|
|
351
|
+
const account = unwrapApiData(accountResponse);
|
|
352
|
+
const output = {
|
|
353
|
+
kind: "mrscraper-cli-status-summary",
|
|
354
|
+
source_endpoints: ["/subscription-accounts"],
|
|
355
|
+
status_code: accountResponse.status_code,
|
|
356
|
+
data: {
|
|
357
|
+
account: summarizeSubscriptionAccount(account && typeof account === "object" && !Array.isArray(account)
|
|
358
|
+
? account
|
|
359
|
+
: {}),
|
|
360
|
+
},
|
|
361
|
+
};
|
|
362
|
+
if (!input.domain)
|
|
363
|
+
return output;
|
|
364
|
+
output.source_endpoints.push("/analytic/statuses");
|
|
365
|
+
const domain = normalizeDomain(input.domain);
|
|
366
|
+
const now = dependencies.now?.() || new Date();
|
|
367
|
+
const end = parseStatusDate(input.to, now, "now");
|
|
368
|
+
const start = parseStatusDate(input.from, end, "24h");
|
|
369
|
+
if (start >= end)
|
|
370
|
+
throw new Error("from must be earlier than to");
|
|
371
|
+
const startDate = formatApiDate(start);
|
|
372
|
+
const endDate = formatApiDate(end);
|
|
373
|
+
const analyticsResponse = await getAnalyticStatusesApi({
|
|
374
|
+
token,
|
|
375
|
+
domain,
|
|
376
|
+
startDate,
|
|
377
|
+
endDate,
|
|
378
|
+
action: input.action || "",
|
|
379
|
+
apiTokenName: input.api_token_name || "",
|
|
380
|
+
fetchFn: dependencies.fetchFn,
|
|
381
|
+
});
|
|
382
|
+
const data = output.data;
|
|
383
|
+
if (analyticsResponse.error) {
|
|
384
|
+
output.error = "Account loaded, but analytics could not be loaded";
|
|
385
|
+
data.analytics = analyticsResponse;
|
|
386
|
+
}
|
|
387
|
+
else {
|
|
388
|
+
const analytics = unwrapApiData(analyticsResponse);
|
|
389
|
+
data.analytics = {
|
|
390
|
+
domain,
|
|
391
|
+
from: `${startDate} UTC`,
|
|
392
|
+
to: `${endDate} UTC`,
|
|
393
|
+
...(analytics &&
|
|
394
|
+
typeof analytics === "object" &&
|
|
395
|
+
!Array.isArray(analytics)
|
|
396
|
+
? analytics
|
|
397
|
+
: { data: analytics }),
|
|
398
|
+
};
|
|
399
|
+
}
|
|
400
|
+
return output;
|
|
401
|
+
}
|
|
402
|
+
export const rerunInputSchema = z.object({
|
|
403
|
+
target: z
|
|
404
|
+
.string()
|
|
405
|
+
.describe("Target URL for a single rerun, or comma/newline-separated target URLs when bulk=true."),
|
|
406
|
+
type: z
|
|
407
|
+
.enum(["ai", "manual"])
|
|
408
|
+
.describe("How the saved scraper was created: ai for an AI scraper or manual for a dashboard-built step workflow."),
|
|
409
|
+
bulk: z
|
|
410
|
+
.boolean()
|
|
411
|
+
.default(false)
|
|
412
|
+
.describe("Independently select target count: false for one URL or true to submit all parsed targets as one asynchronous bulk job."),
|
|
413
|
+
scraper_id: z
|
|
414
|
+
.string()
|
|
415
|
+
.nullable()
|
|
416
|
+
.optional()
|
|
417
|
+
.describe("Required saved scraper UUID when bulk=false. Use the scraperId returned by scrape for an AI scraper, or a dashboard workflow's UUID for manual."),
|
|
418
|
+
id: z
|
|
419
|
+
.string()
|
|
420
|
+
.nullable()
|
|
421
|
+
.optional()
|
|
422
|
+
.describe("Required saved scraper UUID when bulk=true. This identifies the configuration to run and is not a result ID."),
|
|
423
|
+
max_depth: z
|
|
424
|
+
.number()
|
|
425
|
+
.int()
|
|
426
|
+
.positive()
|
|
427
|
+
.nullable()
|
|
428
|
+
.optional()
|
|
429
|
+
.describe("Maximum crawl depth for a single AI rerun; defaults to 2 when omitted."),
|
|
430
|
+
max_pages: z
|
|
431
|
+
.number()
|
|
432
|
+
.int()
|
|
433
|
+
.positive()
|
|
434
|
+
.nullable()
|
|
435
|
+
.optional()
|
|
436
|
+
.describe("Maximum pages for a single AI rerun; defaults to 50 when omitted."),
|
|
437
|
+
limit: z
|
|
438
|
+
.number()
|
|
439
|
+
.int()
|
|
440
|
+
.positive()
|
|
441
|
+
.nullable()
|
|
442
|
+
.optional()
|
|
443
|
+
.describe("Maximum results for a single AI rerun; defaults to 1000 when omitted."),
|
|
444
|
+
include_patterns: z
|
|
445
|
+
.string()
|
|
446
|
+
.nullable()
|
|
447
|
+
.optional()
|
|
448
|
+
.describe("URL include regular expression for a single AI rerun; defaults to an empty string."),
|
|
449
|
+
exclude_patterns: z
|
|
450
|
+
.string()
|
|
451
|
+
.nullable()
|
|
452
|
+
.optional()
|
|
453
|
+
.describe("URL exclude regular expression for a single AI rerun; defaults to an empty string."),
|
|
454
|
+
});
|
|
455
|
+
export async function rerunTool(token, input, dependencies = {}) {
|
|
456
|
+
const aiOptionEntries = [
|
|
457
|
+
["max_depth", input.max_depth],
|
|
458
|
+
["max_pages", input.max_pages],
|
|
459
|
+
["limit", input.limit],
|
|
460
|
+
["include_patterns", input.include_patterns],
|
|
461
|
+
["exclude_patterns", input.exclude_patterns],
|
|
462
|
+
];
|
|
463
|
+
const explicitAiOptions = aiOptionEntries
|
|
464
|
+
.filter(([, value]) => value !== undefined && value !== null)
|
|
465
|
+
.map(([name]) => name);
|
|
466
|
+
let response;
|
|
467
|
+
if (input.bulk) {
|
|
468
|
+
if (!input.id)
|
|
469
|
+
throw new Error("id is required when bulk is true");
|
|
470
|
+
if (input.scraper_id !== undefined && input.scraper_id !== null) {
|
|
471
|
+
throw new Error("scraper_id is only accepted for single reruns; use id when bulk is true");
|
|
472
|
+
}
|
|
473
|
+
if (explicitAiOptions.length) {
|
|
474
|
+
throw new Error(`${explicitAiOptions.join(", ")} ${explicitAiOptions.length === 1 ? "is" : "are"} not accepted by bulk rerun endpoints`);
|
|
475
|
+
}
|
|
476
|
+
const urls = parseBulkUrls(input.target);
|
|
477
|
+
if (!urls.length)
|
|
478
|
+
throw new Error("No URLs found in the bulk target");
|
|
479
|
+
response =
|
|
480
|
+
input.type === "ai"
|
|
481
|
+
? await bulkRerunAiScraperApi({
|
|
482
|
+
token,
|
|
483
|
+
scraperId: input.id,
|
|
484
|
+
urls,
|
|
485
|
+
fetchFn: dependencies.fetchFn,
|
|
486
|
+
})
|
|
487
|
+
: await bulkRerunManualScraperApi({
|
|
488
|
+
token,
|
|
489
|
+
scraperId: input.id,
|
|
490
|
+
urls,
|
|
491
|
+
fetchFn: dependencies.fetchFn,
|
|
492
|
+
});
|
|
493
|
+
}
|
|
494
|
+
else {
|
|
495
|
+
if (!input.scraper_id) {
|
|
496
|
+
throw new Error("scraper_id is required unless bulk is true");
|
|
497
|
+
}
|
|
498
|
+
if (input.id !== undefined && input.id !== null) {
|
|
499
|
+
throw new Error("id is only accepted when bulk is true; use scraper_id for a single rerun");
|
|
500
|
+
}
|
|
501
|
+
if (input.type === "manual" && explicitAiOptions.length) {
|
|
502
|
+
throw new Error(`${explicitAiOptions.join(", ")} ${explicitAiOptions.length === 1 ? "is" : "are"} only accepted by single AI reruns`);
|
|
503
|
+
}
|
|
504
|
+
const url = input.target.trim();
|
|
505
|
+
if (!url)
|
|
506
|
+
throw new Error("target URL must not be empty");
|
|
507
|
+
response =
|
|
508
|
+
input.type === "manual"
|
|
509
|
+
? await rerunManualScraperApi({
|
|
510
|
+
token,
|
|
511
|
+
scraperId: input.scraper_id,
|
|
512
|
+
url,
|
|
513
|
+
fetchFn: dependencies.fetchFn,
|
|
514
|
+
})
|
|
515
|
+
: await rerunAiScraperApi({
|
|
516
|
+
token,
|
|
517
|
+
scraperId: input.scraper_id,
|
|
518
|
+
url,
|
|
519
|
+
maxDepth: input.max_depth ?? 2,
|
|
520
|
+
maxPages: input.max_pages ?? 50,
|
|
521
|
+
limit: input.limit ?? 1000,
|
|
522
|
+
includePatterns: input.include_patterns ?? "",
|
|
523
|
+
excludePatterns: input.exclude_patterns ?? "",
|
|
524
|
+
fetchFn: dependencies.fetchFn,
|
|
525
|
+
});
|
|
526
|
+
}
|
|
527
|
+
return response;
|
|
528
|
+
}
|
|
529
|
+
export const resultsInputSchema = z.object({
|
|
530
|
+
sort_field: z
|
|
531
|
+
.string()
|
|
532
|
+
.min(1)
|
|
533
|
+
.default("updatedAt")
|
|
534
|
+
.describe("Stored-result field used as the sort key; defaults to updatedAt."),
|
|
535
|
+
sort_order: z
|
|
536
|
+
.string()
|
|
537
|
+
.trim()
|
|
538
|
+
.toLowerCase()
|
|
539
|
+
.pipe(z.enum(["asc", "desc"]))
|
|
540
|
+
.default("desc")
|
|
541
|
+
.describe("Sort direction for sort_field: asc or desc, accepted case-insensitively; defaults to desc."),
|
|
542
|
+
page_size: z
|
|
543
|
+
.number()
|
|
544
|
+
.int()
|
|
545
|
+
.positive()
|
|
546
|
+
.default(10)
|
|
547
|
+
.describe("Positive number of stored result records requested per page; defaults to 10."),
|
|
548
|
+
page: z
|
|
549
|
+
.number()
|
|
550
|
+
.int()
|
|
551
|
+
.positive()
|
|
552
|
+
.default(1)
|
|
553
|
+
.describe("One-based page number used to move through the stored result list; defaults to 1."),
|
|
554
|
+
search: z
|
|
555
|
+
.string()
|
|
556
|
+
.nullable()
|
|
557
|
+
.optional()
|
|
558
|
+
.describe("Optional free-text filter used to narrow the stored result list."),
|
|
559
|
+
date_range_column: z
|
|
560
|
+
.string()
|
|
561
|
+
.nullable()
|
|
562
|
+
.optional()
|
|
563
|
+
.describe("Stored-result date column to which start_at and end_at are applied."),
|
|
564
|
+
start_at: z
|
|
565
|
+
.string()
|
|
566
|
+
.nullable()
|
|
567
|
+
.optional()
|
|
568
|
+
.describe("Optional inclusive ISO 8601 start bound applied to date_range_column."),
|
|
569
|
+
end_at: z
|
|
570
|
+
.string()
|
|
571
|
+
.nullable()
|
|
572
|
+
.optional()
|
|
573
|
+
.describe("Optional inclusive ISO 8601 end bound applied to date_range_column."),
|
|
574
|
+
});
|
|
575
|
+
export async function resultsTool(token, input, dependencies = {}) {
|
|
576
|
+
return getAllResultsApi({
|
|
577
|
+
token,
|
|
578
|
+
sortField: input.sort_field,
|
|
579
|
+
sortOrder: input.sort_order.toUpperCase(),
|
|
580
|
+
pageSize: input.page_size,
|
|
581
|
+
page: input.page,
|
|
582
|
+
search: input.search ?? null,
|
|
583
|
+
dateRangeColumn: input.date_range_column ?? null,
|
|
584
|
+
startAt: input.start_at ?? null,
|
|
585
|
+
endAt: input.end_at ?? null,
|
|
586
|
+
fetchFn: dependencies.fetchFn,
|
|
587
|
+
});
|
|
588
|
+
}
|
|
589
|
+
export const resultInputSchema = z.object({
|
|
590
|
+
result_id: z
|
|
591
|
+
.string()
|
|
592
|
+
.min(1)
|
|
593
|
+
.describe("Required UUID of the stored result to retrieve, including a bulkResultId returned by an asynchronous bulk rerun."),
|
|
594
|
+
});
|
|
595
|
+
export async function resultTool(token, input, dependencies = {}) {
|
|
596
|
+
const resultId = input.result_id.trim();
|
|
597
|
+
if (!resultId)
|
|
598
|
+
throw new Error("result_id must not be empty");
|
|
599
|
+
return getResultByIdApi(token, resultId, dependencies);
|
|
600
|
+
}
|
|
601
|
+
const readAnnotations = {
|
|
602
|
+
readOnlyHint: true,
|
|
603
|
+
openWorldHint: true,
|
|
604
|
+
destructiveHint: false,
|
|
605
|
+
};
|
|
606
|
+
const writeAnnotations = {
|
|
607
|
+
readOnlyHint: false,
|
|
608
|
+
openWorldHint: true,
|
|
609
|
+
destructiveHint: false,
|
|
610
|
+
};
|
|
611
|
+
export const TOOL_DESCRIPTIONS = {
|
|
612
|
+
fetch: "Use this when you already know one page URL and need its page response through Web Unblocker, including pages that reject ordinary HTTP requests. Use scrape instead for AI-extracted fields or records, and serp when you first need to discover URLs with Google. The url argument is required. Enable browser_rendering for JavaScript-driven content; wait_for_selector then waits for a CSS selector. geo_code changes proxy location, while home_page, block_resources, max_retries, token_cap, and timeout tune loading and retry behavior.",
|
|
613
|
+
scrape: "Use this when you know a starting URL and need AI-extracted fields, repeated listing records, or a map of site URLs. Choose agent=general for defined fields on a page, agent=listing for repeated records across pages, or agent=map for URL discovery. The general and listing modes require prompt and optionally accept schema_prompt guidance and proxy_country; max_pages applies to listing and map, while max_depth, limit, include_patterns, and exclude_patterns are map-only. A successful run creates a saved scraper and returns scraperId, which rerun can reuse on the same or another URL. Use fetch instead when the page response itself is sufficient.",
|
|
614
|
+
serp: "Use this when the task starts with a Google query or Google search URL and you need to discover relevant result pages before fetching or extracting them. The query_or_url argument is required; region, language, and page control localization and pagination. format=json returns parsed results, format=html returns result-page HTML, render_js includes dynamic features, and client_timeout only controls how long this MCP request waits. raw is a deprecated alias for format=html. Follow useful result URLs with fetch or scrape.",
|
|
615
|
+
status: "Use this to check the current MrScraper account's subscription and usage before or after web-data work. With no arguments it returns the account summary. Supply domain only when request-outcome analytics are also needed; from and to select that analytics window, while action and api_token_name narrow those domain outcomes. This tool reports account and request health, not scrape-job progress; use result for a known asynchronous result ID.",
|
|
616
|
+
rerun: "Use this only when you already have a saved scraper UUID and want to apply that configuration to the same or new target URLs. Set type=ai for a scraper created by scrape; type=manual selects a dashboard-built step workflow and requires the conversation's compliance acknowledgment. For one URL, leave bulk=false and pass target with scraper_id. A single AI rerun also accepts max_depth, max_pages, limit, include_patterns, and exclude_patterns; manual and bulk reruns reject those controls. For multiple URLs, set bulk=true, pass comma- or newline-separated target URLs and id; bulk jobs are asynchronous, so retain bulkResultId and inspect it with result until completion. Use scrape to create a new AI configuration.",
|
|
617
|
+
results: "Use this to browse or locate stored scrape results when you do not yet know the exact result UUID. sort_field and sort_order control ordering; page_size and page control pagination; search narrows the list; and date_range_column with start_at or end_at applies a date window. This returns a result list rather than one complete record. Once an ID is known, use result for that record.",
|
|
618
|
+
result: "Use this when you already know the exact stored result UUID and need the complete record or current job state. Pass that UUID as result_id; this may also be the bulkResultId returned by an asynchronous bulk rerun. Call result again as needed until an asynchronous job completes. Use results first when the UUID is unknown.",
|
|
619
|
+
};
|
|
620
|
+
export function registerTools(server, getToken, dependencies = {}) {
|
|
621
|
+
server.registerTool("fetch", {
|
|
622
|
+
description: TOOL_DESCRIPTIONS.fetch,
|
|
623
|
+
inputSchema: fetchInputSchema,
|
|
624
|
+
outputSchema: fetchOutputSchema,
|
|
625
|
+
annotations: readAnnotations,
|
|
626
|
+
}, async (input) => asStructured(await fetchTool(getToken(), input, dependencies)));
|
|
627
|
+
server.registerTool("scrape", {
|
|
628
|
+
description: TOOL_DESCRIPTIONS.scrape,
|
|
629
|
+
inputSchema: scrapeInputSchema,
|
|
630
|
+
outputSchema: scrapeOutputSchema,
|
|
631
|
+
annotations: writeAnnotations,
|
|
632
|
+
}, async (input) => asStructured(await scrapeTool(getToken(), input, dependencies)));
|
|
633
|
+
server.registerTool("serp", {
|
|
634
|
+
description: TOOL_DESCRIPTIONS.serp,
|
|
635
|
+
inputSchema: serpInputSchema,
|
|
636
|
+
outputSchema: serpOutputSchema,
|
|
637
|
+
annotations: readAnnotations,
|
|
638
|
+
}, async (input) => asStructured(await serpTool(getToken(), input, dependencies)));
|
|
639
|
+
server.registerTool("status", {
|
|
640
|
+
description: TOOL_DESCRIPTIONS.status,
|
|
641
|
+
inputSchema: statusInputSchema,
|
|
642
|
+
outputSchema: statusOutputSchema,
|
|
643
|
+
annotations: readAnnotations,
|
|
644
|
+
}, async (input) => asStructured(await statusTool(getToken(), input, dependencies)));
|
|
645
|
+
server.registerTool("rerun", {
|
|
646
|
+
description: TOOL_DESCRIPTIONS.rerun,
|
|
647
|
+
inputSchema: rerunInputSchema,
|
|
648
|
+
outputSchema: rerunOutputSchema,
|
|
649
|
+
annotations: writeAnnotations,
|
|
650
|
+
}, async (input) => asStructured(await rerunTool(getToken(), input, dependencies)));
|
|
651
|
+
server.registerTool("results", {
|
|
652
|
+
description: TOOL_DESCRIPTIONS.results,
|
|
653
|
+
inputSchema: resultsInputSchema,
|
|
654
|
+
outputSchema: resultsOutputSchema,
|
|
655
|
+
annotations: readAnnotations,
|
|
656
|
+
}, async (input) => asStructured(await resultsTool(getToken(), input, dependencies)));
|
|
657
|
+
server.registerTool("result", {
|
|
658
|
+
description: TOOL_DESCRIPTIONS.result,
|
|
659
|
+
inputSchema: resultInputSchema,
|
|
660
|
+
outputSchema: resultOutputSchema,
|
|
661
|
+
annotations: readAnnotations,
|
|
662
|
+
}, async (input) => asStructured(await resultTool(getToken(), input, dependencies)));
|
|
663
|
+
}
|
|
664
|
+
//# sourceMappingURL=tools.js.map
|