@mrscraper/mcp 0.1.3 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/tools.js CHANGED
@@ -1,745 +1,292 @@
1
1
  import { z } from "zod";
2
- import { bulkRerunAiScraperApi, bulkRerunManualScraperApi, createAiScraperApi, fetchContentApi, getAllResultsApi, getAnalyticStatusesApi, getResultByIdApi, getSubscriptionAccountApi, googleSerpSyncApi, parseBulkUrls, rerunAiScraperApi, rerunManualScraperApi, } from "./api.js";
2
+ import { getSubscriptionAccountApi, scraperApi, webSearchApi } from "./api.js";
3
+ import { summarizeSubscriptionAccount } from "./status.js";
3
4
  import { widgetMeta } from "./widgets/index.js";
4
- import { formatApiDate, parseStatusDate, summarizeSubscriptionAccount, } from "./status.js";
5
- const jsonValueSchema = z.json();
6
- const httpUrlSchema = z.url({ protocol: /^https?$/ });
7
- const headersSchema = z
8
- .record(z.string(), z.string())
9
- .describe("Safe response headers; credentials and cookies are removed.");
10
- const apiResponseSchema = z
11
- .object({
5
+ const apiResponseSchema = z.object({
12
6
  status_code: z
13
7
  .number()
14
8
  .int()
15
9
  .nullable()
16
- .describe("HTTP status returned by the MrScraper service."),
17
- data: jsonValueSchema.describe("Sanitized response payload returned by MrScraper."),
18
- headers: headersSchema,
10
+ .describe("Upstream HTTP status, or null for a network failure."),
11
+ data: z.json().describe("Response body as parsed JSON or text."),
12
+ headers: z
13
+ .record(z.string(), z.string())
14
+ .describe("Response headers with credentials and cookies removed."),
19
15
  error: z.string().optional().describe("Request failure message, if any."),
20
- })
21
- .meta({
22
- title: "MrScraper API response",
23
- description: "A credential-safe response envelope from MrScraper.",
24
16
  });
25
- export const fetchOutputSchema = apiResponseSchema.meta({
26
- title: "Fetch response",
27
- description: "The response envelope returned by the Web Unblocker API.",
17
+ export const scraperAPIOutputSchema = apiResponseSchema.meta({
18
+ title: "Scraper API response",
19
+ description: "Page content as HTML, Markdown, a screenshot, or a combination.",
28
20
  });
29
- export const scrapeOutputSchema = apiResponseSchema.meta({
30
- title: "Scrape response",
31
- description: "The AI scraper response envelope. A successful run contains scraperId for reproducing the saved configuration with rerun.",
21
+ export const webSearchOutputSchema = apiResponseSchema.meta({
22
+ title: "Web search response",
23
+ description: "Parsed Google search results or result-page HTML.",
32
24
  });
33
25
  export const statusOutputSchema = z
34
26
  .object({
35
27
  kind: z.literal("mrscraper-cli-status-summary").optional(),
36
28
  source_endpoints: z.array(z.string()).optional(),
37
29
  status_code: z.number().int().nullable(),
38
- data: jsonValueSchema,
39
- headers: headersSchema.optional(),
30
+ data: z.json(),
31
+ headers: z.record(z.string(), z.string()).optional(),
40
32
  error: z.string().optional(),
41
33
  })
42
34
  .meta({
43
35
  title: "Status response",
44
- description: "Account identity, subscription usage, and optional domain request-outcome analytics.",
45
- });
46
- const serpOutputSchema = apiResponseSchema.meta({
47
- title: "SERP response",
48
- description: "Parsed Google results or raw result-page HTML.",
36
+ description: "Account identity, subscription, token limit, token usage, and remaining tokens.",
49
37
  });
50
- const rerunOutputSchema = apiResponseSchema.meta({
51
- title: "Rerun response",
52
- description: "The MrScraper response for a saved scraper rerun.",
53
- });
54
- const resultsOutputSchema = apiResponseSchema.meta({
55
- title: "Results response",
56
- description: "Paginated stored MrScraper results.",
57
- });
58
- const resultOutputSchema = apiResponseSchema.meta({
59
- title: "Result response",
60
- description: "One stored MrScraper result.",
61
- });
62
- function isApiFailure(result) {
63
- return Boolean(result.error ||
64
- (typeof result.status_code === "number" && result.status_code >= 400));
65
- }
66
- function asStructured(result) {
67
- return {
68
- content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
69
- structuredContent: result,
70
- ...(isApiFailure(result) ? { isError: true } : {}),
71
- };
72
- }
73
- function buildExtractionMessage(prompt, schemaPrompt) {
74
- const instruction = prompt.trim();
75
- if (!instruction) {
76
- throw new Error("prompt is required for general and listing agents");
77
- }
78
- if (schemaPrompt === null || schemaPrompt === undefined)
79
- return instruction;
80
- return `${instruction}\n\nBest-effort output guidance: return JSON matching this JSON Schema. The MrScraper API does not validate this schema:\n${JSON.stringify(schemaPrompt, null, 2)}`;
81
- }
82
- function unwrapApiData(response) {
83
- const body = response.data;
84
- if (body &&
85
- typeof body === "object" &&
86
- !Array.isArray(body) &&
87
- "data" in body) {
88
- return body.data;
89
- }
90
- return body;
91
- }
92
- function normalizeDomain(value) {
93
- const candidate = value.trim();
94
- if (!candidate)
95
- throw new Error("domain must not be empty");
96
- try {
97
- return new URL(/^https?:\/\//i.test(candidate) ? candidate : `https://${candidate}`).hostname;
98
- }
99
- catch {
100
- throw new Error(`Invalid domain: ${value}`);
101
- }
102
- }
103
- export const fetchInputSchema = z.object({
104
- url: httpUrlSchema.describe("Required absolute HTTP or HTTPS URL of the known page to retrieve through Web Unblocker."),
105
- browser_rendering: z
106
- .boolean()
107
- .default(false)
108
- .describe("Choose browser loading and execute page JavaScript. This is independent of super_mode: either value may work better for a site, and some pages fail in a browser but load without one. Required for wait_for_selector."),
109
- super_mode: z
38
+ export const scraperAPIInputSchema = z.object({
39
+ url: z
40
+ .url({ protocol: /^https?$/ })
41
+ .describe("Required absolute HTTP or HTTPS page URL."),
42
+ html: z.boolean().default(true).describe("Request HTML page content."),
43
+ markdown: z
110
44
  .boolean()
111
45
  .default(false)
112
- .describe("Choose real-device routing independently of browser_rendering. Try it with either browser_rendering value when the current combination fails or returns worse content."),
46
+ .describe("Request Markdown page content."),
47
+ timeout: z
48
+ .number()
49
+ .int()
50
+ .min(1)
51
+ .max(300)
52
+ .default(120)
53
+ .describe("Page timeout in seconds (1–300); transport allows 30 additional seconds."),
113
54
  geo_code: z
114
55
  .string()
115
- .nullable()
116
- .optional()
117
- .describe("Optional proxy-routing country code sent as geoCode when the page must be viewed from a specific location."),
118
- wait_for_selector: z
56
+ .default("US")
57
+ .describe("Proxy geolocation country code, such as US, ID, or SG."),
58
+ proxy_country: z
119
59
  .string()
120
60
  .nullable()
121
61
  .optional()
122
- .describe("CSS selector to wait for before returning the page. Requires browser_rendering=true and is useful for content that appears after JavaScript runs."),
123
- home_page: z
124
- .boolean()
125
- .default(false)
126
- .describe("Visit the site's root page before the target URL, which can help establish cookies or session state."),
62
+ .describe("Compatibility country option sent as proxyCountry."),
127
63
  block_resources: z
128
64
  .boolean()
129
- .default(false)
130
- .describe("Ask Web Unblocker to block nonessential page resources during loading when only page content is needed."),
131
- max_retries: z
132
- .number()
133
- .int()
134
- .min(0)
135
- .default(3)
136
- .describe("Maximum Web Unblocker retry count. Use 0 to disable retries; the default is 3."),
137
- token_cap: z
138
- .number()
139
- .int()
140
- .positive()
141
65
  .nullable()
142
66
  .optional()
143
- .describe("Optional maximum retry-token budget consumed across this fetch request's attempts."),
144
- timeout: z
145
- .number()
146
- .int()
147
- .positive()
148
- .default(30)
149
- .describe("API page-load timeout in seconds; the MCP server allows 30 seconds more for transport."),
150
- });
151
- export async function fetchTool(token, input, dependencies = {}) {
152
- if (input.wait_for_selector && !input.browser_rendering) {
153
- throw new Error("wait_for_selector requires browser_rendering");
154
- }
155
- return fetchContentApi({
156
- token,
157
- url: input.url,
158
- browserRendering: input.browser_rendering,
159
- superMode: input.super_mode,
160
- timeout: input.timeout,
161
- geoCode: input.geo_code ?? null,
162
- waitForSelector: input.wait_for_selector ?? null,
163
- homePage: input.home_page,
164
- blockResources: input.block_resources,
165
- maxRetries: input.max_retries,
166
- tokenCap: input.token_cap ?? null,
167
- fetchFn: dependencies.fetchFn,
168
- });
169
- }
170
- export const scrapeInputSchema = z.object({
171
- url: httpUrlSchema.describe("Required absolute HTTP or HTTPS starting URL for the AI scraper."),
172
- prompt: z
173
- .string()
67
+ .default(false)
68
+ .describe("Block images, CSS, and fonts during loading."),
69
+ browser_rendering: z
70
+ .boolean()
174
71
  .nullable()
175
72
  .optional()
176
- .describe("Natural-language description of the fields or repeated records to extract. Required for general and listing; not accepted by map."),
177
- schema_prompt: z
178
- .record(z.string(), jsonValueSchema)
73
+ .describe("True forces browser rendering, false disables it; omit or use null for automatic fallback."),
74
+ super: z
75
+ .boolean()
76
+ .default(false)
77
+ .describe("Use real-device routing independently of browser rendering."),
78
+ use_proxy: z
79
+ .boolean()
80
+ .default(true)
81
+ .describe("Allow the service to use a configured proxy."),
82
+ proxy: z
83
+ .string()
179
84
  .nullable()
180
85
  .optional()
181
- .describe("Optional JSON Schema appended to prompt as best-effort output-shape guidance for general or listing extraction."),
182
- agent: z
183
- .enum(["general", "listing", "map"])
184
- .default("general")
185
- .describe("Extraction mode: general for defined page fields, listing for repeated records across pages, or map for discovering site URLs."),
186
- mode: z
187
- .enum(["Cheap", "Super"])
86
+ .describe("Custom proxy in username:password@host:port form."),
87
+ wait_for_selector: z
88
+ .string()
188
89
  .nullable()
189
90
  .optional()
190
- .describe("Optional backend execution tier: Cheap or Super. This is separate from agent; omit it to preserve the backend default."),
191
- proxy_country: z
91
+ .describe("CSS selector to wait for before returning page content."),
92
+ home_page: z
93
+ .boolean()
94
+ .default(false)
95
+ .describe("Visit the site's root before the target URL; sent as warmStart=true."),
96
+ screenshot: z
97
+ .boolean()
98
+ .default(false)
99
+ .describe("Capture a screenshot; always enables browser rendering."),
100
+ bypass_proxy: z
101
+ .boolean()
102
+ .default(false)
103
+ .describe("Enable the API's proxy-bypass/resource-blocking behavior."),
104
+ cookie_jar: z
192
105
  .string()
193
106
  .nullable()
194
107
  .optional()
195
- .describe("Optional proxy country for general or listing extraction when content varies by location; not accepted by map."),
196
- max_pages: z
197
- .number()
198
- .int()
199
- .positive()
108
+ .describe("Cookie configuration string, such as domain=example.com&country=us&max_used=100&max_age=50m."),
109
+ headers: z
110
+ .union([z.record(z.string(), z.string()), z.string()])
200
111
  .nullable()
201
112
  .optional()
202
- .describe("Maximum pages processed by listing or map. Not accepted by general; omit it to use the service default."),
203
- max_depth: z
113
+ .describe("Target page request headers as an object or JSON string."),
114
+ max_retries: z
204
115
  .number()
205
116
  .int()
206
- .positive()
207
- .nullable()
208
- .optional()
209
- .describe("Maximum link depth followed by the map agent; not accepted by general or listing."),
210
- limit: z
117
+ .min(0)
118
+ .default(3)
119
+ .describe("Maximum Web Unblocker retries; 0 disables retries."),
120
+ token_cap: z
211
121
  .number()
212
122
  .int()
213
- .positive()
123
+ .min(0)
214
124
  .nullable()
215
125
  .optional()
216
- .describe("Maximum number of discovered URLs returned by the map agent; not accepted by general or listing."),
217
- include_patterns: z
218
- .string()
126
+ .describe("Optional nonnegative retry-token budget."),
127
+ stream: z
128
+ .boolean()
219
129
  .nullable()
220
130
  .optional()
221
- .describe("Regular expression selecting URLs the map agent may include; not accepted by general or listing."),
222
- exclude_patterns: z
131
+ .describe("Ask the API to stream; the MCP server buffers the response into the normal envelope."),
132
+ discard_token_secret: z
223
133
  .string()
224
- .nullable()
225
- .optional()
226
- .describe("Regular expression removing matching URLs from map-agent results; not accepted by general or listing."),
134
+ .default("")
135
+ .describe("Optional value for the x-discard-token-secret upstream header."),
227
136
  });
228
- export async function scrapeTool(token, input, dependencies = {}) {
229
- const supplied = (value) => value !== undefined && value !== null;
230
- const agent = input.agent;
231
- const mapOnlyOptions = [
232
- ["max_depth", input.max_depth],
233
- ["limit", input.limit],
234
- ["include_patterns", input.include_patterns],
235
- ["exclude_patterns", input.exclude_patterns],
236
- ];
237
- if (agent === "map") {
238
- if (supplied(input.prompt)) {
239
- throw new Error("prompt is not accepted by the map agent");
240
- }
241
- if (supplied(input.schema_prompt)) {
242
- throw new Error("schema_prompt is not accepted by the map agent");
243
- }
244
- if (supplied(input.proxy_country)) {
245
- throw new Error("proxy_country is not accepted by the map agent");
246
- }
247
- }
248
- else {
249
- if (!input.prompt?.trim()) {
250
- throw new Error("prompt is required for general and listing agents");
251
- }
252
- const invalidMapOptions = mapOnlyOptions
253
- .filter(([, value]) => supplied(value))
254
- .map(([name]) => name);
255
- if (invalidMapOptions.length) {
256
- throw new Error(`${invalidMapOptions.join(", ")} ${invalidMapOptions.length === 1 ? "is" : "are"} only accepted by the map agent`);
257
- }
258
- if (agent === "general" && supplied(input.max_pages)) {
259
- throw new Error("max_pages is only accepted by listing and map agents");
260
- }
261
- }
262
- return createAiScraperApi({
263
- token,
264
- url: input.url,
265
- message: agent === "map"
266
- ? undefined
267
- : buildExtractionMessage(input.prompt, input.schema_prompt),
268
- agent,
269
- mode: input.mode ?? undefined,
270
- proxyCountry: input.proxy_country ?? null,
271
- maxPages: input.max_pages ?? undefined,
272
- maxDepth: input.max_depth ?? undefined,
273
- limit: input.limit ?? undefined,
274
- includePatterns: input.include_patterns ?? undefined,
275
- excludePatterns: input.exclude_patterns ?? undefined,
276
- fetchFn: dependencies.fetchFn,
277
- });
278
- }
279
- export const serpInputSchema = z.object({
280
- query_or_url: z
137
+ export const webSearchInputSchema = z.object({
138
+ url: z
281
139
  .string()
140
+ .trim()
282
141
  .min(1)
283
- .describe("Required Google search query text or complete Google search URL from which to retrieve results."),
142
+ .describe("Required search query text or complete Google search URL."),
143
+ raw: z
144
+ .boolean()
145
+ .default(true)
146
+ .describe("SDK compatibility default: true requests HTML and overrides format. Set false for parsed JSON results."),
284
147
  region: z
285
148
  .string()
286
149
  .nullable()
287
150
  .optional()
288
- .describe("Optional country code used to localize the Google results returned by the service."),
151
+ .describe("Country code; overrides gl from a Google search URL."),
289
152
  language: z
290
153
  .string()
291
154
  .nullable()
292
155
  .optional()
293
- .describe("Optional language code used to localize the Google results returned by the service."),
156
+ .describe("Language code; overrides hl from a Google search URL."),
294
157
  page: z
295
158
  .number()
296
159
  .int()
297
160
  .positive()
298
161
  .nullable()
299
162
  .optional()
300
- .describe("Optional one-based Google results page number for retrieving later result pages."),
163
+ .describe("One-based result page; overrides the page derived from start in a Google URL."),
301
164
  format: z
302
165
  .enum(["json", "html"])
303
166
  .default("json")
304
- .describe("Response representation: json returns parsed search results; html returns the result page's raw HTML."),
167
+ .describe("Response representation when raw=false."),
305
168
  render_js: z
306
169
  .boolean()
307
170
  .default(false)
308
- .describe("Render the Google results page with JavaScript when dynamic features such as AI Overview are needed."),
309
- raw: z
310
- .boolean()
311
- .default(false)
312
- .describe("Deprecated alias for format=html; takes precedence over format."),
313
- client_timeout: z
171
+ .describe("Render JavaScript-dependent search features."),
172
+ timeout: z
314
173
  .number()
315
- .int()
316
174
  .positive()
317
- .default(120)
318
- .describe("Local upstream HTTP timeout in seconds; it is not included in the SERP request body."),
175
+ .default(600)
176
+ .describe("Local upstream HTTP deadline in seconds; omitted from the JSON body."),
319
177
  });
320
- export async function serpTool(token, input, dependencies = {}) {
321
- return googleSerpSyncApi({
178
+ export async function scraperAPITool(token, input, dependencies = {}) {
179
+ return scraperApi({
322
180
  token,
323
- queryOrUrl: input.query_or_url,
324
- region: input.region ?? null,
325
- language: input.language ?? null,
326
- page: input.page ?? null,
181
+ url: input.url,
182
+ html: input.html,
183
+ markdown: input.markdown,
184
+ timeout: input.timeout,
185
+ geoCode: input.geo_code,
186
+ proxyCountry: input.proxy_country,
187
+ blockResources: input.block_resources,
188
+ browserRendering: input.browser_rendering,
189
+ super: input.super,
190
+ useProxy: input.use_proxy,
191
+ proxy: input.proxy,
192
+ waitForSelector: input.wait_for_selector,
193
+ homePage: input.home_page,
194
+ screenshot: input.screenshot,
195
+ bypassProxy: input.bypass_proxy,
196
+ cookieJar: input.cookie_jar,
197
+ headers: input.headers,
198
+ maxRetries: input.max_retries,
199
+ tokenCap: input.token_cap,
200
+ stream: input.stream,
201
+ discardTokenSecret: input.discard_token_secret,
202
+ fetchFn: dependencies.fetchFn,
203
+ });
204
+ }
205
+ export async function webSearchTool(token, input, dependencies = {}) {
206
+ return webSearchApi({
207
+ token,
208
+ url: input.url,
209
+ raw: input.raw,
210
+ region: input.region,
211
+ language: input.language,
212
+ page: input.page,
327
213
  format: input.format,
328
214
  renderJs: input.render_js,
329
- raw: input.raw,
330
- timeout: input.client_timeout,
215
+ timeout: input.timeout,
331
216
  fetchFn: dependencies.fetchFn,
332
217
  });
333
218
  }
334
- export const statusInputSchema = z.object({
335
- domain: z
336
- .string()
337
- .nullable()
338
- .optional()
339
- .describe("Optional hostname or URL. When supplied, status includes request-outcome analytics for its normalized hostname; omit it for account usage only."),
340
- from: z
341
- .string()
342
- .default("24h")
343
- .describe("Analytics range start as ISO 8601, now, or a relative duration such as 24h or 7d. Used only when domain is supplied."),
344
- to: z
345
- .string()
346
- .default("now")
347
- .describe("Analytics range end as ISO 8601, now, or a relative duration. Used only when domain is supplied."),
348
- action: z
349
- .string()
350
- .nullable()
351
- .optional()
352
- .describe("Optional exact MrScraper action filter applied to domain analytics; used only when domain is supplied."),
353
- api_token_name: z
354
- .string()
355
- .nullable()
356
- .optional()
357
- .describe("Optional API token name used to filter domain analytics; used only when domain is supplied."),
358
- });
359
- export async function statusTool(token, input, dependencies = {}) {
360
- const accountResponse = await getSubscriptionAccountApi(token, dependencies);
361
- if (isApiFailure(accountResponse))
362
- return accountResponse;
363
- const account = unwrapApiData(accountResponse);
364
- const output = {
219
+ export const statusInputSchema = z.object({});
220
+ function isApiFailure(result) {
221
+ return Boolean(result.error ||
222
+ (typeof result.status_code === "number" && result.status_code >= 400));
223
+ }
224
+ export async function statusTool(token, _input = {}, dependencies = {}) {
225
+ const response = await getSubscriptionAccountApi(token, {
226
+ fetchFn: dependencies.fetchFn,
227
+ });
228
+ if (isApiFailure(response))
229
+ return response;
230
+ const body = response.data;
231
+ const account = body && typeof body === "object" && !Array.isArray(body) && "data" in body
232
+ ? body.data
233
+ : body;
234
+ return {
365
235
  kind: "mrscraper-cli-status-summary",
366
236
  source_endpoints: ["/subscription-accounts"],
367
- status_code: accountResponse.status_code,
237
+ status_code: response.status_code,
368
238
  data: {
369
239
  account: summarizeSubscriptionAccount(account && typeof account === "object" && !Array.isArray(account)
370
240
  ? account
371
241
  : {}),
372
242
  },
373
243
  };
374
- if (!input.domain)
375
- return output;
376
- output.source_endpoints.push("/analytic/statuses");
377
- const domain = normalizeDomain(input.domain);
378
- const now = dependencies.now?.() || new Date();
379
- const end = parseStatusDate(input.to, now, "now");
380
- const start = parseStatusDate(input.from, end, "24h");
381
- if (start >= end)
382
- throw new Error("from must be earlier than to");
383
- const startDate = formatApiDate(start);
384
- const endDate = formatApiDate(end);
385
- const analyticsResponse = await getAnalyticStatusesApi({
386
- token,
387
- domain,
388
- startDate,
389
- endDate,
390
- action: input.action || "",
391
- apiTokenName: input.api_token_name || "",
392
- fetchFn: dependencies.fetchFn,
393
- });
394
- const data = output.data;
395
- if (analyticsResponse.error) {
396
- output.error = "Account loaded, but analytics could not be loaded";
397
- data.analytics = analyticsResponse;
398
- }
399
- else {
400
- const analytics = unwrapApiData(analyticsResponse);
401
- data.analytics = {
402
- domain,
403
- from: `${startDate} UTC`,
404
- to: `${endDate} UTC`,
405
- ...(analytics &&
406
- typeof analytics === "object" &&
407
- !Array.isArray(analytics)
408
- ? analytics
409
- : { data: analytics }),
410
- };
411
- }
412
- return output;
413
244
  }
414
- export const rerunInputSchema = z.object({
415
- target: z
416
- .string()
417
- .describe("Target URL for a single rerun, or comma/newline-separated target URLs when bulk=true."),
418
- type: z
419
- .enum(["ai", "manual"])
420
- .describe("How the saved scraper was created: ai for an AI scraper or manual for a dashboard-built step workflow."),
421
- bulk: z
422
- .boolean()
423
- .default(false)
424
- .describe("Independently select target count: false for one URL or true to submit all parsed targets as one asynchronous bulk job."),
425
- scraper_id: z
426
- .string()
427
- .nullable()
428
- .optional()
429
- .describe("Required saved scraper UUID when bulk=false. Use the scraperId returned by scrape for an AI scraper, or a dashboard workflow's UUID for manual."),
430
- id: z
431
- .string()
432
- .nullable()
433
- .optional()
434
- .describe("Required saved scraper UUID when bulk=true. This identifies the configuration to run and is not a result ID."),
435
- max_depth: z
436
- .number()
437
- .int()
438
- .positive()
439
- .nullable()
440
- .optional()
441
- .describe("Maximum crawl depth for a single AI rerun; omit it to preserve the saved scraper or backend default."),
442
- max_pages: z
443
- .number()
444
- .int()
445
- .positive()
446
- .nullable()
447
- .optional()
448
- .describe("Maximum pages for a single AI rerun; omit it to preserve the saved scraper or backend default."),
449
- limit: z
450
- .number()
451
- .int()
452
- .positive()
453
- .nullable()
454
- .optional()
455
- .describe("Maximum results for a single AI rerun; omit it to preserve the saved scraper or backend default."),
456
- include_patterns: z
457
- .string()
458
- .nullable()
459
- .optional()
460
- .describe("URL include regular expression for a single AI rerun; omit it to preserve the saved scraper or backend default."),
461
- exclude_patterns: z
462
- .string()
463
- .nullable()
464
- .optional()
465
- .describe("URL exclude regular expression for a single AI rerun; omit it to preserve the saved scraper or backend default."),
466
- proxy_country: z
467
- .string()
468
- .nullable()
469
- .optional()
470
- .describe("Proxy country code for a single AI rerun; omit it to preserve the saved scraper or backend default."),
471
- max_retry: z
472
- .number()
473
- .int()
474
- .min(0)
475
- .nullable()
476
- .optional()
477
- .describe("Maximum retry count for a single AI rerun; omit it to preserve the saved scraper or backend default."),
478
- timeout: z
479
- .number()
480
- .int()
481
- .positive()
482
- .nullable()
483
- .optional()
484
- .describe("Timeout in seconds for a single AI rerun, used by listing reruns; omit it to preserve the backend default."),
485
- });
486
- export async function rerunTool(token, input, dependencies = {}) {
487
- const aiOptionEntries = [
488
- ["max_depth", input.max_depth],
489
- ["max_pages", input.max_pages],
490
- ["limit", input.limit],
491
- ["include_patterns", input.include_patterns],
492
- ["exclude_patterns", input.exclude_patterns],
493
- ["proxy_country", input.proxy_country],
494
- ["max_retry", input.max_retry],
495
- ["timeout", input.timeout],
496
- ];
497
- const explicitAiOptions = aiOptionEntries
498
- .filter(([, value]) => value !== undefined && value !== null)
499
- .map(([name]) => name);
500
- let response;
501
- if (input.bulk) {
502
- if (!input.id)
503
- throw new Error("id is required when bulk is true");
504
- if (input.scraper_id !== undefined && input.scraper_id !== null) {
505
- throw new Error("scraper_id is only accepted for single reruns; use id when bulk is true");
506
- }
507
- if (explicitAiOptions.length) {
508
- throw new Error(`${explicitAiOptions.join(", ")} ${explicitAiOptions.length === 1 ? "is" : "are"} not accepted by bulk rerun endpoints`);
509
- }
510
- const urls = parseBulkUrls(input.target);
511
- if (!urls.length)
512
- throw new Error("No URLs found in the bulk target");
513
- response =
514
- input.type === "ai"
515
- ? await bulkRerunAiScraperApi({
516
- token,
517
- scraperId: input.id,
518
- urls,
519
- fetchFn: dependencies.fetchFn,
520
- })
521
- : await bulkRerunManualScraperApi({
522
- token,
523
- scraperId: input.id,
524
- urls,
525
- fetchFn: dependencies.fetchFn,
526
- });
527
- }
528
- else {
529
- if (!input.scraper_id) {
530
- throw new Error("scraper_id is required unless bulk is true");
531
- }
532
- if (input.id !== undefined && input.id !== null) {
533
- throw new Error("id is only accepted when bulk is true; use scraper_id for a single rerun");
534
- }
535
- if (input.type === "manual" && explicitAiOptions.length) {
536
- throw new Error(`${explicitAiOptions.join(", ")} ${explicitAiOptions.length === 1 ? "is" : "are"} only accepted by single AI reruns`);
537
- }
538
- const url = input.target.trim();
539
- if (!url)
540
- throw new Error("target URL must not be empty");
541
- response =
542
- input.type === "manual"
543
- ? await rerunManualScraperApi({
544
- token,
545
- scraperId: input.scraper_id,
546
- url,
547
- fetchFn: dependencies.fetchFn,
548
- })
549
- : await rerunAiScraperApi({
550
- token,
551
- scraperId: input.scraper_id,
552
- url,
553
- maxDepth: input.max_depth ?? undefined,
554
- maxPages: input.max_pages ?? undefined,
555
- limit: input.limit ?? undefined,
556
- includePatterns: input.include_patterns ?? undefined,
557
- excludePatterns: input.exclude_patterns ?? undefined,
558
- proxyCountry: input.proxy_country ?? undefined,
559
- maxRetry: input.max_retry ?? undefined,
560
- timeout: input.timeout ?? undefined,
561
- fetchFn: dependencies.fetchFn,
562
- });
563
- }
564
- return response;
565
- }
566
- export const resultsInputSchema = z.object({
567
- sort_field: z
568
- .string()
569
- .min(1)
570
- .default("updatedAt")
571
- .describe("Stored-result field used as the sort key; defaults to updatedAt."),
572
- sort_order: z
573
- .string()
574
- .trim()
575
- .toLowerCase()
576
- .pipe(z.enum(["asc", "desc"]))
577
- .default("desc")
578
- .describe("Sort direction for sort_field: asc or desc, accepted case-insensitively; defaults to desc."),
579
- page_size: z
580
- .number()
581
- .int()
582
- .positive()
583
- .default(10)
584
- .describe("Positive number of stored result records requested per page; defaults to 10."),
585
- page: z
586
- .number()
587
- .int()
588
- .positive()
589
- .default(1)
590
- .describe("One-based page number used to move through the stored result list; defaults to 1."),
591
- search: z
592
- .string()
593
- .nullable()
594
- .optional()
595
- .describe("Optional free-text filter used to narrow the stored result list."),
596
- date_range_column: z
597
- .string()
598
- .nullable()
599
- .optional()
600
- .describe("Stored-result date column to which start_at and end_at are applied."),
601
- start_at: z
602
- .string()
603
- .nullable()
604
- .optional()
605
- .describe("Optional inclusive ISO 8601 start bound applied to date_range_column."),
606
- end_at: z
607
- .string()
608
- .nullable()
609
- .optional()
610
- .describe("Optional inclusive ISO 8601 end bound applied to date_range_column."),
611
- scraper_id: z
612
- .string()
613
- .min(1)
614
- .nullable()
615
- .optional()
616
- .describe("Optional exact saved scraper UUID filter, sent as filters[scraperId]."),
617
- status: z
618
- .enum(["Draft", "Finished", "Running", "Failed", "Cancelled"])
619
- .nullable()
620
- .optional()
621
- .describe("Optional exact result status filter: Draft, Finished, Running, Failed, or Cancelled."),
622
- type: z
623
- .string()
624
- .min(1)
625
- .nullable()
626
- .optional()
627
- .describe("Optional exact result type filter, such as AI, Manual, Rerun-AI, or Bulk-AI."),
628
- url: httpUrlSchema
629
- .nullable()
630
- .optional()
631
- .describe("Optional exact stored target URL filter, including its path and query string."),
632
- });
633
- export async function resultsTool(token, input, dependencies = {}) {
634
- return getAllResultsApi({
635
- token,
636
- sortField: input.sort_field,
637
- sortOrder: input.sort_order.toUpperCase(),
638
- pageSize: input.page_size,
639
- page: input.page,
640
- search: input.search ?? null,
641
- dateRangeColumn: input.date_range_column ?? null,
642
- startAt: input.start_at ?? null,
643
- endAt: input.end_at ?? null,
644
- scraperId: input.scraper_id ?? null,
645
- status: input.status ?? null,
646
- type: input.type ?? null,
647
- url: input.url ?? null,
648
- fetchFn: dependencies.fetchFn,
649
- });
650
- }
651
- export const resultInputSchema = z.object({
652
- result_id: z
653
- .string()
654
- .min(1)
655
- .describe("Required UUID of the stored result to retrieve, including a bulkResultId returned by an asynchronous bulk rerun."),
656
- include_html: z
657
- .boolean()
658
- .default(true)
659
- .describe("Include stored HTML in the result response. Set false for a smaller response when polling status or reading extracted data only."),
660
- });
661
- export async function resultTool(token, input, dependencies = {}) {
662
- const resultId = input.result_id.trim();
663
- if (!resultId)
664
- throw new Error("result_id must not be empty");
665
- return getResultByIdApi(token, resultId, {
666
- includeHtml: input.include_html,
667
- fetchFn: dependencies.fetchFn,
668
- });
245
+ function asStructured(result) {
246
+ return {
247
+ content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
248
+ structuredContent: result,
249
+ ...(isApiFailure(result) ? { isError: true } : {}),
250
+ };
669
251
  }
670
- const readAnnotations = {
252
+ const annotations = {
671
253
  readOnlyHint: true,
672
- openWorldHint: false,
673
- destructiveHint: false,
674
- };
675
- const writeAnnotations = {
676
- readOnlyHint: false,
677
- openWorldHint: false,
254
+ openWorldHint: true,
678
255
  destructiveHint: false,
679
256
  };
680
257
  export const TOOL_DESCRIPTIONS = {
681
- fetch: "Use this when a public page URL is known; fetch is the default first content-acquisition step for agent-led work. It returns the raw page response through Web Unblocker, preserving source details for reading, analysis, verification, and follow-up transformations. Keep that response as the source of truth. If the user needs fields, JSON, a table, or extraction across many similar pages, fetch and inspect representative pages, define one reusable local extractor, fetch the remaining pages, and apply the extractor locally. For 100 similarly structured pages, run the fetches concurrently when safe and feed the saved responses into one local batch extraction; this is often faster than 100 separate backend-LLM extractions and preserves every raw page. Do not call scrape merely because the requested output is structured. Use scrape only when the user explicitly requests MrScraper-managed extraction or after fetch-led exploration has established a stable output schema and a clear benefit. Use serp when no target URL is known. The url argument is required. browser_rendering selects browser loading and JavaScript execution; super_mode independently selects real-device routing. The four false/false, true/false, false/true, and true/true combinations can return different results. Start with both false, inspect the response, and change one axis at a time when needed. Browser rendering is not strictly stronger: some sites fail or return worse content with browser_rendering=true, so retry the same super_mode value with browser_rendering=false. Try super_mode with either loader when routing may be the problem; use both true for real-device browser loading. Do not repeat an identical combination, and stop once the response is usable unless comparison is requested. wait_for_selector requires browser_rendering=true. geo_code changes proxy location, while home_page, block_resources, max_retries, token_cap, and timeout tune loading and retry behavior.",
682
- scrape: "Use this when the user explicitly wants MrScraper-managed structured extraction or bounded URL discovery within a known site. Do not use general or listing for the first exploration of a known page: they send page content through a backend LLM, which can omit source details, narrow the result to the prompt, and repeat model work across pages. Start with fetch, inspect the raw content, understand the site, and define a stable output schema. Prefer one reusable local extractor across saved fetch responses; requesting fields, JSON, or a table does not by itself justify scrape. This remains true for 100 same-layout pages: safe concurrent fetches plus one local batch extraction are often faster than 100 separate backend-LLM runs and preserve every source response. Use general or listing only when managed extraction was explicitly requested or still has a clear benefit after that exploration. Choose agent=general for defined page fields, agent=listing for repeated records across pages, or agent=map for bounded URL discovery within the known site; map is separate from LLM extraction and does not require a schema. mode=Cheap or mode=Super selects the backend execution tier independently of agent; omit mode to preserve the backend default. General and listing require prompt and optionally accept schema_prompt and proxy_country. max_pages applies to listing and map; max_depth, limit, include_patterns, and exclude_patterns are map-only. A successful run returns scraperId for reuse with rerun.",
683
- serp: "Use this when the task starts with a Google query or Google search URL rather than a known target page. After selecting relevant result URLs, use fetch for every page whose content will inform the answer and preserve each raw response. Analyze or transform those responses locally, using one reusable extractor when pages share a structure. Use scrape only when the user explicitly requests MrScraper-managed extraction or after fetch-led exploration has established a stable schema and clear benefit; stop after serp when the user only needs URL discovery. The query_or_url argument is required; region, language, and page control localization and pagination. format=json returns parsed results, format=html returns result-page HTML, render_js includes dynamic features, and client_timeout controls how long this MCP request waits. raw is a deprecated alias for format=html.",
684
- status: "Use this to check the current MrScraper account's identity, subscription, and usage before or after web-data work. With no arguments it returns an account summary including the user's name, email, and verification state. Supply domain only when request-outcome analytics are also needed; from and to select that analytics window, while action and api_token_name narrow those domain outcomes. This tool reports account and request health, not scrape-job progress; use result for a known asynchronous result ID.",
685
- rerun: "Use this only when you already have a saved scraper UUID and want to apply that configuration to the same or new target URLs. Set type=ai for a scraper created by scrape; type=manual selects a dashboard-built step workflow and requires the conversation's compliance acknowledgment. For one URL, leave bulk=false and pass target with scraper_id. A single AI rerun also accepts max_depth, max_pages, limit, include_patterns, exclude_patterns, proxy_country, max_retry, and timeout; omit controls to preserve saved scraper and backend defaults. Manual and bulk reruns reject those controls. For multiple URLs, set bulk=true, pass comma- or newline-separated target URLs and id; bulk jobs are asynchronous, so retain bulkResultId and inspect it with result until completion. Use scrape to create a new AI configuration.",
686
- results: "Use this to browse or locate stored scrape results when you do not yet know the exact result UUID. sort_field and sort_order control ordering; page_size and page control pagination; search narrows the list; and date_range_column with start_at or end_at applies a date window. scraper_id, status, type, and url apply exact backend filters. This returns a result list rather than one complete record. Once an ID is known, use result for that record.",
687
- result: "Use this when you already know the exact stored result UUID and need the complete record or current job state. Pass that UUID as result_id; this may also be the bulkResultId returned by an asynchronous bulk rerun. include_html defaults to true; set it false for a smaller response while polling status or when extracted data is sufficient. Call result again as needed until an asynchronous job completes. Use results first when the UUID is unknown.",
258
+ scraperAPI: "Retrieves a known page through MrScraper's Scraper API as HTML, Markdown, a screenshot, or a combination. Matches the Python SDK's scraper_api options. Omit browser_rendering for automatic browser fallback; screenshot=true always enables it. super enables real-device routing independently. home_page maps to warmStart. Proxy, cookie, target-header, retry, and timeout options control page loading. Returns status_code, data, and sanitized headers.",
259
+ webSearch: "Searches Google from query text or a Google search URL passed as url. Matches the Python SDK's web_search options. Set raw=false and format=json for parsed results; the SDK-compatible default raw=true returns HTML. Google URLs supply q, gl, hl, and start; explicit region, language, and page override URL-derived values. render_js enables dynamic features; timeout controls the local request deadline. Returns status_code, data, and sanitized headers.",
260
+ status: "Returns the connected MrScraper account's identity (name, email, and verification state), subscription status, token limit, token usage, remaining tokens, and rate limit. Use it to check the account and remaining quota, for example before a large batch of requests. It takes no input and reports account usage, not the progress of a request.",
688
261
  };
689
262
  export function registerTools(server, getToken, dependencies = {}) {
690
- server.registerTool("fetch", {
691
- title: "Fetch Page",
692
- description: TOOL_DESCRIPTIONS.fetch,
693
- inputSchema: fetchInputSchema,
694
- outputSchema: fetchOutputSchema,
695
- annotations: readAnnotations,
696
- }, async (input) => asStructured(await fetchTool(getToken(), input, dependencies)));
697
- server.registerTool("scrape", {
698
- title: "Extract Data",
699
- description: TOOL_DESCRIPTIONS.scrape,
700
- inputSchema: scrapeInputSchema,
701
- outputSchema: scrapeOutputSchema,
702
- _meta: widgetMeta("records", "Extracting data…", "Extraction complete."),
703
- annotations: writeAnnotations,
704
- }, async (input) => asStructured(await scrapeTool(getToken(), input, dependencies)));
705
- server.registerTool("serp", {
706
- title: "Google Search",
707
- description: TOOL_DESCRIPTIONS.serp,
708
- inputSchema: serpInputSchema,
709
- outputSchema: serpOutputSchema,
710
- _meta: widgetMeta("serp", "Searching Google…", "Search complete."),
711
- annotations: readAnnotations,
712
- }, async (input) => asStructured(await serpTool(getToken(), input, dependencies)));
263
+ server.registerTool("scraperAPI", {
264
+ title: "Scraper API",
265
+ annotations: { title: "Scraper API", ...annotations },
266
+ description: TOOL_DESCRIPTIONS.scraperAPI,
267
+ inputSchema: scraperAPIInputSchema,
268
+ outputSchema: scraperAPIOutputSchema,
269
+ }, async (input) => asStructured(await scraperAPITool(getToken(), input, dependencies)));
270
+ server.registerTool("webSearch", {
271
+ title: "Web Search",
272
+ annotations: { title: "Web Search", ...annotations },
273
+ description: TOOL_DESCRIPTIONS.webSearch,
274
+ inputSchema: webSearchInputSchema,
275
+ outputSchema: webSearchOutputSchema,
276
+ _meta: widgetMeta("webSearch", "Searching Google…", "Search complete."),
277
+ }, async (input) => asStructured(await webSearchTool(getToken(), input, dependencies)));
713
278
  server.registerTool("status", {
714
279
  title: "Account Status",
280
+ annotations: {
281
+ title: "Account Status",
282
+ readOnlyHint: true,
283
+ openWorldHint: false,
284
+ destructiveHint: false,
285
+ },
715
286
  description: TOOL_DESCRIPTIONS.status,
716
287
  inputSchema: statusInputSchema,
717
288
  outputSchema: statusOutputSchema,
718
289
  _meta: widgetMeta("status", "Checking your account…", "Account status loaded."),
719
- annotations: readAnnotations,
720
290
  }, async (input) => asStructured(await statusTool(getToken(), input, dependencies)));
721
- server.registerTool("rerun", {
722
- title: "Rerun Saved Scraper",
723
- description: TOOL_DESCRIPTIONS.rerun,
724
- inputSchema: rerunInputSchema,
725
- outputSchema: rerunOutputSchema,
726
- annotations: writeAnnotations,
727
- }, async (input) => asStructured(await rerunTool(getToken(), input, dependencies)));
728
- server.registerTool("results", {
729
- title: "List Results",
730
- description: TOOL_DESCRIPTIONS.results,
731
- inputSchema: resultsInputSchema,
732
- outputSchema: resultsOutputSchema,
733
- _meta: widgetMeta("records", "Loading results…", "Results loaded."),
734
- annotations: readAnnotations,
735
- }, async (input) => asStructured(await resultsTool(getToken(), input, dependencies)));
736
- server.registerTool("result", {
737
- title: "Get Result",
738
- description: TOOL_DESCRIPTIONS.result,
739
- inputSchema: resultInputSchema,
740
- outputSchema: resultOutputSchema,
741
- _meta: widgetMeta("records", "Loading result…", "Result loaded."),
742
- annotations: readAnnotations,
743
- }, async (input) => asStructured(await resultTool(getToken(), input, dependencies)));
744
291
  }
745
292
  //# sourceMappingURL=tools.js.map