@mrscraper/mcp 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/README.md +230 -353
  2. package/dist/api.d.ts +26 -75
  3. package/dist/api.d.ts.map +1 -1
  4. package/dist/api.js +72 -171
  5. package/dist/api.js.map +1 -1
  6. package/dist/app.js +1 -1
  7. package/dist/app.js.map +1 -1
  8. package/dist/config.d.ts +5 -12
  9. package/dist/config.d.ts.map +1 -1
  10. package/dist/config.js +6 -25
  11. package/dist/config.js.map +1 -1
  12. package/dist/http.d.ts.map +1 -1
  13. package/dist/http.js +5 -1
  14. package/dist/http.js.map +1 -1
  15. package/dist/index.d.ts +0 -2
  16. package/dist/index.d.ts.map +1 -1
  17. package/dist/index.js +0 -2
  18. package/dist/index.js.map +1 -1
  19. package/dist/scopes.d.ts.map +1 -1
  20. package/dist/scopes.js +2 -6
  21. package/dist/scopes.js.map +1 -1
  22. package/dist/server.d.ts.map +1 -1
  23. package/dist/server.js +3 -16
  24. package/dist/server.js.map +1 -1
  25. package/dist/status.d.ts +0 -2
  26. package/dist/status.d.ts.map +1 -1
  27. package/dist/status.js +2 -30
  28. package/dist/status.js.map +1 -1
  29. package/dist/tools.d.ts +30 -80
  30. package/dist/tools.d.ts.map +1 -1
  31. package/dist/tools.js +184 -569
  32. package/dist/tools.js.map +1 -1
  33. package/dist/widgets/bundles.generated.d.ts.map +1 -1
  34. package/dist/widgets/bundles.generated.js +1 -2
  35. package/dist/widgets/bundles.generated.js.map +1 -1
  36. package/dist/widgets/index.d.ts +1 -7
  37. package/dist/widgets/index.d.ts.map +1 -1
  38. package/dist/widgets/index.js +1 -7
  39. package/dist/widgets/index.js.map +1 -1
  40. package/package.json +4 -4
  41. package/dist/compliance.d.ts +0 -4
  42. package/dist/compliance.d.ts.map +0 -1
  43. package/dist/compliance.js +0 -10
  44. package/dist/compliance.js.map +0 -1
package/dist/tools.js CHANGED
@@ -1,677 +1,292 @@
1
1
  import { z } from "zod";
2
- import { bulkRerunAiScraperApi, bulkRerunManualScraperApi, createAiScraperApi, fetchContentApi, getAllResultsApi, getAnalyticStatusesApi, getResultByIdApi, getSubscriptionAccountApi, googleSerpSyncApi, parseBulkUrls, rerunAiScraperApi, rerunManualScraperApi, } from "./api.js";
2
+ import { getSubscriptionAccountApi, scraperApi, webSearchApi } from "./api.js";
3
+ import { summarizeSubscriptionAccount } from "./status.js";
3
4
  import { widgetMeta } from "./widgets/index.js";
4
- import { formatApiDate, parseStatusDate, summarizeSubscriptionAccount, } from "./status.js";
5
- const jsonValueSchema = z.json();
6
- const httpUrlSchema = z.url({ protocol: /^https?$/ });
7
- const headersSchema = z
8
- .record(z.string(), z.string())
9
- .describe("Safe response headers; credentials and cookies are removed.");
10
- const apiResponseSchema = z
11
- .object({
5
+ const apiResponseSchema = z.object({
12
6
  status_code: z
13
7
  .number()
14
8
  .int()
15
9
  .nullable()
16
- .describe("HTTP status returned by the MrScraper service."),
17
- data: jsonValueSchema.describe("Sanitized response payload returned by MrScraper."),
18
- headers: headersSchema,
10
+ .describe("Upstream HTTP status, or null for a network failure."),
11
+ data: z.json().describe("Response body as parsed JSON or text."),
12
+ headers: z
13
+ .record(z.string(), z.string())
14
+ .describe("Response headers with credentials and cookies removed."),
19
15
  error: z.string().optional().describe("Request failure message, if any."),
20
- })
21
- .meta({
22
- title: "MrScraper API response",
23
- description: "A credential-safe response envelope from MrScraper.",
24
16
  });
25
- export const fetchOutputSchema = apiResponseSchema.meta({
26
- title: "Fetch response",
27
- description: "The response envelope returned by the Web Unblocker API.",
17
+ export const scraperAPIOutputSchema = apiResponseSchema.meta({
18
+ title: "Scraper API response",
19
+ description: "Page content as HTML, Markdown, a screenshot, or a combination.",
28
20
  });
29
- export const scrapeOutputSchema = apiResponseSchema.meta({
30
- title: "Scrape response",
31
- description: "The AI scraper response envelope. A successful run contains scraperId for reproducing the saved configuration with rerun.",
21
+ export const webSearchOutputSchema = apiResponseSchema.meta({
22
+ title: "Web search response",
23
+ description: "Parsed Google search results or result-page HTML.",
32
24
  });
33
25
  export const statusOutputSchema = z
34
26
  .object({
35
27
  kind: z.literal("mrscraper-cli-status-summary").optional(),
36
28
  source_endpoints: z.array(z.string()).optional(),
37
29
  status_code: z.number().int().nullable(),
38
- data: jsonValueSchema,
39
- headers: headersSchema.optional(),
30
+ data: z.json(),
31
+ headers: z.record(z.string(), z.string()).optional(),
40
32
  error: z.string().optional(),
41
33
  })
42
34
  .meta({
43
35
  title: "Status response",
44
- description: "Account identity, subscription usage, and optional domain request-outcome analytics.",
45
- });
46
- const serpOutputSchema = apiResponseSchema.meta({
47
- title: "SERP response",
48
- description: "Parsed Google results or raw result-page HTML.",
49
- });
50
- const rerunOutputSchema = apiResponseSchema.meta({
51
- title: "Rerun response",
52
- description: "The MrScraper response for a saved scraper rerun.",
53
- });
54
- const resultsOutputSchema = apiResponseSchema.meta({
55
- title: "Results response",
56
- description: "Paginated stored MrScraper results.",
57
- });
58
- const resultOutputSchema = apiResponseSchema.meta({
59
- title: "Result response",
60
- description: "One stored MrScraper result.",
36
+ description: "Account identity, subscription, token limit, token usage, and remaining tokens.",
61
37
  });
62
- function isApiFailure(result) {
63
- return Boolean(result.error ||
64
- (typeof result.status_code === "number" && result.status_code >= 400));
65
- }
66
- function asStructured(result) {
67
- return {
68
- content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
69
- structuredContent: result,
70
- ...(isApiFailure(result) ? { isError: true } : {}),
71
- };
72
- }
73
- function buildExtractionMessage(prompt, schemaPrompt) {
74
- const instruction = prompt.trim();
75
- if (!instruction) {
76
- throw new Error("prompt is required for general and listing agents");
77
- }
78
- if (schemaPrompt === null || schemaPrompt === undefined)
79
- return instruction;
80
- return `${instruction}\n\nBest-effort output guidance: return JSON matching this JSON Schema. The MrScraper API does not validate this schema:\n${JSON.stringify(schemaPrompt, null, 2)}`;
81
- }
82
- function unwrapApiData(response) {
83
- const body = response.data;
84
- if (body &&
85
- typeof body === "object" &&
86
- !Array.isArray(body) &&
87
- "data" in body) {
88
- return body.data;
89
- }
90
- return body;
91
- }
92
- function normalizeDomain(value) {
93
- const candidate = value.trim();
94
- if (!candidate)
95
- throw new Error("domain must not be empty");
96
- try {
97
- return new URL(/^https?:\/\//i.test(candidate) ? candidate : `https://${candidate}`).hostname;
98
- }
99
- catch {
100
- throw new Error(`Invalid domain: ${value}`);
101
- }
102
- }
103
- export const fetchInputSchema = z.object({
104
- url: httpUrlSchema.describe("Required absolute HTTP or HTTPS URL of the known page to retrieve through Web Unblocker."),
105
- browser_rendering: z
38
+ export const scraperAPIInputSchema = z.object({
39
+ url: z
40
+ .url({ protocol: /^https?$/ })
41
+ .describe("Required absolute HTTP or HTTPS page URL."),
42
+ html: z.boolean().default(true).describe("Request HTML page content."),
43
+ markdown: z
106
44
  .boolean()
107
45
  .default(false)
108
- .describe("Execute page JavaScript in a browser. Enable this for client-rendered content or before using wait_for_selector."),
46
+ .describe("Request Markdown page content."),
47
+ timeout: z
48
+ .number()
49
+ .int()
50
+ .min(1)
51
+ .max(300)
52
+ .default(120)
53
+ .describe("Page timeout in seconds (1–300); transport allows 30 additional seconds."),
109
54
  geo_code: z
110
55
  .string()
111
- .nullable()
112
- .optional()
113
- .describe("Optional proxy-routing country code sent as geoCode when the page must be viewed from a specific location."),
114
- wait_for_selector: z
56
+ .default("US")
57
+ .describe("Proxy geolocation country code, such as US, ID, or SG."),
58
+ proxy_country: z
115
59
  .string()
116
60
  .nullable()
117
61
  .optional()
118
- .describe("CSS selector to wait for before returning the page. Requires browser_rendering=true and is useful for content that appears after JavaScript runs."),
119
- home_page: z
120
- .boolean()
121
- .default(false)
122
- .describe("Visit the site's root page before the target URL, which can help establish cookies or session state."),
62
+ .describe("Compatibility country option sent as proxyCountry."),
123
63
  block_resources: z
124
64
  .boolean()
65
+ .nullable()
66
+ .optional()
125
67
  .default(false)
126
- .describe("Ask Web Unblocker to block nonessential page resources during loading when only page content is needed."),
127
- max_retries: z
128
- .number()
129
- .int()
130
- .min(0)
131
- .default(3)
132
- .describe("Maximum Web Unblocker retry count. Use 0 to disable retries; the default is 3."),
133
- token_cap: z
134
- .number()
135
- .int()
136
- .positive()
68
+ .describe("Block images, CSS, and fonts during loading."),
69
+ browser_rendering: z
70
+ .boolean()
137
71
  .nullable()
138
72
  .optional()
139
- .describe("Optional maximum retry-token budget consumed across this fetch request's attempts."),
140
- timeout: z
141
- .number()
142
- .int()
143
- .positive()
144
- .default(30)
145
- .describe("API page-load timeout in seconds; the MCP server allows 30 seconds more for transport."),
146
- });
147
- export async function fetchTool(token, input, dependencies = {}) {
148
- if (input.wait_for_selector && !input.browser_rendering) {
149
- throw new Error("wait_for_selector requires browser_rendering");
150
- }
151
- return fetchContentApi({
152
- token,
153
- url: input.url,
154
- browserRendering: input.browser_rendering,
155
- timeout: input.timeout,
156
- geoCode: input.geo_code ?? null,
157
- waitForSelector: input.wait_for_selector ?? null,
158
- homePage: input.home_page,
159
- blockResources: input.block_resources,
160
- maxRetries: input.max_retries,
161
- tokenCap: input.token_cap ?? null,
162
- fetchFn: dependencies.fetchFn,
163
- });
164
- }
165
- export const scrapeInputSchema = z.object({
166
- url: httpUrlSchema.describe("Required absolute HTTP or HTTPS starting URL for the AI scraper."),
167
- prompt: z
73
+ .describe("True forces browser rendering, false disables it; omit or use null for automatic fallback."),
74
+ super: z
75
+ .boolean()
76
+ .default(false)
77
+ .describe("Use real-device routing independently of browser rendering."),
78
+ use_proxy: z
79
+ .boolean()
80
+ .default(true)
81
+ .describe("Allow the service to use a configured proxy."),
82
+ proxy: z
168
83
  .string()
169
84
  .nullable()
170
85
  .optional()
171
- .describe("Natural-language description of the fields or repeated records to extract. Required for general and listing; not accepted by map."),
172
- schema_prompt: z
173
- .record(z.string(), jsonValueSchema)
86
+ .describe("Custom proxy in username:password@host:port form."),
87
+ wait_for_selector: z
88
+ .string()
174
89
  .nullable()
175
90
  .optional()
176
- .describe("Optional JSON Schema appended to prompt as best-effort output-shape guidance for general or listing extraction."),
177
- agent: z
178
- .enum(["general", "listing", "map"])
179
- .default("general")
180
- .describe("Extraction mode: general for defined page fields, listing for repeated records across pages, or map for discovering site URLs."),
181
- proxy_country: z
91
+ .describe("CSS selector to wait for before returning page content."),
92
+ home_page: z
93
+ .boolean()
94
+ .default(false)
95
+ .describe("Visit the site's root before the target URL; sent as warmStart=true."),
96
+ screenshot: z
97
+ .boolean()
98
+ .default(false)
99
+ .describe("Capture a screenshot; always enables browser rendering."),
100
+ bypass_proxy: z
101
+ .boolean()
102
+ .default(false)
103
+ .describe("Enable the API's proxy-bypass/resource-blocking behavior."),
104
+ cookie_jar: z
182
105
  .string()
183
106
  .nullable()
184
107
  .optional()
185
- .describe("Optional proxy country for general or listing extraction when content varies by location; not accepted by map."),
186
- max_pages: z
187
- .number()
188
- .int()
189
- .positive()
108
+ .describe("Cookie configuration string, such as domain=example.com&country=us&max_used=100&max_age=50m."),
109
+ headers: z
110
+ .union([z.record(z.string(), z.string()), z.string()])
190
111
  .nullable()
191
112
  .optional()
192
- .describe("Maximum pages processed by listing or map. Not accepted by general; omit it to use the service default."),
193
- max_depth: z
113
+ .describe("Target page request headers as an object or JSON string."),
114
+ max_retries: z
194
115
  .number()
195
116
  .int()
196
- .positive()
197
- .nullable()
198
- .optional()
199
- .describe("Maximum link depth followed by the map agent; not accepted by general or listing."),
200
- limit: z
117
+ .min(0)
118
+ .default(3)
119
+ .describe("Maximum Web Unblocker retries; 0 disables retries."),
120
+ token_cap: z
201
121
  .number()
202
122
  .int()
203
- .positive()
123
+ .min(0)
204
124
  .nullable()
205
125
  .optional()
206
- .describe("Maximum number of discovered URLs returned by the map agent; not accepted by general or listing."),
207
- include_patterns: z
208
- .string()
126
+ .describe("Optional nonnegative retry-token budget."),
127
+ stream: z
128
+ .boolean()
209
129
  .nullable()
210
130
  .optional()
211
- .describe("Regular expression selecting URLs the map agent may include; not accepted by general or listing."),
212
- exclude_patterns: z
131
+ .describe("Ask the API to stream; the MCP server buffers the response into the normal envelope."),
132
+ discard_token_secret: z
213
133
  .string()
214
- .nullable()
215
- .optional()
216
- .describe("Regular expression removing matching URLs from map-agent results; not accepted by general or listing."),
134
+ .default("")
135
+ .describe("Optional value for the x-discard-token-secret upstream header."),
217
136
  });
218
- export async function scrapeTool(token, input, dependencies = {}) {
219
- const supplied = (value) => value !== undefined && value !== null;
220
- const agent = input.agent;
221
- const mapOnlyOptions = [
222
- ["max_depth", input.max_depth],
223
- ["limit", input.limit],
224
- ["include_patterns", input.include_patterns],
225
- ["exclude_patterns", input.exclude_patterns],
226
- ];
227
- if (agent === "map") {
228
- if (supplied(input.prompt)) {
229
- throw new Error("prompt is not accepted by the map agent");
230
- }
231
- if (supplied(input.schema_prompt)) {
232
- throw new Error("schema_prompt is not accepted by the map agent");
233
- }
234
- if (supplied(input.proxy_country)) {
235
- throw new Error("proxy_country is not accepted by the map agent");
236
- }
237
- }
238
- else {
239
- if (!input.prompt?.trim()) {
240
- throw new Error("prompt is required for general and listing agents");
241
- }
242
- const invalidMapOptions = mapOnlyOptions
243
- .filter(([, value]) => supplied(value))
244
- .map(([name]) => name);
245
- if (invalidMapOptions.length) {
246
- throw new Error(`${invalidMapOptions.join(", ")} ${invalidMapOptions.length === 1 ? "is" : "are"} only accepted by the map agent`);
247
- }
248
- if (agent === "general" && supplied(input.max_pages)) {
249
- throw new Error("max_pages is only accepted by listing and map agents");
250
- }
251
- }
252
- return createAiScraperApi({
253
- token,
254
- url: input.url,
255
- message: agent === "map"
256
- ? undefined
257
- : buildExtractionMessage(input.prompt, input.schema_prompt),
258
- agent,
259
- proxyCountry: input.proxy_country ?? null,
260
- maxPages: input.max_pages ?? undefined,
261
- maxDepth: input.max_depth ?? undefined,
262
- limit: input.limit ?? undefined,
263
- includePatterns: input.include_patterns ?? undefined,
264
- excludePatterns: input.exclude_patterns ?? undefined,
265
- fetchFn: dependencies.fetchFn,
266
- });
267
- }
268
- export const serpInputSchema = z.object({
269
- query_or_url: z
137
+ export const webSearchInputSchema = z.object({
138
+ url: z
270
139
  .string()
140
+ .trim()
271
141
  .min(1)
272
- .describe("Required Google search query text or complete Google search URL from which to retrieve results."),
142
+ .describe("Required search query text or complete Google search URL."),
143
+ raw: z
144
+ .boolean()
145
+ .default(true)
146
+ .describe("SDK compatibility default: true requests HTML and overrides format. Set false for parsed JSON results."),
273
147
  region: z
274
148
  .string()
275
149
  .nullable()
276
150
  .optional()
277
- .describe("Optional country code used to localize the Google results returned by the service."),
151
+ .describe("Country code; overrides gl from a Google search URL."),
278
152
  language: z
279
153
  .string()
280
154
  .nullable()
281
155
  .optional()
282
- .describe("Optional language code used to localize the Google results returned by the service."),
156
+ .describe("Language code; overrides hl from a Google search URL."),
283
157
  page: z
284
158
  .number()
285
159
  .int()
286
160
  .positive()
287
161
  .nullable()
288
162
  .optional()
289
- .describe("Optional one-based Google results page number for retrieving later result pages."),
163
+ .describe("One-based result page; overrides the page derived from start in a Google URL."),
290
164
  format: z
291
165
  .enum(["json", "html"])
292
166
  .default("json")
293
- .describe("Response representation: json returns parsed search results; html returns the result page's raw HTML."),
167
+ .describe("Response representation when raw=false."),
294
168
  render_js: z
295
169
  .boolean()
296
170
  .default(false)
297
- .describe("Render the Google results page with JavaScript when dynamic features such as AI Overview are needed."),
298
- raw: z
299
- .boolean()
300
- .default(false)
301
- .describe("Deprecated alias for format=html; takes precedence over format."),
302
- client_timeout: z
171
+ .describe("Render JavaScript-dependent search features."),
172
+ timeout: z
303
173
  .number()
304
- .int()
305
174
  .positive()
306
- .default(120)
307
- .describe("Local upstream HTTP timeout in seconds; it is not included in the SERP request body."),
175
+ .default(600)
176
+ .describe("Local upstream HTTP deadline in seconds; omitted from the JSON body."),
308
177
  });
309
- export async function serpTool(token, input, dependencies = {}) {
310
- return googleSerpSyncApi({
178
+ export async function scraperAPITool(token, input, dependencies = {}) {
179
+ return scraperApi({
311
180
  token,
312
- queryOrUrl: input.query_or_url,
313
- region: input.region ?? null,
314
- language: input.language ?? null,
315
- page: input.page ?? null,
181
+ url: input.url,
182
+ html: input.html,
183
+ markdown: input.markdown,
184
+ timeout: input.timeout,
185
+ geoCode: input.geo_code,
186
+ proxyCountry: input.proxy_country,
187
+ blockResources: input.block_resources,
188
+ browserRendering: input.browser_rendering,
189
+ super: input.super,
190
+ useProxy: input.use_proxy,
191
+ proxy: input.proxy,
192
+ waitForSelector: input.wait_for_selector,
193
+ homePage: input.home_page,
194
+ screenshot: input.screenshot,
195
+ bypassProxy: input.bypass_proxy,
196
+ cookieJar: input.cookie_jar,
197
+ headers: input.headers,
198
+ maxRetries: input.max_retries,
199
+ tokenCap: input.token_cap,
200
+ stream: input.stream,
201
+ discardTokenSecret: input.discard_token_secret,
202
+ fetchFn: dependencies.fetchFn,
203
+ });
204
+ }
205
+ export async function webSearchTool(token, input, dependencies = {}) {
206
+ return webSearchApi({
207
+ token,
208
+ url: input.url,
209
+ raw: input.raw,
210
+ region: input.region,
211
+ language: input.language,
212
+ page: input.page,
316
213
  format: input.format,
317
214
  renderJs: input.render_js,
318
- raw: input.raw,
319
- timeout: input.client_timeout,
215
+ timeout: input.timeout,
320
216
  fetchFn: dependencies.fetchFn,
321
217
  });
322
218
  }
323
- export const statusInputSchema = z.object({
324
- domain: z
325
- .string()
326
- .nullable()
327
- .optional()
328
- .describe("Optional hostname or URL. When supplied, status includes request-outcome analytics for its normalized hostname; omit it for account usage only."),
329
- from: z
330
- .string()
331
- .default("24h")
332
- .describe("Analytics range start as ISO 8601, now, or a relative duration such as 24h or 7d. Used only when domain is supplied."),
333
- to: z
334
- .string()
335
- .default("now")
336
- .describe("Analytics range end as ISO 8601, now, or a relative duration. Used only when domain is supplied."),
337
- action: z
338
- .string()
339
- .nullable()
340
- .optional()
341
- .describe("Optional exact MrScraper action filter applied to domain analytics; used only when domain is supplied."),
342
- api_token_name: z
343
- .string()
344
- .nullable()
345
- .optional()
346
- .describe("Optional API token name used to filter domain analytics; used only when domain is supplied."),
347
- });
348
- export async function statusTool(token, input, dependencies = {}) {
349
- const accountResponse = await getSubscriptionAccountApi(token, dependencies);
350
- if (isApiFailure(accountResponse))
351
- return accountResponse;
352
- const account = unwrapApiData(accountResponse);
353
- const output = {
219
+ export const statusInputSchema = z.object({});
220
+ function isApiFailure(result) {
221
+ return Boolean(result.error ||
222
+ (typeof result.status_code === "number" && result.status_code >= 400));
223
+ }
224
+ export async function statusTool(token, _input = {}, dependencies = {}) {
225
+ const response = await getSubscriptionAccountApi(token, {
226
+ fetchFn: dependencies.fetchFn,
227
+ });
228
+ if (isApiFailure(response))
229
+ return response;
230
+ const body = response.data;
231
+ const account = body && typeof body === "object" && !Array.isArray(body) && "data" in body
232
+ ? body.data
233
+ : body;
234
+ return {
354
235
  kind: "mrscraper-cli-status-summary",
355
236
  source_endpoints: ["/subscription-accounts"],
356
- status_code: accountResponse.status_code,
237
+ status_code: response.status_code,
357
238
  data: {
358
239
  account: summarizeSubscriptionAccount(account && typeof account === "object" && !Array.isArray(account)
359
240
  ? account
360
241
  : {}),
361
242
  },
362
243
  };
363
- if (!input.domain)
364
- return output;
365
- output.source_endpoints.push("/analytic/statuses");
366
- const domain = normalizeDomain(input.domain);
367
- const now = dependencies.now?.() || new Date();
368
- const end = parseStatusDate(input.to, now, "now");
369
- const start = parseStatusDate(input.from, end, "24h");
370
- if (start >= end)
371
- throw new Error("from must be earlier than to");
372
- const startDate = formatApiDate(start);
373
- const endDate = formatApiDate(end);
374
- const analyticsResponse = await getAnalyticStatusesApi({
375
- token,
376
- domain,
377
- startDate,
378
- endDate,
379
- action: input.action || "",
380
- apiTokenName: input.api_token_name || "",
381
- fetchFn: dependencies.fetchFn,
382
- });
383
- const data = output.data;
384
- if (analyticsResponse.error) {
385
- output.error = "Account loaded, but analytics could not be loaded";
386
- data.analytics = analyticsResponse;
387
- }
388
- else {
389
- const analytics = unwrapApiData(analyticsResponse);
390
- data.analytics = {
391
- domain,
392
- from: `${startDate} UTC`,
393
- to: `${endDate} UTC`,
394
- ...(analytics &&
395
- typeof analytics === "object" &&
396
- !Array.isArray(analytics)
397
- ? analytics
398
- : { data: analytics }),
399
- };
400
- }
401
- return output;
402
- }
403
- export const rerunInputSchema = z.object({
404
- target: z
405
- .string()
406
- .describe("Target URL for a single rerun, or comma/newline-separated target URLs when bulk=true."),
407
- type: z
408
- .enum(["ai", "manual"])
409
- .describe("How the saved scraper was created: ai for an AI scraper or manual for a dashboard-built step workflow."),
410
- bulk: z
411
- .boolean()
412
- .default(false)
413
- .describe("Independently select target count: false for one URL or true to submit all parsed targets as one asynchronous bulk job."),
414
- scraper_id: z
415
- .string()
416
- .nullable()
417
- .optional()
418
- .describe("Required saved scraper UUID when bulk=false. Use the scraperId returned by scrape for an AI scraper, or a dashboard workflow's UUID for manual."),
419
- id: z
420
- .string()
421
- .nullable()
422
- .optional()
423
- .describe("Required saved scraper UUID when bulk=true. This identifies the configuration to run and is not a result ID."),
424
- max_depth: z
425
- .number()
426
- .int()
427
- .positive()
428
- .nullable()
429
- .optional()
430
- .describe("Maximum crawl depth for a single AI rerun; defaults to 2 when omitted."),
431
- max_pages: z
432
- .number()
433
- .int()
434
- .positive()
435
- .nullable()
436
- .optional()
437
- .describe("Maximum pages for a single AI rerun; defaults to 50 when omitted."),
438
- limit: z
439
- .number()
440
- .int()
441
- .positive()
442
- .nullable()
443
- .optional()
444
- .describe("Maximum results for a single AI rerun; defaults to 1000 when omitted."),
445
- include_patterns: z
446
- .string()
447
- .nullable()
448
- .optional()
449
- .describe("URL include regular expression for a single AI rerun; defaults to an empty string."),
450
- exclude_patterns: z
451
- .string()
452
- .nullable()
453
- .optional()
454
- .describe("URL exclude regular expression for a single AI rerun; defaults to an empty string."),
455
- });
456
- export async function rerunTool(token, input, dependencies = {}) {
457
- const aiOptionEntries = [
458
- ["max_depth", input.max_depth],
459
- ["max_pages", input.max_pages],
460
- ["limit", input.limit],
461
- ["include_patterns", input.include_patterns],
462
- ["exclude_patterns", input.exclude_patterns],
463
- ];
464
- const explicitAiOptions = aiOptionEntries
465
- .filter(([, value]) => value !== undefined && value !== null)
466
- .map(([name]) => name);
467
- let response;
468
- if (input.bulk) {
469
- if (!input.id)
470
- throw new Error("id is required when bulk is true");
471
- if (input.scraper_id !== undefined && input.scraper_id !== null) {
472
- throw new Error("scraper_id is only accepted for single reruns; use id when bulk is true");
473
- }
474
- if (explicitAiOptions.length) {
475
- throw new Error(`${explicitAiOptions.join(", ")} ${explicitAiOptions.length === 1 ? "is" : "are"} not accepted by bulk rerun endpoints`);
476
- }
477
- const urls = parseBulkUrls(input.target);
478
- if (!urls.length)
479
- throw new Error("No URLs found in the bulk target");
480
- response =
481
- input.type === "ai"
482
- ? await bulkRerunAiScraperApi({
483
- token,
484
- scraperId: input.id,
485
- urls,
486
- fetchFn: dependencies.fetchFn,
487
- })
488
- : await bulkRerunManualScraperApi({
489
- token,
490
- scraperId: input.id,
491
- urls,
492
- fetchFn: dependencies.fetchFn,
493
- });
494
- }
495
- else {
496
- if (!input.scraper_id) {
497
- throw new Error("scraper_id is required unless bulk is true");
498
- }
499
- if (input.id !== undefined && input.id !== null) {
500
- throw new Error("id is only accepted when bulk is true; use scraper_id for a single rerun");
501
- }
502
- if (input.type === "manual" && explicitAiOptions.length) {
503
- throw new Error(`${explicitAiOptions.join(", ")} ${explicitAiOptions.length === 1 ? "is" : "are"} only accepted by single AI reruns`);
504
- }
505
- const url = input.target.trim();
506
- if (!url)
507
- throw new Error("target URL must not be empty");
508
- response =
509
- input.type === "manual"
510
- ? await rerunManualScraperApi({
511
- token,
512
- scraperId: input.scraper_id,
513
- url,
514
- fetchFn: dependencies.fetchFn,
515
- })
516
- : await rerunAiScraperApi({
517
- token,
518
- scraperId: input.scraper_id,
519
- url,
520
- maxDepth: input.max_depth ?? 2,
521
- maxPages: input.max_pages ?? 50,
522
- limit: input.limit ?? 1000,
523
- includePatterns: input.include_patterns ?? "",
524
- excludePatterns: input.exclude_patterns ?? "",
525
- fetchFn: dependencies.fetchFn,
526
- });
527
- }
528
- return response;
529
- }
530
- export const resultsInputSchema = z.object({
531
- sort_field: z
532
- .string()
533
- .min(1)
534
- .default("updatedAt")
535
- .describe("Stored-result field used as the sort key; defaults to updatedAt."),
536
- sort_order: z
537
- .string()
538
- .trim()
539
- .toLowerCase()
540
- .pipe(z.enum(["asc", "desc"]))
541
- .default("desc")
542
- .describe("Sort direction for sort_field: asc or desc, accepted case-insensitively; defaults to desc."),
543
- page_size: z
544
- .number()
545
- .int()
546
- .positive()
547
- .default(10)
548
- .describe("Positive number of stored result records requested per page; defaults to 10."),
549
- page: z
550
- .number()
551
- .int()
552
- .positive()
553
- .default(1)
554
- .describe("One-based page number used to move through the stored result list; defaults to 1."),
555
- search: z
556
- .string()
557
- .nullable()
558
- .optional()
559
- .describe("Optional free-text filter used to narrow the stored result list."),
560
- date_range_column: z
561
- .string()
562
- .nullable()
563
- .optional()
564
- .describe("Stored-result date column to which start_at and end_at are applied."),
565
- start_at: z
566
- .string()
567
- .nullable()
568
- .optional()
569
- .describe("Optional inclusive ISO 8601 start bound applied to date_range_column."),
570
- end_at: z
571
- .string()
572
- .nullable()
573
- .optional()
574
- .describe("Optional inclusive ISO 8601 end bound applied to date_range_column."),
575
- });
576
- export async function resultsTool(token, input, dependencies = {}) {
577
- return getAllResultsApi({
578
- token,
579
- sortField: input.sort_field,
580
- sortOrder: input.sort_order.toUpperCase(),
581
- pageSize: input.page_size,
582
- page: input.page,
583
- search: input.search ?? null,
584
- dateRangeColumn: input.date_range_column ?? null,
585
- startAt: input.start_at ?? null,
586
- endAt: input.end_at ?? null,
587
- fetchFn: dependencies.fetchFn,
588
- });
589
244
  }
590
- export const resultInputSchema = z.object({
591
- result_id: z
592
- .string()
593
- .min(1)
594
- .describe("Required UUID of the stored result to retrieve, including a bulkResultId returned by an asynchronous bulk rerun."),
595
- });
596
- export async function resultTool(token, input, dependencies = {}) {
597
- const resultId = input.result_id.trim();
598
- if (!resultId)
599
- throw new Error("result_id must not be empty");
600
- return getResultByIdApi(token, resultId, dependencies);
245
+ function asStructured(result) {
246
+ return {
247
+ content: [{ type: "text", text: JSON.stringify(result, null, 2) }],
248
+ structuredContent: result,
249
+ ...(isApiFailure(result) ? { isError: true } : {}),
250
+ };
601
251
  }
602
- const readAnnotations = {
252
+ const annotations = {
603
253
  readOnlyHint: true,
604
- openWorldHint: false,
605
- destructiveHint: false,
606
- };
607
- const writeAnnotations = {
608
- readOnlyHint: false,
609
- openWorldHint: false,
254
+ openWorldHint: true,
610
255
  destructiveHint: false,
611
256
  };
612
257
  export const TOOL_DESCRIPTIONS = {
613
- fetch: "Use this when a public page URL is known; fetch is the default first content-acquisition step for agent-led work. It returns the raw page response through Web Unblocker, preserving source details for reading, analysis, verification, and follow-up transformations. Keep that response as the source of truth. If the user needs fields, JSON, a table, or extraction across many similar pages, fetch and inspect representative pages, define one reusable local extractor, fetch the remaining pages, and apply the extractor locally. For 100 similarly structured pages, run the fetches concurrently when safe and feed the saved responses into one local batch extraction; this is often faster than 100 separate backend-LLM extractions and preserves every raw page. Do not call scrape merely because the requested output is structured. Use scrape only when the user explicitly requests MrScraper-managed extraction or after fetch-led exploration has established a stable output schema and a clear benefit. Use serp when no target URL is known. The url argument is required. Enable browser_rendering for JavaScript-driven content; wait_for_selector then waits for a CSS selector. geo_code changes proxy location, while home_page, block_resources, max_retries, token_cap, and timeout tune loading and retry behavior.",
614
- scrape: "Use this when the user explicitly wants MrScraper-managed structured extraction or bounded URL discovery within a known site. Do not use general or listing for the first exploration of a known page: they send page content through a backend LLM, which can omit source details, narrow the result to the prompt, and repeat model work across pages. Start with fetch, inspect the raw content, understand the site, and define a stable output schema. Prefer one reusable local extractor across saved fetch responses; requesting fields, JSON, or a table does not by itself justify scrape. This remains true for 100 same-layout pages: safe concurrent fetches plus one local batch extraction are often faster than 100 separate backend-LLM runs and preserve every source response. Use general or listing only when managed extraction was explicitly requested or still has a clear benefit after that exploration. Choose agent=general for defined page fields, agent=listing for repeated records across pages, or agent=map for bounded URL discovery within the known site; map is separate from LLM extraction and does not require a schema. General and listing require prompt and optionally accept schema_prompt and proxy_country. max_pages applies to listing and map; max_depth, limit, include_patterns, and exclude_patterns are map-only. A successful run returns scraperId for reuse with rerun.",
615
- serp: "Use this when the task starts with a Google query or Google search URL rather than a known target page. After selecting relevant result URLs, use fetch for every page whose content will inform the answer and preserve each raw response. Analyze or transform those responses locally, using one reusable extractor when pages share a structure. Use scrape only when the user explicitly requests MrScraper-managed extraction or after fetch-led exploration has established a stable schema and clear benefit; stop after serp when the user only needs URL discovery. The query_or_url argument is required; region, language, and page control localization and pagination. format=json returns parsed results, format=html returns result-page HTML, render_js includes dynamic features, and client_timeout controls how long this MCP request waits. raw is a deprecated alias for format=html.",
616
- status: "Use this to check the current MrScraper account's identity, subscription, and usage before or after web-data work. With no arguments it returns an account summary including the user's name, email, and verification state. Supply domain only when request-outcome analytics are also needed; from and to select that analytics window, while action and api_token_name narrow those domain outcomes. This tool reports account and request health, not scrape-job progress; use result for a known asynchronous result ID.",
617
- rerun: "Use this only when you already have a saved scraper UUID and want to apply that configuration to the same or new target URLs. Set type=ai for a scraper created by scrape; type=manual selects a dashboard-built step workflow and requires the conversation's compliance acknowledgment. For one URL, leave bulk=false and pass target with scraper_id. A single AI rerun also accepts max_depth, max_pages, limit, include_patterns, and exclude_patterns; manual and bulk reruns reject those controls. For multiple URLs, set bulk=true, pass comma- or newline-separated target URLs and id; bulk jobs are asynchronous, so retain bulkResultId and inspect it with result until completion. Use scrape to create a new AI configuration.",
618
- results: "Use this to browse or locate stored scrape results when you do not yet know the exact result UUID. sort_field and sort_order control ordering; page_size and page control pagination; search narrows the list; and date_range_column with start_at or end_at applies a date window. This returns a result list rather than one complete record. Once an ID is known, use result for that record.",
619
- result: "Use this when you already know the exact stored result UUID and need the complete record or current job state. Pass that UUID as result_id; this may also be the bulkResultId returned by an asynchronous bulk rerun. Call result again as needed until an asynchronous job completes. Use results first when the UUID is unknown.",
258
+ scraperAPI: "Retrieves a known page through MrScraper's Scraper API as HTML, Markdown, a screenshot, or a combination. Matches the Python SDK's scraper_api options. Omit browser_rendering for automatic browser fallback; screenshot=true always enables it. super enables real-device routing independently. home_page maps to warmStart. Proxy, cookie, target-header, retry, and timeout options control page loading. Returns status_code, data, and sanitized headers.",
259
+ webSearch: "Searches Google from query text or a Google search URL passed as url. Matches the Python SDK's web_search options. Set raw=false and format=json for parsed results; the SDK-compatible default raw=true returns HTML. Google URLs supply q, gl, hl, and start; explicit region, language, and page override URL-derived values. render_js enables dynamic features; timeout controls the local request deadline. Returns status_code, data, and sanitized headers.",
260
+ status: "Returns the connected MrScraper account's identity (name, email, and verification state), subscription status, token limit, token usage, remaining tokens, and rate limit. Use it to check the account and remaining quota, for example before a large batch of requests. It takes no input and reports account usage, not the progress of a request.",
620
261
  };
621
262
  export function registerTools(server, getToken, dependencies = {}) {
622
- server.registerTool("fetch", {
623
- title: "Fetch Page",
624
- description: TOOL_DESCRIPTIONS.fetch,
625
- inputSchema: fetchInputSchema,
626
- outputSchema: fetchOutputSchema,
627
- annotations: readAnnotations,
628
- }, async (input) => asStructured(await fetchTool(getToken(), input, dependencies)));
629
- server.registerTool("scrape", {
630
- title: "Extract Data",
631
- description: TOOL_DESCRIPTIONS.scrape,
632
- inputSchema: scrapeInputSchema,
633
- outputSchema: scrapeOutputSchema,
634
- _meta: widgetMeta("records", "Extracting data…", "Extraction complete."),
635
- annotations: writeAnnotations,
636
- }, async (input) => asStructured(await scrapeTool(getToken(), input, dependencies)));
637
- server.registerTool("serp", {
638
- title: "Google Search",
639
- description: TOOL_DESCRIPTIONS.serp,
640
- inputSchema: serpInputSchema,
641
- outputSchema: serpOutputSchema,
642
- _meta: widgetMeta("serp", "Searching Google…", "Search complete."),
643
- annotations: readAnnotations,
644
- }, async (input) => asStructured(await serpTool(getToken(), input, dependencies)));
263
+ server.registerTool("scraperAPI", {
264
+ title: "Scraper API",
265
+ annotations: { title: "Scraper API", ...annotations },
266
+ description: TOOL_DESCRIPTIONS.scraperAPI,
267
+ inputSchema: scraperAPIInputSchema,
268
+ outputSchema: scraperAPIOutputSchema,
269
+ }, async (input) => asStructured(await scraperAPITool(getToken(), input, dependencies)));
270
+ server.registerTool("webSearch", {
271
+ title: "Web Search",
272
+ annotations: { title: "Web Search", ...annotations },
273
+ description: TOOL_DESCRIPTIONS.webSearch,
274
+ inputSchema: webSearchInputSchema,
275
+ outputSchema: webSearchOutputSchema,
276
+ _meta: widgetMeta("webSearch", "Searching Google…", "Search complete."),
277
+ }, async (input) => asStructured(await webSearchTool(getToken(), input, dependencies)));
645
278
  server.registerTool("status", {
646
279
  title: "Account Status",
280
+ annotations: {
281
+ title: "Account Status",
282
+ readOnlyHint: true,
283
+ openWorldHint: false,
284
+ destructiveHint: false,
285
+ },
647
286
  description: TOOL_DESCRIPTIONS.status,
648
287
  inputSchema: statusInputSchema,
649
288
  outputSchema: statusOutputSchema,
650
289
  _meta: widgetMeta("status", "Checking your account…", "Account status loaded."),
651
- annotations: readAnnotations,
652
290
  }, async (input) => asStructured(await statusTool(getToken(), input, dependencies)));
653
- server.registerTool("rerun", {
654
- title: "Rerun Saved Scraper",
655
- description: TOOL_DESCRIPTIONS.rerun,
656
- inputSchema: rerunInputSchema,
657
- outputSchema: rerunOutputSchema,
658
- annotations: writeAnnotations,
659
- }, async (input) => asStructured(await rerunTool(getToken(), input, dependencies)));
660
- server.registerTool("results", {
661
- title: "List Results",
662
- description: TOOL_DESCRIPTIONS.results,
663
- inputSchema: resultsInputSchema,
664
- outputSchema: resultsOutputSchema,
665
- _meta: widgetMeta("records", "Loading results…", "Results loaded."),
666
- annotations: readAnnotations,
667
- }, async (input) => asStructured(await resultsTool(getToken(), input, dependencies)));
668
- server.registerTool("result", {
669
- title: "Get Result",
670
- description: TOOL_DESCRIPTIONS.result,
671
- inputSchema: resultInputSchema,
672
- outputSchema: resultOutputSchema,
673
- _meta: widgetMeta("records", "Loading result…", "Result loaded."),
674
- annotations: readAnnotations,
675
- }, async (input) => asStructured(await resultTool(getToken(), input, dependencies)));
676
291
  }
677
292
  //# sourceMappingURL=tools.js.map