@mrscraper/ai-sdk 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +335 -25
- package/dist/index.d.ts +34 -32
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +69 -96
- package/dist/index.js.map +1 -1
- package/dist/inputs.d.ts +113 -0
- package/dist/inputs.d.ts.map +1 -0
- package/dist/inputs.js +108 -0
- package/dist/inputs.js.map +1 -0
- package/dist/model-output.d.ts +12 -0
- package/dist/model-output.d.ts.map +1 -0
- package/dist/model-output.js +23 -0
- package/dist/model-output.js.map +1 -0
- package/package.json +3 -3
- package/dist/results.d.ts +0 -71
- package/dist/results.d.ts.map +0 -1
- package/dist/results.js +0 -78
- package/dist/results.js.map +0 -1
- package/dist/search.d.ts +0 -22
- package/dist/search.d.ts.map +0 -1
- package/dist/search.js +0 -61
- package/dist/search.js.map +0 -1
package/README.md
CHANGED
|
@@ -1,20 +1,47 @@
|
|
|
1
1
|
# MrScraper tools for the AI SDK
|
|
2
2
|
|
|
3
3
|
[MrScraper](https://mrscraper.com) tools for the [Vercel AI SDK](https://ai-sdk.dev):
|
|
4
|
-
retrieve public web pages as
|
|
5
|
-
|
|
4
|
+
retrieve public web pages as HTML, Markdown, or screenshots, extract brand
|
|
5
|
+
details and structured JSON, search the web, and check your MrScraper account
|
|
6
|
+
status.
|
|
6
7
|
|
|
7
|
-
##
|
|
8
|
+
## Overview
|
|
9
|
+
|
|
10
|
+
The seven MrScraper services as AI SDK tools. Tool names, descriptions, input
|
|
11
|
+
fields, and defaults follow the MrScraper Integrations Standard, shared by the
|
|
12
|
+
MrScraper CLI, SDKs, and [MCP server](https://github.com/mrscraper-com/mrscraper-mcp).
|
|
13
|
+
Requests go through [`@mrscraper/sdk`](https://www.npmjs.com/package/@mrscraper/sdk) 6.
|
|
14
|
+
See the [API Reference](https://docs.mrscraper.com/docs/api/overview).
|
|
15
|
+
|
|
16
|
+
| Tool | Factory | Description |
|
|
17
|
+
| --- | --- | --- |
|
|
18
|
+
| `status` | `statusTool()` | Return the raw account status API response without summarizing, redacting body fields, or adding calculated fields. |
|
|
19
|
+
| `scrapeHtml` | `scrapeHtmlTool()` | Retrieve the raw HTML of a page. Browser rendering is optional. |
|
|
20
|
+
| `scrapeMarkdown` | `scrapeMarkdownTool()` | Retrieve a page as Markdown. Including HTML also enables browser rendering and returns both outputs as JSON. |
|
|
21
|
+
| `getScreenshot` | `getScreenshotTool()` | Capture a base64 screenshot of the full page or top viewport. Browser rendering is always enabled. |
|
|
22
|
+
| `scrapeBrand` | `scrapeBrandTool()` | Extract a website's colors, typography, logos, and social links as JSON. |
|
|
23
|
+
| `scrapeJson` | `scrapeJsonTool()` | Extract structured JSON from a page using a natural-language prompt. Append Json Schema: and a JSON schema to shape the output. |
|
|
24
|
+
| `webSearch` | `webSearchTool()` | Search the web and return ranked results with titles, URLs, descriptions, and sources. |
|
|
25
|
+
|
|
26
|
+
`mrscraperTools()` returns all seven, keyed by tool name.
|
|
27
|
+
|
|
28
|
+
## Installation
|
|
8
29
|
|
|
9
30
|
```bash
|
|
10
31
|
npm install @mrscraper/ai-sdk ai zod
|
|
11
32
|
```
|
|
12
33
|
|
|
13
|
-
Requires AI SDK 7 and Node.js 18 or later.
|
|
14
|
-
|
|
15
|
-
|
|
34
|
+
Requires AI SDK 7 and Node.js 18 or later.
|
|
35
|
+
|
|
36
|
+
## Authentication
|
|
16
37
|
|
|
17
|
-
|
|
38
|
+
Get an API key at [app.mrscraper.com/api-tokens](https://app.mrscraper.com/api-tokens)
|
|
39
|
+
and set it as `MRSCRAPER_API_KEY`, or pass `apiKey` to `mrscraperTools()` or a
|
|
40
|
+
tool factory. The tools use `apiKey`, then `MRSCRAPER_API_KEY`, then
|
|
41
|
+
`MRSCRAPER_API_TOKEN`. The key is sent in request headers. It is not a tool
|
|
42
|
+
input, so the model never sees it.
|
|
43
|
+
|
|
44
|
+
## Quickstart
|
|
18
45
|
|
|
19
46
|
```ts
|
|
20
47
|
import { generateText, isStepCount } from "ai";
|
|
@@ -22,8 +49,7 @@ import { mrscraperTools } from "@mrscraper/ai-sdk";
|
|
|
22
49
|
|
|
23
50
|
const { text } = await generateText({
|
|
24
51
|
model: "openai/gpt-5",
|
|
25
|
-
prompt:
|
|
26
|
-
"Read https://www.scrapethissite.com/pages/simple/ and list the five countries with the largest area.",
|
|
52
|
+
prompt: "Read https://example.com as Markdown and summarize it in two sentences.",
|
|
27
53
|
tools: mrscraperTools(),
|
|
28
54
|
stopWhen: isStepCount(5),
|
|
29
55
|
});
|
|
@@ -34,43 +60,327 @@ console.log(text);
|
|
|
34
60
|
Pick individual tools when you only need some of them:
|
|
35
61
|
|
|
36
62
|
```ts
|
|
37
|
-
import {
|
|
63
|
+
import { scrapeMarkdownTool, webSearchTool } from "@mrscraper/ai-sdk";
|
|
38
64
|
|
|
39
65
|
const tools = {
|
|
40
|
-
|
|
66
|
+
scrapeMarkdown: scrapeMarkdownTool({ apiKey: process.env.MRSCRAPER_API_KEY }),
|
|
41
67
|
webSearch: webSearchTool(),
|
|
42
68
|
};
|
|
43
69
|
```
|
|
44
70
|
|
|
45
|
-
|
|
71
|
+
Examples use `example.com` as the sample target. Replace it for your use case.
|
|
72
|
+
Live scraping and search calls consume account quota.
|
|
46
73
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
74
|
+
## Shared parameters
|
|
75
|
+
|
|
76
|
+
### Page controls
|
|
77
|
+
|
|
78
|
+
`scrapeHtml`, `scrapeMarkdown`, `getScreenshot`, and `scrapeJson` share these
|
|
79
|
+
inputs. Page defaults are 300 seconds, US geolocation/proxy country,
|
|
80
|
+
real-device routing off, and dashboard saving off. Omitted retry limits mean
|
|
81
|
+
no limit.
|
|
82
|
+
|
|
83
|
+
| Parameter | Type | Default | Description |
|
|
84
|
+
| --- | --- | --- | --- |
|
|
85
|
+
| `url` | string | Required | Absolute HTTP or HTTPS URL of the page to process. |
|
|
86
|
+
| `timeout` | number (integer) | `300` | Page load timeout in seconds, from 1 to 300. The HTTP request allows 30 additional seconds. |
|
|
87
|
+
| `geoCode` | string | `"us"` | Two-letter country code to load the page from, sent in lowercase. |
|
|
88
|
+
| `proxyCountry` | string | `"us"` | Two-letter country code of the proxy server, sent in lowercase. |
|
|
89
|
+
| `super` | boolean | `false` | Whether to route the request through real devices. |
|
|
90
|
+
| `waitUntil` | string: `domcontentloaded`, `load`, `networkidle0`, `networkidle2` | `"domcontentloaded"` | Navigation completion event to wait for: domcontentloaded, load, networkidle0, or networkidle2. |
|
|
91
|
+
| `proxy` | string | Omitted | Custom proxy URL used to load the page. |
|
|
92
|
+
| `maxRetries` | number (integer) | Omitted | Maximum number of retries, as a nonnegative integer. Omit this parameter for no limit. |
|
|
93
|
+
| `saveResult` | boolean | `false` | Whether to save the result to the dashboard Result tab. |
|
|
94
|
+
|
|
95
|
+
### Loading controls
|
|
96
|
+
|
|
97
|
+
`scrapeHtml`, `scrapeMarkdown`, and `getScreenshot` also accept these inputs.
|
|
98
|
+
`scrapeJson` does not.
|
|
99
|
+
|
|
100
|
+
| Parameter | Type | Default | Description |
|
|
101
|
+
| --- | --- | --- | --- |
|
|
102
|
+
| `blockResources` | boolean | `false` | Whether to block images, fonts, and stylesheets while loading the page. |
|
|
103
|
+
| `waitForSelector` | string | Omitted | Wait for the page to contain the specified CSS selector before returning the result. |
|
|
104
|
+
| `returnCookie` | boolean | `false` | Whether to include browser session cookies in the response. |
|
|
105
|
+
|
|
106
|
+
Browser rendering is optional for HTML, Markdown, and JSON; screenshots always
|
|
107
|
+
enable it. Markdown with HTML also enables browser rendering.
|
|
108
|
+
|
|
109
|
+
## Service reference
|
|
110
|
+
|
|
111
|
+
### Status
|
|
112
|
+
|
|
113
|
+
Tool `status`, factory `statusTool()`.
|
|
114
|
+
|
|
115
|
+
Return the raw account status API response without summarizing, redacting body fields, or adding calculated fields.
|
|
116
|
+
|
|
117
|
+
This service takes no service parameters. Authentication is configured
|
|
118
|
+
separately. The HTTP deadline is fixed at 600 seconds. It is not a service
|
|
119
|
+
parameter.
|
|
120
|
+
|
|
121
|
+
Response: The API body contains message and data. Account details remain
|
|
122
|
+
nested in data, including subscriptionItem and scraperAllowance. The response
|
|
123
|
+
envelope therefore exposes account details at data.data. Sensitive response
|
|
124
|
+
headers are removed.
|
|
125
|
+
|
|
126
|
+
### Scrape HTML
|
|
127
|
+
|
|
128
|
+
Tool `scrapeHtml`, factory `scrapeHtmlTool()`.
|
|
129
|
+
|
|
130
|
+
Retrieve the raw HTML of a page. Browser rendering is optional.
|
|
131
|
+
|
|
132
|
+
Inputs: [page controls](#page-controls), [loading controls](#loading-controls), and:
|
|
133
|
+
|
|
134
|
+
| Parameter | Type | Default | Description |
|
|
135
|
+
| --- | --- | --- | --- |
|
|
136
|
+
| `browserRendering` | boolean | `false` | Whether to render the page in a browser before returning the result. |
|
|
137
|
+
| `warmStart` | boolean | `false` | Whether to enable warm start for more reliable scraping, at the cost of additional latency. |
|
|
138
|
+
|
|
139
|
+
Example input:
|
|
140
|
+
|
|
141
|
+
```json
|
|
142
|
+
{
|
|
143
|
+
"url": "https://example.com",
|
|
144
|
+
"browserRendering": true,
|
|
145
|
+
"warmStart": true,
|
|
146
|
+
"waitForSelector": "body",
|
|
147
|
+
"timeout": 120
|
|
148
|
+
}
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
Response: The default response body is HTML text. Additional controls can
|
|
152
|
+
cause the API to return JSON; the returned structure is preserved.
|
|
153
|
+
|
|
154
|
+
### Scrape Markdown
|
|
155
|
+
|
|
156
|
+
Tool `scrapeMarkdown`, factory `scrapeMarkdownTool()`.
|
|
157
|
+
|
|
158
|
+
Retrieve a page as Markdown. Including HTML also enables browser rendering and returns both outputs as JSON.
|
|
159
|
+
|
|
160
|
+
Inputs: [page controls](#page-controls), [loading controls](#loading-controls), and:
|
|
161
|
+
|
|
162
|
+
| Parameter | Type | Default | Description |
|
|
163
|
+
| --- | --- | --- | --- |
|
|
164
|
+
| `browserRendering` | boolean | `false` | Whether to render the page in a browser before returning the result. |
|
|
165
|
+
| `warmStart` | boolean | `false` | Whether to enable warm start for more reliable scraping, at the cost of additional latency. |
|
|
166
|
+
| `html` | boolean | `false` | Whether to include the page HTML in the response. |
|
|
167
|
+
|
|
168
|
+
Example input:
|
|
169
|
+
|
|
170
|
+
```json
|
|
171
|
+
{
|
|
172
|
+
"url": "https://example.com",
|
|
173
|
+
"html": true,
|
|
174
|
+
"warmStart": true
|
|
175
|
+
}
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
Response: The default response body is Markdown text. Enabling HTML returns
|
|
179
|
+
JSON containing both HTML and Markdown.
|
|
180
|
+
|
|
181
|
+
### Get screenshot
|
|
182
|
+
|
|
183
|
+
Tool `getScreenshot`, factory `getScreenshotTool()`.
|
|
184
|
+
|
|
185
|
+
Capture a base64 screenshot of the full page or top viewport. Browser rendering is always enabled.
|
|
186
|
+
|
|
187
|
+
Inputs: [page controls](#page-controls), [loading controls](#loading-controls), and:
|
|
188
|
+
|
|
189
|
+
| Parameter | Type | Default | Description |
|
|
190
|
+
| --- | --- | --- | --- |
|
|
191
|
+
| `screenshot` | string: `full`, `top` | `"full"` | Screenshot area to capture: full for the entire page or top for the top viewport. |
|
|
192
|
+
| `html` | boolean | `true` | Whether to include the page HTML in the response. |
|
|
193
|
+
| `markdown` | boolean | `false` | Whether to include the page Markdown in the response. |
|
|
194
|
+
|
|
195
|
+
Example input:
|
|
196
|
+
|
|
197
|
+
```json
|
|
198
|
+
{
|
|
199
|
+
"url": "https://example.com",
|
|
200
|
+
"screenshot": "top",
|
|
201
|
+
"html": false,
|
|
202
|
+
"markdown": true
|
|
203
|
+
}
|
|
204
|
+
```
|
|
205
|
+
|
|
206
|
+
Response: The response body contains a base64 screenshot and requested page
|
|
207
|
+
outputs. The API can still include HTML when HTML output is disabled.
|
|
208
|
+
|
|
209
|
+
### Scrape brand
|
|
210
|
+
|
|
211
|
+
Tool `scrapeBrand`, factory `scrapeBrandTool()`.
|
|
212
|
+
|
|
213
|
+
Extract a website's colors, typography, logos, and social links as JSON.
|
|
214
|
+
|
|
215
|
+
| Parameter | Type | Default | Description |
|
|
216
|
+
| --- | --- | --- | --- |
|
|
217
|
+
| `url` | string | Required | Absolute HTTP or HTTPS URL of the page to process. |
|
|
218
|
+
|
|
219
|
+
The HTTP deadline is fixed at 120 seconds. It is not a service parameter. The
|
|
220
|
+
request uses `action=brand-scraping` and `bundle=none`; page controls are not
|
|
221
|
+
accepted.
|
|
222
|
+
|
|
223
|
+
Example input:
|
|
224
|
+
|
|
225
|
+
```json
|
|
226
|
+
{
|
|
227
|
+
"url": "https://example.com"
|
|
228
|
+
}
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
Response: The API body contains message and a data object with job_id,
|
|
232
|
+
status, url, final_url, brand, and error. Brand details are available at
|
|
233
|
+
data.data.brand in the response envelope.
|
|
234
|
+
|
|
235
|
+
### Scrape JSON
|
|
236
|
+
|
|
237
|
+
Tool `scrapeJson`, factory `scrapeJsonTool()`.
|
|
238
|
+
|
|
239
|
+
Extract structured JSON from a page using a natural-language prompt. Append Json Schema: and a JSON schema to shape the output.
|
|
240
|
+
|
|
241
|
+
Inputs: [page controls](#page-controls) and:
|
|
242
|
+
|
|
243
|
+
| Parameter | Type | Default | Description |
|
|
244
|
+
| --- | --- | --- | --- |
|
|
245
|
+
| `prompt` | string | `"Extract all data as complete as possible."` | Instructions describing the data to extract. Append Json Schema: and a JSON schema to shape the output. |
|
|
246
|
+
| `browserRendering` | boolean | `false` | Whether to render the page in a browser before returning the result. |
|
|
247
|
+
| `html` | boolean | `false` | Whether to include the page HTML in the response. |
|
|
248
|
+
| `screenshot` | boolean | `false` | Whether to include a base64 screenshot in the response. |
|
|
249
|
+
|
|
250
|
+
The request uses `agent=general` and GET. If the encoded URL exceeds 8,000
|
|
251
|
+
characters, it switches to POST with `url`, `prompt`, and `agent` in the body;
|
|
252
|
+
page controls remain in the query.
|
|
253
|
+
|
|
254
|
+
Example input:
|
|
255
|
+
|
|
256
|
+
```json
|
|
257
|
+
{
|
|
258
|
+
"url": "https://example.com",
|
|
259
|
+
"prompt": "Extract the first 2 items with title and price. Json Schema: {\"type\":\"array\",\"items\":{\"type\":\"object\",\"properties\":{\"title\":{\"type\":\"string\"},\"price\":{\"type\":\"string\"}},\"required\":[\"title\",\"price\"]}}",
|
|
260
|
+
"browserRendering": true
|
|
261
|
+
}
|
|
262
|
+
```
|
|
263
|
+
|
|
264
|
+
Response: Extraction results are in the API body's data field and follow the
|
|
265
|
+
requested schema. Response metadata includes thread_id, token_usage, status,
|
|
266
|
+
error, runtime, and cookies. html, markdown, and screenshot can be empty
|
|
267
|
+
strings when not requested.
|
|
268
|
+
|
|
269
|
+
### Web search
|
|
270
|
+
|
|
271
|
+
Tool `webSearch`, factory `webSearchTool()`.
|
|
52
272
|
|
|
53
|
-
|
|
54
|
-
text, so a screenshot costs image tokens rather than hundreds of thousands of
|
|
55
|
-
text tokens. Other results are sent as JSON.
|
|
273
|
+
Search the web and return ranked results with titles, URLs, descriptions, and sources.
|
|
56
274
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
275
|
+
| Parameter | Type | Default | Description |
|
|
276
|
+
| --- | --- | --- | --- |
|
|
277
|
+
| `query` | string | Required | Search query text, trimmed before sending, containing 1 to 1000 characters. |
|
|
278
|
+
| `country` | string | `"us"` | Two-letter country code to search from, sent in lowercase. |
|
|
279
|
+
| `numResults` | number (integer) | `10` | Number of search results to request, as an integer from 10 to 100. Fewer results may be returned. |
|
|
60
280
|
|
|
61
|
-
|
|
281
|
+
The HTTP deadline is fixed at 120 seconds. It is not a service parameter.
|
|
282
|
+
|
|
283
|
+
Example input:
|
|
284
|
+
|
|
285
|
+
```json
|
|
286
|
+
{
|
|
287
|
+
"query": "site:example.com documentation",
|
|
288
|
+
"country": "id",
|
|
289
|
+
"numResults": 20
|
|
290
|
+
}
|
|
291
|
+
```
|
|
292
|
+
|
|
293
|
+
Response: The API body contains success, message, a data array, and
|
|
294
|
+
tokenUsage. Each result includes rank, title, url, description, and source.
|
|
295
|
+
|
|
296
|
+
## Response format
|
|
297
|
+
|
|
298
|
+
Each tool returns the `@mrscraper/sdk` response envelope (`MrScraperResponse`)
|
|
299
|
+
containing `status_code`, `data`, and `headers`. The `data` field contains the
|
|
300
|
+
API response body; its field names are not converted. Status preserves the
|
|
301
|
+
complete body without adding summary fields.
|
|
302
|
+
|
|
303
|
+
```json
|
|
304
|
+
{
|
|
305
|
+
"status_code": 200,
|
|
306
|
+
"data": "<html><body>Example page content...</body></html>",
|
|
307
|
+
"headers": {
|
|
308
|
+
"content-type": "text/html; charset=utf-8"
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
```
|
|
312
|
+
|
|
313
|
+
The AI SDK sends tool results to the model as JSON. `getScreenshot`, and
|
|
314
|
+
`scrapeJson` when a screenshot is requested, send the base64 screenshot as an
|
|
315
|
+
image part instead, so a screenshot costs image tokens rather than hundreds of
|
|
316
|
+
thousands of text tokens. Your code still receives the base64 string in
|
|
317
|
+
`data.screenshot`.
|
|
318
|
+
|
|
319
|
+
## Error handling
|
|
320
|
+
|
|
321
|
+
Authentication, HTTP, network, and timeout failures throw `MrScraperError`
|
|
322
|
+
from `@mrscraper/sdk`, with the API key redacted. Its `status` property is the
|
|
323
|
+
HTTP status when available. The AI SDK reports a thrown error to the model as
|
|
324
|
+
a tool error. Inputs that do not match a tool's schema, including fields the
|
|
325
|
+
tool does not define, are rejected before any request.
|
|
326
|
+
|
|
327
|
+
## Integration features
|
|
328
|
+
|
|
329
|
+
### Configuration
|
|
330
|
+
|
|
331
|
+
`mrscraperTools()` and each factory accept `{ apiKey }`. TypeScript exports
|
|
332
|
+
include `MrScraperToolOptions`, an input type per tool (`StatusInput`,
|
|
333
|
+
`ScrapeHtmlInput`, `ScrapeMarkdownInput`, `GetScreenshotInput`,
|
|
334
|
+
`ScrapeBrandInput`, `ScrapeJsonInput`, `WebSearchInput`), and the SDK's
|
|
335
|
+
`MrScraperResponse`, `PageData`, `WebSearchResponse`, and `WebSearchResult`.
|
|
336
|
+
|
|
337
|
+
### Data and permissions
|
|
62
338
|
|
|
63
339
|
The tools send target URLs, search queries, and request options to MrScraper's
|
|
64
340
|
API. Use them only for public or otherwise authorized content, and follow the
|
|
65
341
|
target site's terms. See the [Privacy Policy](https://mrscraper.com/privacy-policy)
|
|
66
342
|
and [Acceptable Use Policy](https://mrscraper.com/acceptable-use-policy).
|
|
67
343
|
|
|
68
|
-
|
|
344
|
+
### Links
|
|
69
345
|
|
|
70
346
|
- Documentation: https://docs.mrscraper.com/docs/integrations/vercel-ai-sdk
|
|
71
347
|
- MrScraper MCP server (for MCP clients): https://github.com/mrscraper-com/mrscraper-mcp
|
|
72
348
|
- Support: support@mrscraper.com
|
|
73
349
|
|
|
350
|
+
## Migration
|
|
351
|
+
|
|
352
|
+
Upgrading from 0.2:
|
|
353
|
+
|
|
354
|
+
- `scraperAPI` (`scraperApiTool()`) is removed, and with it the `formats`
|
|
355
|
+
input. Use `scrapeMarkdown` for Markdown (with `html: true` for both),
|
|
356
|
+
`scrapeHtml` for raw HTML, or `getScreenshot` for a screenshot.
|
|
357
|
+
- Browser rendering no longer switches on automatically. It is off unless
|
|
358
|
+
`browserRendering` is true; `getScreenshot` always renders. The page timeout
|
|
359
|
+
defaults to 300 seconds (was 120).
|
|
360
|
+
- `webSearch`: rename `region` to `country`. `query` is search text; a search
|
|
361
|
+
results URL is no longer read for its `q` parameter.
|
|
362
|
+
- `accountStatusTool()` is now `statusTool()`. It returns the raw account
|
|
363
|
+
status response, with account details at `data.data`, instead of summary
|
|
364
|
+
fields such as `token_remaining`.
|
|
365
|
+
- Every tool returns `{ status_code, data, headers }` with the API body in
|
|
366
|
+
`data`. Search results are in `data.data`, each with `url` (was `link`).
|
|
367
|
+
The `PageResult`, `PageFormat`, `SearchResult`, `SearchResultItem`,
|
|
368
|
+
`ScraperApiInput`, and `AccountSummary` types are removed.
|
|
369
|
+
- `mrscraperTools()` adds `scrapeBrand` and `scrapeJson`. Pick individual
|
|
370
|
+
tools if the model should not use them.
|
|
371
|
+
|
|
372
|
+
## Development and testing
|
|
373
|
+
|
|
374
|
+
The automated suite stubs `fetch` and makes no API calls.
|
|
375
|
+
|
|
376
|
+
```bash
|
|
377
|
+
npm ci
|
|
378
|
+
npm run typecheck
|
|
379
|
+
npm test
|
|
380
|
+
npm run build
|
|
381
|
+
npm pack --dry-run
|
|
382
|
+
```
|
|
383
|
+
|
|
74
384
|
## License
|
|
75
385
|
|
|
76
386
|
[MIT](LICENSE)
|
package/dist/index.d.ts
CHANGED
|
@@ -1,43 +1,45 @@
|
|
|
1
|
-
import { type
|
|
1
|
+
import { type MrScraperResponse, type PageData, type WebSearchResponse } from "@mrscraper/sdk";
|
|
2
2
|
import { type Tool } from "ai";
|
|
3
|
-
import { type
|
|
4
|
-
export type {
|
|
5
|
-
export type {
|
|
6
|
-
/** Input of the scraperAPI tool after validation. */
|
|
7
|
-
export interface ScraperApiInput {
|
|
8
|
-
url: string;
|
|
9
|
-
formats: PageFormat[];
|
|
10
|
-
browserRendering?: boolean;
|
|
11
|
-
waitForSelector?: string;
|
|
12
|
-
geoCode?: string;
|
|
13
|
-
timeout?: number;
|
|
14
|
-
}
|
|
15
|
-
/** Input of the webSearch tool after validation. */
|
|
16
|
-
export interface WebSearchInput {
|
|
17
|
-
query: string;
|
|
18
|
-
region?: string;
|
|
19
|
-
numResults?: number;
|
|
20
|
-
}
|
|
3
|
+
import { type GetScreenshotInput, type ScrapeBrandInput, type ScrapeHtmlInput, type ScrapeJsonInput, type ScrapeMarkdownInput, type StatusInput, type WebSearchInput } from "./inputs.js";
|
|
4
|
+
export type { MrScraperResponse, PageData, WebSearchResponse, WebSearchResult } from "@mrscraper/sdk";
|
|
5
|
+
export type { GetScreenshotInput, ScrapeBrandInput, ScrapeHtmlInput, ScrapeJsonInput, ScrapeMarkdownInput, StatusInput, WebSearchInput, } from "./inputs.js";
|
|
21
6
|
export interface MrScraperToolOptions {
|
|
22
7
|
/**
|
|
23
|
-
*
|
|
24
|
-
*
|
|
8
|
+
* API key used to authenticate the request. Defaults to the
|
|
9
|
+
* MRSCRAPER_API_KEY environment variable, then MRSCRAPER_API_TOKEN. Get one
|
|
10
|
+
* at https://app.mrscraper.com/api-tokens.
|
|
25
11
|
*/
|
|
26
12
|
apiKey?: string;
|
|
27
13
|
}
|
|
14
|
+
/** Return the raw account status API response without summarizing, redacting body fields, or adding calculated fields. */
|
|
15
|
+
export declare function statusTool(options?: MrScraperToolOptions): Tool<StatusInput, MrScraperResponse>;
|
|
16
|
+
/** Retrieve the raw HTML of a page. Browser rendering is optional. */
|
|
17
|
+
export declare function scrapeHtmlTool(options?: MrScraperToolOptions): Tool<ScrapeHtmlInput, MrScraperResponse<PageData>>;
|
|
18
|
+
/** Retrieve a page as Markdown. Including HTML also enables browser rendering and returns both outputs as JSON. */
|
|
19
|
+
export declare function scrapeMarkdownTool(options?: MrScraperToolOptions): Tool<ScrapeMarkdownInput, MrScraperResponse<PageData>>;
|
|
20
|
+
/**
|
|
21
|
+
* Capture a base64 screenshot of the full page or top viewport. Browser
|
|
22
|
+
* rendering is always enabled. The model receives the screenshot as an image.
|
|
23
|
+
*/
|
|
24
|
+
export declare function getScreenshotTool(options?: MrScraperToolOptions): Tool<GetScreenshotInput, MrScraperResponse<PageData>>;
|
|
25
|
+
/** Extract a website's colors, typography, logos, and social links as JSON. */
|
|
26
|
+
export declare function scrapeBrandTool(options?: MrScraperToolOptions): Tool<ScrapeBrandInput, MrScraperResponse>;
|
|
28
27
|
/**
|
|
29
|
-
*
|
|
30
|
-
*
|
|
28
|
+
* Extract structured JSON from a page using a natural-language prompt. Append
|
|
29
|
+
* Json Schema: and a JSON schema to shape the output. A requested screenshot
|
|
30
|
+
* reaches the model as an image.
|
|
31
31
|
*/
|
|
32
|
-
export declare function
|
|
33
|
-
/**
|
|
34
|
-
export declare function webSearchTool(options?: MrScraperToolOptions): Tool<WebSearchInput,
|
|
35
|
-
/**
|
|
36
|
-
export declare function accountStatusTool(options?: MrScraperToolOptions): Tool<Record<string, never>, AccountSummary>;
|
|
37
|
-
/** All MrScraper tools, keyed by the names the MrScraper MCP server uses. */
|
|
32
|
+
export declare function scrapeJsonTool(options?: MrScraperToolOptions): Tool<ScrapeJsonInput, MrScraperResponse>;
|
|
33
|
+
/** Search the web and return ranked results with titles, URLs, descriptions, and sources. */
|
|
34
|
+
export declare function webSearchTool(options?: MrScraperToolOptions): Tool<WebSearchInput, MrScraperResponse<WebSearchResponse>>;
|
|
35
|
+
/** All seven MrScraper tools, keyed by the names the Node SDK and MCP server use. */
|
|
38
36
|
export declare function mrscraperTools(options?: MrScraperToolOptions): {
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
37
|
+
status: Tool<StatusInput, MrScraperResponse>;
|
|
38
|
+
scrapeHtml: Tool<ScrapeHtmlInput, MrScraperResponse<PageData>>;
|
|
39
|
+
scrapeMarkdown: Tool<ScrapeMarkdownInput, MrScraperResponse<PageData>>;
|
|
40
|
+
getScreenshot: Tool<GetScreenshotInput, MrScraperResponse<PageData>>;
|
|
41
|
+
scrapeBrand: Tool<ScrapeBrandInput, MrScraperResponse>;
|
|
42
|
+
scrapeJson: Tool<ScrapeJsonInput, MrScraperResponse>;
|
|
43
|
+
webSearch: Tool<WebSearchInput, MrScraperResponse<WebSearchResponse>>;
|
|
42
44
|
};
|
|
43
45
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EASL,KAAK,iBAAiB,EACtB,KAAK,QAAQ,EACb,KAAK,iBAAiB,EACvB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAQ,KAAK,IAAI,EAAE,MAAM,IAAI,CAAC;AAErC,OAAO,EAQL,KAAK,kBAAkB,EACvB,KAAK,gBAAgB,EACrB,KAAK,eAAe,EACpB,KAAK,eAAe,EACpB,KAAK,mBAAmB,EACxB,KAAK,WAAW,EAChB,KAAK,cAAc,EACpB,MAAM,aAAa,CAAC;AAGrB,YAAY,EAAE,iBAAiB,EAAE,QAAQ,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACtG,YAAY,EACV,kBAAkB,EAClB,gBAAgB,EAChB,eAAe,EACf,eAAe,EACf,mBAAmB,EACnB,WAAW,EACX,cAAc,GACf,MAAM,aAAa,CAAC;AAErB,MAAM,WAAW,oBAAoB;IACnC;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAYD,0HAA0H;AAC1H,wBAAgB,UAAU,CAAC,OAAO,GAAE,oBAAyB,GAAG,IAAI,CAAC,WAAW,EAAE,iBAAiB,CAAC,CAOnG;AAED,sEAAsE;AACtE,wBAAgB,cAAc,CAAC,OAAO,GAAE,oBAAyB,GAAG,IAAI,CAAC,eAAe,EAAE,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAMrH;AAED,mHAAmH;AACnH,wBAAgB,kBAAkB,CAChC,OAAO,GAAE,oBAAyB,GACjC,IAAI,CAAC,mBAAmB,EAAE,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAOxD;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,GAAE,oBAAyB,GACjC,IAAI,CAAC,kBAAkB,EAAE,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAQvD;AAED,+EAA+E;AAC/E,wBAAgB,eAAe,CAAC,OAAO,GAAE,oBAAyB,GAAG,IAAI,CAAC,gBAAgB,EAAE,iBAAiB,CAAC,CAM7G;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,OAAO,GAAE,oBAAyB,GAAG,IAAI,CAAC,eAAe,EAAE,iBAAiB,CAAC,CAQ3G;AAED,6FAA6F;AAC7F,wBAAgB,aAAa,CAC3B,OAAO,GAAE,oBAAyB,GACjC,IAAI,CAAC,cAAc,EAAE,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,CAM5D;AAED,qFAAqF;AACrF,wBAAgB,cAAc,CAAC,OAAO,GAAE,oBAAyB,GAAG;IAClE,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,iBAAiB,CAAC,CAAC;IAC7C,UAAU,EAAE,IAAI,CAAC,eAAe,EAAE,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC/D,cAAc,EAAE,IAAI,CAAC,mBAAmB,EAAE,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC;IACvE,aAAa,EAAE,IAAI,CAAC,kBAAkB,EAAE,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC;IACrE,WAAW,EAAE,IAAI,CAAC,gBAAgB,EAAE,iBAAiB,CAAC,CAAC;IACvD,UAAU,EAAE,IAAI,CAAC,eAAe,EAAE,iBAAiB,CAAC,CAAC;IACrD,SAAS,EAAE,IAAI,CAAC,cAAc,EAAE,iBAAiB,CAAC,iBAAiB,CAAC,CAAC,CAAC;CACvE,CAUA"}
|