@decodo/mcp-server 1.2.3 → 1.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +142 -31
- package/build/auth/constants.js +12 -0
- package/build/auth/constants.js.map +1 -0
- package/build/auth/credential.js +27 -0
- package/build/auth/credential.js.map +1 -0
- package/build/auth/detect-credential-type.js +14 -0
- package/build/auth/detect-credential-type.js.map +1 -0
- package/build/auth/index.js +12 -0
- package/build/auth/index.js.map +1 -0
- package/build/{clients → auth}/types.js.map +1 -1
- package/build/clients/retry.js +17 -19
- package/build/clients/retry.js.map +1 -1
- package/build/clients/scraper-api-client.js +99 -42
- package/build/clients/scraper-api-client.js.map +1 -1
- package/build/constants.js +1 -34
- package/build/constants.js.map +1 -1
- package/build/index.js +8 -4
- package/build/index.js.map +1 -1
- package/build/logger.js +16 -0
- package/build/logger.js.map +1 -0
- package/build/server/cors.js +30 -0
- package/build/server/cors.js.map +1 -0
- package/build/server/sapi-base-server.js +0 -1
- package/build/server/sapi-base-server.js.map +1 -1
- package/build/server/sapi-http-server.js.map +1 -1
- package/build/server/sapi-stdio-server.js.map +1 -1
- package/build/server.js +9 -7
- package/build/server.js.map +1 -1
- package/build/tools/amazon-bestsellers/amazon-bestsellers-tool.js +5 -3
- package/build/tools/amazon-bestsellers/amazon-bestsellers-tool.js.map +1 -1
- package/build/tools/amazon-pricing/amazon-pricing-tool.js +5 -2
- package/build/tools/amazon-pricing/amazon-pricing-tool.js.map +1 -1
- package/build/tools/amazon-product/amazon-product-tool.js +5 -2
- package/build/tools/amazon-product/amazon-product-tool.js.map +1 -1
- package/build/tools/amazon-search/amazon-search-tool.js +5 -2
- package/build/tools/amazon-search/amazon-search-tool.js.map +1 -1
- package/build/tools/amazon-sellers/amazon-sellers-tool.js +5 -2
- package/build/tools/amazon-sellers/amazon-sellers-tool.js.map +1 -1
- package/build/tools/bing-search/bing-search-tool.js +11 -9
- package/build/tools/bing-search/bing-search-tool.js.map +1 -1
- package/build/tools/chatgpt/chatgpt-tool.js +5 -1
- package/build/tools/chatgpt/chatgpt-tool.js.map +1 -1
- package/build/tools/google-ads/google-ads-tool.js +5 -1
- package/build/tools/google-ads/google-ads-tool.js.map +1 -1
- package/build/tools/google-ai-mode/google-ai-mode-tool.js +13 -7
- package/build/tools/google-ai-mode/google-ai-mode-tool.js.map +1 -1
- package/build/tools/google-lens/google-lens-tool.js +5 -1
- package/build/tools/google-lens/google-lens-tool.js.map +1 -1
- package/build/tools/google-search/google-search-tool.js +5 -1
- package/build/tools/google-search/google-search-tool.js.map +1 -1
- package/build/tools/google-travel-hotels/google-travel-hotels-tool.js +5 -1
- package/build/tools/google-travel-hotels/google-travel-hotels-tool.js.map +1 -1
- package/build/tools/perplexity/perplexity-tool.js +5 -1
- package/build/tools/perplexity/perplexity-tool.js.map +1 -1
- package/build/tools/reddit-post/reddit-post-tool.js +5 -1
- package/build/tools/reddit-post/reddit-post-tool.js.map +1 -1
- package/build/tools/reddit-subreddit/reddit-subreddit-tool.js +5 -1
- package/build/tools/reddit-subreddit/reddit-subreddit-tool.js.map +1 -1
- package/build/tools/reddit-user/reddit-user-tool.js +5 -1
- package/build/tools/reddit-user/reddit-user-tool.js.map +1 -1
- package/build/tools/scrape-as-markdown/scrape-as-markdown-tool.js +3 -0
- package/build/tools/scrape-as-markdown/scrape-as-markdown-tool.js.map +1 -1
- package/build/tools/screenshot/screenshot-tool.js +3 -0
- package/build/tools/screenshot/screenshot-tool.js.map +1 -1
- package/build/tools/target-product/target-product-tool.js +5 -3
- package/build/tools/target-product/target-product-tool.js.map +1 -1
- package/build/tools/target-search/target-search-tool.js +5 -3
- package/build/tools/target-search/target-search-tool.js.map +1 -1
- package/build/tools/tiktok-post/tiktok-post-tool.js +5 -1
- package/build/tools/tiktok-post/tiktok-post-tool.js.map +1 -1
- package/build/tools/tiktok-shop-product/tiktok-shop-product-tool.js +5 -1
- package/build/tools/tiktok-shop-product/tiktok-shop-product-tool.js.map +1 -1
- package/build/tools/tiktok-shop-search/tiktok-shop-search-tool.js +5 -2
- package/build/tools/tiktok-shop-search/tiktok-shop-search-tool.js.map +1 -1
- package/build/tools/tiktok-shop-url/tiktok-shop-url-tool.js +5 -1
- package/build/tools/tiktok-shop-url/tiktok-shop-url-tool.js.map +1 -1
- package/build/tools/tool.js.map +1 -1
- package/build/tools/walmart-product/walmart-product-tool.js +18 -2
- package/build/tools/walmart-product/walmart-product-tool.js.map +1 -1
- package/build/tools/walmart-search/walmart-search-tool.js +18 -3
- package/build/tools/walmart-search/walmart-search-tool.js.map +1 -1
- package/build/tools/youtube-channel/youtube-channel-tool.js +5 -1
- package/build/tools/youtube-channel/youtube-channel-tool.js.map +1 -1
- package/build/tools/youtube-metadata/youtube-metadata-tool.js +5 -1
- package/build/tools/youtube-metadata/youtube-metadata-tool.js.map +1 -1
- package/build/tools/youtube-search/youtube-search-tool.js +5 -1
- package/build/tools/youtube-search/youtube-search-tool.js.map +1 -1
- package/build/tools/youtube-subtitles/youtube-subtitles-tool.js +5 -1
- package/build/tools/youtube-subtitles/youtube-subtitles-tool.js.map +1 -1
- package/package.json +5 -2
- /package/build/{clients → auth}/types.js +0 -0
package/README.md
CHANGED
|
@@ -6,60 +6,145 @@
|
|
|
6
6
|
<p align="center">
|
|
7
7
|
<a href="https://dashboard.decodo.com/integrations?utm_source=github&utm_medium=social&utm_campaign=mcp_server"> <img src="https://github.com/user-attachments/assets/a1e52a9e-3da1-4081-b3c6-053aafb8f196"/></a>
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
9
|
+
Connect LLMs and AI agents to live web data using MCP (Model Context Protocol). The Decodo MCP
|
|
10
|
+
Server lets you scrape websites, search engines, eCommerce platforms, and social media directly from
|
|
11
|
+
AI tools like Claude, Cursor, and Windsurf, all without the need to build scraping infrastructure
|
|
12
|
+
from scratch.
|
|
12
13
|
|
|
13
|
-
|
|
14
|
+
- Structured outputs in JSON, Markdown, and screenshots
|
|
15
|
+
- Server-side JavaScript rendering and anti-bot handling
|
|
16
|
+
- 125M+ IPs across 195+ locations
|
|
14
17
|
|
|
15
|
-
|
|
16
|
-
- Geographic flexibility. Access content regardless of regional restrictions.
|
|
17
|
-
- Enhanced privacy. Browse and collect data while maintaining anonymity.
|
|
18
|
-
- Reliable scraping. Advanced techniques to avoid detection and blocks.
|
|
19
|
-
- Simple integration. Seamless setup with popular MCP clients like Claude Desktop, Cursor, and
|
|
20
|
-
Windsurf.
|
|
18
|
+
## What is Decodo MCP server?
|
|
21
19
|
|
|
22
|
-
|
|
20
|
+
The Decodo MCP Server is a web scraping layer for AI agents. It connects MCP-compatible clients to
|
|
21
|
+
Decodo's Web Scraping API, enabling:
|
|
23
22
|
|
|
24
|
-
|
|
25
|
-
|
|
23
|
+
- Web scraping for LLMs
|
|
24
|
+
- Real-time data retrieval for RAG
|
|
25
|
+
- AI agent browsing and research
|
|
26
|
+
- Structured data extraction from dynamic websites
|
|
26
27
|
|
|
27
|
-
|
|
28
|
-
|
|
28
|
+
Instead of maintaining proxies, parsers, and retry logic, you get a single integration point for
|
|
29
|
+
reliable web data access.
|
|
29
30
|
|
|
30
|
-
##
|
|
31
|
+
## Why use MCP for web scraping?
|
|
31
32
|
|
|
32
|
-
|
|
33
|
+
Model Context Protocol (MCP) is the emerging standard for connecting AI agents to external tools and
|
|
34
|
+
data sources. With MCP:
|
|
35
|
+
|
|
36
|
+
- Agents can call tools dynamically
|
|
37
|
+
- Integrations stay standardized
|
|
38
|
+
- Workflows scale across environments
|
|
39
|
+
|
|
40
|
+
The Decodo MCP Server gives your agents reliable, production-ready web access through this standard.
|
|
41
|
+
|
|
42
|
+
## Key features
|
|
43
|
+
|
|
44
|
+
**Web scraping for AI agents, no infrastructure required.** Scrape any website, including
|
|
45
|
+
JavaScript-heavy pages, without handling proxy rotation, CAPTCHA solving, or anti-bot systems.
|
|
46
|
+
|
|
47
|
+
**Structured outputs for LLM workflows.** Markdown (LLM-ready), JSON (for structured pipelines), and
|
|
48
|
+
screenshots (for visual context), built for RAG pipelines, AI research agents, and automation flows.
|
|
49
|
+
|
|
50
|
+
**Built-in support for popular targets.** Ready-made tools for Google and Bing (SERPs), Amazon,
|
|
51
|
+
Walmart, and Target (eCommerce), Reddit, TikTok, and YouTube (social media), and ChatGPT and
|
|
52
|
+
Perplexity (AI search).
|
|
53
|
+
|
|
54
|
+
**Global proxy infrastructure.** 125M+ residential IPs, 195+ geo-locations, and a 99.99% success
|
|
55
|
+
rate on even the most protected targets.
|
|
56
|
+
|
|
57
|
+
**Modular MCP toolsets.** Enable only what you need: `web`, `search`, `ecommerce`, `social_media`,
|
|
58
|
+
`ai` for cleaner tool selection and better agent performance.
|
|
59
|
+
|
|
60
|
+
**Fast time to value.** From API key to first scrape in minutes, no setup overhead.
|
|
61
|
+
|
|
62
|
+
## Use cases
|
|
63
|
+
|
|
64
|
+
Use the Decodo MCP Server when you need web scraping for AI agents, structured data extraction at
|
|
65
|
+
scale, reliable access to dynamic websites, real-time data for RAG, or an alternative to building
|
|
66
|
+
scraping infrastructure from scratch. Common scenarios:
|
|
67
|
+
|
|
68
|
+
- **AI-powered web scraping** – give LLMs the ability to collect fresh data instead of relying on
|
|
69
|
+
static training data.
|
|
70
|
+
- **RAG with live data** – pull real-time Google, Bing, and AI search results into retrieval
|
|
71
|
+
pipelines.
|
|
72
|
+
- **eCommerce intelligence** – track product prices, listings, and sellers across marketplaces
|
|
73
|
+
without getting blocked.
|
|
74
|
+
- **Social media data collection** – gather posts, channels, and engagement data from Reddit,
|
|
75
|
+
TikTok, and YouTube.
|
|
76
|
+
- **Travel and price aggregation** – build tools that collect live pricing and availability across
|
|
77
|
+
websites.
|
|
78
|
+
|
|
79
|
+
## Quick start
|
|
80
|
+
|
|
81
|
+
1. **Create a free account** at [dashboard.decodo.com](https://dashboard.decodo.com/) – up to 2K
|
|
82
|
+
free requests, no credit card required.
|
|
83
|
+
2. **Get your authentication token.** Obtain a Web Scraping API basic authentication token from the dashboard.
|
|
84
|
+
3. **Download Node.js 18+** from https://nodejs.org.
|
|
85
|
+
4. **Get MCP client** like Claude Desktop, Cursor, Windsurf or other MCP-compatible
|
|
86
|
+
tools.
|
|
87
|
+
5. **Configure the MCP server** in your AI client (see configuration examples below).
|
|
88
|
+
|
|
89
|
+
## Connecting to [Decodo's MCP server](https://mcp.decodo.com/mcp)
|
|
90
|
+
|
|
91
|
+
Open your preferred MCP client and add the following configuration (see examples for Claude Code, Cursor, Windsurf bellow):
|
|
33
92
|
|
|
34
93
|
```
|
|
35
94
|
{
|
|
36
|
-
"
|
|
37
|
-
"
|
|
38
|
-
|
|
39
|
-
"
|
|
95
|
+
"mcpServers": {
|
|
96
|
+
"Decodo": {
|
|
97
|
+
"url": "https://mcp.decodo.com/mcp",
|
|
98
|
+
"headers": {
|
|
99
|
+
"Authorization": "Basic <basic_auth_token>"
|
|
100
|
+
}
|
|
40
101
|
}
|
|
41
102
|
}
|
|
42
103
|
}
|
|
43
104
|
```
|
|
44
105
|
|
|
45
|
-
|
|
106
|
+
### Claude Desktop
|
|
107
|
+
1. Open Claude Desktop → Settings → Developer → Edit Config.
|
|
108
|
+
2. Add to claude_desktop_config.json:
|
|
46
109
|
|
|
47
110
|
```
|
|
48
111
|
{
|
|
49
112
|
"mcpServers": {
|
|
50
|
-
"Decodo
|
|
113
|
+
"Decodo": {
|
|
51
114
|
"command": "npx",
|
|
52
|
-
"args": ["@decodo/mcp-server"],
|
|
115
|
+
"args": ["-y", "@decodo/mcp-server"],
|
|
53
116
|
"env": {
|
|
54
|
-
"SCRAPER_API_TOKEN": "<
|
|
117
|
+
"SCRAPER_API_TOKEN": "<basic_auth_token>",
|
|
55
118
|
"TOOLSETS": "web,ai"
|
|
56
119
|
}
|
|
57
120
|
}
|
|
58
121
|
}
|
|
59
122
|
}
|
|
60
123
|
```
|
|
124
|
+
3. Save and restart Claude Desktop.
|
|
125
|
+
|
|
126
|
+
### Cursor
|
|
127
|
+
1. Open Settings → MCP.
|
|
128
|
+
2. Click Add a new global MCP server (opens mcp.json).
|
|
129
|
+
3. Add the same configuration as above.
|
|
130
|
+
4. Save — look for a green status indicator next to Decodo.
|
|
131
|
+
|
|
132
|
+
### Windsurf
|
|
133
|
+
1. Open Settings → Windsurf Settings.
|
|
134
|
+
2. Scroll to Cascade → Add custom server + (opens mcp_config.json).
|
|
135
|
+
3. Add the same configuration as above.
|
|
136
|
+
4. Save and restart Windsurf.
|
|
137
|
+
|
|
138
|
+
## Test your setup
|
|
139
|
+
Once connected, try this prompt in your client:
|
|
140
|
+
|
|
141
|
+
▎ "Scrape the titles of the top 5 articles from Hacker News"
|
|
142
|
+
|
|
143
|
+
You should get a structured list back within seconds. If you see an auth error, double-check your token from the dashboard.
|
|
144
|
+
## Optional: enable specific toolsets
|
|
145
|
+
|
|
61
146
|
|
|
62
|
-
|
|
147
|
+
# Running the MCP server locally
|
|
63
148
|
|
|
64
149
|
<details>
|
|
65
150
|
|
|
@@ -191,11 +276,11 @@ The following parameters are inferred from user prompts:
|
|
|
191
276
|
| `prompt` | Prompt to send to AI tools (`chatgpt`, `perplexity`). |
|
|
192
277
|
| `search` | Activates ChatGPT's web search functionality (`chatgpt` only). |
|
|
193
278
|
| `xhr` | When true, includes XHR or fetch responses in the scrape result where supported (e.g. `tiktok_post`). |
|
|
194
|
-
| `deviceType` | Device type to emulate
|
|
279
|
+
| `deviceType` | Device type to emulate (`desktop`, `mobile`, `tablet`). Parsed targets are stricter: Amazon tools are desktop-only so they omit it, and `bing_search`/`google_ai_mode` accept `desktop` and `mobile` only. |
|
|
195
280
|
| `domain` | Domain to use for the request (e.g., `amazon.com`, `amazon.co.uk`, `bing.com`). |
|
|
196
281
|
| `pageFrom` | Starting page number for pagination. |
|
|
197
282
|
| `deliveryZip` | ZIP code for delivery location (Target, Walmart). |
|
|
198
|
-
| `storeId` | Store ID for local inventory (
|
|
283
|
+
| `storeId` | Store ID for local inventory (Walmart). |
|
|
199
284
|
| `country` | Country for TikTok Shop requests. |
|
|
200
285
|
| `limit` | Maximum number of results to return (e.g., YouTube channel videos). |
|
|
201
286
|
| `language_code` | Language code for subtitles (e.g., `en`, `es`). |
|
|
@@ -232,13 +317,39 @@ If your agent has a big context window, tell it to return `full content`:
|
|
|
232
317
|
Scrape hacker news, return full content.
|
|
233
318
|
```
|
|
234
319
|
|
|
320
|
+
## Use with Decodo agent skills
|
|
321
|
+
|
|
322
|
+
This server gives your agent the scraping *tools*. [`Decodo/agent-skills`](https://github.com/Decodo/agent-skills)
|
|
323
|
+
teaches it **when** to reach for them, **which** surface to use, and **how** to call it — so the agent
|
|
324
|
+
picks the right tool and parameters on its own instead of guessing.
|
|
325
|
+
|
|
326
|
+
The skills route across the `decodo` CLI, this hosted MCP server, and the raw HTTP API, and fall back
|
|
327
|
+
to the MCP server automatically when no shell is available. Pairing the two gives your agent both the
|
|
328
|
+
execution layer (this server) and the decision layer (the skills) in a single setup.
|
|
329
|
+
|
|
330
|
+
**Quick start** (Claude Code):
|
|
331
|
+
|
|
332
|
+
```text
|
|
333
|
+
/plugin marketplace add Decodo/agent-skills
|
|
334
|
+
/plugin install decodo@decodo-skills
|
|
335
|
+
```
|
|
336
|
+
|
|
337
|
+
See the [agent-skills README](https://github.com/Decodo/agent-skills#readme) for manual install and
|
|
338
|
+
other agents (Cursor, Codex, Gemini CLI, Windsurf).
|
|
339
|
+
|
|
235
340
|
## Related repositories
|
|
236
341
|
|
|
237
|
-
[Web Scraping API](https://github.com/Decodo/Web-Scraping-API)
|
|
342
|
+
[Web Scraping API](https://github.com/Decodo/Web-Scraping-API),
|
|
343
|
+
[Decodo agent skills](https://github.com/Decodo/agent-skills),
|
|
344
|
+
[Decodo OpenClaw skill](https://github.com/Decodo/decodo-openclaw-skill)
|
|
345
|
+
|
|
346
|
+
## Try it
|
|
238
347
|
|
|
239
|
-
|
|
348
|
+
Plug Decodo MCP Server into your AI workflow in just a few clicks and equip your AI agents with
|
|
349
|
+
real-time data from any website.
|
|
240
350
|
|
|
241
|
-
[
|
|
351
|
+
[Start for free](https://dashboard.decodo.com/) | [Docs](https://help.decodo.com/docs/introduction)
|
|
352
|
+
| [Discord](https://discord.gg/Ja8dqKgvbZ)
|
|
242
353
|
|
|
243
354
|
## License
|
|
244
355
|
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.AUTH_SCHEME = exports.AUTH_TYPE = void 0;
|
|
4
|
+
exports.AUTH_TYPE = {
|
|
5
|
+
TOKEN: 'token',
|
|
6
|
+
API_KEY: 'apiKey',
|
|
7
|
+
};
|
|
8
|
+
exports.AUTH_SCHEME = {
|
|
9
|
+
BASIC: 'Basic',
|
|
10
|
+
BEARER: 'Bearer',
|
|
11
|
+
};
|
|
12
|
+
//# sourceMappingURL=constants.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"constants.js","sourceRoot":"","sources":["../../src/auth/constants.ts"],"names":[],"mappings":";;;AAAa,QAAA,SAAS,GAAG;IACvB,KAAK,EAAE,OAAO;IACd,OAAO,EAAE,QAAQ;CACT,CAAC;AAEE,QAAA,WAAW,GAAG;IACzB,KAAK,EAAE,OAAO;IACd,MAAM,EAAE,QAAQ;CACR,CAAC"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.credentialFromAuthHeader = exports.credentialFromValue = void 0;
|
|
4
|
+
const constants_1 = require("./constants");
|
|
5
|
+
const detect_credential_type_1 = require("./detect-credential-type");
|
|
6
|
+
const TYPE_BY_SCHEME = {
|
|
7
|
+
[constants_1.AUTH_SCHEME.BASIC]: constants_1.AUTH_TYPE.TOKEN,
|
|
8
|
+
[constants_1.AUTH_SCHEME.BEARER]: constants_1.AUTH_TYPE.API_KEY,
|
|
9
|
+
};
|
|
10
|
+
const toCredential = (type, value) => {
|
|
11
|
+
const trimmed = value.trim();
|
|
12
|
+
if (!type || !trimmed) {
|
|
13
|
+
return;
|
|
14
|
+
}
|
|
15
|
+
return { type, value: trimmed };
|
|
16
|
+
};
|
|
17
|
+
const credentialFromValue = (value) => toCredential((0, detect_credential_type_1.detectCredentialType)(value), value);
|
|
18
|
+
exports.credentialFromValue = credentialFromValue;
|
|
19
|
+
const credentialFromAuthHeader = (header) => {
|
|
20
|
+
const [scheme, value, ...rest] = header.trim().split(/\s+/);
|
|
21
|
+
if (!scheme || !value || rest.length > 0) {
|
|
22
|
+
return;
|
|
23
|
+
}
|
|
24
|
+
return toCredential(TYPE_BY_SCHEME[scheme], value);
|
|
25
|
+
};
|
|
26
|
+
exports.credentialFromAuthHeader = credentialFromAuthHeader;
|
|
27
|
+
//# sourceMappingURL=credential.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"credential.js","sourceRoot":"","sources":["../../src/auth/credential.ts"],"names":[],"mappings":";;;AAAA,2CAAqD;AACrD,qEAAgE;AAGhE,MAAM,cAAc,GAAyC;IAC3D,CAAC,uBAAW,CAAC,KAAK,CAAC,EAAE,qBAAS,CAAC,KAAK;IACpC,CAAC,uBAAW,CAAC,MAAM,CAAC,EAAE,qBAAS,CAAC,OAAO;CACxC,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,IAA0B,EAAE,KAAa,EAA8B,EAAE;IAC7F,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAE7B,IAAI,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QACtB,OAAO;IACT,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;AAClC,CAAC,CAAC;AAEK,MAAM,mBAAmB,GAAG,CAAC,KAAa,EAA8B,EAAE,CAC/E,YAAY,CAAC,IAAA,6CAAoB,EAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;AADtC,QAAA,mBAAmB,uBACmB;AAE5C,MAAM,wBAAwB,GAAG,CAAC,MAAc,EAA8B,EAAE;IACrF,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAE5D,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzC,OAAO;IACT,CAAC;IAED,OAAO,YAAY,CAAC,cAAc,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,CAAC;AACrD,CAAC,CAAC;AARW,QAAA,wBAAwB,4BAQnC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.detectCredentialType = void 0;
|
|
4
|
+
const constants_1 = require("./constants");
|
|
5
|
+
const PRINTABLE_ASCII = /^[\x20-\x7e]+$/;
|
|
6
|
+
const detectCredentialType = (value) => {
|
|
7
|
+
const decoded = Buffer.from(value, 'base64').toString('utf8');
|
|
8
|
+
if (PRINTABLE_ASCII.test(decoded) && decoded.includes(':')) {
|
|
9
|
+
return constants_1.AUTH_TYPE.TOKEN;
|
|
10
|
+
}
|
|
11
|
+
return constants_1.AUTH_TYPE.API_KEY;
|
|
12
|
+
};
|
|
13
|
+
exports.detectCredentialType = detectCredentialType;
|
|
14
|
+
//# sourceMappingURL=detect-credential-type.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"detect-credential-type.js","sourceRoot":"","sources":["../../src/auth/detect-credential-type.ts"],"names":[],"mappings":";;;AAAA,2CAAwC;AAGxC,MAAM,eAAe,GAAG,gBAAgB,CAAC;AAElC,MAAM,oBAAoB,GAAG,CAAC,KAAa,EAAY,EAAE;IAC9D,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAE9D,IAAI,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3D,OAAO,qBAAS,CAAC,KAAK,CAAC;IACzB,CAAC;IAED,OAAO,qBAAS,CAAC,OAAO,CAAC;AAC3B,CAAC,CAAC;AARW,QAAA,oBAAoB,wBAQ/B"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.detectCredentialType = exports.credentialFromValue = exports.credentialFromAuthHeader = exports.AUTH_TYPE = exports.AUTH_SCHEME = void 0;
|
|
4
|
+
var constants_1 = require("./constants");
|
|
5
|
+
Object.defineProperty(exports, "AUTH_SCHEME", { enumerable: true, get: function () { return constants_1.AUTH_SCHEME; } });
|
|
6
|
+
Object.defineProperty(exports, "AUTH_TYPE", { enumerable: true, get: function () { return constants_1.AUTH_TYPE; } });
|
|
7
|
+
var credential_1 = require("./credential");
|
|
8
|
+
Object.defineProperty(exports, "credentialFromAuthHeader", { enumerable: true, get: function () { return credential_1.credentialFromAuthHeader; } });
|
|
9
|
+
Object.defineProperty(exports, "credentialFromValue", { enumerable: true, get: function () { return credential_1.credentialFromValue; } });
|
|
10
|
+
var detect_credential_type_1 = require("./detect-credential-type");
|
|
11
|
+
Object.defineProperty(exports, "detectCredentialType", { enumerable: true, get: function () { return detect_credential_type_1.detectCredentialType; } });
|
|
12
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/auth/index.ts"],"names":[],"mappings":";;;AAAA,yCAAqD;AAA5C,wGAAA,WAAW,OAAA;AAAE,sGAAA,SAAS,OAAA;AAC/B,2CAA6E;AAApE,sHAAA,wBAAwB,OAAA;AAAE,iHAAA,mBAAmB,OAAA;AACtD,mEAAgE;AAAvD,8HAAA,oBAAoB,OAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/auth/types.ts"],"names":[],"mappings":""}
|
package/build/clients/retry.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
var _a;
|
|
3
3
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
-
exports.sleep = exports.getRetryDelay = exports.isRetryable = exports.BASE_RETRY_DELAY_MS = exports.WAITING_INTERVAL_MS = exports.WAITING_INITIAL_DELAY_MS = exports.RETRYABLE_NETWORK_CODES = exports.RETRYABLE_STATUS_CODES = exports.MAX_RETRIES = void 0;
|
|
4
|
+
exports.sleep = exports.getRetryDelay = exports.isRetryable = exports.getNetworkErrorCode = exports.BASE_RETRY_DELAY_MS = exports.WAITING_INTERVAL_MS = exports.WAITING_INITIAL_DELAY_MS = exports.RETRYABLE_NETWORK_CODES = exports.RETRYABLE_STATUS_CODES = exports.MAX_RETRIES = void 0;
|
|
5
|
+
const sdk_ts_1 = require("@decodo/sdk-ts");
|
|
5
6
|
exports.MAX_RETRIES = Math.max(0, parseInt((_a = process.env.MAX_RETRIES) !== null && _a !== void 0 ? _a : '2', 10) || 2);
|
|
6
7
|
exports.RETRYABLE_STATUS_CODES = new Set([429, 502, 503, 504]);
|
|
7
8
|
exports.RETRYABLE_NETWORK_CODES = new Set([
|
|
@@ -13,29 +14,26 @@ exports.RETRYABLE_NETWORK_CODES = new Set([
|
|
|
13
14
|
exports.WAITING_INITIAL_DELAY_MS = 3000;
|
|
14
15
|
exports.WAITING_INTERVAL_MS = 5000;
|
|
15
16
|
exports.BASE_RETRY_DELAY_MS = 1000;
|
|
17
|
+
const getNetworkErrorCode = (error) => {
|
|
18
|
+
if (!(error instanceof TypeError)) {
|
|
19
|
+
return undefined;
|
|
20
|
+
}
|
|
21
|
+
const { cause } = error;
|
|
22
|
+
return typeof (cause === null || cause === void 0 ? void 0 : cause.code) === 'string' ? cause.code : undefined;
|
|
23
|
+
};
|
|
24
|
+
exports.getNetworkErrorCode = getNetworkErrorCode;
|
|
16
25
|
const isRetryable = (error) => {
|
|
17
26
|
var _a;
|
|
18
|
-
if (error.
|
|
19
|
-
return
|
|
27
|
+
if (error instanceof sdk_ts_1.TimeoutError) {
|
|
28
|
+
return true;
|
|
29
|
+
}
|
|
30
|
+
if (error instanceof sdk_ts_1.DecodoError) {
|
|
31
|
+
return exports.RETRYABLE_STATUS_CODES.has(error.statusCode);
|
|
20
32
|
}
|
|
21
|
-
return exports.RETRYABLE_NETWORK_CODES.has((_a =
|
|
33
|
+
return exports.RETRYABLE_NETWORK_CODES.has((_a = (0, exports.getNetworkErrorCode)(error)) !== null && _a !== void 0 ? _a : '');
|
|
22
34
|
};
|
|
23
35
|
exports.isRetryable = isRetryable;
|
|
24
|
-
const getRetryDelay = ({ attempt,
|
|
25
|
-
var _a;
|
|
26
|
-
if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 429) {
|
|
27
|
-
const retryAfter = error.response.headers['retry-after'];
|
|
28
|
-
if (retryAfter) {
|
|
29
|
-
const seconds = Number(retryAfter);
|
|
30
|
-
if (!isNaN(seconds)) {
|
|
31
|
-
return seconds * 1000;
|
|
32
|
-
}
|
|
33
|
-
const date = Date.parse(retryAfter);
|
|
34
|
-
if (!isNaN(date)) {
|
|
35
|
-
return Math.max(0, date - Date.now());
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
}
|
|
36
|
+
const getRetryDelay = ({ attempt, baseDelayMs = exports.BASE_RETRY_DELAY_MS, }) => {
|
|
39
37
|
const baseMs = baseDelayMs * Math.pow(2, attempt);
|
|
40
38
|
const jitterMs = Math.random() * 500;
|
|
41
39
|
return baseMs + jitterMs;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"retry.js","sourceRoot":"","sources":["../../src/clients/retry.ts"],"names":[],"mappings":";;;;
|
|
1
|
+
{"version":3,"file":"retry.js","sourceRoot":"","sources":["../../src/clients/retry.ts"],"names":[],"mappings":";;;;AAAA,2CAA2D;AAE9C,QAAA,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAA,OAAO,CAAC,GAAG,CAAC,WAAW,mCAAI,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;AAC7E,QAAA,sBAAsB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AACvD,QAAA,uBAAuB,GAAG,IAAI,GAAG,CAAC;IAC7C,YAAY;IACZ,WAAW;IACX,cAAc;IACd,WAAW;CACZ,CAAC,CAAC;AACU,QAAA,wBAAwB,GAAG,IAAI,CAAC;AAChC,QAAA,mBAAmB,GAAG,IAAI,CAAC;AAE3B,QAAA,mBAAmB,GAAG,IAAI,CAAC;AAEjC,MAAM,mBAAmB,GAAG,CAAC,KAAc,EAAsB,EAAE;IACxE,IAAI,CAAC,CAAC,KAAK,YAAY,SAAS,CAAC,EAAE,CAAC;QAClC,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,EAAE,KAAK,EAAE,GAAG,KAAmD,CAAC;IAEtE,OAAO,OAAO,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,IAAI,CAAA,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;AAClE,CAAC,CAAC;AARW,QAAA,mBAAmB,uBAQ9B;AAEK,MAAM,WAAW,GAAG,CAAC,KAAc,EAAW,EAAE;;IACrD,IAAI,KAAK,YAAY,qBAAY,EAAE,CAAC;QAClC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,KAAK,YAAY,oBAAW,EAAE,CAAC;QACjC,OAAO,8BAAsB,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACtD,CAAC;IAED,OAAO,+BAAuB,CAAC,GAAG,CAAC,MAAA,IAAA,2BAAmB,EAAC,KAAK,CAAC,mCAAI,EAAE,CAAC,CAAC;AACvE,CAAC,CAAC;AAVW,QAAA,WAAW,eAUtB;AAEK,MAAM,aAAa,GAAG,CAAC,EAC5B,OAAO,EACP,WAAW,GAAG,2BAAmB,GAIlC,EAAU,EAAE;IACX,MAAM,MAAM,GAAG,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC;IACrC,OAAO,MAAM,GAAG,QAAQ,CAAC;AAC3B,CAAC,CAAC;AAVW,QAAA,aAAa,iBAUxB;AAEK,MAAM,KAAK,GAAG,CAAC,EAAU,EAAiB,EAAE,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAAvF,QAAA,KAAK,SAAkF"}
|
|
@@ -10,78 +10,135 @@ var __rest = (this && this.__rest) || function (s, e) {
|
|
|
10
10
|
}
|
|
11
11
|
return t;
|
|
12
12
|
};
|
|
13
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
14
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
15
|
-
};
|
|
16
13
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
14
|
exports.ScraperApiClient = void 0;
|
|
18
|
-
const
|
|
15
|
+
const sdk_ts_1 = require("@decodo/sdk-ts");
|
|
16
|
+
const auth_1 = require("../auth");
|
|
19
17
|
const utils_1 = require("../utils");
|
|
18
|
+
const logger_1 = require("../logger");
|
|
20
19
|
const retry_1 = require("./retry");
|
|
20
|
+
const INTEGRATION_HEADER = 'mcp';
|
|
21
|
+
const REQUEST_TIMEOUT_MS = 180000;
|
|
22
|
+
const API_PARAM_ALIASES = new Map([
|
|
23
|
+
['deviceType', 'device_type'],
|
|
24
|
+
['pageFrom', 'page_from'],
|
|
25
|
+
['deliveryZip', 'delivery_zip'],
|
|
26
|
+
]);
|
|
21
27
|
class ScraperApiClient {
|
|
22
28
|
constructor({ maxRetries = retry_1.MAX_RETRIES, delayMs = retry_1.BASE_RETRY_DELAY_MS, } = {}) {
|
|
23
29
|
this.transformScrapingParams = ({ scrapingParams, }) => {
|
|
24
|
-
|
|
25
|
-
const
|
|
30
|
+
var _a;
|
|
31
|
+
const { jsRender, headless, target } = scrapingParams, rest = __rest(scrapingParams, ["jsRender", "headless", "target"]);
|
|
32
|
+
const transformed = Object.assign({ target: target !== null && target !== void 0 ? target : sdk_ts_1.Target.Universal }, (headless ? { headless } : jsRender && { headless: 'html' }));
|
|
33
|
+
for (const [key, value] of Object.entries(rest)) {
|
|
34
|
+
if (key === 'tokenLimit') {
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
transformed[(_a = API_PARAM_ALIASES.get(key)) !== null && _a !== void 0 ? _a : key] = value;
|
|
38
|
+
}
|
|
26
39
|
return transformed;
|
|
27
40
|
};
|
|
28
41
|
this.transformResponse = ({ res }) => {
|
|
29
|
-
const
|
|
30
|
-
|
|
42
|
+
const [result] = res.results;
|
|
43
|
+
if (!result) {
|
|
44
|
+
throw new Error('Scraper API returned no results.');
|
|
45
|
+
}
|
|
46
|
+
return { data: result.content };
|
|
47
|
+
};
|
|
48
|
+
this.sdkError = ({ error, target, authType, startMs, }) => {
|
|
49
|
+
const latencyMs = Date.now() - startMs;
|
|
50
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
51
|
+
if (error instanceof sdk_ts_1.DecodoError) {
|
|
52
|
+
const sdkMessage = error instanceof sdk_ts_1.AuthenticationError ? 'Authentication failed.' : error.message;
|
|
53
|
+
(0, logger_1.log)('error', 'tool_call', {
|
|
54
|
+
outcome: 'error',
|
|
55
|
+
target,
|
|
56
|
+
auth_type: authType,
|
|
57
|
+
error_type: 'upstream_api',
|
|
58
|
+
upstream_status: error.statusCode,
|
|
59
|
+
message: sdkMessage,
|
|
60
|
+
latency_ms: latencyMs,
|
|
61
|
+
});
|
|
62
|
+
return new Error(`Scraper API request failed (${error.statusCode}): ${sdkMessage}`);
|
|
63
|
+
}
|
|
64
|
+
const errorCode = error instanceof sdk_ts_1.TimeoutError ? 'ETIMEDOUT' : (0, retry_1.getNetworkErrorCode)(error);
|
|
65
|
+
if (errorCode) {
|
|
66
|
+
(0, logger_1.log)('error', 'tool_call', {
|
|
67
|
+
outcome: 'error',
|
|
68
|
+
target,
|
|
69
|
+
auth_type: authType,
|
|
70
|
+
error_type: 'network',
|
|
71
|
+
error_code: errorCode,
|
|
72
|
+
message,
|
|
73
|
+
latency_ms: latencyMs,
|
|
74
|
+
});
|
|
75
|
+
return new Error(`Scraper API request failed: network error ${errorCode}`);
|
|
76
|
+
}
|
|
77
|
+
(0, logger_1.log)('error', 'tool_call', {
|
|
78
|
+
outcome: 'error',
|
|
79
|
+
target,
|
|
80
|
+
auth_type: authType,
|
|
81
|
+
error_type: 'unexpected',
|
|
82
|
+
message,
|
|
83
|
+
latency_ms: latencyMs,
|
|
84
|
+
});
|
|
85
|
+
return error;
|
|
31
86
|
};
|
|
87
|
+
this.sdkCredentials = (auth) => auth.type === auth_1.AUTH_TYPE.API_KEY ? { apiKey: auth.value } : { token: auth.value };
|
|
32
88
|
this.scrape = async ({ auth, scrapingParams, extra, }) => {
|
|
33
|
-
var _a, _b, _c, _d
|
|
89
|
+
var _a, _b, _c, _d;
|
|
34
90
|
const notifier = new utils_1.ProgressNotifier(extra);
|
|
91
|
+
const startMs = Date.now();
|
|
35
92
|
try {
|
|
36
93
|
await notifier.notify('Submitting request to Decodo API...', 0, 1);
|
|
37
94
|
notifier.startWaitingNotifications(retry_1.WAITING_INITIAL_DELAY_MS, retry_1.WAITING_INTERVAL_MS);
|
|
38
|
-
const
|
|
39
|
-
const
|
|
95
|
+
const params = this.transformScrapingParams({ scrapingParams });
|
|
96
|
+
const { target } = params;
|
|
97
|
+
const { webScrapingApi } = new sdk_ts_1.DecodoClient({
|
|
98
|
+
webScrapingApi: Object.assign(Object.assign({}, this.sdkCredentials(auth)), { integrationHeader: INTEGRATION_HEADER }),
|
|
99
|
+
timeoutMs: REQUEST_TIMEOUT_MS,
|
|
100
|
+
});
|
|
40
101
|
let lastError;
|
|
41
102
|
for (let attempt = 0; attempt <= this.maxRetries; attempt++) {
|
|
42
103
|
try {
|
|
43
|
-
const res = await
|
|
44
|
-
url: `${url}/v2/scrape`,
|
|
45
|
-
method: 'POST',
|
|
46
|
-
headers: {
|
|
47
|
-
authorization: `Basic ${auth}`,
|
|
48
|
-
'x-integration': 'mcp',
|
|
49
|
-
},
|
|
50
|
-
timeout: 180000,
|
|
51
|
-
data: Object.assign({}, transformedParams),
|
|
52
|
-
});
|
|
104
|
+
const res = await webScrapingApi.scrape(params);
|
|
53
105
|
notifier.stopWaitingNotifications();
|
|
54
106
|
await notifier.notify('Processing response...', 0.9, 1);
|
|
55
|
-
|
|
107
|
+
const result = this.transformResponse({ res });
|
|
108
|
+
(0, logger_1.log)('info', 'tool_call', {
|
|
109
|
+
outcome: 'success',
|
|
110
|
+
target,
|
|
111
|
+
auth_type: auth.type,
|
|
112
|
+
upstream_status: (_b = (_a = res.results[0]) === null || _a === void 0 ? void 0 : _a.status_code) !== null && _b !== void 0 ? _b : null,
|
|
113
|
+
latency_ms: Date.now() - startMs,
|
|
114
|
+
attempt,
|
|
115
|
+
});
|
|
116
|
+
return result;
|
|
56
117
|
}
|
|
57
118
|
catch (error) {
|
|
58
119
|
lastError = error;
|
|
59
|
-
if (attempt <
|
|
60
|
-
|
|
61
|
-
const
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
120
|
+
if (attempt < this.maxRetries && (0, retry_1.isRetryable)(error)) {
|
|
121
|
+
const delayMs = (0, retry_1.getRetryDelay)({ attempt, baseDelayMs: this.delayMs });
|
|
122
|
+
const reason = error instanceof sdk_ts_1.DecodoError
|
|
123
|
+
? `status ${error.statusCode}`
|
|
124
|
+
: `network error ${(_c = (0, retry_1.getNetworkErrorCode)(error)) !== null && _c !== void 0 ? _c : 'unknown'}`;
|
|
125
|
+
(0, logger_1.log)('warn', 'tool_call_retry', {
|
|
126
|
+
target,
|
|
127
|
+
attempt: attempt + 1,
|
|
128
|
+
max_retries: this.maxRetries,
|
|
129
|
+
reason,
|
|
130
|
+
error_code: (_d = (0, retry_1.getNetworkErrorCode)(error)) !== null && _d !== void 0 ? _d : null,
|
|
131
|
+
upstream_status: error instanceof sdk_ts_1.DecodoError ? error.statusCode : null,
|
|
132
|
+
delay_ms: Math.round(delayMs),
|
|
133
|
+
});
|
|
134
|
+
await notifier.notify(`Retrying (${attempt + 1}/${this.maxRetries})...`, 0.1, 1);
|
|
67
135
|
await (0, retry_1.sleep)(delayMs);
|
|
68
136
|
continue;
|
|
69
137
|
}
|
|
70
138
|
break;
|
|
71
139
|
}
|
|
72
140
|
}
|
|
73
|
-
|
|
74
|
-
const status = (_a = lastError.response) === null || _a === void 0 ? void 0 : _a.status;
|
|
75
|
-
let errorMessage = (_d = (_c = (_b = lastError.response) === null || _b === void 0 ? void 0 : _b.data) === null || _c === void 0 ? void 0 : _c.message) !== null && _d !== void 0 ? _d : lastError.message;
|
|
76
|
-
if (status === 401) {
|
|
77
|
-
errorMessage = 'Authentication failed.';
|
|
78
|
-
}
|
|
79
|
-
if (status === 429) {
|
|
80
|
-
errorMessage = JSON.stringify((_e = lastError.response) === null || _e === void 0 ? void 0 : _e.data);
|
|
81
|
-
}
|
|
82
|
-
throw new Error(`Scraper API request failed (${status}): ${errorMessage}`);
|
|
83
|
-
}
|
|
84
|
-
throw lastError;
|
|
141
|
+
throw this.sdkError({ error: lastError, target, authType: auth.type, startMs });
|
|
85
142
|
}
|
|
86
143
|
finally {
|
|
87
144
|
notifier.stopWaitingNotifications();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scraper-api-client.js","sourceRoot":"","sources":["../../src/clients/scraper-api-client.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"scraper-api-client.js","sourceRoot":"","sources":["../../src/clients/scraper-api-client.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAAA,2CAMwB;AAGxB,kCAAoC;AAEpC,oCAA2D;AAC3D,sCAAgC;AAChC,mCASiB;AAEjB,MAAM,kBAAkB,GAAG,KAAK,CAAC;AACjC,MAAM,kBAAkB,GAAG,MAAO,CAAC;AAEnC,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IAChC,CAAC,YAAY,EAAE,aAAa,CAAC;IAC7B,CAAC,UAAU,EAAE,WAAW,CAAC;IACzB,CAAC,aAAa,EAAE,cAAc,CAAC;CAChC,CAAC,CAAC;AAEH,MAAa,gBAAgB;IAK3B,YAAY,EACV,UAAU,GAAG,mBAAW,EACxB,OAAO,GAAG,2BAAmB,MAI3B,EAAE;QAKN,4BAAuB,GAAG,CAAC,EACzB,cAAc,GAGf,EAAiB,EAAE;;YAClB,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,KAAc,cAAc,EAAvB,IAAI,UAAK,cAAc,EAAxD,kCAAuC,CAAiB,CAAC;YAE/D,MAAM,WAAW,mBACf,MAAM,EAAE,MAAM,aAAN,MAAM,cAAN,MAAM,GAAI,eAAM,CAAC,SAAS,IAC/B,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAChE,CAAC;YAEF,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBAChD,IAAI,GAAG,KAAK,YAAY,EAAE,CAAC;oBACzB,SAAS;gBACX,CAAC;gBAED,WAAW,CAAC,MAAA,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,mCAAI,GAAG,CAAC,GAAG,KAAK,CAAC;YACzD,CAAC;YAED,OAAO,WAAuC,CAAC;QACjD,CAAC,CAAC;QAEF,sBAAiB,GAAG,CAAI,EAAE,GAAG,EAAyB,EAAE,EAAE;YACxD,MAAM,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC;YAE7B,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;YACtD,CAAC;YAED,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,OAAY,EAAE,CAAC;QACvC,CAAC,CAAC;QAEM,aAAQ,GAAG,CAAC,EAClB,KAAK,EACL,MAAM,EACN,QAAQ,EACR,OAAO,GAMR,EAAW,EAAE;YACZ,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;YACvC,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAEvE,IAAI,KAAK,YAAY,oBAAW,EAAE,CAAC;gBACjC,MAAM,UAAU,GACd,KAAK,YAAY,4BAAmB,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC;gBAElF,IAAA,YAAG,EAAC,OAAO,EAAE,WAAW,EAAE;oBACxB,OAAO,EAAE,OAAO;oBAChB,MAAM;oBACN,SAAS,EAAE,QAAQ;oBACnB,UAAU,EAAE,cAAc;oBAC1B,eAAe,EAAE,KAAK,CAAC,UAAU;oBACjC,OAAO,EAAE,UAAU;oBACnB,UAAU,EAAE,SAAS;iBACtB,CAAC,CAAC;gBAEH,OAAO,IAAI,KAAK,CAAC,+BAA+B,KAAK,CAAC,UAAU,MAAM,UAAU,EAAE,CAAC,CAAC;YACtF,CAAC;YAED,MAAM,SAAS,GAAG,KAAK,YAAY,qBAAY,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAA,2BAAmB,EAAC,KAAK,CAAC,CAAC;YAE3F,IAAI,SAAS,EAAE,CAAC;gBACd,IAAA,YAAG,EAAC,OAAO,EAAE,WAAW,EAAE;oBACxB,OAAO,EAAE,OAAO;oBAChB,MAAM;oBACN,SAAS,EAAE,QAAQ;oBACnB,UAAU,EAAE,SAAS;oBACrB,UAAU,EAAE,SAAS;oBACrB,OAAO;oBACP,UAAU,EAAE,SAAS;iBACtB,CAAC,CAAC;gBAEH,OAAO,IAAI,KAAK,CAAC,6CAA6C,SAAS,EAAE,CAAC,CAAC;YAC7E,CAAC;YAED,IAAA,YAAG,EAAC,OAAO,EAAE,WAAW,EAAE;gBACxB,OAAO,EAAE,OAAO;gBAChB,MAAM;gBACN,SAAS,EAAE,QAAQ;gBACnB,UAAU,EAAE,YAAY;gBACxB,OAAO;gBACP,UAAU,EAAE,SAAS;aACtB,CAAC,CAAC;YAEH,OAAO,KAAK,CAAC;QACf,CAAC,CAAC;QAEM,mBAAc,GAAG,CAAC,IAAoB,EAAE,EAAE,CAChD,IAAI,CAAC,IAAI,KAAK,gBAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;QAEnF,WAAM,GAAG,KAAK,EAAc,EAC1B,IAAI,EACJ,cAAc,EACd,KAAK,GAKN,EAAE,EAAE;;YACH,MAAM,QAAQ,GAAG,IAAI,wBAAgB,CAAC,KAAK,CAAC,CAAC;YAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAE3B,IAAI,CAAC;gBACH,MAAM,QAAQ,CAAC,MAAM,CAAC,qCAAqC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;gBAEnE,QAAQ,CAAC,yBAAyB,CAAC,gCAAwB,EAAE,2BAAmB,CAAC,CAAC;gBAElF,MAAM,MAAM,GAAG,IAAI,CAAC,uBAAuB,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC;gBAChE,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;gBAE1B,MAAM,EAAE,cAAc,EAAE,GAAG,IAAI,qBAAY,CAAC;oBAC1C,cAAc,kCACT,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,KAC5B,iBAAiB,EAAE,kBAAkB,GACtC;oBACD,SAAS,EAAE,kBAAkB;iBAC9B,CAAC,CAAC;gBAEH,IAAI,SAAkB,CAAC;gBAEvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;oBAC5D,IAAI,CAAC;wBACH,MAAM,GAAG,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;wBAEhD,QAAQ,CAAC,wBAAwB,EAAE,CAAC;wBAEpC,MAAM,QAAQ,CAAC,MAAM,CAAC,wBAAwB,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;wBAExD,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAI,EAAE,GAAG,EAAE,CAAC,CAAC;wBAElD,IAAA,YAAG,EAAC,MAAM,EAAE,WAAW,EAAE;4BACvB,OAAO,EAAE,SAAS;4BAClB,MAAM;4BACN,SAAS,EAAE,IAAI,CAAC,IAAI;4BACpB,eAAe,EAAE,MAAA,MAAA,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,0CAAE,WAAW,mCAAI,IAAI;4BACpD,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;4BAChC,OAAO;yBACR,CAAC,CAAC;wBAEH,OAAO,MAAM,CAAC;oBAChB,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,SAAS,GAAG,KAAK,CAAC;wBAElB,IAAI,OAAO,GAAG,IAAI,CAAC,UAAU,IAAI,IAAA,mBAAW,EAAC,KAAK,CAAC,EAAE,CAAC;4BACpD,MAAM,OAAO,GAAG,IAAA,qBAAa,EAAC,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;4BACtE,MAAM,MAAM,GACV,KAAK,YAAY,oBAAW;gCAC1B,CAAC,CAAC,UAAU,KAAK,CAAC,UAAU,EAAE;gCAC9B,CAAC,CAAC,iBAAiB,MAAA,IAAA,2BAAmB,EAAC,KAAK,CAAC,mCAAI,SAAS,EAAE,CAAC;4BAEjE,IAAA,YAAG,EAAC,MAAM,EAAE,iBAAiB,EAAE;gCAC7B,MAAM;gCACN,OAAO,EAAE,OAAO,GAAG,CAAC;gCACpB,WAAW,EAAE,IAAI,CAAC,UAAU;gCAC5B,MAAM;gCACN,UAAU,EAAE,MAAA,IAAA,2BAAmB,EAAC,KAAK,CAAC,mCAAI,IAAI;gCAC9C,eAAe,EAAE,KAAK,YAAY,oBAAW,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI;gCACvE,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;6BAC9B,CAAC,CAAC;4BAEH,MAAM,QAAQ,CAAC,MAAM,CAAC,aAAa,OAAO,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;4BAEjF,MAAM,IAAA,aAAK,EAAC,OAAO,CAAC,CAAC;4BACrB,SAAS;wBACX,CAAC;wBAED,MAAM;oBACR,CAAC;gBACH,CAAC;gBAED,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;YAClF,CAAC;oBAAS,CAAC;gBACT,QAAQ,CAAC,wBAAwB,EAAE,CAAC;YACtC,CAAC;QACH,CAAC,CAAC;QAvLA,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;CAsLF;AApMD,4CAoMC"}
|