@decodo/mcp-server 1.1.1 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -16
- package/build/clients/retry.js +46 -0
- package/build/clients/retry.js.map +1 -0
- package/build/clients/scraper-api-client.js +56 -71
- package/build/clients/scraper-api-client.js.map +1 -1
- package/build/index.js +4 -6
- package/build/index.js.map +1 -1
- package/build/server/sapi-base-server.js +3 -2
- package/build/server/sapi-base-server.js.map +1 -1
- package/build/tools/amazon-bestsellers/amazon-bestsellers-tool.js +2 -2
- package/build/tools/amazon-bestsellers/amazon-bestsellers-tool.js.map +1 -1
- package/build/tools/amazon-pricing/amazon-pricing-tool.js +2 -2
- package/build/tools/amazon-pricing/amazon-pricing-tool.js.map +1 -1
- package/build/tools/amazon-product/amazon-product-tool.js +2 -2
- package/build/tools/amazon-product/amazon-product-tool.js.map +1 -1
- package/build/tools/amazon-search/amazon-search-tool.js +2 -2
- package/build/tools/amazon-search/amazon-search-tool.js.map +1 -1
- package/build/tools/amazon-sellers/amazon-sellers-tool.js +2 -2
- package/build/tools/amazon-sellers/amazon-sellers-tool.js.map +1 -1
- package/build/tools/bing-search/bing-search-tool.js +2 -2
- package/build/tools/bing-search/bing-search-tool.js.map +1 -1
- package/build/tools/chatgpt/chatgpt-tool.js +2 -2
- package/build/tools/chatgpt/chatgpt-tool.js.map +1 -1
- package/build/tools/google-ads/google-ads-tool.js +2 -2
- package/build/tools/google-ads/google-ads-tool.js.map +1 -1
- package/build/tools/google-ai-mode/google-ai-mode-tool.js +2 -2
- package/build/tools/google-ai-mode/google-ai-mode-tool.js.map +1 -1
- package/build/tools/google-lens/google-lens-tool.js +2 -2
- package/build/tools/google-lens/google-lens-tool.js.map +1 -1
- package/build/tools/google-search/google-search-tool.js +2 -2
- package/build/tools/google-search/google-search-tool.js.map +1 -1
- package/build/tools/google-travel-hotels/google-travel-hotels-tool.js +2 -2
- package/build/tools/google-travel-hotels/google-travel-hotels-tool.js.map +1 -1
- package/build/tools/perplexity/perplexity-tool.js +2 -2
- package/build/tools/perplexity/perplexity-tool.js.map +1 -1
- package/build/tools/reddit-post/reddit-post-tool.js +2 -2
- package/build/tools/reddit-post/reddit-post-tool.js.map +1 -1
- package/build/tools/reddit-subreddit/reddit-subreddit-tool.js +2 -2
- package/build/tools/reddit-subreddit/reddit-subreddit-tool.js.map +1 -1
- package/build/tools/reddit-user/reddit-user-tool.js +2 -2
- package/build/tools/reddit-user/reddit-user-tool.js.map +1 -1
- package/build/tools/scrape-as-markdown/scrape-as-markdown-tool.js +2 -2
- package/build/tools/scrape-as-markdown/scrape-as-markdown-tool.js.map +1 -1
- package/build/tools/screenshot/screenshot-tool.js +2 -1
- package/build/tools/screenshot/screenshot-tool.js.map +1 -1
- package/build/tools/target-product/target-product-tool.js +2 -2
- package/build/tools/target-product/target-product-tool.js.map +1 -1
- package/build/tools/target-search/target-search-tool.js +2 -2
- package/build/tools/target-search/target-search-tool.js.map +1 -1
- package/build/tools/tiktok-post/tiktok-post-tool.js +2 -2
- package/build/tools/tiktok-post/tiktok-post-tool.js.map +1 -1
- package/build/tools/tiktok-shop-product/tiktok-shop-product-tool.js +2 -2
- package/build/tools/tiktok-shop-product/tiktok-shop-product-tool.js.map +1 -1
- package/build/tools/tiktok-shop-search/tiktok-shop-search-tool.js +2 -2
- package/build/tools/tiktok-shop-search/tiktok-shop-search-tool.js.map +1 -1
- package/build/tools/tiktok-shop-url/tiktok-shop-url-tool.js +2 -2
- package/build/tools/tiktok-shop-url/tiktok-shop-url-tool.js.map +1 -1
- package/build/tools/walmart-product/walmart-product-tool.js +2 -2
- package/build/tools/walmart-product/walmart-product-tool.js.map +1 -1
- package/build/tools/walmart-search/walmart-search-tool.js +2 -2
- package/build/tools/walmart-search/walmart-search-tool.js.map +1 -1
- package/build/tools/youtube-channel/youtube-channel-tool.js +2 -2
- package/build/tools/youtube-channel/youtube-channel-tool.js.map +1 -1
- package/build/tools/youtube-metadata/youtube-metadata-tool.js +2 -2
- package/build/tools/youtube-metadata/youtube-metadata-tool.js.map +1 -1
- package/build/tools/youtube-search/youtube-search-tool.js +2 -2
- package/build/tools/youtube-search/youtube-search-tool.js.map +1 -1
- package/build/tools/youtube-subtitles/youtube-subtitles-tool.js +2 -2
- package/build/tools/youtube-subtitles/youtube-subtitles-tool.js.map +1 -1
- package/build/utils/progress.js +285 -0
- package/build/utils/progress.js.map +1 -0
- package/build/utils.js +4 -1
- package/build/utils.js.map +1 -1
- package/build/version.js +8 -0
- package/build/version.js.map +1 -0
- package/package.json +7 -6
package/README.md
CHANGED
|
@@ -20,7 +20,7 @@ services, streamlining access to our tools and capabilities.
|
|
|
20
20
|
- Simple integration. Seamless setup with popular MCP clients like Claude Desktop, Cursor, and
|
|
21
21
|
Windsurf.
|
|
22
22
|
|
|
23
|
-
##
|
|
23
|
+
## Authenticating
|
|
24
24
|
|
|
25
25
|
1. Go to [decodo.com](https://decodo.com/scraping/web) and start a Web Scraping API plan (free plan
|
|
26
26
|
available).
|
|
@@ -28,6 +28,8 @@ services, streamlining access to our tools and capabilities.
|
|
|
28
28
|
2. Once your plan has started, obtain a Web Scraping API basic authentication token from the
|
|
29
29
|
[dashboard](https://dashboard.decodo.com/).
|
|
30
30
|
|
|
31
|
+
## Connecting to [Decodo's MCP server](https://mcp.decodo.com/mcp) (Claude Code, Cursor)
|
|
32
|
+
|
|
31
33
|
3. Open your preferred MCP client and add the following configuration:
|
|
32
34
|
|
|
33
35
|
```
|
|
@@ -41,6 +43,23 @@ services, streamlining access to our tools and capabilities.
|
|
|
41
43
|
}
|
|
42
44
|
```
|
|
43
45
|
|
|
46
|
+
## Claude Desktop
|
|
47
|
+
|
|
48
|
+
```
|
|
49
|
+
{
|
|
50
|
+
"mcpServers": {
|
|
51
|
+
"Decodo MCP Server": {
|
|
52
|
+
"command": "npx",
|
|
53
|
+
"args": ["@decodo/mcp"],
|
|
54
|
+
"env": {
|
|
55
|
+
"SCRAPER_API_TOKEN": "<web_scraping_api_base64_token>",
|
|
56
|
+
"TOOLSETS": "web,ai"
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
```
|
|
62
|
+
|
|
44
63
|
## Running the MCP server locally
|
|
45
64
|
|
|
46
65
|
<details>
|
|
@@ -56,7 +75,7 @@ services, streamlining access to our tools and capabilities.
|
|
|
56
75
|
1. Clone this repository:
|
|
57
76
|
|
|
58
77
|
```
|
|
59
|
-
git clone https://github.com/Decodo/
|
|
78
|
+
git clone https://github.com/Decodo/mcp-server
|
|
60
79
|
```
|
|
61
80
|
|
|
62
81
|
2. Run the following commands in the terminal:
|
|
@@ -90,8 +109,7 @@ this:
|
|
|
90
109
|
"command": "node",
|
|
91
110
|
"args": ["/Users/your.user/projects/decodo-mcp/build/index.js"],
|
|
92
111
|
"env": {
|
|
93
|
-
"
|
|
94
|
-
"SCRAPER_API_PASSWORD": "your_password"
|
|
112
|
+
"SCRAPER_API_TOKEN": "<web_scraping_api_base64_token>"
|
|
95
113
|
}
|
|
96
114
|
}
|
|
97
115
|
}
|
|
@@ -133,31 +151,31 @@ The server exposes the following tools:
|
|
|
133
151
|
| `scrape_as_markdown` | Scrapes any target URL, expects a URL to be given via prompt. Returns results in Markdown. | Scrape peacock.com from a US IP address and tell me the pricing. |
|
|
134
152
|
| `screenshot` | Captures a screenshot of any webpage and returns it as a PNG image. | Take a screenshot of github.com from a US IP address. |
|
|
135
153
|
| `google_search` | Scrapes Google Search for a given query, and returns parsed results. | Scrape Google Search for shoes and tell me the top position. |
|
|
136
|
-
| `google_ads` | Scrapes Google Ads search results
|
|
137
|
-
| `google_lens` | Scrapes Google Lens image search results
|
|
138
|
-
| `google_ai_mode` | Scrapes Google AI Mode (Search with AI) results
|
|
154
|
+
| `google_ads` | Scrapes Google Ads search results. | Scrape Google Ads for laptop and show me the top ads. |
|
|
155
|
+
| `google_lens` | Scrapes Google Lens image search results. | Search Google Lens for this image: https://example.com/image.jpg |
|
|
156
|
+
| `google_ai_mode` | Scrapes Google AI Mode (Search with AI) results. | Ask Google AI Mode: What are the top three dog breeds? |
|
|
139
157
|
| `google_travel_hotels` | Scrapes Google Travel Hotels search results. | Search Google Travel Hotels for hotels in Paris. |
|
|
140
158
|
| `amazon_search` | Scrapes Amazon Search for a given query, and returns parsed results. | Scrape Amazon Search for wireless keyboard. |
|
|
141
|
-
| `amazon_product` | Scrapes Amazon Product page
|
|
142
|
-
| `amazon_pricing` | Scrapes Amazon Product pricing information
|
|
143
|
-
| `amazon_sellers` | Scrapes Amazon Seller information
|
|
144
|
-
| `amazon_bestsellers` | Scrapes Amazon Bestsellers list
|
|
159
|
+
| `amazon_product` | Scrapes Amazon Product page. | Scrape Amazon product B09H74FXNW and show me the details. |
|
|
160
|
+
| `amazon_pricing` | Scrapes Amazon Product pricing information. | Get pricing for Amazon product B09H74FXNW. |
|
|
161
|
+
| `amazon_sellers` | Scrapes Amazon Seller information. | Get information about Amazon seller A1R0Z7FJGTKESH. |
|
|
162
|
+
| `amazon_bestsellers` | Scrapes Amazon Bestsellers list. | Show me Amazon bestsellers in electronics. |
|
|
145
163
|
| `walmart_search` | Scrapes Walmart Search for a given query, and returns parsed results. | Scrape Walmart Search for camping tent. |
|
|
146
|
-
| `walmart_product` | Scrapes Walmart Product page
|
|
164
|
+
| `walmart_product` | Scrapes Walmart Product page. | Scrape Walmart product 15296401808. |
|
|
147
165
|
| `target_search` | Scrapes Target Search for a given query, and returns parsed results. | Scrape Target Search for kitchen appliances. |
|
|
148
|
-
| `target_product` | Scrapes Target Product page
|
|
166
|
+
| `target_product` | Scrapes Target Product page. | Scrape Target product 92186007. |
|
|
149
167
|
| `tiktok_post` | Scrapes a TikTok post URL for structured data (e.g. engagement, caption, hashtags). | Scrape this TikTok post: https://www.tiktok.com/@nba/video/7393013274725403950 |
|
|
150
168
|
| `tiktok_shop_search` | Scrapes TikTok Shop Search for a given query, and returns parsed results. | Scrape TikTok Shop Search for phone cases. |
|
|
151
169
|
| `tiktok_shop_product` | Scrapes TikTok Shop Product page. | Scrape TikTok Shop product 1731541214379741272. |
|
|
152
170
|
| `tiktok_shop_url` | Scrapes TikTok Shop page by URL. | Scrape this TikTok Shop URL: https://www.tiktok.com/shop/s?q=HEADPHONES |
|
|
153
171
|
| `youtube_metadata` | Scrapes YouTube video metadata. | Get metadata for YouTube video dFu9aKJoqGg. |
|
|
154
|
-
| `youtube_channel` | Scrapes YouTube channel videos
|
|
172
|
+
| `youtube_channel` | Scrapes YouTube channel videos. | Scrape YouTube channel @decodo_official. |
|
|
155
173
|
| `youtube_subtitles` | Scrapes YouTube video subtitles. | Get subtitles for YouTube video L8zSWbQN-v8. |
|
|
156
174
|
| `youtube_search` | Search YouTube videos. | Search YouTube for "How to care for chinchillas". |
|
|
157
175
|
| `reddit_post` | Scrapes a specific Reddit post. | Scrape the following Reddit post: https://www.reddit.com/r/horseracing/comments/1nsrn3/ |
|
|
158
|
-
| `reddit_subreddit` | Scrapes Reddit subreddit results
|
|
176
|
+
| `reddit_subreddit` | Scrapes Reddit subreddit results. | Scrape the top 5 posts on r/Python this week. |
|
|
159
177
|
| `reddit_user` | Scrapes a Reddit user profile and their posts/comments. | Scrape this Reddit user: https://www.reddit.com/user/IWasRightOnce/ |
|
|
160
|
-
| `bing_search` | Scrapes Bing Search results
|
|
178
|
+
| `bing_search` | Scrapes Bing Search results. | Search Bing for laptop reviews. |
|
|
161
179
|
| `chatgpt` | Search and interact with ChatGPT for AI-powered responses and conversations. | Ask ChatGPT to explain quantum computing in simple terms. |
|
|
162
180
|
| `perplexity` | Search and interact with Perplexity for AI-powered responses and conversations. | Ask Perplexity what the latest trends in web development are. |
|
|
163
181
|
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var _a;
|
|
3
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
+
exports.sleep = exports.getRetryDelay = exports.isRetryable = exports.BASE_RETRY_DELAY_MS = exports.WAITING_INTERVAL_MS = exports.WAITING_INITIAL_DELAY_MS = exports.RETRYABLE_NETWORK_CODES = exports.RETRYABLE_STATUS_CODES = exports.MAX_RETRIES = void 0;
|
|
5
|
+
exports.MAX_RETRIES = Math.max(0, parseInt((_a = process.env.MAX_RETRIES) !== null && _a !== void 0 ? _a : '2', 10) || 2);
|
|
6
|
+
exports.RETRYABLE_STATUS_CODES = new Set([429, 502, 503, 504]);
|
|
7
|
+
exports.RETRYABLE_NETWORK_CODES = new Set([
|
|
8
|
+
'ECONNRESET',
|
|
9
|
+
'ETIMEDOUT',
|
|
10
|
+
'ECONNABORTED',
|
|
11
|
+
'ENOTFOUND',
|
|
12
|
+
]);
|
|
13
|
+
exports.WAITING_INITIAL_DELAY_MS = 3000;
|
|
14
|
+
exports.WAITING_INTERVAL_MS = 5000;
|
|
15
|
+
exports.BASE_RETRY_DELAY_MS = 1000;
|
|
16
|
+
const isRetryable = (error) => {
|
|
17
|
+
var _a;
|
|
18
|
+
if (error.response) {
|
|
19
|
+
return exports.RETRYABLE_STATUS_CODES.has(error.response.status);
|
|
20
|
+
}
|
|
21
|
+
return exports.RETRYABLE_NETWORK_CODES.has((_a = error.code) !== null && _a !== void 0 ? _a : '');
|
|
22
|
+
};
|
|
23
|
+
exports.isRetryable = isRetryable;
|
|
24
|
+
const getRetryDelay = ({ attempt, error, baseDelayMs = exports.BASE_RETRY_DELAY_MS, }) => {
|
|
25
|
+
var _a;
|
|
26
|
+
if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 429) {
|
|
27
|
+
const retryAfter = error.response.headers['retry-after'];
|
|
28
|
+
if (retryAfter) {
|
|
29
|
+
const seconds = Number(retryAfter);
|
|
30
|
+
if (!isNaN(seconds)) {
|
|
31
|
+
return seconds * 1000;
|
|
32
|
+
}
|
|
33
|
+
const date = Date.parse(retryAfter);
|
|
34
|
+
if (!isNaN(date)) {
|
|
35
|
+
return Math.max(0, date - Date.now());
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
const baseMs = baseDelayMs * Math.pow(2, attempt);
|
|
40
|
+
const jitterMs = Math.random() * 500;
|
|
41
|
+
return baseMs + jitterMs;
|
|
42
|
+
};
|
|
43
|
+
exports.getRetryDelay = getRetryDelay;
|
|
44
|
+
const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
|
45
|
+
exports.sleep = sleep;
|
|
46
|
+
//# sourceMappingURL=retry.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"retry.js","sourceRoot":"","sources":["../../src/clients/retry.ts"],"names":[],"mappings":";;;;AAEa,QAAA,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,MAAA,OAAO,CAAC,GAAG,CAAC,WAAW,mCAAI,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;AAC7E,QAAA,sBAAsB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AACvD,QAAA,uBAAuB,GAAG,IAAI,GAAG,CAAC;IAC7C,YAAY;IACZ,WAAW;IACX,cAAc;IACd,WAAW;CACZ,CAAC,CAAC;AACU,QAAA,wBAAwB,GAAG,IAAI,CAAC;AAChC,QAAA,mBAAmB,GAAG,IAAI,CAAC;AAE3B,QAAA,mBAAmB,GAAG,IAAI,CAAC;AAEjC,MAAM,WAAW,GAAG,CAAC,KAAiB,EAAW,EAAE;;IACxD,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;QACnB,OAAO,8BAAsB,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC3D,CAAC;IACD,OAAO,+BAAuB,CAAC,GAAG,CAAC,MAAA,KAAK,CAAC,IAAI,mCAAI,EAAE,CAAC,CAAC;AACvD,CAAC,CAAC;AALW,QAAA,WAAW,eAKtB;AAEK,MAAM,aAAa,GAAG,CAAC,EAC5B,OAAO,EACP,KAAK,EACL,WAAW,GAAG,2BAAmB,GAKlC,EAAU,EAAE;;IACX,IAAI,CAAA,MAAA,KAAK,CAAC,QAAQ,0CAAE,MAAM,MAAK,GAAG,EAAE,CAAC;QACnC,MAAM,UAAU,GAAG,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;QACzD,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;YACnC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;gBACpB,OAAO,OAAO,GAAG,IAAI,CAAC;YACxB,CAAC;YAED,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACpC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC;IACrC,OAAO,MAAM,GAAG,QAAQ,CAAC;AAC3B,CAAC,CAAC;AA3BW,QAAA,aAAa,iBA2BxB;AAEK,MAAM,KAAK,GAAG,CAAC,EAAU,EAAiB,EAAE,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAAvF,QAAA,KAAK,SAAkF"}
|
|
@@ -13,42 +13,13 @@ var __rest = (this && this.__rest) || function (s, e) {
|
|
|
13
13
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
14
14
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
15
15
|
};
|
|
16
|
-
var _a;
|
|
17
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
17
|
exports.ScraperApiClient = void 0;
|
|
19
18
|
const axios_1 = __importDefault(require("axios"));
|
|
20
|
-
const
|
|
21
|
-
const
|
|
22
|
-
const RETRYABLE_NETWORK_CODES = new Set(['ECONNRESET', 'ETIMEDOUT', 'ECONNABORTED', 'ENOTFOUND']);
|
|
23
|
-
const isRetryable = (error) => {
|
|
24
|
-
var _a;
|
|
25
|
-
if (error.response) {
|
|
26
|
-
return RETRYABLE_STATUS_CODES.has(error.response.status);
|
|
27
|
-
}
|
|
28
|
-
return RETRYABLE_NETWORK_CODES.has((_a = error.code) !== null && _a !== void 0 ? _a : '');
|
|
29
|
-
};
|
|
30
|
-
const getRetryDelay = (attempt, error) => {
|
|
31
|
-
var _a;
|
|
32
|
-
if (((_a = error.response) === null || _a === void 0 ? void 0 : _a.status) === 429) {
|
|
33
|
-
const retryAfter = error.response.headers['retry-after'];
|
|
34
|
-
if (retryAfter) {
|
|
35
|
-
const seconds = Number(retryAfter);
|
|
36
|
-
if (!isNaN(seconds)) {
|
|
37
|
-
return seconds * 1000;
|
|
38
|
-
}
|
|
39
|
-
const date = Date.parse(retryAfter);
|
|
40
|
-
if (!isNaN(date)) {
|
|
41
|
-
return Math.max(0, date - Date.now());
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
const baseMs = 1000 * Math.pow(2, attempt);
|
|
46
|
-
const jitterMs = Math.random() * 500;
|
|
47
|
-
return baseMs + jitterMs;
|
|
48
|
-
};
|
|
49
|
-
const sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
|
19
|
+
const utils_1 = require("../utils");
|
|
20
|
+
const retry_1 = require("./retry");
|
|
50
21
|
class ScraperApiClient {
|
|
51
|
-
constructor() {
|
|
22
|
+
constructor({ maxRetries = retry_1.MAX_RETRIES, delayMs = retry_1.BASE_RETRY_DELAY_MS, } = {}) {
|
|
52
23
|
this.transformScrapingParams = ({ scrapingParams, }) => {
|
|
53
24
|
const { jsRender, headless } = scrapingParams, rest = __rest(scrapingParams, ["jsRender", "headless"]);
|
|
54
25
|
const transformed = Object.assign(Object.assign({}, (headless ? { headless } : jsRender && { headless: 'html' })), rest);
|
|
@@ -58,52 +29,66 @@ class ScraperApiClient {
|
|
|
58
29
|
const content = res.data.results[0].content;
|
|
59
30
|
return Object.assign(Object.assign({}, res), { data: content });
|
|
60
31
|
};
|
|
61
|
-
this.scrape = async ({ auth, scrapingParams, }) => {
|
|
32
|
+
this.scrape = async ({ auth, scrapingParams, extra, }) => {
|
|
62
33
|
var _a, _b, _c, _d, _e;
|
|
63
|
-
const
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
34
|
+
const notifier = new utils_1.ProgressNotifier(extra);
|
|
35
|
+
try {
|
|
36
|
+
await notifier.notify('Submitting request to Decodo API...', 0, 1);
|
|
37
|
+
notifier.startWaitingNotifications(retry_1.WAITING_INITIAL_DELAY_MS, retry_1.WAITING_INTERVAL_MS);
|
|
38
|
+
const transformedParams = this.transformScrapingParams({ scrapingParams });
|
|
39
|
+
const url = process.env.DECODO_SAPI_HOST || 'https://scraper-api.decodo.com';
|
|
40
|
+
let lastError;
|
|
41
|
+
for (let attempt = 0; attempt <= this.maxRetries; attempt++) {
|
|
42
|
+
try {
|
|
43
|
+
const res = await axios_1.default.request({
|
|
44
|
+
url: `${url}/v2/scrape`,
|
|
45
|
+
method: 'POST',
|
|
46
|
+
headers: {
|
|
47
|
+
authorization: `Basic ${auth}`,
|
|
48
|
+
'x-integration': 'mcp',
|
|
49
|
+
},
|
|
50
|
+
timeout: 180000,
|
|
51
|
+
data: Object.assign({}, transformedParams),
|
|
52
|
+
});
|
|
53
|
+
notifier.stopWaitingNotifications();
|
|
54
|
+
await notifier.notify('Processing response...', 0.9, 1);
|
|
55
|
+
return this.transformResponse({ res });
|
|
56
|
+
}
|
|
57
|
+
catch (error) {
|
|
58
|
+
lastError = error;
|
|
59
|
+
if (attempt < retry_1.MAX_RETRIES && axios_1.default.isAxiosError(error) && (0, retry_1.isRetryable)(error)) {
|
|
60
|
+
// const delayMs = getRetryDelay(attempt, error);
|
|
61
|
+
const delayMs = (0, retry_1.getRetryDelay)({ attempt, error, baseDelayMs: this.delayMs });
|
|
62
|
+
const reason = error.response
|
|
63
|
+
? `status ${error.response.status}`
|
|
64
|
+
: `network error ${error.code}`;
|
|
65
|
+
console.error(`[scraper-api-client] Retry ${attempt + 1}/${retry_1.MAX_RETRIES} after ${reason}, waiting ${Math.round(delayMs)}ms`);
|
|
66
|
+
await notifier.notify(`Retrying (${attempt + 1}/${retry_1.MAX_RETRIES})...`, 0.1, 1);
|
|
67
|
+
await (0, retry_1.sleep)(delayMs);
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
break;
|
|
71
|
+
}
|
|
79
72
|
}
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
? `status ${error.response.status}`
|
|
86
|
-
: `network error ${error.code}`;
|
|
87
|
-
console.error(`[scraper-api-client] Retry ${attempt + 1}/${MAX_RETRIES} after ${reason}, waiting ${Math.round(delayMs)}ms`);
|
|
88
|
-
await sleep(delayMs);
|
|
89
|
-
continue;
|
|
73
|
+
if (axios_1.default.isAxiosError(lastError)) {
|
|
74
|
+
const status = (_a = lastError.response) === null || _a === void 0 ? void 0 : _a.status;
|
|
75
|
+
let errorMessage = (_d = (_c = (_b = lastError.response) === null || _b === void 0 ? void 0 : _b.data) === null || _c === void 0 ? void 0 : _c.message) !== null && _d !== void 0 ? _d : lastError.message;
|
|
76
|
+
if (status === 401) {
|
|
77
|
+
errorMessage = 'Authentication failed.';
|
|
90
78
|
}
|
|
91
|
-
|
|
79
|
+
if (status === 429) {
|
|
80
|
+
errorMessage = JSON.stringify((_e = lastError.response) === null || _e === void 0 ? void 0 : _e.data);
|
|
81
|
+
}
|
|
82
|
+
throw new Error(`Scraper API request failed (${status}): ${errorMessage}`);
|
|
92
83
|
}
|
|
84
|
+
throw lastError;
|
|
93
85
|
}
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
let errorMessage = (_d = (_c = (_b = lastError.response) === null || _b === void 0 ? void 0 : _b.data) === null || _c === void 0 ? void 0 : _c.message) !== null && _d !== void 0 ? _d : lastError.message;
|
|
97
|
-
if (status === 401) {
|
|
98
|
-
errorMessage = 'Authentication failed.';
|
|
99
|
-
}
|
|
100
|
-
if (status === 429) {
|
|
101
|
-
errorMessage = JSON.stringify((_e = lastError.response) === null || _e === void 0 ? void 0 : _e.data);
|
|
102
|
-
}
|
|
103
|
-
throw new Error(`Scraper API request failed (${status}): ${errorMessage}`);
|
|
86
|
+
finally {
|
|
87
|
+
notifier.stopWaitingNotifications();
|
|
104
88
|
}
|
|
105
|
-
throw lastError;
|
|
106
89
|
};
|
|
90
|
+
this.maxRetries = maxRetries;
|
|
91
|
+
this.delayMs = delayMs;
|
|
107
92
|
}
|
|
108
93
|
}
|
|
109
94
|
exports.ScraperApiClient = ScraperApiClient;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"scraper-api-client.js","sourceRoot":"","sources":["../../src/clients/scraper-api-client.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"scraper-api-client.js","sourceRoot":"","sources":["../../src/clients/scraper-api-client.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;AAAA,kDAA6C;AAG7C,oCAA2D;AAC3D,mCAQiB;AAEjB,MAAa,gBAAgB;IAK3B,YAAY,EACV,UAAU,GAAG,mBAAW,EACxB,OAAO,GAAG,2BAAmB,MAI3B,EAAE;QAKN,4BAAuB,GAAG,CAAC,EACzB,cAAc,GAGf,EAAoB,EAAE;YACrB,MAAM,EAAE,QAAQ,EAAE,QAAQ,KAAc,cAAc,EAAvB,IAAI,UAAK,cAAc,EAAhD,wBAA+B,CAAiB,CAAC;YACvD,MAAM,WAAW,mCACZ,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,GAC5D,IAAI,CACR,CAAC;YAEF,OAAO,WAAW,CAAC;QACrB,CAAC,CAAC;QAEF,sBAAiB,GAAG,CAAI,EAAE,GAAG,EAAqD,EAAE,EAAE;YACpF,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;YAE5C,uCAAY,GAAG,KAAE,IAAI,EAAE,OAAO,IAAG;QACnC,CAAC,CAAC;QAEF,WAAM,GAAG,KAAK,EAAc,EAC1B,IAAI,EACJ,cAAc,EACd,KAAK,GAKN,EAAE,EAAE;;YACH,MAAM,QAAQ,GAAG,IAAI,wBAAgB,CAAC,KAAK,CAAC,CAAC;YAE7C,IAAI,CAAC;gBACH,MAAM,QAAQ,CAAC,MAAM,CAAC,qCAAqC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;gBAEnE,QAAQ,CAAC,yBAAyB,CAAC,gCAAwB,EAAE,2BAAmB,CAAC,CAAC;gBAElF,MAAM,iBAAiB,GAAG,IAAI,CAAC,uBAAuB,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC;gBAC3E,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,gCAAgC,CAAC;gBAE7E,IAAI,SAAkB,CAAC;gBAEvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;oBAC5D,IAAI,CAAC;wBACH,MAAM,GAAG,GAAG,MAAM,eAAK,CAAC,OAAO,CAA4B;4BACzD,GAAG,EAAE,GAAG,GAAG,YAAY;4BACvB,MAAM,EAAE,MAAM;4BACd,OAAO,EAAE;gCACP,aAAa,EAAE,SAAS,IAAI,EAAE;gCAC9B,eAAe,EAAE,KAAK;6BACvB;4BACD,OAAO,EAAE,MAAM;4BACf,IAAI,oBACC,iBAAiB,CACrB;yBACF,CAAC,CAAC;wBAEH,QAAQ,CAAC,wBAAwB,EAAE,CAAC;wBAEpC,MAAM,QAAQ,CAAC,MAAM,CAAC,wBAAwB,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;wBAExD,OAAO,IAAI,CAAC,iBAAiB,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;oBACzC,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,SAAS,GAAG,KAAK,CAAC;wBAElB,IAAI,OAAO,GAAG,mBAAW,IAAI,eAAK,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,IAAA,mBAAW,EAAC,KAAK,CAAC,EAAE,CAAC;4BAC7E,iDAAiD;4BACjD,MAAM,OAAO,GAAG,IAAA,qBAAa,EAAC,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;4BAC7E,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ;gCAC3B,CAAC,CAAC,UAAU,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE;gCACnC,CAAC,CAAC,iBAAiB,KAAK,CAAC,IAAI,EAAE,CAAC;4BAElC,OAAO,CAAC,KAAK,CACX,8BACE,OAAO,GAAG,CACZ,IAAI,mBAAW,UAAU,MAAM,aAAa,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CACpE,CAAC;4BAEF,MAAM,QAAQ,CAAC,MAAM,CAAC,aAAa,OAAO,GAAG,CAAC,IAAI,mBAAW,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;4BAE7E,MAAM,IAAA,aAAK,EAAC,OAAO,CAAC,CAAC;4BACrB,SAAS;wBACX,CAAC;wBAED,MAAM;oBACR,CAAC;gBACH,CAAC;gBAED,IAAI,eAAK,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE,CAAC;oBAClC,MAAM,MAAM,GAAG,MAAA,SAAS,CAAC,QAAQ,0CAAE,MAAM,CAAC;oBAC1C,IAAI,YAAY,GAAG,MAAA,MAAA,MAAA,SAAS,CAAC,QAAQ,0CAAE,IAAI,0CAAE,OAAO,mCAAI,SAAS,CAAC,OAAO,CAAC;oBAE1E,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;wBACnB,YAAY,GAAG,wBAAwB,CAAC;oBAC1C,CAAC;oBACD,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;wBACnB,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,MAAA,SAAS,CAAC,QAAQ,0CAAE,IAAI,CAAC,CAAC;oBAC1D,CAAC;oBAED,MAAM,IAAI,KAAK,CAAC,+BAA+B,MAAM,MAAM,YAAY,EAAE,CAAC,CAAC;gBAC7E,CAAC;gBACD,MAAM,SAAS,CAAC;YAClB,CAAC;oBAAS,CAAC;gBACT,QAAQ,CAAC,wBAAwB,EAAE,CAAC;YACtC,CAAC;QACH,CAAC,CAAC;QA5GA,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;CA2GF;AAzHD,4CAyHC"}
|
package/build/index.js
CHANGED
|
@@ -6,7 +6,7 @@ const stdio_js_1 = require("@modelcontextprotocol/sdk/server/stdio.js");
|
|
|
6
6
|
const sapi_stdio_server_1 = require("./server/sapi-stdio-server");
|
|
7
7
|
const utils_1 = require("./utils");
|
|
8
8
|
const parseEnvsOrExit = () => {
|
|
9
|
-
const envs = ['
|
|
9
|
+
const envs = ['SCRAPER_API_TOKEN'];
|
|
10
10
|
for (const envKey of envs) {
|
|
11
11
|
if (!process.env[envKey]) {
|
|
12
12
|
console.error(`env ${envKey} missing`);
|
|
@@ -14,17 +14,15 @@ const parseEnvsOrExit = () => {
|
|
|
14
14
|
}
|
|
15
15
|
}
|
|
16
16
|
return {
|
|
17
|
-
|
|
18
|
-
sapiPassword: process.env['SCRAPER_API_PASSWORD'],
|
|
17
|
+
sapiAuth: process.env['SCRAPER_API_TOKEN'],
|
|
19
18
|
};
|
|
20
19
|
};
|
|
21
20
|
const main = async () => {
|
|
22
21
|
const transport = new stdio_js_1.StdioServerTransport();
|
|
23
22
|
// if there are no envs, some MCP clients will fail silently
|
|
24
|
-
const {
|
|
25
|
-
const auth = Buffer.from(`${sapiUsername}:${sapiPassword}`).toString('base64');
|
|
23
|
+
const { sapiAuth } = parseEnvsOrExit();
|
|
26
24
|
const sapiMcpServer = new sapi_stdio_server_1.ScraperAPIStdioServer({
|
|
27
|
-
auth,
|
|
25
|
+
auth: sapiAuth,
|
|
28
26
|
toolsets: (0, utils_1.resolveToolsets)(process.env.TOOLSETS),
|
|
29
27
|
});
|
|
30
28
|
await sapiMcpServer.connect(transport);
|
package/build/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AACA,yBAAuB;AAEvB,wEAAiF;AACjF,kEAAmE;AACnE,mCAA0C;AAE1C,MAAM,eAAe,GAAG,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AACA,yBAAuB;AAEvB,wEAAiF;AACjF,kEAAmE;AACnE,mCAA0C;AAE1C,MAAM,eAAe,GAAG,GAAG,EAAE;IAC3B,MAAM,IAAI,GAAG,CAAC,mBAAmB,CAAC,CAAC;IAEnC,KAAK,MAAM,MAAM,IAAI,IAAI,EAAE,CAAC;QAC1B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,KAAK,CAAC,OAAO,MAAM,UAAU,CAAC,CAAC;YACvC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAED,OAAO;QACL,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAW;KACrD,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE;IACtB,MAAM,SAAS,GAAG,IAAI,+BAAoB,EAAE,CAAC;IAE7C,4DAA4D;IAC5D,MAAM,EAAE,QAAQ,EAAE,GAAG,eAAe,EAAE,CAAC;IAEvC,MAAM,aAAa,GAAG,IAAI,yCAAqB,CAAC;QAC9C,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,IAAA,uBAAe,EAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;KAChD,CAAC,CAAC;IACH,MAAM,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAEvC,OAAO,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;AAC/C,CAAC,CAAC;AAEF,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,KAAK,CAAC,CAAC;IAC/C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
|
@@ -3,15 +3,16 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.ScraperAPIBaseServer = void 0;
|
|
4
4
|
const mcp_js_1 = require("@modelcontextprotocol/sdk/server/mcp.js");
|
|
5
5
|
const scraper_api_client_1 = require("../clients/scraper-api-client");
|
|
6
|
+
const version_1 = require("../version");
|
|
6
7
|
const tools_1 = require("../tools");
|
|
7
8
|
class ScraperAPIBaseServer {
|
|
8
9
|
constructor({ auth, toolsets = [] }) {
|
|
9
10
|
this.auth = '';
|
|
10
11
|
this.server = new mcp_js_1.McpServer({
|
|
11
12
|
name: 'decodo',
|
|
12
|
-
version:
|
|
13
|
+
version: version_1.PACKAGE_VERSION,
|
|
13
14
|
});
|
|
14
|
-
this.sapiClient = new scraper_api_client_1.ScraperApiClient();
|
|
15
|
+
this.sapiClient = new scraper_api_client_1.ScraperApiClient({});
|
|
15
16
|
this.auth = auth;
|
|
16
17
|
this.registerTools({ toolsets });
|
|
17
18
|
this.registerResources();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sapi-base-server.js","sourceRoot":"","sources":["../../src/server/sapi-base-server.ts"],"names":[],"mappings":";;;AAAA,oEAAoE;AAEpE,sEAAiE;AACjE,oCA+BkB;AAKlB,MAAa,oBAAoB;IAO/B,YAAY,EAAE,IAAI,EAAE,QAAQ,GAAG,EAAE,EAAyC;QAF1E,SAAI,GAAW,EAAE,CAAC;QAGhB,IAAI,CAAC,MAAM,GAAG,IAAI,kBAAS,CAAC;YAC1B,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"sapi-base-server.js","sourceRoot":"","sources":["../../src/server/sapi-base-server.ts"],"names":[],"mappings":";;;AAAA,oEAAoE;AAEpE,sEAAiE;AACjE,wCAA6C;AAC7C,oCA+BkB;AAKlB,MAAa,oBAAoB;IAO/B,YAAY,EAAE,IAAI,EAAE,QAAQ,GAAG,EAAE,EAAyC;QAF1E,SAAI,GAAW,EAAE,CAAC;QAGhB,IAAI,CAAC,MAAM,GAAG,IAAI,kBAAS,CAAC;YAC1B,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,yBAAe;SACzB,CAAC,CAAC;QACH,IAAI,CAAC,UAAU,GAAG,IAAI,qCAAgB,CAAC,EAAE,CAAC,CAAC;QAE3C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QAEjB,IAAI,CAAC,aAAa,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;QAEjC,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC3B,CAAC;IAED,OAAO,CAAC,SAA+D;QACrE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACjC,CAAC;IAmCD,aAAa,CAAC,EAAE,QAAQ,EAA2B;QACjD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACxB,OAAO;QACT,CAAC;QAED,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,oBAAoB,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,OAAO,CAAC,CAAC;YACrF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YACvF,CAAC;QACH,CAAC;IACH,CAAC;IAED,gBAAgB;QACd,KAAK,MAAM,IAAI,IAAI,oBAAoB,CAAC,QAAQ,EAAE,CAAC;YACjD,IAAI,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACvF,CAAC;IACH,CAAC;IAED,iBAAiB;QACf,gDAAgD;IAClD,CAAC;;AAhFH,oDAiFC;AAxDQ,6BAAQ,GAAW;IACxB,IAAI,4BAAoB,EAAE;IAC1B,IAAI,sBAAc,EAAE;IACpB,IAAI,wBAAgB,EAAE;IACtB,IAAI,qBAAa,EAAE;IACnB,IAAI,sBAAc,EAAE;IACpB,IAAI,wBAAgB,EAAE;IACtB,IAAI,8BAAsB,EAAE;IAC5B,IAAI,wBAAgB,EAAE;IACtB,IAAI,yBAAiB,EAAE;IACvB,IAAI,yBAAiB,EAAE;IACvB,IAAI,yBAAiB,EAAE;IACvB,IAAI,6BAAqB,EAAE;IAC3B,IAAI,yBAAiB,EAAE;IACvB,IAAI,0BAAkB,EAAE;IACxB,IAAI,wBAAgB,EAAE;IACtB,IAAI,yBAAiB,EAAE;IACvB,IAAI,sBAAc,EAAE;IACpB,IAAI,4BAAoB,EAAE;IAC1B,IAAI,6BAAqB,EAAE;IAC3B,IAAI,yBAAiB,EAAE;IACvB,IAAI,2BAAmB,EAAE;IACzB,IAAI,0BAAkB,EAAE;IACxB,IAAI,4BAAoB,EAAE;IAC1B,IAAI,yBAAiB,EAAE;IACvB,IAAI,sBAAc,EAAE;IACpB,IAAI,2BAAmB,EAAE;IACzB,IAAI,sBAAc,EAAE;IACpB,IAAI,sBAAc,EAAE;IACpB,IAAI,mBAAW,EAAE;IACjB,IAAI,sBAAc,EAAE;CACrB,AA/Bc,CA+Bb"}
|
|
@@ -36,9 +36,9 @@ class AmazonBestsellersTool extends tool_1.Tool {
|
|
|
36
36
|
readOnlyHint: true,
|
|
37
37
|
openWorldHint: true,
|
|
38
38
|
},
|
|
39
|
-
}, async (scrapingParams) => {
|
|
39
|
+
}, async (scrapingParams, extra) => {
|
|
40
40
|
const params = Object.assign(Object.assign({}, scrapingParams), { target: constants_1.SCRAPER_API_TARGETS.AMAZON_BESTSELLERS, parse: true });
|
|
41
|
-
const { data } = await sapiClient.scrape({ auth, scrapingParams: params });
|
|
41
|
+
const { data } = await sapiClient.scrape({ auth, scrapingParams: params, extra });
|
|
42
42
|
return {
|
|
43
43
|
content: [
|
|
44
44
|
{
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"amazon-bestsellers-tool.js","sourceRoot":"","sources":["../../../src/tools/amazon-bestsellers/amazon-bestsellers-tool.ts"],"names":[],"mappings":";;;;;;AAAA,8CAAoB;AAEpB,+CAA+D;AAC/D,mDAAoD;AACpD,kCAAqD;
|
|
1
|
+
{"version":3,"file":"amazon-bestsellers-tool.js","sourceRoot":"","sources":["../../../src/tools/amazon-bestsellers/amazon-bestsellers-tool.ts"],"names":[],"mappings":";;;;;;AAAA,8CAAoB;AAEpB,+CAA+D;AAC/D,mDAAoD;AACpD,kCAAqD;AAGrD,MAAM,SAAS,GAAG,aAAC;KAChB,MAAM,EAAE;KACR,QAAQ,CAAC,gDAAgD,CAAC;KAC1D,QAAQ,EAAE,CAAC;AAEd,MAAM,WAAW,GAAG,aAAC;KAClB,MAAM,EAAE;KACR,QAAQ,CAAC,qCAAqC,CAAC;KAC/C,QAAQ,EAAE,CAAC;AAEd,MAAa,qBAAsB,SAAQ,WAAI;IAA/C;;QACE,YAAO,GAAG,mBAAO,CAAC,SAAS,CAAC;QAE5B,sBAAiB,GAAG,CAAC,EAAE,IAAI,EAAoB,EAAE,EAAE;YACjD,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,CAAC,CAAC;QAEF,aAAQ,GAAG,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAwB,EAAE,EAAE;YAChE,MAAM,CAAC,YAAY,CACjB,oBAAoB,EACpB;gBACE,WAAW,EAAE,uDAAuD;gBACpE,WAAW,EAAE;oBACX,KAAK,EAAE,aAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sDAAsD,CAAC;oBAClF,MAAM,EAAE,SAAS;oBACjB,UAAU,EAAE,yBAAa;oBACzB,QAAQ,EAAE,WAAW;iBACtB;gBACD,WAAW,EAAE;oBACX,YAAY,EAAE,IAAI;oBAClB,aAAa,EAAE,IAAI;iBACpB;aACF,EACD,KAAK,EAAE,cAAiC,EAAE,KAAoB,EAAE,EAAE;gBAChE,MAAM,MAAM,GAAG,gCACV,cAAc,KACjB,MAAM,EAAE,+BAAmB,CAAC,kBAAkB,EAC9C,KAAK,EAAE,IAAI,GACe,CAAC;gBAE7B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,UAAU,CAAC,MAAM,CAAS,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;gBAE1F,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;yBAC3B;qBACF;iBACF,CAAC;YACJ,CAAC,CACF,CAAC;QACJ,CAAC,CAAC;IACJ,CAAC;CAAA;AA3CD,sDA2CC"}
|
|
@@ -46,9 +46,9 @@ class AmazonPricingTool extends tool_1.Tool {
|
|
|
46
46
|
readOnlyHint: true,
|
|
47
47
|
openWorldHint: true,
|
|
48
48
|
},
|
|
49
|
-
}, async (scrapingParams) => {
|
|
49
|
+
}, async (scrapingParams, extra) => {
|
|
50
50
|
const params = Object.assign(Object.assign({}, scrapingParams), { target: constants_1.SCRAPER_API_TARGETS.AMAZON_PRICING, parse: true });
|
|
51
|
-
const { data } = await sapiClient.scrape({ auth, scrapingParams: params });
|
|
51
|
+
const { data } = await sapiClient.scrape({ auth, scrapingParams: params, extra });
|
|
52
52
|
const { data: text } = this.transformResponse({ data });
|
|
53
53
|
return {
|
|
54
54
|
content: [
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"amazon-pricing-tool.js","sourceRoot":"","sources":["../../../src/tools/amazon-pricing/amazon-pricing-tool.ts"],"names":[],"mappings":";;;;;;AAAA,8CAAoB;AAEpB,+CAA+D;AAC/D,
|
|
1
|
+
{"version":3,"file":"amazon-pricing-tool.js","sourceRoot":"","sources":["../../../src/tools/amazon-pricing/amazon-pricing-tool.ts"],"names":[],"mappings":";;;;;;AAAA,8CAAoB;AAEpB,+CAA+D;AAC/D,uCAAuE;AACvE,mDAAiE;AACjE,kCAAqD;AAErD,MAAM,SAAS,GAAG,aAAC;KAChB,MAAM,EAAE;KACR,QAAQ,CAAC,gDAAgD,CAAC;KAC1D,QAAQ,EAAE,CAAC;AAEd,MAAM,MAAM,GAAG,aAAC;KACb,MAAM,EAAE;KACR,QAAQ,CAAC,mDAAmD,CAAC;KAC7D,QAAQ,EAAE,CAAC;AAEd,MAAM,WAAW,GAAG,aAAC;KAClB,MAAM,EAAE;KACR,QAAQ,CAAC,qCAAqC,CAAC;KAC/C,QAAQ,EAAE,CAAC;AAEd,MAAa,iBAAkB,SAAQ,WAAI;IAA3C;;QACE,YAAO,GAAG,mBAAO,CAAC,SAAS,CAAC;QAI5B,sBAAiB,GAAG,CAAC,EAAE,IAAI,EAAoB,EAAE,EAAE;YACjD,KAAK,MAAM,aAAa,IAAI,iBAAiB,CAAC,2BAA2B,EAAE,CAAC;gBAC1E,IAAI,GAAG,IAAA,iCAAyB,EAAC,EAAE,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,aAAa,EAAE,CAAC,CAAC;YAC9E,CAAC;YAED,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,CAAC,CAAC;QAEF,aAAQ,GAAG,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAwB,EAAE,EAAE;YAChE,MAAM,CAAC,YAAY,CACjB,gBAAgB,EAChB;gBACE,WAAW,EAAE,kEAAkE;gBAC/E,WAAW,EAAE;oBACX,KAAK,EAAE,aAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,0CAA0C,CAAC;oBACtE,QAAQ,EAAE,uBAAW;oBACrB,MAAM,EAAE,SAAS;oBACjB,UAAU,EAAE,yBAAa;oBACzB,QAAQ,EAAE,WAAW;oBACrB,GAAG,EAAE,MAAM;iBACZ;gBACD,WAAW,EAAE;oBACX,YAAY,EAAE,IAAI;oBAClB,aAAa,EAAE,IAAI;iBACpB;aACF,EACD,KAAK,EAAE,cAAiC,EAAE,KAAoB,EAAE,EAAE;gBAChE,MAAM,MAAM,GAAG,gCACV,cAAc,KACjB,MAAM,EAAE,+BAAmB,CAAC,cAAc,EAC1C,KAAK,EAAE,IAAI,GACe,CAAC;gBAE7B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,UAAU,CAAC,MAAM,CAAS,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;gBAE1F,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,iBAAiB,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;gBAExD,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI;yBACL;qBACF;iBACF,CAAC;YACJ,CAAC,CACF,CAAC;QACJ,CAAC,CAAC;IACJ,CAAC;;AArDD,8CAqDC;AAlDgB,6CAA2B,GAAG,CAAC,aAAa,CAAC,AAAlB,CAAmB"}
|
|
@@ -41,9 +41,9 @@ class AmazonProductTool extends tool_1.Tool {
|
|
|
41
41
|
readOnlyHint: true,
|
|
42
42
|
openWorldHint: true,
|
|
43
43
|
},
|
|
44
|
-
}, async (scrapingParams) => {
|
|
44
|
+
}, async (scrapingParams, extra) => {
|
|
45
45
|
const params = Object.assign(Object.assign({}, scrapingParams), { target: constants_1.SCRAPER_API_TARGETS.AMAZON_PRODUCT, parse: true });
|
|
46
|
-
const { data } = await sapiClient.scrape({ auth, scrapingParams: params });
|
|
46
|
+
const { data } = await sapiClient.scrape({ auth, scrapingParams: params, extra });
|
|
47
47
|
const { data: text } = this.transformResponse({ data });
|
|
48
48
|
return {
|
|
49
49
|
content: [
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"amazon-product-tool.js","sourceRoot":"","sources":["../../../src/tools/amazon-product/amazon-product-tool.ts"],"names":[],"mappings":";;;;;;AAAA,8CAAoB;AAEpB,+CAA+D;AAC/D,
|
|
1
|
+
{"version":3,"file":"amazon-product-tool.js","sourceRoot":"","sources":["../../../src/tools/amazon-product/amazon-product-tool.ts"],"names":[],"mappings":";;;;;;AAAA,8CAAoB;AAEpB,+CAA+D;AAC/D,uCAAuE;AACvE,mDAAiE;AACjE,kCAAqD;AAErD,MAAM,SAAS,GAAG,aAAC;KAChB,MAAM,EAAE;KACR,QAAQ,CAAC,gDAAgD,CAAC;KAC1D,QAAQ,EAAE,CAAC;AAEd,MAAM,MAAM,GAAG,aAAC;KACb,MAAM,EAAE;KACR,QAAQ,CAAC,mDAAmD,CAAC;KAC7D,QAAQ,EAAE,CAAC;AAEd,MAAa,iBAAkB,SAAQ,WAAI;IAA3C;;QACE,YAAO,GAAG,mBAAO,CAAC,SAAS,CAAC;QAI5B,sBAAiB,GAAG,CAAC,EAAE,IAAI,EAAoB,EAAE,EAAE;YACjD,KAAK,MAAM,aAAa,IAAI,iBAAiB,CAAC,2BAA2B,EAAE,CAAC;gBAC1E,IAAI,GAAG,IAAA,iCAAyB,EAAC,EAAE,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,aAAa,EAAE,CAAC,CAAC;YAC9E,CAAC;YAED,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,CAAC,CAAC;QAEF,aAAQ,GAAG,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAwB,EAAE,EAAE;YAChE,MAAM,CAAC,YAAY,CACjB,gBAAgB,EAChB;gBACE,WAAW,EAAE,mDAAmD;gBAChE,WAAW,EAAE;oBACX,KAAK,EAAE,aAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,0CAA0C,CAAC;oBACtE,QAAQ,EAAE,uBAAW;oBACrB,MAAM,EAAE,SAAS;oBACjB,UAAU,EAAE,yBAAa;oBACzB,GAAG,EAAE,MAAM;iBACZ;gBACD,WAAW,EAAE;oBACX,YAAY,EAAE,IAAI;oBAClB,aAAa,EAAE,IAAI;iBACpB;aACF,EACD,KAAK,EAAE,cAAiC,EAAE,KAAoB,EAAE,EAAE;gBAChE,MAAM,MAAM,GAAG,gCACV,cAAc,KACjB,MAAM,EAAE,+BAAmB,CAAC,cAAc,EAC1C,KAAK,EAAE,IAAI,GACe,CAAC;gBAE7B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,UAAU,CAAC,MAAM,CAAS,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;gBAE1F,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,iBAAiB,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;gBAExD,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI;yBACL;qBACF;iBACF,CAAC;YACJ,CAAC,CACF,CAAC;QACJ,CAAC,CAAC;IACJ,CAAC;;AApDD,8CAoDC;AAjDgB,6CAA2B,GAAG,CAAC,eAAe,EAAE,aAAa,CAAC,AAAnC,CAAoC"}
|
|
@@ -42,9 +42,9 @@ class AmazonSearchTool extends tool_1.Tool {
|
|
|
42
42
|
readOnlyHint: true,
|
|
43
43
|
openWorldHint: true,
|
|
44
44
|
},
|
|
45
|
-
}, async (scrapingParams) => {
|
|
45
|
+
}, async (scrapingParams, extra) => {
|
|
46
46
|
const params = Object.assign(Object.assign({}, scrapingParams), { target: constants_1.SCRAPER_API_TARGETS.AMAZON_SEARCH, parse: true });
|
|
47
|
-
const { data } = await sapiClient.scrape({ auth, scrapingParams: params });
|
|
47
|
+
const { data } = await sapiClient.scrape({ auth, scrapingParams: params, extra });
|
|
48
48
|
const { data: text } = this.transformResponse({ data });
|
|
49
49
|
return {
|
|
50
50
|
content: [
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"amazon-search-tool.js","sourceRoot":"","sources":["../../../src/tools/amazon-search/amazon-search-tool.ts"],"names":[],"mappings":";;;;;;AAAA,8CAAoB;AAEpB,+CAA+D;AAC/D,
|
|
1
|
+
{"version":3,"file":"amazon-search-tool.js","sourceRoot":"","sources":["../../../src/tools/amazon-search/amazon-search-tool.ts"],"names":[],"mappings":";;;;;;AAAA,8CAAoB;AAEpB,+CAA+D;AAC/D,uCAAuE;AACvE,mDAAyE;AACzE,kCAAqD;AAErD,MAAM,SAAS,GAAG,aAAC;KAChB,MAAM,EAAE;KACR,QAAQ,CAAC,gDAAgD,CAAC;KAC1D,QAAQ,EAAE,CAAC;AAEd,MAAM,WAAW,GAAG,aAAC;KAClB,MAAM,EAAE;KACR,QAAQ,CAAC,qCAAqC,CAAC;KAC/C,QAAQ,EAAE,CAAC;AAEd,MAAa,gBAAiB,SAAQ,WAAI;IAA1C;;QACE,YAAO,GAAG,mBAAO,CAAC,SAAS,CAAC;QAI5B,sBAAiB,GAAG,CAAC,EAAE,IAAI,EAAoB,EAAE,EAAE;YACjD,KAAK,MAAM,aAAa,IAAI,gBAAgB,CAAC,2BAA2B,EAAE,CAAC;gBACzE,IAAI,GAAG,IAAA,iCAAyB,EAAC,EAAE,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,aAAa,EAAE,CAAC,CAAC;YAC9E,CAAC;YAED,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,CAAC,CAAC;QAEF,aAAQ,GAAG,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAwB,EAAE,EAAE;YAChE,MAAM,CAAC,YAAY,CACjB,eAAe,EACf;gBACE,WAAW,EAAE,qDAAqD;gBAClE,WAAW,EAAE;oBACX,KAAK,EAAE,aAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,8DAA8D,CAAC;oBAC1F,GAAG,EAAE,kBAAM;oBACX,QAAQ,EAAE,uBAAW;oBACrB,MAAM,EAAE,SAAS;oBACjB,UAAU,EAAE,yBAAa;oBACzB,QAAQ,EAAE,WAAW;iBACtB;gBACD,WAAW,EAAE;oBACX,YAAY,EAAE,IAAI;oBAClB,aAAa,EAAE,IAAI;iBACpB;aACF,EACD,KAAK,EAAE,cAAiC,EAAE,KAAoB,EAAE,EAAE;gBAChE,MAAM,MAAM,GAAG,gCACV,cAAc,KACjB,MAAM,EAAE,+BAAmB,CAAC,aAAa,EACzC,KAAK,EAAE,IAAI,GACe,CAAC;gBAE7B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,UAAU,CAAC,MAAM,CAAS,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;gBAE1F,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,iBAAiB,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;gBAExD,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI;yBACL;qBACF;iBACF,CAAC;YACJ,CAAC,CACF,CAAC;QACJ,CAAC,CAAC;IACJ,CAAC;;AArDD,4CAqDC;AAlDgB,4CAA2B,GAAG,CAAC,WAAW,EAAE,iBAAiB,EAAE,aAAa,CAAC,AAAlD,CAAmD"}
|
|
@@ -37,9 +37,9 @@ class AmazonSellersTool extends tool_1.Tool {
|
|
|
37
37
|
readOnlyHint: true,
|
|
38
38
|
openWorldHint: true,
|
|
39
39
|
},
|
|
40
|
-
}, async (scrapingParams) => {
|
|
40
|
+
}, async (scrapingParams, extra) => {
|
|
41
41
|
const params = Object.assign(Object.assign({}, scrapingParams), { target: constants_1.SCRAPER_API_TARGETS.AMAZON_SELLERS, parse: true });
|
|
42
|
-
const { data } = await sapiClient.scrape({ auth, scrapingParams: params });
|
|
42
|
+
const { data } = await sapiClient.scrape({ auth, scrapingParams: params, extra });
|
|
43
43
|
return {
|
|
44
44
|
content: [
|
|
45
45
|
{
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"amazon-sellers-tool.js","sourceRoot":"","sources":["../../../src/tools/amazon-sellers/amazon-sellers-tool.ts"],"names":[],"mappings":";;;;;;AAAA,8CAAoB;AAEpB,+CAA+D;AAC/D,mDAAiE;AACjE,kCAAqD;
|
|
1
|
+
{"version":3,"file":"amazon-sellers-tool.js","sourceRoot":"","sources":["../../../src/tools/amazon-sellers/amazon-sellers-tool.ts"],"names":[],"mappings":";;;;;;AAAA,8CAAoB;AAEpB,+CAA+D;AAC/D,mDAAiE;AACjE,kCAAqD;AAGrD,MAAM,SAAS,GAAG,aAAC;KAChB,MAAM,EAAE;KACR,QAAQ,CAAC,gDAAgD,CAAC;KAC1D,QAAQ,EAAE,CAAC;AAEd,MAAM,MAAM,GAAG,aAAC;KACb,MAAM,EAAE;KACR,QAAQ,CAAC,mDAAmD,CAAC;KAC7D,QAAQ,EAAE,CAAC;AAEd,MAAa,iBAAkB,SAAQ,WAAI;IAA3C;;QACE,YAAO,GAAG,mBAAO,CAAC,SAAS,CAAC;QAE5B,sBAAiB,GAAG,CAAC,EAAE,IAAI,EAAoB,EAAE,EAAE;YACjD,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,CAAC,CAAC;QAEF,aAAQ,GAAG,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAwB,EAAE,EAAE;YAChE,MAAM,CAAC,YAAY,CACjB,gBAAgB,EAChB;gBACE,WAAW,EAAE,yDAAyD;gBACtE,WAAW,EAAE;oBACX,KAAK,EAAE,aAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,2CAA2C,CAAC;oBACvE,QAAQ,EAAE,uBAAW;oBACrB,MAAM,EAAE,SAAS;oBACjB,UAAU,EAAE,yBAAa;oBACzB,GAAG,EAAE,MAAM;iBACZ;gBACD,WAAW,EAAE;oBACX,YAAY,EAAE,IAAI;oBAClB,aAAa,EAAE,IAAI;iBACpB;aACF,EACD,KAAK,EAAE,cAAiC,EAAE,KAAoB,EAAE,EAAE;gBAChE,MAAM,MAAM,GAAG,gCACV,cAAc,KACjB,MAAM,EAAE,+BAAmB,CAAC,cAAc,EAC1C,KAAK,EAAE,IAAI,GACe,CAAC;gBAE7B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,UAAU,CAAC,MAAM,CAAS,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;gBAE1F,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;yBAC3B;qBACF;iBACF,CAAC;YACJ,CAAC,CACF,CAAC;QACJ,CAAC,CAAC;IACJ,CAAC;CAAA;AA5CD,8CA4CC"}
|
|
@@ -43,9 +43,9 @@ class BingSearchTool extends tool_1.Tool {
|
|
|
43
43
|
readOnlyHint: true,
|
|
44
44
|
openWorldHint: true,
|
|
45
45
|
},
|
|
46
|
-
}, async (scrapingParams) => {
|
|
46
|
+
}, async (scrapingParams, extra) => {
|
|
47
47
|
const params = Object.assign(Object.assign({}, scrapingParams), { target: constants_1.SCRAPER_API_TARGETS.BING_SEARCH, parse: true });
|
|
48
|
-
const { data } = await sapiClient.scrape({ auth, scrapingParams: params });
|
|
48
|
+
const { data } = await sapiClient.scrape({ auth, scrapingParams: params, extra });
|
|
49
49
|
const { data: text } = this.transformResponse({ data });
|
|
50
50
|
return {
|
|
51
51
|
content: [
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bing-search-tool.js","sourceRoot":"","sources":["../../../src/tools/bing-search/bing-search-tool.ts"],"names":[],"mappings":";;;;;;AAAA,8CAAoB;AAEpB,+CAA+D;AAC/D,
|
|
1
|
+
{"version":3,"file":"bing-search-tool.js","sourceRoot":"","sources":["../../../src/tools/bing-search/bing-search-tool.ts"],"names":[],"mappings":";;;;;;AAAA,8CAAoB;AAEpB,+CAA+D;AAC/D,uCAAuE;AACvE,mDAAoF;AACpF,kCAAqD;AAErD,MAAM,SAAS,GAAG,aAAC;KAChB,MAAM,EAAE;KACR,QAAQ,CAAC,0CAA0C,CAAC;KACpD,QAAQ,EAAE,CAAC;AAEd,MAAM,WAAW,GAAG,aAAC;KAClB,MAAM,EAAE;KACR,QAAQ,CAAC,qCAAqC,CAAC;KAC/C,QAAQ,EAAE,CAAC;AAEd,MAAa,cAAe,SAAQ,WAAI;IAAxC;;QACE,YAAO,GAAG,mBAAO,CAAC,MAAM,CAAC;QAIzB,sBAAiB,GAAG,CAAC,EAAE,IAAI,EAAoB,EAAE,EAAE;YACjD,KAAK,MAAM,aAAa,IAAI,cAAc,CAAC,2BAA2B,EAAE,CAAC;gBACvE,IAAI,GAAG,IAAA,iCAAyB,EAAC,EAAE,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,aAAa,EAAE,CAAC,CAAC;YAC9E,CAAC;YAED,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,CAAC,CAAC;QAEF,aAAQ,GAAG,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAwB,EAAE,EAAE;YAChE,MAAM,CAAC,YAAY,CACjB,aAAa,EACb;gBACE,WAAW,EAAE,mDAAmD;gBAChE,WAAW,EAAE;oBACX,KAAK,EAAE,aAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,wCAAwC,CAAC;oBACpE,GAAG,EAAE,kBAAM;oBACX,MAAM,EAAE,qBAAS;oBACjB,QAAQ,EAAE,uBAAW;oBACrB,MAAM,EAAE,SAAS;oBACjB,UAAU,EAAE,yBAAa;oBACzB,QAAQ,EAAE,WAAW;iBACtB;gBACD,WAAW,EAAE;oBACX,YAAY,EAAE,IAAI;oBAClB,aAAa,EAAE,IAAI;iBACpB;aACF,EACD,KAAK,EAAE,cAAiC,EAAE,KAAoB,EAAE,EAAE;gBAChE,MAAM,MAAM,GAAG,gCACV,cAAc,KACjB,MAAM,EAAE,+BAAmB,CAAC,WAAW,EACvC,KAAK,EAAE,IAAI,GACe,CAAC;gBAE7B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,UAAU,CAAC,MAAM,CAAS,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;gBAE1F,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,iBAAiB,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;gBAExD,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI;yBACL;qBACF;iBACF,CAAC;YACJ,CAAC,CACF,CAAC;QACJ,CAAC,CAAC;IACJ,CAAC;;AAtDD,wCAsDC;AAnDgB,0CAA2B,GAAG,CAAC,KAAK,CAAC,AAAV,CAAW"}
|
|
@@ -27,9 +27,9 @@ class ChatGPTTool extends tool_1.Tool {
|
|
|
27
27
|
readOnlyHint: true,
|
|
28
28
|
openWorldHint: true,
|
|
29
29
|
},
|
|
30
|
-
}, async (scrapingParams) => {
|
|
30
|
+
}, async (scrapingParams, extra) => {
|
|
31
31
|
const params = Object.assign(Object.assign({}, scrapingParams), { target: constants_1.SCRAPER_API_TARGETS.CHATGPT, parse: true });
|
|
32
|
-
const { data } = await sapiClient.scrape({ auth, scrapingParams: params });
|
|
32
|
+
const { data } = await sapiClient.scrape({ auth, scrapingParams: params, extra });
|
|
33
33
|
const { data: text } = this.transformResponse({ data });
|
|
34
34
|
return {
|
|
35
35
|
content: [
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chatgpt-tool.js","sourceRoot":"","sources":["../../../src/tools/chatgpt/chatgpt-tool.ts"],"names":[],"mappings":";;;;;;AAAA,8CAAoB;AAEpB,+CAA+D;AAC/D,mDAA6C;AAC7C,kCAAqD;
|
|
1
|
+
{"version":3,"file":"chatgpt-tool.js","sourceRoot":"","sources":["../../../src/tools/chatgpt/chatgpt-tool.ts"],"names":[],"mappings":";;;;;;AAAA,8CAAoB;AAEpB,+CAA+D;AAC/D,mDAA6C;AAC7C,kCAAqD;AAGrD,MAAa,WAAY,SAAQ,WAAI;IAArC;;QACE,YAAO,GAAG,mBAAO,CAAC,EAAE,CAAC;QAErB,sBAAiB,GAAG,CAAC,EAAE,IAAI,EAAoB,EAAE,EAAE;YACjD,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;QACjD,CAAC,CAAC;QAEF,aAAQ,GAAG,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAwB,EAAE,EAAE;YAChE,MAAM,CAAC,YAAY,CACjB,SAAS,EACT;gBACE,WAAW,EAAE,6EAA6E;gBAC1F,WAAW,EAAE;oBACX,MAAM,EAAE,aAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,2BAA2B,CAAC;oBACxD,MAAM,EAAE,aAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,8CAA8C,CAAC,CAAC,QAAQ,EAAE;oBACvF,GAAG,EAAE,kBAAM;iBACZ;gBACD,WAAW,EAAE;oBACX,YAAY,EAAE,IAAI;oBAClB,aAAa,EAAE,IAAI;iBACpB;aACF,EACD,KAAK,EAAE,cAAiC,EAAE,KAAoB,EAAE,EAAE;gBAChE,MAAM,MAAM,GAAG,gCACV,cAAc,KACjB,MAAM,EAAE,+BAAmB,CAAC,OAAO,EACnC,KAAK,EAAE,IAAI,GACe,CAAC;gBAE7B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,UAAU,CAAC,MAAM,CAAS,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;gBAE1F,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,iBAAiB,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;gBAExD,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI;yBACL;qBACF;iBACF,CAAC;YACJ,CAAC,CACF,CAAC;QACJ,CAAC,CAAC;IACJ,CAAC;CAAA;AA5CD,kCA4CC"}
|
|
@@ -38,9 +38,9 @@ class GoogleAdsTool extends tool_1.Tool {
|
|
|
38
38
|
readOnlyHint: true,
|
|
39
39
|
openWorldHint: true,
|
|
40
40
|
},
|
|
41
|
-
}, async (scrapingParams) => {
|
|
41
|
+
}, async (scrapingParams, extra) => {
|
|
42
42
|
const params = Object.assign(Object.assign({}, scrapingParams), { target: constants_1.SCRAPER_API_TARGETS.GOOGLE_ADS, parse: true });
|
|
43
|
-
const { data } = await sapiClient.scrape({ auth, scrapingParams: params });
|
|
43
|
+
const { data } = await sapiClient.scrape({ auth, scrapingParams: params, extra });
|
|
44
44
|
const { data: text } = this.transformResponse({ data });
|
|
45
45
|
return {
|
|
46
46
|
content: [
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"google-ads-tool.js","sourceRoot":"","sources":["../../../src/tools/google-ads/google-ads-tool.ts"],"names":[],"mappings":";;;;;;AAAA,8CAAoB;AAEpB,+CAA+D;AAC/D,
|
|
1
|
+
{"version":3,"file":"google-ads-tool.js","sourceRoot":"","sources":["../../../src/tools/google-ads/google-ads-tool.ts"],"names":[],"mappings":";;;;;;AAAA,8CAAoB;AAEpB,+CAA+D;AAC/D,uCAAuE;AACvE,mDAAoF;AACpF,kCAAqD;AAErD,MAAM,WAAW,GAAG,aAAC;KAClB,MAAM,EAAE;KACR,QAAQ,CAAC,qCAAqC,CAAC;KAC/C,QAAQ,EAAE,CAAC;AAEd,MAAa,aAAc,SAAQ,WAAI;IAAvC;;QACE,YAAO,GAAG,mBAAO,CAAC,MAAM,CAAC;QAIzB,sBAAiB,GAAG,CAAC,EAAE,IAAI,EAAoB,EAAE,EAAE;YACjD,KAAK,MAAM,aAAa,IAAI,aAAa,CAAC,2BAA2B,EAAE,CAAC;gBACtE,IAAI,GAAG,IAAA,iCAAyB,EAAC,EAAE,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,aAAa,EAAE,CAAC,CAAC;YAC9E,CAAC;YAED,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,CAAC,CAAC;QAEF,aAAQ,GAAG,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAwB,EAAE,EAAE;YAChE,MAAM,CAAC,YAAY,CACjB,YAAY,EACZ;gBACE,WAAW,EAAE,yDAAyD;gBACtE,WAAW,EAAE;oBACX,KAAK,EAAE,aAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,8CAA8C,CAAC;oBAC1E,GAAG,EAAE,kBAAM;oBACX,MAAM,EAAE,qBAAS;oBACjB,QAAQ,EAAE,uBAAW;oBACrB,UAAU,EAAE,yBAAa;oBACzB,QAAQ,EAAE,WAAW;iBACtB;gBACD,WAAW,EAAE;oBACX,YAAY,EAAE,IAAI;oBAClB,aAAa,EAAE,IAAI;iBACpB;aACF,EACD,KAAK,EAAE,cAAiC,EAAE,KAAoB,EAAE,EAAE;gBAChE,MAAM,MAAM,GAAG,gCACV,cAAc,KACjB,MAAM,EAAE,+BAAmB,CAAC,UAAU,EACtC,KAAK,EAAE,IAAI,GACe,CAAC;gBAE7B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,UAAU,CAAC,MAAM,CAAS,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;gBAE1F,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,iBAAiB,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;gBAExD,OAAO;oBACL,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI;yBACL;qBACF;iBACF,CAAC;YACJ,CAAC,CACF,CAAC;QACJ,CAAC,CAAC;IACJ,CAAC;;AArDD,sCAqDC;AAlDgB,yCAA2B,GAAG,CAAC,KAAK,CAAC,AAAV,CAAW"}
|