mcp-researchpowerpack 3.9.4 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +116 -208
- package/dist/index.js +280 -337
- package/dist/index.js.map +7 -1
- package/dist/mcp-use.json +7 -0
- package/dist/src/clients/reddit.js +278 -0
- package/dist/src/clients/reddit.js.map +7 -0
- package/dist/src/clients/scraper.js +326 -0
- package/dist/src/clients/scraper.js.map +7 -0
- package/dist/src/clients/search.js +217 -0
- package/dist/src/clients/search.js.map +7 -0
- package/dist/src/config/index.js +138 -0
- package/dist/src/config/index.js.map +7 -0
- package/dist/src/prompts/deep-research.js +41 -0
- package/dist/src/prompts/deep-research.js.map +7 -0
- package/dist/src/prompts/reddit-sentiment.js +47 -0
- package/dist/src/prompts/reddit-sentiment.js.map +7 -0
- package/dist/src/schemas/reddit.js +21 -0
- package/dist/src/schemas/reddit.js.map +7 -0
- package/dist/src/schemas/scrape-links.js +26 -0
- package/dist/src/schemas/scrape-links.js.map +7 -0
- package/dist/src/schemas/start-research.js +17 -0
- package/dist/src/schemas/start-research.js.map +7 -0
- package/dist/src/schemas/web-search.js +53 -0
- package/dist/src/schemas/web-search.js.map +7 -0
- package/dist/src/services/llm-processor.js +675 -0
- package/dist/src/services/llm-processor.js.map +7 -0
- package/dist/src/services/markdown-cleaner.js +62 -0
- package/dist/src/services/markdown-cleaner.js.map +7 -0
- package/dist/src/services/workflow-state.js +116 -0
- package/dist/src/services/workflow-state.js.map +7 -0
- package/dist/src/tools/mcp-helpers.js +50 -0
- package/dist/src/tools/mcp-helpers.js.map +7 -0
- package/dist/src/tools/reddit.js +277 -0
- package/dist/src/tools/reddit.js.map +7 -0
- package/dist/src/tools/registry.js +18 -0
- package/dist/src/tools/registry.js.map +7 -0
- package/dist/src/tools/scrape.js +334 -0
- package/dist/src/tools/scrape.js.map +7 -0
- package/dist/src/tools/search.js +423 -0
- package/dist/src/tools/search.js.map +7 -0
- package/dist/src/tools/start-research.js +199 -0
- package/dist/src/tools/start-research.js.map +7 -0
- package/dist/src/tools/utils.js +29 -0
- package/dist/src/tools/utils.js.map +7 -0
- package/dist/src/utils/bootstrap-guard.js +27 -0
- package/dist/src/utils/bootstrap-guard.js.map +7 -0
- package/dist/src/utils/concurrency.js +62 -0
- package/dist/src/utils/concurrency.js.map +7 -0
- package/dist/src/utils/content-extractor.js +61 -0
- package/dist/src/utils/content-extractor.js.map +7 -0
- package/dist/src/utils/errors.js +211 -0
- package/dist/src/utils/errors.js.map +7 -0
- package/dist/src/utils/logger.js +25 -0
- package/dist/src/utils/logger.js.map +7 -0
- package/dist/src/utils/markdown-formatter.js +15 -0
- package/dist/src/utils/markdown-formatter.js.map +7 -0
- package/dist/src/utils/reddit-keyword-guard.js +29 -0
- package/dist/src/utils/reddit-keyword-guard.js.map +7 -0
- package/dist/src/utils/response.js +81 -0
- package/dist/src/utils/response.js.map +7 -0
- package/dist/src/utils/retry.js +13 -0
- package/dist/src/utils/retry.js.map +7 -0
- package/dist/src/utils/sanitize.js +10 -0
- package/dist/src/utils/sanitize.js.map +7 -0
- package/dist/src/utils/source-type.js +41 -0
- package/dist/src/utils/source-type.js.map +7 -0
- package/dist/src/utils/url-aggregator.js +227 -0
- package/dist/src/utils/url-aggregator.js.map +7 -0
- package/dist/src/utils/workflow-key.js +14 -0
- package/dist/src/utils/workflow-key.js.map +7 -0
- package/dist/src/version.js +32 -0
- package/dist/src/version.js.map +7 -0
- package/package.json +33 -28
- package/dist/clients/reddit.d.ts +0 -69
- package/dist/clients/reddit.d.ts.map +0 -1
- package/dist/clients/reddit.js +0 -369
- package/dist/clients/reddit.js.map +0 -1
- package/dist/clients/research.d.ts +0 -67
- package/dist/clients/research.d.ts.map +0 -1
- package/dist/clients/research.js +0 -290
- package/dist/clients/research.js.map +0 -1
- package/dist/clients/scraper.d.ts +0 -72
- package/dist/clients/scraper.d.ts.map +0 -1
- package/dist/clients/scraper.js +0 -338
- package/dist/clients/scraper.js.map +0 -1
- package/dist/clients/search.d.ts +0 -57
- package/dist/clients/search.d.ts.map +0 -1
- package/dist/clients/search.js +0 -223
- package/dist/clients/search.js.map +0 -1
- package/dist/config/index.d.ts +0 -78
- package/dist/config/index.d.ts.map +0 -1
- package/dist/config/index.js +0 -201
- package/dist/config/index.js.map +0 -1
- package/dist/config/loader.d.ts +0 -40
- package/dist/config/loader.d.ts.map +0 -1
- package/dist/config/loader.js +0 -322
- package/dist/config/loader.js.map +0 -1
- package/dist/config/types.d.ts +0 -81
- package/dist/config/types.d.ts.map +0 -1
- package/dist/config/types.js +0 -6
- package/dist/config/types.js.map +0 -1
- package/dist/config/yaml/tools.yaml +0 -146
- package/dist/index.d.ts +0 -7
- package/dist/index.d.ts.map +0 -1
- package/dist/schemas/deep-research.d.ts +0 -64
- package/dist/schemas/deep-research.d.ts.map +0 -1
- package/dist/schemas/deep-research.js +0 -224
- package/dist/schemas/deep-research.js.map +0 -1
- package/dist/schemas/scrape-links.d.ts +0 -32
- package/dist/schemas/scrape-links.d.ts.map +0 -1
- package/dist/schemas/scrape-links.js +0 -34
- package/dist/schemas/scrape-links.js.map +0 -1
- package/dist/schemas/web-search.d.ts +0 -22
- package/dist/schemas/web-search.d.ts.map +0 -1
- package/dist/schemas/web-search.js +0 -21
- package/dist/schemas/web-search.js.map +0 -1
- package/dist/services/file-attachment.d.ts +0 -30
- package/dist/services/file-attachment.d.ts.map +0 -1
- package/dist/services/file-attachment.js +0 -207
- package/dist/services/file-attachment.js.map +0 -1
- package/dist/services/llm-processor.d.ts +0 -29
- package/dist/services/llm-processor.d.ts.map +0 -1
- package/dist/services/llm-processor.js +0 -244
- package/dist/services/llm-processor.js.map +0 -1
- package/dist/services/markdown-cleaner.d.ts +0 -8
- package/dist/services/markdown-cleaner.d.ts.map +0 -1
- package/dist/services/markdown-cleaner.js +0 -74
- package/dist/services/markdown-cleaner.js.map +0 -1
- package/dist/tools/definitions.d.ts +0 -16
- package/dist/tools/definitions.d.ts.map +0 -1
- package/dist/tools/definitions.js +0 -17
- package/dist/tools/definitions.js.map +0 -1
- package/dist/tools/reddit.d.ts +0 -14
- package/dist/tools/reddit.d.ts.map +0 -1
- package/dist/tools/reddit.js +0 -265
- package/dist/tools/reddit.js.map +0 -1
- package/dist/tools/registry.d.ts +0 -71
- package/dist/tools/registry.d.ts.map +0 -1
- package/dist/tools/registry.js +0 -252
- package/dist/tools/registry.js.map +0 -1
- package/dist/tools/research.d.ts +0 -14
- package/dist/tools/research.d.ts.map +0 -1
- package/dist/tools/research.js +0 -196
- package/dist/tools/research.js.map +0 -1
- package/dist/tools/scrape.d.ts +0 -14
- package/dist/tools/scrape.d.ts.map +0 -1
- package/dist/tools/scrape.js +0 -234
- package/dist/tools/scrape.js.map +0 -1
- package/dist/tools/search.d.ts +0 -10
- package/dist/tools/search.d.ts.map +0 -1
- package/dist/tools/search.js +0 -158
- package/dist/tools/search.js.map +0 -1
- package/dist/tools/utils.d.ts +0 -105
- package/dist/tools/utils.d.ts.map +0 -1
- package/dist/tools/utils.js +0 -159
- package/dist/tools/utils.js.map +0 -1
- package/dist/utils/concurrency.d.ts +0 -28
- package/dist/utils/concurrency.d.ts.map +0 -1
- package/dist/utils/concurrency.js +0 -92
- package/dist/utils/concurrency.js.map +0 -1
- package/dist/utils/errors.d.ts +0 -95
- package/dist/utils/errors.d.ts.map +0 -1
- package/dist/utils/errors.js +0 -390
- package/dist/utils/errors.js.map +0 -1
- package/dist/utils/logger.d.ts +0 -39
- package/dist/utils/logger.d.ts.map +0 -1
- package/dist/utils/logger.js +0 -57
- package/dist/utils/logger.js.map +0 -1
- package/dist/utils/markdown-formatter.d.ts +0 -5
- package/dist/utils/markdown-formatter.d.ts.map +0 -1
- package/dist/utils/markdown-formatter.js +0 -15
- package/dist/utils/markdown-formatter.js.map +0 -1
- package/dist/utils/response.d.ts +0 -93
- package/dist/utils/response.d.ts.map +0 -1
- package/dist/utils/response.js +0 -170
- package/dist/utils/response.js.map +0 -1
- package/dist/utils/retry.d.ts +0 -43
- package/dist/utils/retry.d.ts.map +0 -1
- package/dist/utils/retry.js +0 -57
- package/dist/utils/retry.js.map +0 -1
- package/dist/utils/url-aggregator.d.ts +0 -90
- package/dist/utils/url-aggregator.d.ts.map +0 -1
- package/dist/utils/url-aggregator.js +0 -538
- package/dist/utils/url-aggregator.js.map +0 -1
- package/dist/version.d.ts +0 -29
- package/dist/version.d.ts.map +0 -1
- package/dist/version.js +0 -55
- package/dist/version.js.map +0 -1
package/dist/clients/scraper.js
DELETED
|
@@ -1,338 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Web Scraper Client
|
|
3
|
-
* Generic interface for URL scraping with automatic fallback modes
|
|
4
|
-
* Implements robust error handling that NEVER crashes
|
|
5
|
-
*/
|
|
6
|
-
import { parseEnv } from '../config/index.js';
|
|
7
|
-
import { classifyError, fetchWithTimeout, sleep, ErrorCode, } from '../utils/errors.js';
|
|
8
|
-
import { calculateBackoff } from '../utils/retry.js';
|
|
9
|
-
import { pMapSettled } from '../utils/concurrency.js';
|
|
10
|
-
import { mcpLog } from '../utils/logger.js';
|
|
11
|
-
// ── Constants ──
|
|
12
|
-
const SCRAPE_MODES = ['basic', 'javascript', 'javascript_geo'];
|
|
13
|
-
const CREDIT_COSTS = { basic: 1, javascript: 5, javascript_geo: 5 };
|
|
14
|
-
const DEFAULT_SCRAPE_CONCURRENCY = 10;
|
|
15
|
-
const SCRAPE_BATCH_SIZE = 30;
|
|
16
|
-
const MAX_RETRIES = 2;
|
|
17
|
-
/** Overall timeout for all fallback attempts on a single URL */
|
|
18
|
-
const FALLBACK_OVERALL_TIMEOUT_MS = 45_000;
|
|
19
|
-
// Status codes that indicate we should retry (no credit consumed)
|
|
20
|
-
const RETRYABLE_STATUS_CODES = new Set([429, 502, 503, 504, 510]);
|
|
21
|
-
// Status codes that are permanent failures (don't retry)
|
|
22
|
-
const PERMANENT_FAILURE_CODES = new Set([400, 401, 403]);
|
|
23
|
-
const FALLBACK_ATTEMPTS = [
|
|
24
|
-
{ mode: 'basic', description: 'basic mode' },
|
|
25
|
-
{ mode: 'javascript', description: 'javascript rendering' },
|
|
26
|
-
{ mode: 'javascript', country: 'us', description: 'javascript + US geo-targeting' },
|
|
27
|
-
];
|
|
28
|
-
export class ScraperClient {
|
|
29
|
-
apiKey;
|
|
30
|
-
baseURL = 'https://api.scrape.do';
|
|
31
|
-
constructor(apiKey) {
|
|
32
|
-
const env = parseEnv();
|
|
33
|
-
this.apiKey = apiKey || env.SCRAPER_API_KEY;
|
|
34
|
-
if (!this.apiKey) {
|
|
35
|
-
throw new Error('Web scraping capability is not configured. Please set up the required API credentials.');
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
/**
|
|
39
|
-
* Scrape a single URL with retry logic
|
|
40
|
-
* NEVER throws - always returns a ScrapeResponse (possibly with error)
|
|
41
|
-
*/
|
|
42
|
-
async scrape(request, maxRetries = MAX_RETRIES) {
|
|
43
|
-
const { url, mode = 'basic', timeout = 15, country } = request;
|
|
44
|
-
const credits = CREDIT_COSTS[mode] ?? 1;
|
|
45
|
-
// Validate URL first
|
|
46
|
-
try {
|
|
47
|
-
new URL(url);
|
|
48
|
-
}
|
|
49
|
-
catch {
|
|
50
|
-
return {
|
|
51
|
-
content: `Invalid URL: ${url}`,
|
|
52
|
-
statusCode: 400,
|
|
53
|
-
credits: 0,
|
|
54
|
-
error: { code: ErrorCode.INVALID_INPUT, message: `Invalid URL: ${url}`, retryable: false },
|
|
55
|
-
};
|
|
56
|
-
}
|
|
57
|
-
const params = new URLSearchParams({
|
|
58
|
-
url: url,
|
|
59
|
-
token: this.apiKey,
|
|
60
|
-
timeout: String(timeout * 1000),
|
|
61
|
-
});
|
|
62
|
-
if (mode === 'javascript') {
|
|
63
|
-
params.append('render', 'true');
|
|
64
|
-
}
|
|
65
|
-
if (country) {
|
|
66
|
-
params.append('geoCode', country.toUpperCase());
|
|
67
|
-
}
|
|
68
|
-
const apiUrl = `${this.baseURL}?${params.toString()}`;
|
|
69
|
-
let lastError;
|
|
70
|
-
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
71
|
-
try {
|
|
72
|
-
// Use AbortController for timeout
|
|
73
|
-
const timeoutMs = (timeout + 5) * 1000; // Add 5s buffer over scrape timeout
|
|
74
|
-
const response = await fetchWithTimeout(apiUrl, {
|
|
75
|
-
method: 'GET',
|
|
76
|
-
headers: { Accept: 'text/html,application/json' },
|
|
77
|
-
timeoutMs,
|
|
78
|
-
});
|
|
79
|
-
// Safely read response body
|
|
80
|
-
let content;
|
|
81
|
-
try {
|
|
82
|
-
content = await response.text();
|
|
83
|
-
}
|
|
84
|
-
catch (readError) {
|
|
85
|
-
content = `Failed to read response: ${readError instanceof Error ? readError.message : String(readError)}`;
|
|
86
|
-
}
|
|
87
|
-
// SUCCESS: 2xx - Successful API call
|
|
88
|
-
if (response.ok) {
|
|
89
|
-
return {
|
|
90
|
-
content,
|
|
91
|
-
statusCode: response.status,
|
|
92
|
-
credits,
|
|
93
|
-
headers: Object.fromEntries(response.headers.entries()),
|
|
94
|
-
};
|
|
95
|
-
}
|
|
96
|
-
// 404 - Target not found (permanent, but not an error for our purposes)
|
|
97
|
-
if (response.status === 404) {
|
|
98
|
-
return {
|
|
99
|
-
content: '404 - Page not found',
|
|
100
|
-
statusCode: 404,
|
|
101
|
-
credits,
|
|
102
|
-
};
|
|
103
|
-
}
|
|
104
|
-
// Permanent failures - don't retry
|
|
105
|
-
if (PERMANENT_FAILURE_CODES.has(response.status)) {
|
|
106
|
-
const errorMsg = response.status === 401
|
|
107
|
-
? 'No credits remaining or subscription suspended'
|
|
108
|
-
: `Request failed with status ${response.status}`;
|
|
109
|
-
return {
|
|
110
|
-
content: `Error: ${errorMsg}`,
|
|
111
|
-
statusCode: response.status,
|
|
112
|
-
credits: 0,
|
|
113
|
-
error: {
|
|
114
|
-
code: response.status === 401 ? ErrorCode.AUTH_ERROR : ErrorCode.INVALID_INPUT,
|
|
115
|
-
message: errorMsg,
|
|
116
|
-
retryable: false,
|
|
117
|
-
statusCode: response.status,
|
|
118
|
-
},
|
|
119
|
-
};
|
|
120
|
-
}
|
|
121
|
-
// Retryable status codes
|
|
122
|
-
if (RETRYABLE_STATUS_CODES.has(response.status)) {
|
|
123
|
-
lastError = {
|
|
124
|
-
code: response.status === 429 ? ErrorCode.RATE_LIMITED : ErrorCode.SERVICE_UNAVAILABLE,
|
|
125
|
-
message: `Server returned ${response.status}`,
|
|
126
|
-
retryable: true,
|
|
127
|
-
statusCode: response.status,
|
|
128
|
-
};
|
|
129
|
-
if (attempt < maxRetries - 1) {
|
|
130
|
-
const delayMs = calculateBackoff(attempt);
|
|
131
|
-
mcpLog('warning', `${response.status} on attempt ${attempt + 1}/${maxRetries}. Retrying in ${delayMs}ms`, 'scraper');
|
|
132
|
-
await sleep(delayMs);
|
|
133
|
-
continue;
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
// Other non-success status - treat as retryable
|
|
137
|
-
lastError = classifyError({ status: response.status, message: content });
|
|
138
|
-
if (attempt < maxRetries - 1 && lastError.retryable) {
|
|
139
|
-
const delayMs = calculateBackoff(attempt);
|
|
140
|
-
mcpLog('warning', `Status ${response.status}. Retrying in ${delayMs}ms`, 'scraper');
|
|
141
|
-
await sleep(delayMs);
|
|
142
|
-
continue;
|
|
143
|
-
}
|
|
144
|
-
// Final attempt failed
|
|
145
|
-
return {
|
|
146
|
-
content: `Error: ${lastError.message}`,
|
|
147
|
-
statusCode: response.status,
|
|
148
|
-
credits: 0,
|
|
149
|
-
error: lastError,
|
|
150
|
-
};
|
|
151
|
-
}
|
|
152
|
-
catch (error) {
|
|
153
|
-
lastError = classifyError(error);
|
|
154
|
-
// Non-retryable errors - return immediately
|
|
155
|
-
if (!lastError.retryable) {
|
|
156
|
-
return {
|
|
157
|
-
content: `Error: ${lastError.message}`,
|
|
158
|
-
statusCode: lastError.statusCode || 500,
|
|
159
|
-
credits: 0,
|
|
160
|
-
error: lastError,
|
|
161
|
-
};
|
|
162
|
-
}
|
|
163
|
-
// Retryable error - continue if attempts remaining
|
|
164
|
-
if (attempt < maxRetries - 1) {
|
|
165
|
-
const delayMs = calculateBackoff(attempt);
|
|
166
|
-
mcpLog('warning', `${lastError.code}: ${lastError.message}. Retry ${attempt + 1}/${maxRetries} in ${delayMs}ms`, 'scraper');
|
|
167
|
-
await sleep(delayMs);
|
|
168
|
-
continue;
|
|
169
|
-
}
|
|
170
|
-
}
|
|
171
|
-
}
|
|
172
|
-
// All retries exhausted
|
|
173
|
-
return {
|
|
174
|
-
content: `Error: Failed after ${maxRetries} attempts. ${lastError?.message || 'Unknown error'}`,
|
|
175
|
-
statusCode: lastError?.statusCode || 500,
|
|
176
|
-
credits: 0,
|
|
177
|
-
error: lastError || { code: ErrorCode.UNKNOWN_ERROR, message: 'All retries exhausted', retryable: false },
|
|
178
|
-
};
|
|
179
|
-
}
|
|
180
|
-
/**
|
|
181
|
-
* Scrape with automatic fallback through different modes
|
|
182
|
-
* NEVER throws - always returns a ScrapeResponse
|
|
183
|
-
*/
|
|
184
|
-
async scrapeWithFallback(url, options = {}) {
|
|
185
|
-
const attemptResults = [];
|
|
186
|
-
let lastResult = null;
|
|
187
|
-
const deadline = Date.now() + FALLBACK_OVERALL_TIMEOUT_MS;
|
|
188
|
-
for (const attempt of FALLBACK_ATTEMPTS) {
|
|
189
|
-
// Check overall deadline before starting next fallback
|
|
190
|
-
if (Date.now() >= deadline) {
|
|
191
|
-
mcpLog('warning', `Overall fallback timeout reached for ${url} after ${attemptResults.length} attempt(s)`, 'scraper');
|
|
192
|
-
break;
|
|
193
|
-
}
|
|
194
|
-
const result = await this.tryFallbackAttempt(url, attempt, options);
|
|
195
|
-
if (result.done) {
|
|
196
|
-
if (attemptResults.length > 0) {
|
|
197
|
-
mcpLog('info', `Success with ${attempt.description} after ${attemptResults.length} fallback(s)`, 'scraper');
|
|
198
|
-
}
|
|
199
|
-
return result.response;
|
|
200
|
-
}
|
|
201
|
-
lastResult = result.response;
|
|
202
|
-
attemptResults.push(`${attempt.description}: ${result.response.error?.message || result.response.statusCode}`);
|
|
203
|
-
mcpLog('warning', `Failed with ${attempt.description} (${result.response.statusCode}), trying next fallback...`, 'scraper');
|
|
204
|
-
}
|
|
205
|
-
// All fallbacks exhausted or deadline reached
|
|
206
|
-
const errorMessage = `Failed after ${attemptResults.length} fallback attempt(s): ${attemptResults.join('; ')}`;
|
|
207
|
-
return {
|
|
208
|
-
content: `Error: ${errorMessage}`,
|
|
209
|
-
statusCode: lastResult?.statusCode || 500,
|
|
210
|
-
credits: 0,
|
|
211
|
-
error: {
|
|
212
|
-
code: ErrorCode.SERVICE_UNAVAILABLE,
|
|
213
|
-
message: errorMessage,
|
|
214
|
-
retryable: false,
|
|
215
|
-
},
|
|
216
|
-
};
|
|
217
|
-
}
|
|
218
|
-
/**
|
|
219
|
-
* Execute a single fallback attempt and determine whether to continue.
|
|
220
|
-
* Returns { done: true } on success/terminal or { done: false } to try the next mode.
|
|
221
|
-
*/
|
|
222
|
-
async tryFallbackAttempt(url, attempt, options) {
|
|
223
|
-
const result = await this.scrape({
|
|
224
|
-
url,
|
|
225
|
-
mode: attempt.mode,
|
|
226
|
-
timeout: options.timeout,
|
|
227
|
-
country: attempt.country,
|
|
228
|
-
});
|
|
229
|
-
// Success
|
|
230
|
-
if (result.statusCode >= 200 && result.statusCode < 300 && !result.error) {
|
|
231
|
-
return { done: true, response: result };
|
|
232
|
-
}
|
|
233
|
-
// 404 is a valid response, not an error
|
|
234
|
-
if (result.statusCode === 404) {
|
|
235
|
-
return { done: true, response: result };
|
|
236
|
-
}
|
|
237
|
-
// Non-retryable errors - don't try other modes
|
|
238
|
-
if (result.error && !result.error.retryable) {
|
|
239
|
-
mcpLog('error', `Non-retryable error with ${attempt.description}: ${result.error.message}`, 'scraper');
|
|
240
|
-
return { done: true, response: result };
|
|
241
|
-
}
|
|
242
|
-
return { done: false, response: result };
|
|
243
|
-
}
|
|
244
|
-
/**
|
|
245
|
-
* Scrape multiple URLs with batching
|
|
246
|
-
* NEVER throws - always returns results array
|
|
247
|
-
*/
|
|
248
|
-
async scrapeMultiple(urls, options = {}) {
|
|
249
|
-
if (urls.length === 0) {
|
|
250
|
-
return [];
|
|
251
|
-
}
|
|
252
|
-
if (urls.length <= SCRAPE_BATCH_SIZE) {
|
|
253
|
-
return this.processBatch(urls, options);
|
|
254
|
-
}
|
|
255
|
-
const result = await this.batchScrape(urls, options);
|
|
256
|
-
return result.results;
|
|
257
|
-
}
|
|
258
|
-
/**
|
|
259
|
-
* Batch scrape with progress callback
|
|
260
|
-
* NEVER throws - uses Promise.allSettled internally
|
|
261
|
-
*/
|
|
262
|
-
async batchScrape(urls, options = {}, onBatchComplete) {
|
|
263
|
-
const totalBatches = Math.ceil(urls.length / SCRAPE_BATCH_SIZE);
|
|
264
|
-
const allResults = [];
|
|
265
|
-
let rateLimitHits = 0;
|
|
266
|
-
mcpLog('info', `Starting batch processing: ${urls.length} URLs in ${totalBatches} batch(es)`, 'scraper');
|
|
267
|
-
for (let batchNum = 0; batchNum < totalBatches; batchNum++) {
|
|
268
|
-
const startIdx = batchNum * SCRAPE_BATCH_SIZE;
|
|
269
|
-
const endIdx = Math.min(startIdx + SCRAPE_BATCH_SIZE, urls.length);
|
|
270
|
-
const batchUrls = urls.slice(startIdx, endIdx);
|
|
271
|
-
mcpLog('info', `Processing batch ${batchNum + 1}/${totalBatches} (${batchUrls.length} URLs)`, 'scraper');
|
|
272
|
-
const batchResults = await pMapSettled(batchUrls, url => this.scrapeWithFallback(url, options), DEFAULT_SCRAPE_CONCURRENCY);
|
|
273
|
-
for (let i = 0; i < batchResults.length; i++) {
|
|
274
|
-
const result = batchResults[i];
|
|
275
|
-
if (!result)
|
|
276
|
-
continue;
|
|
277
|
-
const url = batchUrls[i] ?? '';
|
|
278
|
-
if (result.status === 'fulfilled') {
|
|
279
|
-
const scrapeResult = result.value;
|
|
280
|
-
allResults.push({ ...scrapeResult, url });
|
|
281
|
-
// Track rate limits
|
|
282
|
-
if (scrapeResult.error?.code === ErrorCode.RATE_LIMITED) {
|
|
283
|
-
rateLimitHits++;
|
|
284
|
-
}
|
|
285
|
-
}
|
|
286
|
-
else {
|
|
287
|
-
// This shouldn't happen since scrapeWithFallback never throws,
|
|
288
|
-
// but handle it gracefully just in case
|
|
289
|
-
const errorMsg = result.reason instanceof Error ? result.reason.message : String(result.reason);
|
|
290
|
-
mcpLog('error', `Unexpected rejection for ${url}: ${errorMsg}`, 'scraper');
|
|
291
|
-
allResults.push({
|
|
292
|
-
url,
|
|
293
|
-
content: `Error: Unexpected failure - ${errorMsg}`,
|
|
294
|
-
statusCode: 500,
|
|
295
|
-
credits: 0,
|
|
296
|
-
error: classifyError(result.reason),
|
|
297
|
-
});
|
|
298
|
-
}
|
|
299
|
-
}
|
|
300
|
-
// Safe callback invocation
|
|
301
|
-
try {
|
|
302
|
-
onBatchComplete?.(batchNum + 1, totalBatches, allResults.length);
|
|
303
|
-
}
|
|
304
|
-
catch (callbackError) {
|
|
305
|
-
mcpLog('error', `onBatchComplete callback error: ${callbackError}`, 'scraper');
|
|
306
|
-
}
|
|
307
|
-
mcpLog('info', `Completed batch ${batchNum + 1}/${totalBatches} (${allResults.length}/${urls.length} total)`, 'scraper');
|
|
308
|
-
// Adaptive delay between batches — back off harder under rate limiting
|
|
309
|
-
if (batchNum < totalBatches - 1) {
|
|
310
|
-
const batchDelay = rateLimitHits > 0 ? 2000 : 500;
|
|
311
|
-
await sleep(batchDelay);
|
|
312
|
-
}
|
|
313
|
-
}
|
|
314
|
-
return { results: allResults, batchesProcessed: totalBatches, totalAttempted: urls.length, rateLimitHits };
|
|
315
|
-
}
|
|
316
|
-
/**
|
|
317
|
-
* Process a single batch of URLs
|
|
318
|
-
* NEVER throws
|
|
319
|
-
*/
|
|
320
|
-
async processBatch(urls, options) {
|
|
321
|
-
const results = await pMapSettled(urls, url => this.scrapeWithFallback(url, options), DEFAULT_SCRAPE_CONCURRENCY);
|
|
322
|
-
return results.map((result, index) => {
|
|
323
|
-
const url = urls[index] || '';
|
|
324
|
-
if (result.status === 'fulfilled') {
|
|
325
|
-
return { ...result.value, url };
|
|
326
|
-
}
|
|
327
|
-
// Shouldn't happen, but handle gracefully
|
|
328
|
-
return {
|
|
329
|
-
url,
|
|
330
|
-
content: `Error: ${result.reason instanceof Error ? result.reason.message : String(result.reason)}`,
|
|
331
|
-
statusCode: 500,
|
|
332
|
-
credits: 0,
|
|
333
|
-
error: classifyError(result.reason),
|
|
334
|
-
};
|
|
335
|
-
});
|
|
336
|
-
}
|
|
337
|
-
}
|
|
338
|
-
//# sourceMappingURL=scraper.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"scraper.js","sourceRoot":"","sources":["../../src/clients/scraper.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAW,MAAM,oBAAoB,CAAC;AACvD,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,KAAK,EACL,SAAS,GAEV,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,kBAAkB;AAElB,MAAM,YAAY,GAAG,CAAC,OAAO,EAAE,YAAY,EAAE,gBAAgB,CAAU,CAAC;AAGxE,MAAM,YAAY,GAA2B,EAAE,KAAK,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAW,CAAC;AACrG,MAAM,0BAA0B,GAAG,EAAW,CAAC;AAC/C,MAAM,iBAAiB,GAAG,EAAW,CAAC;AACtC,MAAM,WAAW,GAAG,CAAU,CAAC;AAC/B,gEAAgE;AAChE,MAAM,2BAA2B,GAAG,MAAe,CAAC;AA0BpD,kEAAkE;AAClE,MAAM,sBAAsB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAClE,yDAAyD;AACzD,MAAM,uBAAuB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AASzD,MAAM,iBAAiB,GAA+B;IACpD,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE;IAC5C,EAAE,IAAI,EAAE,YAAY,EAAE,WAAW,EAAE,sBAAsB,EAAE;IAC3D,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,+BAA+B,EAAE;CAC3E,CAAC;AAEX,MAAM,OAAO,aAAa;IAChB,MAAM,CAAS;IACf,OAAO,GAAG,uBAAuB,CAAC;IAE1C,YAAY,MAAe;QACzB,MAAM,GAAG,GAAG,QAAQ,EAAE,CAAC;QACvB,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,GAAG,CAAC,eAAe,CAAC;QAE5C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,wFAAwF,CAAC,CAAC;QAC5G,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,OAAsB,EAAE,UAAU,GAAG,WAAW;QAC3D,MAAM,EAAE,GAAG,EAAE,IAAI,GAAG,OAAO,EAAE,OAAO,GAAG,EAAE,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;QAC/D,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAExC,qBAAqB;QACrB,IAAI,CAAC;YACH,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACf,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;gBACL,OAAO,EAAE,gBAAgB,GAAG,EAAE;gBAC9B,UAAU,EAAE,GAAG;gBACf,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC,aAAa,EAAE,OAAO,EAAE,gBAAgB,GAAG,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE;aAC3F,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;YACjC,GAAG,EAAE,GAAG;YACR,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,OAAO,EAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;SAChC,CAAC,CAAC;QAEH,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;YAC1B,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC;QACtD,IAAI,SAAsC,CAAC;QAE3C,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YACtD,IAAI,CAAC;gBACH,kCAAkC;gBAClC,MAAM,SAAS,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,oCAAoC;gBAC5E,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,MAAM,EAAE;oBAC9C,MAAM,EAAE,KAAK;oBACb,OAAO,EAAE,EAAE,MAAM,EAAE,4BAA4B,EAAE;oBACjD,SAAS;iBACV,CAAC,CAAC;gBAEH,4BAA4B;gBAC5B,IAAI,OAAe,CAAC;gBACpB,IAAI,CAAC;oBACH,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAClC,CAAC;gBAAC,OAAO,SAAS,EAAE,CAAC;oBACnB,OAAO,GAAG,4BAA4B,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC7G,CAAC;gBAED,qCAAqC;gBACrC,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;oBAChB,OAAO;wBACL,OAAO;wBACP,UAAU,EAAE,QAAQ,CAAC,MAAM;wBAC3B,OAAO;wBACP,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;qBACxD,CAAC;gBACJ,CAAC;gBAED,wEAAwE;gBACxE,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBAC5B,OAAO;wBACL,OAAO,EAAE,sBAAsB;wBAC/B,UAAU,EAAE,GAAG;wBACf,OAAO;qBACR,CAAC;gBACJ,CAAC;gBAED,mCAAmC;gBACnC,IAAI,uBAAuB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;oBACjD,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,KAAK,GAAG;wBACtC,CAAC,CAAC,gDAAgD;wBAClD,CAAC,CAAC,8BAA8B,QAAQ,CAAC,MAAM,EAAE,CAAC;oBACpD,OAAO;wBACL,OAAO,EAAE,UAAU,QAAQ,EAAE;wBAC7B,UAAU,EAAE,QAAQ,CAAC,MAAM;wBAC3B,OAAO,EAAE,CAAC;wBACV,KAAK,EAAE;4BACL,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,aAAa;4BAC9E,OAAO,EAAE,QAAQ;4BACjB,SAAS,EAAE,KAAK;4BAChB,UAAU,EAAE,QAAQ,CAAC,MAAM;yBAC5B;qBACF,CAAC;gBACJ,CAAC;gBAED,yBAAyB;gBACzB,IAAI,sBAAsB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;oBAChD,SAAS,GAAG;wBACV,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC,mBAAmB;wBACtF,OAAO,EAAE,mBAAmB,QAAQ,CAAC,MAAM,EAAE;wBAC7C,SAAS,EAAE,IAAI;wBACf,UAAU,EAAE,QAAQ,CAAC,MAAM;qBAC5B,CAAC;oBAEF,IAAI,OAAO,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;wBAC7B,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;wBAC1C,MAAM,CAAC,SAAS,EAAE,GAAG,QAAQ,CAAC,MAAM,eAAe,OAAO,GAAG,CAAC,IAAI,UAAU,iBAAiB,OAAO,IAAI,EAAE,SAAS,CAAC,CAAC;wBACrH,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;wBACrB,SAAS;oBACX,CAAC;gBACH,CAAC;gBAED,gDAAgD;gBAChD,SAAS,GAAG,aAAa,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;gBACzE,IAAI,OAAO,GAAG,UAAU,GAAG,CAAC,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;oBACpD,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;oBAC1C,MAAM,CAAC,SAAS,EAAE,UAAU,QAAQ,CAAC,MAAM,iBAAiB,OAAO,IAAI,EAAE,SAAS,CAAC,CAAC;oBACpF,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;oBACrB,SAAS;gBACX,CAAC;gBAED,uBAAuB;gBACvB,OAAO;oBACL,OAAO,EAAE,UAAU,SAAS,CAAC,OAAO,EAAE;oBACtC,UAAU,EAAE,QAAQ,CAAC,MAAM;oBAC3B,OAAO,EAAE,CAAC;oBACV,KAAK,EAAE,SAAS;iBACjB,CAAC;YAEJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;gBAEjC,4CAA4C;gBAC5C,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC;oBACzB,OAAO;wBACL,OAAO,EAAE,UAAU,SAAS,CAAC,OAAO,EAAE;wBACtC,UAAU,EAAE,SAAS,CAAC,UAAU,IAAI,GAAG;wBACvC,OAAO,EAAE,CAAC;wBACV,KAAK,EAAE,SAAS;qBACjB,CAAC;gBACJ,CAAC;gBAED,mDAAmD;gBACnD,IAAI,OAAO,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;oBAC7B,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;oBAC1C,MAAM,CAAC,SAAS,EAAE,GAAG,SAAS,CAAC,IAAI,KAAK,SAAS,CAAC,OAAO,WAAW,OAAO,GAAG,CAAC,IAAI,UAAU,OAAO,OAAO,IAAI,EAAE,SAAS,CAAC,CAAC;oBAC5H,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;oBACrB,SAAS;gBACX,CAAC;YACH,CAAC;QACH,CAAC;QAED,wBAAwB;QACxB,OAAO;YACL,OAAO,EAAE,uBAAuB,UAAU,cAAc,SAAS,EAAE,OAAO,IAAI,eAAe,EAAE;YAC/F,UAAU,EAAE,SAAS,EAAE,UAAU,IAAI,GAAG;YACxC,OAAO,EAAE,CAAC;YACV,KAAK,EAAE,SAAS,IAAI,EAAE,IAAI,EAAE,SAAS,CAAC,aAAa,EAAE,OAAO,EAAE,uBAAuB,EAAE,SAAS,EAAE,KAAK,EAAE;SAC1G,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,kBAAkB,CAAC,GAAW,EAAE,UAAgC,EAAE;QACtE,MAAM,cAAc,GAAa,EAAE,CAAC;QACpC,IAAI,UAAU,GAA0B,IAAI,CAAC;QAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,2BAA2B,CAAC;QAE1D,KAAK,MAAM,OAAO,IAAI,iBAAiB,EAAE,CAAC;YACxC,uDAAuD;YACvD,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,QAAQ,EAAE,CAAC;gBAC3B,MAAM,CAAC,SAAS,EAAE,wCAAwC,GAAG,UAAU,cAAc,CAAC,MAAM,aAAa,EAAE,SAAS,CAAC,CAAC;gBACtH,MAAM;YACR,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;YAEpE,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBAChB,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC9B,MAAM,CAAC,MAAM,EAAE,gBAAgB,OAAO,CAAC,WAAW,UAAU,cAAc,CAAC,MAAM,cAAc,EAAE,SAAS,CAAC,CAAC;gBAC9G,CAAC;gBACD,OAAO,MAAM,CAAC,QAAQ,CAAC;YACzB,CAAC;YAED,UAAU,GAAG,MAAM,CAAC,QAAQ,CAAC;YAC7B,cAAc,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,WAAW,KAAK,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YAC/G,MAAM,CAAC,SAAS,EAAE,eAAe,OAAO,CAAC,WAAW,KAAK,MAAM,CAAC,QAAQ,CAAC,UAAU,4BAA4B,EAAE,SAAS,CAAC,CAAC;QAC9H,CAAC;QAED,8CAA8C;QAC9C,MAAM,YAAY,GAAG,gBAAgB,cAAc,CAAC,MAAM,yBAAyB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/G,OAAO;YACL,OAAO,EAAE,UAAU,YAAY,EAAE;YACjC,UAAU,EAAE,UAAU,EAAE,UAAU,IAAI,GAAG;YACzC,OAAO,EAAE,CAAC;YACV,KAAK,EAAE;gBACL,IAAI,EAAE,SAAS,CAAC,mBAAmB;gBACnC,OAAO,EAAE,YAAY;gBACrB,SAAS,EAAE,KAAK;aACjB;SACF,CAAC;IACJ,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,kBAAkB,CAC9B,GAAW,EACX,OAAwB,EACxB,OAA6B;QAE7B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC;YAC/B,GAAG;YACH,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,OAAO,EAAE,OAAO,CAAC,OAAO;SACzB,CAAC,CAAC;QAEH,UAAU;QACV,IAAI,MAAM,CAAC,UAAU,IAAI,GAAG,IAAI,MAAM,CAAC,UAAU,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YACzE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAC1C,CAAC;QAED,wCAAwC;QACxC,IAAI,MAAM,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;YAC9B,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAC1C,CAAC;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;YAC5C,MAAM,CAAC,OAAO,EAAE,4BAA4B,OAAO,CAAC,WAAW,KAAK,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,SAAS,CAAC,CAAC;YACvG,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAC1C,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;IAC3C,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc,CAAC,IAAc,EAAE,UAAgC,EAAE;QACrE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,IAAI,iBAAiB,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC1C,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACrD,OAAO,MAAM,CAAC,OAAkD,CAAC;IACnE,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,WAAW,CACf,IAAc,EACd,UAAgC,EAAE,EAClC,eAAqF;QAErF,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,iBAAiB,CAAC,CAAC;QAChE,MAAM,UAAU,GAA4C,EAAE,CAAC;QAC/D,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,MAAM,CAAC,MAAM,EAAE,8BAA8B,IAAI,CAAC,MAAM,YAAY,YAAY,YAAY,EAAE,SAAS,CAAC,CAAC;QAEzG,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,YAAY,EAAE,QAAQ,EAAE,EAAE,CAAC;YAC3D,MAAM,QAAQ,GAAG,QAAQ,GAAG,iBAAiB,CAAC;YAC9C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,iBAAiB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACnE,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAE/C,MAAM,CAAC,MAAM,EAAE,oBAAoB,QAAQ,GAAG,CAAC,IAAI,YAAY,KAAK,SAAS,CAAC,MAAM,QAAQ,EAAE,SAAS,CAAC,CAAC;YAEzG,MAAM,YAAY,GAAG,MAAM,WAAW,CACpC,SAAS,EACT,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,OAAO,CAAC,EAC5C,0BAA0B,CAC3B,CAAC;YAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7C,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;gBAC/B,IAAI,CAAC,MAAM;oBAAE,SAAS;gBACtB,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAE/B,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;oBAClC,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC;oBAClC,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,YAAY,EAAE,GAAG,EAAE,CAAC,CAAC;oBAE1C,oBAAoB;oBACpB,IAAI,YAAY,CAAC,KAAK,EAAE,IAAI,KAAK,SAAS,CAAC,YAAY,EAAE,CAAC;wBACxD,aAAa,EAAE,CAAC;oBAClB,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,+DAA+D;oBAC/D,wCAAwC;oBACxC,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;oBAChG,MAAM,CAAC,OAAO,EAAE,4BAA4B,GAAG,KAAK,QAAQ,EAAE,EAAE,SAAS,CAAC,CAAC;oBAE3E,UAAU,CAAC,IAAI,CAAC;wBACd,GAAG;wBACH,OAAO,EAAE,+BAA+B,QAAQ,EAAE;wBAClD,UAAU,EAAE,GAAG;wBACf,OAAO,EAAE,CAAC;wBACV,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC;qBACpC,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,2BAA2B;YAC3B,IAAI,CAAC;gBACH,eAAe,EAAE,CAAC,QAAQ,GAAG,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;YACnE,CAAC;YAAC,OAAO,aAAa,EAAE,CAAC;gBACvB,MAAM,CAAC,OAAO,EAAE,mCAAmC,aAAa,EAAE,EAAE,SAAS,CAAC,CAAC;YACjF,CAAC;YAED,MAAM,CAAC,MAAM,EAAE,mBAAmB,QAAQ,GAAG,CAAC,IAAI,YAAY,KAAK,UAAU,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,SAAS,EAAE,SAAS,CAAC,CAAC;YAEzH,uEAAuE;YACvE,IAAI,QAAQ,GAAG,YAAY,GAAG,CAAC,EAAE,CAAC;gBAChC,MAAM,UAAU,GAAG,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;gBAClD,MAAM,KAAK,CAAC,UAAU,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,YAAY,EAAE,cAAc,EAAE,IAAI,CAAC,MAAM,EAAE,aAAa,EAAE,CAAC;IAC7G,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,YAAY,CAAC,IAAc,EAAE,OAA6B;QACtE,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,0BAA0B,CAAC,CAAC;QAElH,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;YACnC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAE9B,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClC,OAAO,EAAE,GAAG,MAAM,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC;YAClC,CAAC;YAED,0CAA0C;YAC1C,OAAO;gBACL,GAAG;gBACH,OAAO,EAAE,UAAU,MAAM,CAAC,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE;gBACnG,UAAU,EAAE,GAAG;gBACf,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC;aACpC,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
|
package/dist/clients/search.d.ts
DELETED
|
@@ -1,57 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Web Search Client
|
|
3
|
-
* Generic interface for web search via Google (Serper implementation)
|
|
4
|
-
* Implements robust error handling that NEVER crashes
|
|
5
|
-
*/
|
|
6
|
-
import { type StructuredError } from '../utils/errors.js';
|
|
7
|
-
interface SearchResult {
|
|
8
|
-
readonly title: string;
|
|
9
|
-
readonly link: string;
|
|
10
|
-
readonly snippet: string;
|
|
11
|
-
readonly date?: string;
|
|
12
|
-
readonly position: number;
|
|
13
|
-
}
|
|
14
|
-
export interface KeywordSearchResult {
|
|
15
|
-
readonly keyword: string;
|
|
16
|
-
readonly results: SearchResult[];
|
|
17
|
-
readonly totalResults: number;
|
|
18
|
-
readonly related: string[];
|
|
19
|
-
readonly error?: StructuredError;
|
|
20
|
-
}
|
|
21
|
-
interface MultipleSearchResponse {
|
|
22
|
-
readonly searches: KeywordSearchResult[];
|
|
23
|
-
readonly totalKeywords: number;
|
|
24
|
-
readonly executionTime: number;
|
|
25
|
-
readonly error?: StructuredError;
|
|
26
|
-
}
|
|
27
|
-
export interface RedditSearchResult {
|
|
28
|
-
readonly title: string;
|
|
29
|
-
readonly url: string;
|
|
30
|
-
readonly snippet: string;
|
|
31
|
-
readonly date?: string;
|
|
32
|
-
}
|
|
33
|
-
export declare class SearchClient {
|
|
34
|
-
private apiKey;
|
|
35
|
-
constructor(apiKey?: string);
|
|
36
|
-
/**
|
|
37
|
-
* Check if error is retryable
|
|
38
|
-
*/
|
|
39
|
-
private isRetryable;
|
|
40
|
-
/**
|
|
41
|
-
* Search multiple keywords in parallel
|
|
42
|
-
* NEVER throws - always returns a valid response
|
|
43
|
-
*/
|
|
44
|
-
searchMultiple(keywords: string[]): Promise<MultipleSearchResponse>;
|
|
45
|
-
/**
|
|
46
|
-
* Search Reddit via Google (adds site:reddit.com automatically)
|
|
47
|
-
* NEVER throws - returns empty array on failure
|
|
48
|
-
*/
|
|
49
|
-
searchReddit(query: string, dateAfter?: string): Promise<RedditSearchResult[]>;
|
|
50
|
-
/**
|
|
51
|
-
* Search Reddit with multiple queries (bounded concurrency)
|
|
52
|
-
* NEVER throws - searchReddit never throws, pMap preserves order
|
|
53
|
-
*/
|
|
54
|
-
searchRedditMultiple(queries: string[], dateAfter?: string): Promise<Map<string, RedditSearchResult[]>>;
|
|
55
|
-
}
|
|
56
|
-
export {};
|
|
57
|
-
//# sourceMappingURL=search.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/clients/search.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAKL,KAAK,eAAe,EACrB,MAAM,oBAAoB,CAAC;AAc5B,UAAU,YAAY;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,YAAY,EAAE,CAAC;IACjC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,eAAe,CAAC;CAClC;AAED,UAAU,sBAAsB;IAC9B,QAAQ,CAAC,QAAQ,EAAE,mBAAmB,EAAE,CAAC;IACzC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,KAAK,CAAC,EAAE,eAAe,CAAC;CAClC;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;CACxB;AAwHD,qBAAa,YAAY;IACvB,OAAO,CAAC,MAAM,CAAS;gBAEX,MAAM,CAAC,EAAE,MAAM;IAS3B;;OAEG;IACH,OAAO,CAAC,WAAW;IAUnB;;;OAGG;IACG,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAkCzE;;;OAGG;IACG,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAuDpF;;;OAGG;IACG,oBAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,kBAAkB,EAAE,CAAC,CAAC;CAa9G"}
|
package/dist/clients/search.js
DELETED
|
@@ -1,223 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Web Search Client
|
|
3
|
-
* Generic interface for web search via Google (Serper implementation)
|
|
4
|
-
* Implements robust error handling that NEVER crashes
|
|
5
|
-
*/
|
|
6
|
-
import { parseEnv } from '../config/index.js';
|
|
7
|
-
import { classifyError, fetchWithTimeout, sleep, ErrorCode, } from '../utils/errors.js';
|
|
8
|
-
import { calculateBackoff } from '../utils/retry.js';
|
|
9
|
-
import { pMap } from '../utils/concurrency.js';
|
|
10
|
-
import { mcpLog } from '../utils/logger.js';
|
|
11
|
-
// ── Constants ──
|
|
12
|
-
const SERPER_API_URL = 'https://google.serper.dev/search';
|
|
13
|
-
const DEFAULT_RESULTS_PER_KEYWORD = 10;
|
|
14
|
-
const MAX_SEARCH_CONCURRENCY = 8;
|
|
15
|
-
const MAX_RETRIES = 3;
|
|
16
|
-
// ── Retry Configuration ──
|
|
17
|
-
const SEARCH_RETRY_CONFIG = {
|
|
18
|
-
maxRetries: MAX_RETRIES,
|
|
19
|
-
baseDelayMs: 1000,
|
|
20
|
-
maxDelayMs: 10000,
|
|
21
|
-
timeoutMs: 30000,
|
|
22
|
-
};
|
|
23
|
-
const RETRYABLE_SEARCH_CODES = new Set([429, 500, 502, 503, 504]);
|
|
24
|
-
// Pre-compiled regex patterns for Reddit search
|
|
25
|
-
const REDDIT_SITE_REGEX = /site:\s*reddit\.com/i;
|
|
26
|
-
const REDDIT_SUBREDDIT_SUFFIX_REGEX = / : r\/\w+$/;
|
|
27
|
-
const REDDIT_SUFFIX_REGEX = / - Reddit$/;
|
|
28
|
-
// ── Helper: Parse Serper search responses into structured results ──
|
|
29
|
-
function parseSearchResponses(responses, keywords) {
|
|
30
|
-
return responses.map((resp, index) => {
|
|
31
|
-
try {
|
|
32
|
-
const organic = (resp.organic || []);
|
|
33
|
-
const results = organic.map((item, idx) => ({
|
|
34
|
-
title: item.title || 'No title',
|
|
35
|
-
link: item.link || '#',
|
|
36
|
-
snippet: item.snippet || '',
|
|
37
|
-
date: item.date,
|
|
38
|
-
position: item.position || idx + 1,
|
|
39
|
-
}));
|
|
40
|
-
const searchInfo = resp.searchInformation;
|
|
41
|
-
const totalResults = searchInfo?.totalResults
|
|
42
|
-
? parseInt(String(searchInfo.totalResults).replace(/,/g, ''), 10)
|
|
43
|
-
: results.length;
|
|
44
|
-
const relatedSearches = (resp.relatedSearches || []);
|
|
45
|
-
const related = relatedSearches.map((r) => r.query || '');
|
|
46
|
-
return { keyword: keywords[index] || '', results, totalResults, related };
|
|
47
|
-
}
|
|
48
|
-
catch {
|
|
49
|
-
return { keyword: keywords[index] || '', results: [], totalResults: 0, related: [] };
|
|
50
|
-
}
|
|
51
|
-
});
|
|
52
|
-
}
|
|
53
|
-
// ── Helper: Execute search API call with retry ──
|
|
54
|
-
async function executeSearchWithRetry(apiKey, body, isRetryable) {
|
|
55
|
-
let lastError;
|
|
56
|
-
for (let attempt = 0; attempt <= SEARCH_RETRY_CONFIG.maxRetries; attempt++) {
|
|
57
|
-
try {
|
|
58
|
-
if (attempt > 0) {
|
|
59
|
-
mcpLog('warning', `Retry attempt ${attempt}/${SEARCH_RETRY_CONFIG.maxRetries}`, 'search');
|
|
60
|
-
}
|
|
61
|
-
const response = await fetchWithTimeout(SERPER_API_URL, {
|
|
62
|
-
method: 'POST',
|
|
63
|
-
headers: {
|
|
64
|
-
'X-API-KEY': apiKey,
|
|
65
|
-
'Content-Type': 'application/json',
|
|
66
|
-
},
|
|
67
|
-
body: JSON.stringify(body),
|
|
68
|
-
timeoutMs: SEARCH_RETRY_CONFIG.timeoutMs,
|
|
69
|
-
});
|
|
70
|
-
if (!response.ok) {
|
|
71
|
-
const errorText = await response.text().catch(() => '');
|
|
72
|
-
lastError = classifyError({ status: response.status, message: errorText });
|
|
73
|
-
if (isRetryable(response.status) && attempt < SEARCH_RETRY_CONFIG.maxRetries) {
|
|
74
|
-
const delayMs = calculateBackoff(attempt, SEARCH_RETRY_CONFIG.baseDelayMs, SEARCH_RETRY_CONFIG.maxDelayMs);
|
|
75
|
-
mcpLog('warning', `API returned ${response.status}, retrying in ${delayMs}ms...`, 'search');
|
|
76
|
-
await sleep(delayMs);
|
|
77
|
-
continue;
|
|
78
|
-
}
|
|
79
|
-
return { data: undefined, error: lastError };
|
|
80
|
-
}
|
|
81
|
-
try {
|
|
82
|
-
const data = await response.json();
|
|
83
|
-
return { data };
|
|
84
|
-
}
|
|
85
|
-
catch {
|
|
86
|
-
return {
|
|
87
|
-
data: undefined,
|
|
88
|
-
error: { code: ErrorCode.PARSE_ERROR, message: 'Failed to parse search response', retryable: false },
|
|
89
|
-
};
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
catch (error) {
|
|
93
|
-
lastError = classifyError(error);
|
|
94
|
-
if (isRetryable(undefined, error) && attempt < SEARCH_RETRY_CONFIG.maxRetries) {
|
|
95
|
-
const delayMs = calculateBackoff(attempt, SEARCH_RETRY_CONFIG.baseDelayMs, SEARCH_RETRY_CONFIG.maxDelayMs);
|
|
96
|
-
mcpLog('warning', `${lastError.code}: ${lastError.message}, retrying in ${delayMs}ms...`, 'search');
|
|
97
|
-
await sleep(delayMs);
|
|
98
|
-
continue;
|
|
99
|
-
}
|
|
100
|
-
return { data: undefined, error: lastError };
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
return {
|
|
104
|
-
data: undefined,
|
|
105
|
-
error: lastError || { code: ErrorCode.UNKNOWN_ERROR, message: 'Search failed', retryable: false },
|
|
106
|
-
};
|
|
107
|
-
}
|
|
108
|
-
// ── SearchClient ──
|
|
109
|
-
export class SearchClient {
|
|
110
|
-
apiKey;
|
|
111
|
-
constructor(apiKey) {
|
|
112
|
-
const env = parseEnv();
|
|
113
|
-
this.apiKey = apiKey || env.SEARCH_API_KEY || '';
|
|
114
|
-
if (!this.apiKey) {
|
|
115
|
-
throw new Error('Web search capability is not configured. Please set up the required API credentials.');
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
/**
|
|
119
|
-
* Check if error is retryable
|
|
120
|
-
*/
|
|
121
|
-
isRetryable(status, error) {
|
|
122
|
-
if (status && RETRYABLE_SEARCH_CODES.has(status))
|
|
123
|
-
return true;
|
|
124
|
-
if (error == null)
|
|
125
|
-
return false;
|
|
126
|
-
const message = (typeof error === 'object' && 'message' in error && typeof error.message === 'string')
|
|
127
|
-
? error.message.toLowerCase()
|
|
128
|
-
: '';
|
|
129
|
-
return message.includes('timeout') || message.includes('rate limit') || message.includes('connection');
|
|
130
|
-
}
|
|
131
|
-
/**
|
|
132
|
-
* Search multiple keywords in parallel
|
|
133
|
-
* NEVER throws - always returns a valid response
|
|
134
|
-
*/
|
|
135
|
-
async searchMultiple(keywords) {
|
|
136
|
-
const startTime = Date.now();
|
|
137
|
-
if (keywords.length === 0) {
|
|
138
|
-
return {
|
|
139
|
-
searches: [],
|
|
140
|
-
totalKeywords: 0,
|
|
141
|
-
executionTime: 0,
|
|
142
|
-
error: { code: ErrorCode.INVALID_INPUT, message: 'No keywords provided', retryable: false },
|
|
143
|
-
};
|
|
144
|
-
}
|
|
145
|
-
const searchQueries = keywords.map(keyword => ({ q: keyword }));
|
|
146
|
-
const { data, error } = await executeSearchWithRetry(this.apiKey, searchQueries, (status, err) => this.isRetryable(status, err));
|
|
147
|
-
if (error || data === undefined) {
|
|
148
|
-
return {
|
|
149
|
-
searches: [],
|
|
150
|
-
totalKeywords: keywords.length,
|
|
151
|
-
executionTime: Date.now() - startTime,
|
|
152
|
-
error,
|
|
153
|
-
};
|
|
154
|
-
}
|
|
155
|
-
const responses = Array.isArray(data) ? data : [data];
|
|
156
|
-
const searches = parseSearchResponses(responses, keywords);
|
|
157
|
-
return { searches, totalKeywords: keywords.length, executionTime: Date.now() - startTime };
|
|
158
|
-
}
|
|
159
|
-
/**
|
|
160
|
-
* Search Reddit via Google (adds site:reddit.com automatically)
|
|
161
|
-
* NEVER throws - returns empty array on failure
|
|
162
|
-
*/
|
|
163
|
-
async searchReddit(query, dateAfter) {
|
|
164
|
-
if (!query?.trim()) {
|
|
165
|
-
return [];
|
|
166
|
-
}
|
|
167
|
-
let q = query.replace(REDDIT_SITE_REGEX, '').trim() + ' site:reddit.com';
|
|
168
|
-
if (dateAfter) {
|
|
169
|
-
q += ` after:${dateAfter}`;
|
|
170
|
-
}
|
|
171
|
-
for (let attempt = 0; attempt <= SEARCH_RETRY_CONFIG.maxRetries; attempt++) {
|
|
172
|
-
try {
|
|
173
|
-
const res = await fetchWithTimeout(SERPER_API_URL, {
|
|
174
|
-
method: 'POST',
|
|
175
|
-
headers: { 'X-API-KEY': this.apiKey, 'Content-Type': 'application/json' },
|
|
176
|
-
body: JSON.stringify({ q, num: DEFAULT_RESULTS_PER_KEYWORD }),
|
|
177
|
-
timeoutMs: SEARCH_RETRY_CONFIG.timeoutMs,
|
|
178
|
-
});
|
|
179
|
-
if (!res.ok) {
|
|
180
|
-
if (this.isRetryable(res.status) && attempt < SEARCH_RETRY_CONFIG.maxRetries) {
|
|
181
|
-
const delayMs = calculateBackoff(attempt, SEARCH_RETRY_CONFIG.baseDelayMs, SEARCH_RETRY_CONFIG.maxDelayMs);
|
|
182
|
-
mcpLog('warning', `Reddit search ${res.status}, retrying in ${delayMs}ms...`, 'search');
|
|
183
|
-
await sleep(delayMs);
|
|
184
|
-
continue;
|
|
185
|
-
}
|
|
186
|
-
mcpLog('error', `Reddit search failed with status ${res.status}`, 'search');
|
|
187
|
-
return [];
|
|
188
|
-
}
|
|
189
|
-
const data = await res.json();
|
|
190
|
-
return (data.organic || []).map((r) => ({
|
|
191
|
-
title: (r.title || '').replace(REDDIT_SUBREDDIT_SUFFIX_REGEX, '').replace(REDDIT_SUFFIX_REGEX, ''),
|
|
192
|
-
url: r.link || '',
|
|
193
|
-
snippet: r.snippet || '',
|
|
194
|
-
date: r.date,
|
|
195
|
-
}));
|
|
196
|
-
}
|
|
197
|
-
catch (error) {
|
|
198
|
-
const err = classifyError(error);
|
|
199
|
-
if (this.isRetryable(undefined, error) && attempt < SEARCH_RETRY_CONFIG.maxRetries) {
|
|
200
|
-
const delayMs = calculateBackoff(attempt, SEARCH_RETRY_CONFIG.baseDelayMs, SEARCH_RETRY_CONFIG.maxDelayMs);
|
|
201
|
-
mcpLog('warning', `Reddit search ${err.code}, retrying in ${delayMs}ms...`, 'search');
|
|
202
|
-
await sleep(delayMs);
|
|
203
|
-
continue;
|
|
204
|
-
}
|
|
205
|
-
mcpLog('error', `Reddit search failed: ${err.message}`, 'search');
|
|
206
|
-
return [];
|
|
207
|
-
}
|
|
208
|
-
}
|
|
209
|
-
return [];
|
|
210
|
-
}
|
|
211
|
-
/**
|
|
212
|
-
* Search Reddit with multiple queries (bounded concurrency)
|
|
213
|
-
* NEVER throws - searchReddit never throws, pMap preserves order
|
|
214
|
-
*/
|
|
215
|
-
async searchRedditMultiple(queries, dateAfter) {
|
|
216
|
-
if (queries.length === 0) {
|
|
217
|
-
return new Map();
|
|
218
|
-
}
|
|
219
|
-
const results = await pMap(queries, q => this.searchReddit(q, dateAfter), MAX_SEARCH_CONCURRENCY);
|
|
220
|
-
return new Map(queries.map((q, i) => [q, results[i] || []]));
|
|
221
|
-
}
|
|
222
|
-
}
|
|
223
|
-
//# sourceMappingURL=search.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"search.js","sourceRoot":"","sources":["../../src/clients/search.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,KAAK,EACL,SAAS,GAEV,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAC/C,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,kBAAkB;AAElB,MAAM,cAAc,GAAG,kCAA2C,CAAC;AACnE,MAAM,2BAA2B,GAAG,EAAW,CAAC;AAChD,MAAM,sBAAsB,GAAG,CAAU,CAAC;AAC1C,MAAM,WAAW,GAAG,CAAU,CAAC;AAkC/B,4BAA4B;AAE5B,MAAM,mBAAmB,GAAG;IAC1B,UAAU,EAAE,WAAW;IACvB,WAAW,EAAE,IAAI;IACjB,UAAU,EAAE,KAAK;IACjB,SAAS,EAAE,KAAK;CACR,CAAC;AAEX,MAAM,sBAAsB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAElE,gDAAgD;AAChD,MAAM,iBAAiB,GAAG,sBAAsB,CAAC;AACjD,MAAM,6BAA6B,GAAG,YAAY,CAAC;AACnD,MAAM,mBAAmB,GAAG,YAAY,CAAC;AAEzC,sEAAsE;AAEtE,SAAS,oBAAoB,CAC3B,SAAyC,EACzC,QAAkB;IAElB,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QACnC,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAmC,CAAC;YACvE,MAAM,OAAO,GAAmB,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;gBAC1D,KAAK,EAAG,IAAI,CAAC,KAAgB,IAAI,UAAU;gBAC3C,IAAI,EAAG,IAAI,CAAC,IAAe,IAAI,GAAG;gBAClC,OAAO,EAAG,IAAI,CAAC,OAAkB,IAAI,EAAE;gBACvC,IAAI,EAAE,IAAI,CAAC,IAA0B;gBACrC,QAAQ,EAAG,IAAI,CAAC,QAAmB,IAAI,GAAG,GAAG,CAAC;aAC/C,CAAC,CAAC,CAAC;YAEJ,MAAM,UAAU,GAAG,IAAI,CAAC,iBAAwD,CAAC;YACjF,MAAM,YAAY,GAAG,UAAU,EAAE,YAAY;gBAC3C,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;gBACjE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;YAEnB,MAAM,eAAe,GAAG,CAAC,IAAI,CAAC,eAAe,IAAI,EAAE,CAAmC,CAAC;YACvF,MAAM,OAAO,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAE,CAAC,CAAC,KAAgB,IAAI,EAAE,CAAC,CAAC;YAEtE,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC;QAC5E,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QACvF,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,mDAAmD;AAEnD,KAAK,UAAU,sBAAsB,CACnC,MAAc,EACd,IAAa,EACb,WAA0D;IAE1D,IAAI,SAAsC,CAAC;IAE3C,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,mBAAmB,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;QAC3E,IAAI,CAAC;YACH,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChB,MAAM,CAAC,SAAS,EAAE,iBAAiB,OAAO,IAAI,mBAAmB,CAAC,UAAU,EAAE,EAAE,QAAQ,CAAC,CAAC;YAC5F,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,cAAc,EAAE;gBACtD,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,WAAW,EAAE,MAAM;oBACnB,cAAc,EAAE,kBAAkB;iBACnC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;gBAC1B,SAAS,EAAE,mBAAmB,CAAC,SAAS;aACzC,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;gBACxD,SAAS,GAAG,aAAa,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;gBAE3E,IAAI,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,OAAO,GAAG,mBAAmB,CAAC,UAAU,EAAE,CAAC;oBAC7E,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,EAAE,mBAAmB,CAAC,WAAW,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;oBAC3G,MAAM,CAAC,SAAS,EAAE,gBAAgB,QAAQ,CAAC,MAAM,iBAAiB,OAAO,OAAO,EAAE,QAAQ,CAAC,CAAC;oBAC5F,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;oBACrB,SAAS;gBACX,CAAC;gBAED,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;YAC/C,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACnC,OAAO,EAAE,IAAI,EAAE,CAAC;YAClB,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC,WAAW,EAAE,OAAO,EAAE,iCAAiC,EAAE,SAAS,EAAE,KAAK,EAAE;iBACrG,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;YAEjC,IAAI,WAAW,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,OAAO,GAAG,mBAAmB,CAAC,UAAU,EAAE,CAAC;gBAC9E,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,EAAE,mBAAmB,CAAC,WAAW,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;gBAC3G,MAAM,CAAC,SAAS,EAAE,GAAG,SAAS,CAAC,IAAI,KAAK,SAAS,CAAC,OAAO,iBAAiB,OAAO,OAAO,EAAE,QAAQ,CAAC,CAAC;gBACpG,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;gBACrB,SAAS;YACX,CAAC;YAED,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;QAC/C,CAAC;IACH,CAAC;IAED,OAAO;QACL,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,SAAS,IAAI,EAAE,IAAI,EAAE,SAAS,CAAC,aAAa,EAAE,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,KAAK,EAAE;KAClG,CAAC;AACJ,CAAC;AAED,qBAAqB;AAErB,MAAM,OAAO,YAAY;IACf,MAAM,CAAS;IAEvB,YAAY,MAAe;QACzB,MAAM,GAAG,GAAG,QAAQ,EAAE,CAAC;QACvB,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,GAAG,CAAC,cAAc,IAAI,EAAE,CAAC;QAEjD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,sFAAsF,CAAC,CAAC;QAC1G,CAAC;IACH,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,MAAe,EAAE,KAAe;QAClD,IAAI,MAAM,IAAI,sBAAsB,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,OAAO,IAAI,CAAC;QAE9D,IAAI,KAAK,IAAI,IAAI;YAAE,OAAO,KAAK,CAAC;QAChC,MAAM,OAAO,GAAG,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,SAAS,IAAI,KAAK,IAAI,OAAQ,KAA8B,CAAC,OAAO,KAAK,QAAQ,CAAC;YAC9H,CAAC,CAAE,KAA6B,CAAC,OAAO,CAAC,WAAW,EAAE;YACtD,CAAC,CAAC,EAAE,CAAC;QACP,OAAO,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;IACzG,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc,CAAC,QAAkB;QACrC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO;gBACL,QAAQ,EAAE,EAAE;gBACZ,aAAa,EAAE,CAAC;gBAChB,aAAa,EAAE,CAAC;gBAChB,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC,aAAa,EAAE,OAAO,EAAE,sBAAsB,EAAE,SAAS,EAAE,KAAK,EAAE;aAC5F,CAAC;QACJ,CAAC;QAED,MAAM,aAAa,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QAChE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,sBAAsB,CAClD,IAAI,CAAC,MAAM,EACX,aAAa,EACb,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,GAAG,CAAC,CAC/C,CAAC;QAEF,IAAI,KAAK,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YAChC,OAAO;gBACL,QAAQ,EAAE,EAAE;gBACZ,aAAa,EAAE,QAAQ,CAAC,MAAM;gBAC9B,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBACrC,KAAK;aACN,CAAC;QACJ,CAAC;QAED,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACtD,MAAM,QAAQ,GAAG,oBAAoB,CAAC,SAA2C,EAAE,QAAQ,CAAC,CAAC;QAE7F,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,QAAQ,CAAC,MAAM,EAAE,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE,CAAC;IAC7F,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,SAAkB;QAClD,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC;YACnB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,IAAI,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,kBAAkB,CAAC;QAEzE,IAAI,SAAS,EAAE,CAAC;YACd,CAAC,IAAI,UAAU,SAAS,EAAE,CAAC;QAC7B,CAAC;QAED,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,mBAAmB,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YAC3E,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,cAAc,EAAE;oBACjD,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE,EAAE,WAAW,EAAE,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,kBAAkB,EAAE;oBACzE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,2BAA2B,EAAE,CAAC;oBAC7D,SAAS,EAAE,mBAAmB,CAAC,SAAS;iBACzC,CAAC,CAAC;gBAEH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;oBACZ,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,OAAO,GAAG,mBAAmB,CAAC,UAAU,EAAE,CAAC;wBAC7E,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,EAAE,mBAAmB,CAAC,WAAW,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;wBAC3G,MAAM,CAAC,SAAS,EAAE,iBAAiB,GAAG,CAAC,MAAM,iBAAiB,OAAO,OAAO,EAAE,QAAQ,CAAC,CAAC;wBACxF,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;wBACrB,SAAS;oBACX,CAAC;oBACD,MAAM,CAAC,OAAO,EAAE,oCAAoC,GAAG,CAAC,MAAM,EAAE,EAAE,QAAQ,CAAC,CAAC;oBAC5E,OAAO,EAAE,CAAC;gBACZ,CAAC;gBAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAA0F,CAAC;gBACtH,OAAO,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBACtC,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,6BAA6B,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,mBAAmB,EAAE,EAAE,CAAC;oBAClG,GAAG,EAAE,CAAC,CAAC,IAAI,IAAI,EAAE;oBACjB,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,EAAE;oBACxB,IAAI,EAAE,CAAC,CAAC,IAAI;iBACb,CAAC,CAAC,CAAC;YAEN,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,GAAG,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;gBACjC,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,OAAO,GAAG,mBAAmB,CAAC,UAAU,EAAE,CAAC;oBACnF,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,EAAE,mBAAmB,CAAC,WAAW,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;oBAC3G,MAAM,CAAC,SAAS,EAAE,iBAAiB,GAAG,CAAC,IAAI,iBAAiB,OAAO,OAAO,EAAE,QAAQ,CAAC,CAAC;oBACtF,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;oBACrB,SAAS;gBACX,CAAC;gBACD,MAAM,CAAC,OAAO,EAAE,yBAAyB,GAAG,CAAC,OAAO,EAAE,EAAE,QAAQ,CAAC,CAAC;gBAClE,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC;QAED,OAAO,EAAE,CAAC;IACZ,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,oBAAoB,CAAC,OAAiB,EAAE,SAAkB;QAC9D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,IAAI,GAAG,EAAE,CAAC;QACnB,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CACxB,OAAO,EACP,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,EAAE,SAAS,CAAC,EACpC,sBAAsB,CACvB,CAAC;QAEF,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/D,CAAC;CACF"}
|