mcp-researchpowerpack 3.9.4 → 4.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (188) hide show
  1. package/README.md +116 -208
  2. package/dist/index.js +280 -337
  3. package/dist/index.js.map +7 -1
  4. package/dist/mcp-use.json +7 -0
  5. package/dist/src/clients/reddit.js +278 -0
  6. package/dist/src/clients/reddit.js.map +7 -0
  7. package/dist/src/clients/scraper.js +326 -0
  8. package/dist/src/clients/scraper.js.map +7 -0
  9. package/dist/src/clients/search.js +217 -0
  10. package/dist/src/clients/search.js.map +7 -0
  11. package/dist/src/config/index.js +138 -0
  12. package/dist/src/config/index.js.map +7 -0
  13. package/dist/src/prompts/deep-research.js +41 -0
  14. package/dist/src/prompts/deep-research.js.map +7 -0
  15. package/dist/src/prompts/reddit-sentiment.js +47 -0
  16. package/dist/src/prompts/reddit-sentiment.js.map +7 -0
  17. package/dist/src/schemas/reddit.js +21 -0
  18. package/dist/src/schemas/reddit.js.map +7 -0
  19. package/dist/src/schemas/scrape-links.js +26 -0
  20. package/dist/src/schemas/scrape-links.js.map +7 -0
  21. package/dist/src/schemas/start-research.js +17 -0
  22. package/dist/src/schemas/start-research.js.map +7 -0
  23. package/dist/src/schemas/web-search.js +53 -0
  24. package/dist/src/schemas/web-search.js.map +7 -0
  25. package/dist/src/services/llm-processor.js +675 -0
  26. package/dist/src/services/llm-processor.js.map +7 -0
  27. package/dist/src/services/markdown-cleaner.js +62 -0
  28. package/dist/src/services/markdown-cleaner.js.map +7 -0
  29. package/dist/src/services/workflow-state.js +116 -0
  30. package/dist/src/services/workflow-state.js.map +7 -0
  31. package/dist/src/tools/mcp-helpers.js +50 -0
  32. package/dist/src/tools/mcp-helpers.js.map +7 -0
  33. package/dist/src/tools/reddit.js +277 -0
  34. package/dist/src/tools/reddit.js.map +7 -0
  35. package/dist/src/tools/registry.js +18 -0
  36. package/dist/src/tools/registry.js.map +7 -0
  37. package/dist/src/tools/scrape.js +334 -0
  38. package/dist/src/tools/scrape.js.map +7 -0
  39. package/dist/src/tools/search.js +423 -0
  40. package/dist/src/tools/search.js.map +7 -0
  41. package/dist/src/tools/start-research.js +199 -0
  42. package/dist/src/tools/start-research.js.map +7 -0
  43. package/dist/src/tools/utils.js +29 -0
  44. package/dist/src/tools/utils.js.map +7 -0
  45. package/dist/src/utils/bootstrap-guard.js +27 -0
  46. package/dist/src/utils/bootstrap-guard.js.map +7 -0
  47. package/dist/src/utils/concurrency.js +62 -0
  48. package/dist/src/utils/concurrency.js.map +7 -0
  49. package/dist/src/utils/content-extractor.js +61 -0
  50. package/dist/src/utils/content-extractor.js.map +7 -0
  51. package/dist/src/utils/errors.js +211 -0
  52. package/dist/src/utils/errors.js.map +7 -0
  53. package/dist/src/utils/logger.js +25 -0
  54. package/dist/src/utils/logger.js.map +7 -0
  55. package/dist/src/utils/markdown-formatter.js +15 -0
  56. package/dist/src/utils/markdown-formatter.js.map +7 -0
  57. package/dist/src/utils/reddit-keyword-guard.js +29 -0
  58. package/dist/src/utils/reddit-keyword-guard.js.map +7 -0
  59. package/dist/src/utils/response.js +81 -0
  60. package/dist/src/utils/response.js.map +7 -0
  61. package/dist/src/utils/retry.js +13 -0
  62. package/dist/src/utils/retry.js.map +7 -0
  63. package/dist/src/utils/sanitize.js +10 -0
  64. package/dist/src/utils/sanitize.js.map +7 -0
  65. package/dist/src/utils/source-type.js +41 -0
  66. package/dist/src/utils/source-type.js.map +7 -0
  67. package/dist/src/utils/url-aggregator.js +227 -0
  68. package/dist/src/utils/url-aggregator.js.map +7 -0
  69. package/dist/src/utils/workflow-key.js +14 -0
  70. package/dist/src/utils/workflow-key.js.map +7 -0
  71. package/dist/src/version.js +32 -0
  72. package/dist/src/version.js.map +7 -0
  73. package/package.json +33 -28
  74. package/dist/clients/reddit.d.ts +0 -69
  75. package/dist/clients/reddit.d.ts.map +0 -1
  76. package/dist/clients/reddit.js +0 -369
  77. package/dist/clients/reddit.js.map +0 -1
  78. package/dist/clients/research.d.ts +0 -67
  79. package/dist/clients/research.d.ts.map +0 -1
  80. package/dist/clients/research.js +0 -290
  81. package/dist/clients/research.js.map +0 -1
  82. package/dist/clients/scraper.d.ts +0 -72
  83. package/dist/clients/scraper.d.ts.map +0 -1
  84. package/dist/clients/scraper.js +0 -338
  85. package/dist/clients/scraper.js.map +0 -1
  86. package/dist/clients/search.d.ts +0 -57
  87. package/dist/clients/search.d.ts.map +0 -1
  88. package/dist/clients/search.js +0 -223
  89. package/dist/clients/search.js.map +0 -1
  90. package/dist/config/index.d.ts +0 -78
  91. package/dist/config/index.d.ts.map +0 -1
  92. package/dist/config/index.js +0 -201
  93. package/dist/config/index.js.map +0 -1
  94. package/dist/config/loader.d.ts +0 -40
  95. package/dist/config/loader.d.ts.map +0 -1
  96. package/dist/config/loader.js +0 -322
  97. package/dist/config/loader.js.map +0 -1
  98. package/dist/config/types.d.ts +0 -81
  99. package/dist/config/types.d.ts.map +0 -1
  100. package/dist/config/types.js +0 -6
  101. package/dist/config/types.js.map +0 -1
  102. package/dist/config/yaml/tools.yaml +0 -146
  103. package/dist/index.d.ts +0 -7
  104. package/dist/index.d.ts.map +0 -1
  105. package/dist/schemas/deep-research.d.ts +0 -64
  106. package/dist/schemas/deep-research.d.ts.map +0 -1
  107. package/dist/schemas/deep-research.js +0 -224
  108. package/dist/schemas/deep-research.js.map +0 -1
  109. package/dist/schemas/scrape-links.d.ts +0 -32
  110. package/dist/schemas/scrape-links.d.ts.map +0 -1
  111. package/dist/schemas/scrape-links.js +0 -34
  112. package/dist/schemas/scrape-links.js.map +0 -1
  113. package/dist/schemas/web-search.d.ts +0 -22
  114. package/dist/schemas/web-search.d.ts.map +0 -1
  115. package/dist/schemas/web-search.js +0 -21
  116. package/dist/schemas/web-search.js.map +0 -1
  117. package/dist/services/file-attachment.d.ts +0 -30
  118. package/dist/services/file-attachment.d.ts.map +0 -1
  119. package/dist/services/file-attachment.js +0 -207
  120. package/dist/services/file-attachment.js.map +0 -1
  121. package/dist/services/llm-processor.d.ts +0 -29
  122. package/dist/services/llm-processor.d.ts.map +0 -1
  123. package/dist/services/llm-processor.js +0 -244
  124. package/dist/services/llm-processor.js.map +0 -1
  125. package/dist/services/markdown-cleaner.d.ts +0 -8
  126. package/dist/services/markdown-cleaner.d.ts.map +0 -1
  127. package/dist/services/markdown-cleaner.js +0 -74
  128. package/dist/services/markdown-cleaner.js.map +0 -1
  129. package/dist/tools/definitions.d.ts +0 -16
  130. package/dist/tools/definitions.d.ts.map +0 -1
  131. package/dist/tools/definitions.js +0 -17
  132. package/dist/tools/definitions.js.map +0 -1
  133. package/dist/tools/reddit.d.ts +0 -14
  134. package/dist/tools/reddit.d.ts.map +0 -1
  135. package/dist/tools/reddit.js +0 -265
  136. package/dist/tools/reddit.js.map +0 -1
  137. package/dist/tools/registry.d.ts +0 -71
  138. package/dist/tools/registry.d.ts.map +0 -1
  139. package/dist/tools/registry.js +0 -252
  140. package/dist/tools/registry.js.map +0 -1
  141. package/dist/tools/research.d.ts +0 -14
  142. package/dist/tools/research.d.ts.map +0 -1
  143. package/dist/tools/research.js +0 -196
  144. package/dist/tools/research.js.map +0 -1
  145. package/dist/tools/scrape.d.ts +0 -14
  146. package/dist/tools/scrape.d.ts.map +0 -1
  147. package/dist/tools/scrape.js +0 -234
  148. package/dist/tools/scrape.js.map +0 -1
  149. package/dist/tools/search.d.ts +0 -10
  150. package/dist/tools/search.d.ts.map +0 -1
  151. package/dist/tools/search.js +0 -158
  152. package/dist/tools/search.js.map +0 -1
  153. package/dist/tools/utils.d.ts +0 -105
  154. package/dist/tools/utils.d.ts.map +0 -1
  155. package/dist/tools/utils.js +0 -159
  156. package/dist/tools/utils.js.map +0 -1
  157. package/dist/utils/concurrency.d.ts +0 -28
  158. package/dist/utils/concurrency.d.ts.map +0 -1
  159. package/dist/utils/concurrency.js +0 -92
  160. package/dist/utils/concurrency.js.map +0 -1
  161. package/dist/utils/errors.d.ts +0 -95
  162. package/dist/utils/errors.d.ts.map +0 -1
  163. package/dist/utils/errors.js +0 -390
  164. package/dist/utils/errors.js.map +0 -1
  165. package/dist/utils/logger.d.ts +0 -39
  166. package/dist/utils/logger.d.ts.map +0 -1
  167. package/dist/utils/logger.js +0 -57
  168. package/dist/utils/logger.js.map +0 -1
  169. package/dist/utils/markdown-formatter.d.ts +0 -5
  170. package/dist/utils/markdown-formatter.d.ts.map +0 -1
  171. package/dist/utils/markdown-formatter.js +0 -15
  172. package/dist/utils/markdown-formatter.js.map +0 -1
  173. package/dist/utils/response.d.ts +0 -93
  174. package/dist/utils/response.d.ts.map +0 -1
  175. package/dist/utils/response.js +0 -170
  176. package/dist/utils/response.js.map +0 -1
  177. package/dist/utils/retry.d.ts +0 -43
  178. package/dist/utils/retry.d.ts.map +0 -1
  179. package/dist/utils/retry.js +0 -57
  180. package/dist/utils/retry.js.map +0 -1
  181. package/dist/utils/url-aggregator.d.ts +0 -90
  182. package/dist/utils/url-aggregator.d.ts.map +0 -1
  183. package/dist/utils/url-aggregator.js +0 -538
  184. package/dist/utils/url-aggregator.js.map +0 -1
  185. package/dist/version.d.ts +0 -29
  186. package/dist/version.d.ts.map +0 -1
  187. package/dist/version.js +0 -55
  188. package/dist/version.js.map +0 -1
@@ -1,338 +0,0 @@
1
- /**
2
- * Web Scraper Client
3
- * Generic interface for URL scraping with automatic fallback modes
4
- * Implements robust error handling that NEVER crashes
5
- */
6
- import { parseEnv } from '../config/index.js';
7
- import { classifyError, fetchWithTimeout, sleep, ErrorCode, } from '../utils/errors.js';
8
- import { calculateBackoff } from '../utils/retry.js';
9
- import { pMapSettled } from '../utils/concurrency.js';
10
- import { mcpLog } from '../utils/logger.js';
11
- // ── Constants ──
12
- const SCRAPE_MODES = ['basic', 'javascript', 'javascript_geo'];
13
- const CREDIT_COSTS = { basic: 1, javascript: 5, javascript_geo: 5 };
14
- const DEFAULT_SCRAPE_CONCURRENCY = 10;
15
- const SCRAPE_BATCH_SIZE = 30;
16
- const MAX_RETRIES = 2;
17
- /** Overall timeout for all fallback attempts on a single URL */
18
- const FALLBACK_OVERALL_TIMEOUT_MS = 45_000;
19
- // Status codes that indicate we should retry (no credit consumed)
20
- const RETRYABLE_STATUS_CODES = new Set([429, 502, 503, 504, 510]);
21
- // Status codes that are permanent failures (don't retry)
22
- const PERMANENT_FAILURE_CODES = new Set([400, 401, 403]);
23
- const FALLBACK_ATTEMPTS = [
24
- { mode: 'basic', description: 'basic mode' },
25
- { mode: 'javascript', description: 'javascript rendering' },
26
- { mode: 'javascript', country: 'us', description: 'javascript + US geo-targeting' },
27
- ];
28
- export class ScraperClient {
29
- apiKey;
30
- baseURL = 'https://api.scrape.do';
31
- constructor(apiKey) {
32
- const env = parseEnv();
33
- this.apiKey = apiKey || env.SCRAPER_API_KEY;
34
- if (!this.apiKey) {
35
- throw new Error('Web scraping capability is not configured. Please set up the required API credentials.');
36
- }
37
- }
38
- /**
39
- * Scrape a single URL with retry logic
40
- * NEVER throws - always returns a ScrapeResponse (possibly with error)
41
- */
42
- async scrape(request, maxRetries = MAX_RETRIES) {
43
- const { url, mode = 'basic', timeout = 15, country } = request;
44
- const credits = CREDIT_COSTS[mode] ?? 1;
45
- // Validate URL first
46
- try {
47
- new URL(url);
48
- }
49
- catch {
50
- return {
51
- content: `Invalid URL: ${url}`,
52
- statusCode: 400,
53
- credits: 0,
54
- error: { code: ErrorCode.INVALID_INPUT, message: `Invalid URL: ${url}`, retryable: false },
55
- };
56
- }
57
- const params = new URLSearchParams({
58
- url: url,
59
- token: this.apiKey,
60
- timeout: String(timeout * 1000),
61
- });
62
- if (mode === 'javascript') {
63
- params.append('render', 'true');
64
- }
65
- if (country) {
66
- params.append('geoCode', country.toUpperCase());
67
- }
68
- const apiUrl = `${this.baseURL}?${params.toString()}`;
69
- let lastError;
70
- for (let attempt = 0; attempt < maxRetries; attempt++) {
71
- try {
72
- // Use AbortController for timeout
73
- const timeoutMs = (timeout + 5) * 1000; // Add 5s buffer over scrape timeout
74
- const response = await fetchWithTimeout(apiUrl, {
75
- method: 'GET',
76
- headers: { Accept: 'text/html,application/json' },
77
- timeoutMs,
78
- });
79
- // Safely read response body
80
- let content;
81
- try {
82
- content = await response.text();
83
- }
84
- catch (readError) {
85
- content = `Failed to read response: ${readError instanceof Error ? readError.message : String(readError)}`;
86
- }
87
- // SUCCESS: 2xx - Successful API call
88
- if (response.ok) {
89
- return {
90
- content,
91
- statusCode: response.status,
92
- credits,
93
- headers: Object.fromEntries(response.headers.entries()),
94
- };
95
- }
96
- // 404 - Target not found (permanent, but not an error for our purposes)
97
- if (response.status === 404) {
98
- return {
99
- content: '404 - Page not found',
100
- statusCode: 404,
101
- credits,
102
- };
103
- }
104
- // Permanent failures - don't retry
105
- if (PERMANENT_FAILURE_CODES.has(response.status)) {
106
- const errorMsg = response.status === 401
107
- ? 'No credits remaining or subscription suspended'
108
- : `Request failed with status ${response.status}`;
109
- return {
110
- content: `Error: ${errorMsg}`,
111
- statusCode: response.status,
112
- credits: 0,
113
- error: {
114
- code: response.status === 401 ? ErrorCode.AUTH_ERROR : ErrorCode.INVALID_INPUT,
115
- message: errorMsg,
116
- retryable: false,
117
- statusCode: response.status,
118
- },
119
- };
120
- }
121
- // Retryable status codes
122
- if (RETRYABLE_STATUS_CODES.has(response.status)) {
123
- lastError = {
124
- code: response.status === 429 ? ErrorCode.RATE_LIMITED : ErrorCode.SERVICE_UNAVAILABLE,
125
- message: `Server returned ${response.status}`,
126
- retryable: true,
127
- statusCode: response.status,
128
- };
129
- if (attempt < maxRetries - 1) {
130
- const delayMs = calculateBackoff(attempt);
131
- mcpLog('warning', `${response.status} on attempt ${attempt + 1}/${maxRetries}. Retrying in ${delayMs}ms`, 'scraper');
132
- await sleep(delayMs);
133
- continue;
134
- }
135
- }
136
- // Other non-success status - treat as retryable
137
- lastError = classifyError({ status: response.status, message: content });
138
- if (attempt < maxRetries - 1 && lastError.retryable) {
139
- const delayMs = calculateBackoff(attempt);
140
- mcpLog('warning', `Status ${response.status}. Retrying in ${delayMs}ms`, 'scraper');
141
- await sleep(delayMs);
142
- continue;
143
- }
144
- // Final attempt failed
145
- return {
146
- content: `Error: ${lastError.message}`,
147
- statusCode: response.status,
148
- credits: 0,
149
- error: lastError,
150
- };
151
- }
152
- catch (error) {
153
- lastError = classifyError(error);
154
- // Non-retryable errors - return immediately
155
- if (!lastError.retryable) {
156
- return {
157
- content: `Error: ${lastError.message}`,
158
- statusCode: lastError.statusCode || 500,
159
- credits: 0,
160
- error: lastError,
161
- };
162
- }
163
- // Retryable error - continue if attempts remaining
164
- if (attempt < maxRetries - 1) {
165
- const delayMs = calculateBackoff(attempt);
166
- mcpLog('warning', `${lastError.code}: ${lastError.message}. Retry ${attempt + 1}/${maxRetries} in ${delayMs}ms`, 'scraper');
167
- await sleep(delayMs);
168
- continue;
169
- }
170
- }
171
- }
172
- // All retries exhausted
173
- return {
174
- content: `Error: Failed after ${maxRetries} attempts. ${lastError?.message || 'Unknown error'}`,
175
- statusCode: lastError?.statusCode || 500,
176
- credits: 0,
177
- error: lastError || { code: ErrorCode.UNKNOWN_ERROR, message: 'All retries exhausted', retryable: false },
178
- };
179
- }
180
- /**
181
- * Scrape with automatic fallback through different modes
182
- * NEVER throws - always returns a ScrapeResponse
183
- */
184
- async scrapeWithFallback(url, options = {}) {
185
- const attemptResults = [];
186
- let lastResult = null;
187
- const deadline = Date.now() + FALLBACK_OVERALL_TIMEOUT_MS;
188
- for (const attempt of FALLBACK_ATTEMPTS) {
189
- // Check overall deadline before starting next fallback
190
- if (Date.now() >= deadline) {
191
- mcpLog('warning', `Overall fallback timeout reached for ${url} after ${attemptResults.length} attempt(s)`, 'scraper');
192
- break;
193
- }
194
- const result = await this.tryFallbackAttempt(url, attempt, options);
195
- if (result.done) {
196
- if (attemptResults.length > 0) {
197
- mcpLog('info', `Success with ${attempt.description} after ${attemptResults.length} fallback(s)`, 'scraper');
198
- }
199
- return result.response;
200
- }
201
- lastResult = result.response;
202
- attemptResults.push(`${attempt.description}: ${result.response.error?.message || result.response.statusCode}`);
203
- mcpLog('warning', `Failed with ${attempt.description} (${result.response.statusCode}), trying next fallback...`, 'scraper');
204
- }
205
- // All fallbacks exhausted or deadline reached
206
- const errorMessage = `Failed after ${attemptResults.length} fallback attempt(s): ${attemptResults.join('; ')}`;
207
- return {
208
- content: `Error: ${errorMessage}`,
209
- statusCode: lastResult?.statusCode || 500,
210
- credits: 0,
211
- error: {
212
- code: ErrorCode.SERVICE_UNAVAILABLE,
213
- message: errorMessage,
214
- retryable: false,
215
- },
216
- };
217
- }
218
- /**
219
- * Execute a single fallback attempt and determine whether to continue.
220
- * Returns { done: true } on success/terminal or { done: false } to try the next mode.
221
- */
222
- async tryFallbackAttempt(url, attempt, options) {
223
- const result = await this.scrape({
224
- url,
225
- mode: attempt.mode,
226
- timeout: options.timeout,
227
- country: attempt.country,
228
- });
229
- // Success
230
- if (result.statusCode >= 200 && result.statusCode < 300 && !result.error) {
231
- return { done: true, response: result };
232
- }
233
- // 404 is a valid response, not an error
234
- if (result.statusCode === 404) {
235
- return { done: true, response: result };
236
- }
237
- // Non-retryable errors - don't try other modes
238
- if (result.error && !result.error.retryable) {
239
- mcpLog('error', `Non-retryable error with ${attempt.description}: ${result.error.message}`, 'scraper');
240
- return { done: true, response: result };
241
- }
242
- return { done: false, response: result };
243
- }
244
- /**
245
- * Scrape multiple URLs with batching
246
- * NEVER throws - always returns results array
247
- */
248
- async scrapeMultiple(urls, options = {}) {
249
- if (urls.length === 0) {
250
- return [];
251
- }
252
- if (urls.length <= SCRAPE_BATCH_SIZE) {
253
- return this.processBatch(urls, options);
254
- }
255
- const result = await this.batchScrape(urls, options);
256
- return result.results;
257
- }
258
- /**
259
- * Batch scrape with progress callback
260
- * NEVER throws - uses Promise.allSettled internally
261
- */
262
- async batchScrape(urls, options = {}, onBatchComplete) {
263
- const totalBatches = Math.ceil(urls.length / SCRAPE_BATCH_SIZE);
264
- const allResults = [];
265
- let rateLimitHits = 0;
266
- mcpLog('info', `Starting batch processing: ${urls.length} URLs in ${totalBatches} batch(es)`, 'scraper');
267
- for (let batchNum = 0; batchNum < totalBatches; batchNum++) {
268
- const startIdx = batchNum * SCRAPE_BATCH_SIZE;
269
- const endIdx = Math.min(startIdx + SCRAPE_BATCH_SIZE, urls.length);
270
- const batchUrls = urls.slice(startIdx, endIdx);
271
- mcpLog('info', `Processing batch ${batchNum + 1}/${totalBatches} (${batchUrls.length} URLs)`, 'scraper');
272
- const batchResults = await pMapSettled(batchUrls, url => this.scrapeWithFallback(url, options), DEFAULT_SCRAPE_CONCURRENCY);
273
- for (let i = 0; i < batchResults.length; i++) {
274
- const result = batchResults[i];
275
- if (!result)
276
- continue;
277
- const url = batchUrls[i] ?? '';
278
- if (result.status === 'fulfilled') {
279
- const scrapeResult = result.value;
280
- allResults.push({ ...scrapeResult, url });
281
- // Track rate limits
282
- if (scrapeResult.error?.code === ErrorCode.RATE_LIMITED) {
283
- rateLimitHits++;
284
- }
285
- }
286
- else {
287
- // This shouldn't happen since scrapeWithFallback never throws,
288
- // but handle it gracefully just in case
289
- const errorMsg = result.reason instanceof Error ? result.reason.message : String(result.reason);
290
- mcpLog('error', `Unexpected rejection for ${url}: ${errorMsg}`, 'scraper');
291
- allResults.push({
292
- url,
293
- content: `Error: Unexpected failure - ${errorMsg}`,
294
- statusCode: 500,
295
- credits: 0,
296
- error: classifyError(result.reason),
297
- });
298
- }
299
- }
300
- // Safe callback invocation
301
- try {
302
- onBatchComplete?.(batchNum + 1, totalBatches, allResults.length);
303
- }
304
- catch (callbackError) {
305
- mcpLog('error', `onBatchComplete callback error: ${callbackError}`, 'scraper');
306
- }
307
- mcpLog('info', `Completed batch ${batchNum + 1}/${totalBatches} (${allResults.length}/${urls.length} total)`, 'scraper');
308
- // Adaptive delay between batches — back off harder under rate limiting
309
- if (batchNum < totalBatches - 1) {
310
- const batchDelay = rateLimitHits > 0 ? 2000 : 500;
311
- await sleep(batchDelay);
312
- }
313
- }
314
- return { results: allResults, batchesProcessed: totalBatches, totalAttempted: urls.length, rateLimitHits };
315
- }
316
- /**
317
- * Process a single batch of URLs
318
- * NEVER throws
319
- */
320
- async processBatch(urls, options) {
321
- const results = await pMapSettled(urls, url => this.scrapeWithFallback(url, options), DEFAULT_SCRAPE_CONCURRENCY);
322
- return results.map((result, index) => {
323
- const url = urls[index] || '';
324
- if (result.status === 'fulfilled') {
325
- return { ...result.value, url };
326
- }
327
- // Shouldn't happen, but handle gracefully
328
- return {
329
- url,
330
- content: `Error: ${result.reason instanceof Error ? result.reason.message : String(result.reason)}`,
331
- statusCode: 500,
332
- credits: 0,
333
- error: classifyError(result.reason),
334
- };
335
- });
336
- }
337
- }
338
- //# sourceMappingURL=scraper.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"scraper.js","sourceRoot":"","sources":["../../src/clients/scraper.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAW,MAAM,oBAAoB,CAAC;AACvD,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,KAAK,EACL,SAAS,GAEV,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,kBAAkB;AAElB,MAAM,YAAY,GAAG,CAAC,OAAO,EAAE,YAAY,EAAE,gBAAgB,CAAU,CAAC;AAGxE,MAAM,YAAY,GAA2B,EAAE,KAAK,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAW,CAAC;AACrG,MAAM,0BAA0B,GAAG,EAAW,CAAC;AAC/C,MAAM,iBAAiB,GAAG,EAAW,CAAC;AACtC,MAAM,WAAW,GAAG,CAAU,CAAC;AAC/B,gEAAgE;AAChE,MAAM,2BAA2B,GAAG,MAAe,CAAC;AA0BpD,kEAAkE;AAClE,MAAM,sBAAsB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAClE,yDAAyD;AACzD,MAAM,uBAAuB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AASzD,MAAM,iBAAiB,GAA+B;IACpD,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE;IAC5C,EAAE,IAAI,EAAE,YAAY,EAAE,WAAW,EAAE,sBAAsB,EAAE;IAC3D,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,+BAA+B,EAAE;CAC3E,CAAC;AAEX,MAAM,OAAO,aAAa;IAChB,MAAM,CAAS;IACf,OAAO,GAAG,uBAAuB,CAAC;IAE1C,YAAY,MAAe;QACzB,MAAM,GAAG,GAAG,QAAQ,EAAE,CAAC;QACvB,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,GAAG,CAAC,eAAe,CAAC;QAE5C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,wFAAwF,CAAC,CAAC;QAC5G,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,MAAM,CAAC,OAAsB,EAAE,UAAU,GAAG,WAAW;QAC3D,MAAM,EAAE,GAAG,EAAE,IAAI,GAAG,OAAO,EAAE,OAAO,GAAG,EAAE,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;QAC/D,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAExC,qBAAqB;QACrB,IAAI,CAAC;YACH,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QACf,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;gBACL,OAAO,EAAE,gBAAgB,GAAG,EAAE;gBAC9B,UAAU,EAAE,GAAG;gBACf,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC,aAAa,EAAE,OAAO,EAAE,gBAAgB,GAAG,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE;aAC3F,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;YACjC,GAAG,EAAE,GAAG;YACR,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,OAAO,EAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;SAChC,CAAC,CAAC;QAEH,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;YAC1B,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAClC,CAAC;QAED,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC;QACtD,IAAI,SAAsC,CAAC;QAE3C,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YACtD,IAAI,CAAC;gBACH,kCAAkC;gBAClC,MAAM,SAAS,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,oCAAoC;gBAC5E,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,MAAM,EAAE;oBAC9C,MAAM,EAAE,KAAK;oBACb,OAAO,EAAE,EAAE,MAAM,EAAE,4BAA4B,EAAE;oBACjD,SAAS;iBACV,CAAC,CAAC;gBAEH,4BAA4B;gBAC5B,IAAI,OAAe,CAAC;gBACpB,IAAI,CAAC;oBACH,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAClC,CAAC;gBAAC,OAAO,SAAS,EAAE,CAAC;oBACnB,OAAO,GAAG,4BAA4B,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC7G,CAAC;gBAED,qCAAqC;gBACrC,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;oBAChB,OAAO;wBACL,OAAO;wBACP,UAAU,EAAE,QAAQ,CAAC,MAAM;wBAC3B,OAAO;wBACP,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;qBACxD,CAAC;gBACJ,CAAC;gBAED,wEAAwE;gBACxE,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBAC5B,OAAO;wBACL,OAAO,EAAE,sBAAsB;wBAC/B,UAAU,EAAE,GAAG;wBACf,OAAO;qBACR,CAAC;gBACJ,CAAC;gBAED,mCAAmC;gBACnC,IAAI,uBAAuB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;oBACjD,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,KAAK,GAAG;wBACtC,CAAC,CAAC,gDAAgD;wBAClD,CAAC,CAAC,8BAA8B,QAAQ,CAAC,MAAM,EAAE,CAAC;oBACpD,OAAO;wBACL,OAAO,EAAE,UAAU,QAAQ,EAAE;wBAC7B,UAAU,EAAE,QAAQ,CAAC,MAAM;wBAC3B,OAAO,EAAE,CAAC;wBACV,KAAK,EAAE;4BACL,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC,aAAa;4BAC9E,OAAO,EAAE,QAAQ;4BACjB,SAAS,EAAE,KAAK;4BAChB,UAAU,EAAE,QAAQ,CAAC,MAAM;yBAC5B;qBACF,CAAC;gBACJ,CAAC;gBAED,yBAAyB;gBACzB,IAAI,sBAAsB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;oBAChD,SAAS,GAAG;wBACV,IAAI,EAAE,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC,mBAAmB;wBACtF,OAAO,EAAE,mBAAmB,QAAQ,CAAC,MAAM,EAAE;wBAC7C,SAAS,EAAE,IAAI;wBACf,UAAU,EAAE,QAAQ,CAAC,MAAM;qBAC5B,CAAC;oBAEF,IAAI,OAAO,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;wBAC7B,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;wBAC1C,MAAM,CAAC,SAAS,EAAE,GAAG,QAAQ,CAAC,MAAM,eAAe,OAAO,GAAG,CAAC,IAAI,UAAU,iBAAiB,OAAO,IAAI,EAAE,SAAS,CAAC,CAAC;wBACrH,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;wBACrB,SAAS;oBACX,CAAC;gBACH,CAAC;gBAED,gDAAgD;gBAChD,SAAS,GAAG,aAAa,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;gBACzE,IAAI,OAAO,GAAG,UAAU,GAAG,CAAC,IAAI,SAAS,CAAC,SAAS,EAAE,CAAC;oBACpD,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;oBAC1C,MAAM,CAAC,SAAS,EAAE,UAAU,QAAQ,CAAC,MAAM,iBAAiB,OAAO,IAAI,EAAE,SAAS,CAAC,CAAC;oBACpF,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;oBACrB,SAAS;gBACX,CAAC;gBAED,uBAAuB;gBACvB,OAAO;oBACL,OAAO,EAAE,UAAU,SAAS,CAAC,OAAO,EAAE;oBACtC,UAAU,EAAE,QAAQ,CAAC,MAAM;oBAC3B,OAAO,EAAE,CAAC;oBACV,KAAK,EAAE,SAAS;iBACjB,CAAC;YAEJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;gBAEjC,4CAA4C;gBAC5C,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,CAAC;oBACzB,OAAO;wBACL,OAAO,EAAE,UAAU,SAAS,CAAC,OAAO,EAAE;wBACtC,UAAU,EAAE,SAAS,CAAC,UAAU,IAAI,GAAG;wBACvC,OAAO,EAAE,CAAC;wBACV,KAAK,EAAE,SAAS;qBACjB,CAAC;gBACJ,CAAC;gBAED,mDAAmD;gBACnD,IAAI,OAAO,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;oBAC7B,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;oBAC1C,MAAM,CAAC,SAAS,EAAE,GAAG,SAAS,CAAC,IAAI,KAAK,SAAS,CAAC,OAAO,WAAW,OAAO,GAAG,CAAC,IAAI,UAAU,OAAO,OAAO,IAAI,EAAE,SAAS,CAAC,CAAC;oBAC5H,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;oBACrB,SAAS;gBACX,CAAC;YACH,CAAC;QACH,CAAC;QAED,wBAAwB;QACxB,OAAO;YACL,OAAO,EAAE,uBAAuB,UAAU,cAAc,SAAS,EAAE,OAAO,IAAI,eAAe,EAAE;YAC/F,UAAU,EAAE,SAAS,EAAE,UAAU,IAAI,GAAG;YACxC,OAAO,EAAE,CAAC;YACV,KAAK,EAAE,SAAS,IAAI,EAAE,IAAI,EAAE,SAAS,CAAC,aAAa,EAAE,OAAO,EAAE,uBAAuB,EAAE,SAAS,EAAE,KAAK,EAAE;SAC1G,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,kBAAkB,CAAC,GAAW,EAAE,UAAgC,EAAE;QACtE,MAAM,cAAc,GAAa,EAAE,CAAC;QACpC,IAAI,UAAU,GAA0B,IAAI,CAAC;QAC7C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,2BAA2B,CAAC;QAE1D,KAAK,MAAM,OAAO,IAAI,iBAAiB,EAAE,CAAC;YACxC,uDAAuD;YACvD,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,QAAQ,EAAE,CAAC;gBAC3B,MAAM,CAAC,SAAS,EAAE,wCAAwC,GAAG,UAAU,cAAc,CAAC,MAAM,aAAa,EAAE,SAAS,CAAC,CAAC;gBACtH,MAAM;YACR,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;YAEpE,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBAChB,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC9B,MAAM,CAAC,MAAM,EAAE,gBAAgB,OAAO,CAAC,WAAW,UAAU,cAAc,CAAC,MAAM,cAAc,EAAE,SAAS,CAAC,CAAC;gBAC9G,CAAC;gBACD,OAAO,MAAM,CAAC,QAAQ,CAAC;YACzB,CAAC;YAED,UAAU,GAAG,MAAM,CAAC,QAAQ,CAAC;YAC7B,cAAc,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,WAAW,KAAK,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YAC/G,MAAM,CAAC,SAAS,EAAE,eAAe,OAAO,CAAC,WAAW,KAAK,MAAM,CAAC,QAAQ,CAAC,UAAU,4BAA4B,EAAE,SAAS,CAAC,CAAC;QAC9H,CAAC;QAED,8CAA8C;QAC9C,MAAM,YAAY,GAAG,gBAAgB,cAAc,CAAC,MAAM,yBAAyB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/G,OAAO;YACL,OAAO,EAAE,UAAU,YAAY,EAAE;YACjC,UAAU,EAAE,UAAU,EAAE,UAAU,IAAI,GAAG;YACzC,OAAO,EAAE,CAAC;YACV,KAAK,EAAE;gBACL,IAAI,EAAE,SAAS,CAAC,mBAAmB;gBACnC,OAAO,EAAE,YAAY;gBACrB,SAAS,EAAE,KAAK;aACjB;SACF,CAAC;IACJ,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,kBAAkB,CAC9B,GAAW,EACX,OAAwB,EACxB,OAA6B;QAE7B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC;YAC/B,GAAG;YACH,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,OAAO,EAAE,OAAO,CAAC,OAAO;SACzB,CAAC,CAAC;QAEH,UAAU;QACV,IAAI,MAAM,CAAC,UAAU,IAAI,GAAG,IAAI,MAAM,CAAC,UAAU,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YACzE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAC1C,CAAC;QAED,wCAAwC;QACxC,IAAI,MAAM,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;YAC9B,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAC1C,CAAC;QAED,+CAA+C;QAC/C,IAAI,MAAM,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;YAC5C,MAAM,CAAC,OAAO,EAAE,4BAA4B,OAAO,CAAC,WAAW,KAAK,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,SAAS,CAAC,CAAC;YACvG,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAC1C,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;IAC3C,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc,CAAC,IAAc,EAAE,UAAgC,EAAE;QACrE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,IAAI,iBAAiB,EAAE,CAAC;YACrC,OAAO,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC1C,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACrD,OAAO,MAAM,CAAC,OAAkD,CAAC;IACnE,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,WAAW,CACf,IAAc,EACd,UAAgC,EAAE,EAClC,eAAqF;QAErF,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,iBAAiB,CAAC,CAAC;QAChE,MAAM,UAAU,GAA4C,EAAE,CAAC;QAC/D,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,MAAM,CAAC,MAAM,EAAE,8BAA8B,IAAI,CAAC,MAAM,YAAY,YAAY,YAAY,EAAE,SAAS,CAAC,CAAC;QAEzG,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,YAAY,EAAE,QAAQ,EAAE,EAAE,CAAC;YAC3D,MAAM,QAAQ,GAAG,QAAQ,GAAG,iBAAiB,CAAC;YAC9C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,iBAAiB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACnE,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAE/C,MAAM,CAAC,MAAM,EAAE,oBAAoB,QAAQ,GAAG,CAAC,IAAI,YAAY,KAAK,SAAS,CAAC,MAAM,QAAQ,EAAE,SAAS,CAAC,CAAC;YAEzG,MAAM,YAAY,GAAG,MAAM,WAAW,CACpC,SAAS,EACT,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,OAAO,CAAC,EAC5C,0BAA0B,CAC3B,CAAC;YAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC7C,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;gBAC/B,IAAI,CAAC,MAAM;oBAAE,SAAS;gBACtB,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAE/B,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;oBAClC,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC;oBAClC,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,YAAY,EAAE,GAAG,EAAE,CAAC,CAAC;oBAE1C,oBAAoB;oBACpB,IAAI,YAAY,CAAC,KAAK,EAAE,IAAI,KAAK,SAAS,CAAC,YAAY,EAAE,CAAC;wBACxD,aAAa,EAAE,CAAC;oBAClB,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,+DAA+D;oBAC/D,wCAAwC;oBACxC,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;oBAChG,MAAM,CAAC,OAAO,EAAE,4BAA4B,GAAG,KAAK,QAAQ,EAAE,EAAE,SAAS,CAAC,CAAC;oBAE3E,UAAU,CAAC,IAAI,CAAC;wBACd,GAAG;wBACH,OAAO,EAAE,+BAA+B,QAAQ,EAAE;wBAClD,UAAU,EAAE,GAAG;wBACf,OAAO,EAAE,CAAC;wBACV,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC;qBACpC,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,2BAA2B;YAC3B,IAAI,CAAC;gBACH,eAAe,EAAE,CAAC,QAAQ,GAAG,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;YACnE,CAAC;YAAC,OAAO,aAAa,EAAE,CAAC;gBACvB,MAAM,CAAC,OAAO,EAAE,mCAAmC,aAAa,EAAE,EAAE,SAAS,CAAC,CAAC;YACjF,CAAC;YAED,MAAM,CAAC,MAAM,EAAE,mBAAmB,QAAQ,GAAG,CAAC,IAAI,YAAY,KAAK,UAAU,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,SAAS,EAAE,SAAS,CAAC,CAAC;YAEzH,uEAAuE;YACvE,IAAI,QAAQ,GAAG,YAAY,GAAG,CAAC,EAAE,CAAC;gBAChC,MAAM,UAAU,GAAG,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;gBAClD,MAAM,KAAK,CAAC,UAAU,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,YAAY,EAAE,cAAc,EAAE,IAAI,CAAC,MAAM,EAAE,aAAa,EAAE,CAAC;IAC7G,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,YAAY,CAAC,IAAc,EAAE,OAA6B;QACtE,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,kBAAkB,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,0BAA0B,CAAC,CAAC;QAElH,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;YACnC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAE9B,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClC,OAAO,EAAE,GAAG,MAAM,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC;YAClC,CAAC;YAED,0CAA0C;YAC1C,OAAO;gBACL,GAAG;gBACH,OAAO,EAAE,UAAU,MAAM,CAAC,MAAM,YAAY,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE;gBACnG,UAAU,EAAE,GAAG;gBACf,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC;aACpC,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
@@ -1,57 +0,0 @@
1
- /**
2
- * Web Search Client
3
- * Generic interface for web search via Google (Serper implementation)
4
- * Implements robust error handling that NEVER crashes
5
- */
6
- import { type StructuredError } from '../utils/errors.js';
7
- interface SearchResult {
8
- readonly title: string;
9
- readonly link: string;
10
- readonly snippet: string;
11
- readonly date?: string;
12
- readonly position: number;
13
- }
14
- export interface KeywordSearchResult {
15
- readonly keyword: string;
16
- readonly results: SearchResult[];
17
- readonly totalResults: number;
18
- readonly related: string[];
19
- readonly error?: StructuredError;
20
- }
21
- interface MultipleSearchResponse {
22
- readonly searches: KeywordSearchResult[];
23
- readonly totalKeywords: number;
24
- readonly executionTime: number;
25
- readonly error?: StructuredError;
26
- }
27
- export interface RedditSearchResult {
28
- readonly title: string;
29
- readonly url: string;
30
- readonly snippet: string;
31
- readonly date?: string;
32
- }
33
- export declare class SearchClient {
34
- private apiKey;
35
- constructor(apiKey?: string);
36
- /**
37
- * Check if error is retryable
38
- */
39
- private isRetryable;
40
- /**
41
- * Search multiple keywords in parallel
42
- * NEVER throws - always returns a valid response
43
- */
44
- searchMultiple(keywords: string[]): Promise<MultipleSearchResponse>;
45
- /**
46
- * Search Reddit via Google (adds site:reddit.com automatically)
47
- * NEVER throws - returns empty array on failure
48
- */
49
- searchReddit(query: string, dateAfter?: string): Promise<RedditSearchResult[]>;
50
- /**
51
- * Search Reddit with multiple queries (bounded concurrency)
52
- * NEVER throws - searchReddit never throws, pMap preserves order
53
- */
54
- searchRedditMultiple(queries: string[], dateAfter?: string): Promise<Map<string, RedditSearchResult[]>>;
55
- }
56
- export {};
57
- //# sourceMappingURL=search.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/clients/search.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAKL,KAAK,eAAe,EACrB,MAAM,oBAAoB,CAAC;AAc5B,UAAU,YAAY;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,YAAY,EAAE,CAAC;IACjC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,eAAe,CAAC;CAClC;AAED,UAAU,sBAAsB;IAC9B,QAAQ,CAAC,QAAQ,EAAE,mBAAmB,EAAE,CAAC;IACzC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,KAAK,CAAC,EAAE,eAAe,CAAC;CAClC;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;CACxB;AAwHD,qBAAa,YAAY;IACvB,OAAO,CAAC,MAAM,CAAS;gBAEX,MAAM,CAAC,EAAE,MAAM;IAS3B;;OAEG;IACH,OAAO,CAAC,WAAW;IAUnB;;;OAGG;IACG,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAkCzE;;;OAGG;IACG,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAuDpF;;;OAGG;IACG,oBAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,kBAAkB,EAAE,CAAC,CAAC;CAa9G"}
@@ -1,223 +0,0 @@
1
- /**
2
- * Web Search Client
3
- * Generic interface for web search via Google (Serper implementation)
4
- * Implements robust error handling that NEVER crashes
5
- */
6
- import { parseEnv } from '../config/index.js';
7
- import { classifyError, fetchWithTimeout, sleep, ErrorCode, } from '../utils/errors.js';
8
- import { calculateBackoff } from '../utils/retry.js';
9
- import { pMap } from '../utils/concurrency.js';
10
- import { mcpLog } from '../utils/logger.js';
11
- // ── Constants ──
12
- const SERPER_API_URL = 'https://google.serper.dev/search';
13
- const DEFAULT_RESULTS_PER_KEYWORD = 10;
14
- const MAX_SEARCH_CONCURRENCY = 8;
15
- const MAX_RETRIES = 3;
16
- // ── Retry Configuration ──
17
- const SEARCH_RETRY_CONFIG = {
18
- maxRetries: MAX_RETRIES,
19
- baseDelayMs: 1000,
20
- maxDelayMs: 10000,
21
- timeoutMs: 30000,
22
- };
23
- const RETRYABLE_SEARCH_CODES = new Set([429, 500, 502, 503, 504]);
24
- // Pre-compiled regex patterns for Reddit search
25
- const REDDIT_SITE_REGEX = /site:\s*reddit\.com/i;
26
- const REDDIT_SUBREDDIT_SUFFIX_REGEX = / : r\/\w+$/;
27
- const REDDIT_SUFFIX_REGEX = / - Reddit$/;
28
- // ── Helper: Parse Serper search responses into structured results ──
29
- function parseSearchResponses(responses, keywords) {
30
- return responses.map((resp, index) => {
31
- try {
32
- const organic = (resp.organic || []);
33
- const results = organic.map((item, idx) => ({
34
- title: item.title || 'No title',
35
- link: item.link || '#',
36
- snippet: item.snippet || '',
37
- date: item.date,
38
- position: item.position || idx + 1,
39
- }));
40
- const searchInfo = resp.searchInformation;
41
- const totalResults = searchInfo?.totalResults
42
- ? parseInt(String(searchInfo.totalResults).replace(/,/g, ''), 10)
43
- : results.length;
44
- const relatedSearches = (resp.relatedSearches || []);
45
- const related = relatedSearches.map((r) => r.query || '');
46
- return { keyword: keywords[index] || '', results, totalResults, related };
47
- }
48
- catch {
49
- return { keyword: keywords[index] || '', results: [], totalResults: 0, related: [] };
50
- }
51
- });
52
- }
53
- // ── Helper: Execute search API call with retry ──
54
- async function executeSearchWithRetry(apiKey, body, isRetryable) {
55
- let lastError;
56
- for (let attempt = 0; attempt <= SEARCH_RETRY_CONFIG.maxRetries; attempt++) {
57
- try {
58
- if (attempt > 0) {
59
- mcpLog('warning', `Retry attempt ${attempt}/${SEARCH_RETRY_CONFIG.maxRetries}`, 'search');
60
- }
61
- const response = await fetchWithTimeout(SERPER_API_URL, {
62
- method: 'POST',
63
- headers: {
64
- 'X-API-KEY': apiKey,
65
- 'Content-Type': 'application/json',
66
- },
67
- body: JSON.stringify(body),
68
- timeoutMs: SEARCH_RETRY_CONFIG.timeoutMs,
69
- });
70
- if (!response.ok) {
71
- const errorText = await response.text().catch(() => '');
72
- lastError = classifyError({ status: response.status, message: errorText });
73
- if (isRetryable(response.status) && attempt < SEARCH_RETRY_CONFIG.maxRetries) {
74
- const delayMs = calculateBackoff(attempt, SEARCH_RETRY_CONFIG.baseDelayMs, SEARCH_RETRY_CONFIG.maxDelayMs);
75
- mcpLog('warning', `API returned ${response.status}, retrying in ${delayMs}ms...`, 'search');
76
- await sleep(delayMs);
77
- continue;
78
- }
79
- return { data: undefined, error: lastError };
80
- }
81
- try {
82
- const data = await response.json();
83
- return { data };
84
- }
85
- catch {
86
- return {
87
- data: undefined,
88
- error: { code: ErrorCode.PARSE_ERROR, message: 'Failed to parse search response', retryable: false },
89
- };
90
- }
91
- }
92
- catch (error) {
93
- lastError = classifyError(error);
94
- if (isRetryable(undefined, error) && attempt < SEARCH_RETRY_CONFIG.maxRetries) {
95
- const delayMs = calculateBackoff(attempt, SEARCH_RETRY_CONFIG.baseDelayMs, SEARCH_RETRY_CONFIG.maxDelayMs);
96
- mcpLog('warning', `${lastError.code}: ${lastError.message}, retrying in ${delayMs}ms...`, 'search');
97
- await sleep(delayMs);
98
- continue;
99
- }
100
- return { data: undefined, error: lastError };
101
- }
102
- }
103
- return {
104
- data: undefined,
105
- error: lastError || { code: ErrorCode.UNKNOWN_ERROR, message: 'Search failed', retryable: false },
106
- };
107
- }
108
- // ── SearchClient ──
109
- export class SearchClient {
110
- apiKey;
111
- constructor(apiKey) {
112
- const env = parseEnv();
113
- this.apiKey = apiKey || env.SEARCH_API_KEY || '';
114
- if (!this.apiKey) {
115
- throw new Error('Web search capability is not configured. Please set up the required API credentials.');
116
- }
117
- }
118
- /**
119
- * Check if error is retryable
120
- */
121
- isRetryable(status, error) {
122
- if (status && RETRYABLE_SEARCH_CODES.has(status))
123
- return true;
124
- if (error == null)
125
- return false;
126
- const message = (typeof error === 'object' && 'message' in error && typeof error.message === 'string')
127
- ? error.message.toLowerCase()
128
- : '';
129
- return message.includes('timeout') || message.includes('rate limit') || message.includes('connection');
130
- }
131
- /**
132
- * Search multiple keywords in parallel
133
- * NEVER throws - always returns a valid response
134
- */
135
- async searchMultiple(keywords) {
136
- const startTime = Date.now();
137
- if (keywords.length === 0) {
138
- return {
139
- searches: [],
140
- totalKeywords: 0,
141
- executionTime: 0,
142
- error: { code: ErrorCode.INVALID_INPUT, message: 'No keywords provided', retryable: false },
143
- };
144
- }
145
- const searchQueries = keywords.map(keyword => ({ q: keyword }));
146
- const { data, error } = await executeSearchWithRetry(this.apiKey, searchQueries, (status, err) => this.isRetryable(status, err));
147
- if (error || data === undefined) {
148
- return {
149
- searches: [],
150
- totalKeywords: keywords.length,
151
- executionTime: Date.now() - startTime,
152
- error,
153
- };
154
- }
155
- const responses = Array.isArray(data) ? data : [data];
156
- const searches = parseSearchResponses(responses, keywords);
157
- return { searches, totalKeywords: keywords.length, executionTime: Date.now() - startTime };
158
- }
159
- /**
160
- * Search Reddit via Google (adds site:reddit.com automatically)
161
- * NEVER throws - returns empty array on failure
162
- */
163
- async searchReddit(query, dateAfter) {
164
- if (!query?.trim()) {
165
- return [];
166
- }
167
- let q = query.replace(REDDIT_SITE_REGEX, '').trim() + ' site:reddit.com';
168
- if (dateAfter) {
169
- q += ` after:${dateAfter}`;
170
- }
171
- for (let attempt = 0; attempt <= SEARCH_RETRY_CONFIG.maxRetries; attempt++) {
172
- try {
173
- const res = await fetchWithTimeout(SERPER_API_URL, {
174
- method: 'POST',
175
- headers: { 'X-API-KEY': this.apiKey, 'Content-Type': 'application/json' },
176
- body: JSON.stringify({ q, num: DEFAULT_RESULTS_PER_KEYWORD }),
177
- timeoutMs: SEARCH_RETRY_CONFIG.timeoutMs,
178
- });
179
- if (!res.ok) {
180
- if (this.isRetryable(res.status) && attempt < SEARCH_RETRY_CONFIG.maxRetries) {
181
- const delayMs = calculateBackoff(attempt, SEARCH_RETRY_CONFIG.baseDelayMs, SEARCH_RETRY_CONFIG.maxDelayMs);
182
- mcpLog('warning', `Reddit search ${res.status}, retrying in ${delayMs}ms...`, 'search');
183
- await sleep(delayMs);
184
- continue;
185
- }
186
- mcpLog('error', `Reddit search failed with status ${res.status}`, 'search');
187
- return [];
188
- }
189
- const data = await res.json();
190
- return (data.organic || []).map((r) => ({
191
- title: (r.title || '').replace(REDDIT_SUBREDDIT_SUFFIX_REGEX, '').replace(REDDIT_SUFFIX_REGEX, ''),
192
- url: r.link || '',
193
- snippet: r.snippet || '',
194
- date: r.date,
195
- }));
196
- }
197
- catch (error) {
198
- const err = classifyError(error);
199
- if (this.isRetryable(undefined, error) && attempt < SEARCH_RETRY_CONFIG.maxRetries) {
200
- const delayMs = calculateBackoff(attempt, SEARCH_RETRY_CONFIG.baseDelayMs, SEARCH_RETRY_CONFIG.maxDelayMs);
201
- mcpLog('warning', `Reddit search ${err.code}, retrying in ${delayMs}ms...`, 'search');
202
- await sleep(delayMs);
203
- continue;
204
- }
205
- mcpLog('error', `Reddit search failed: ${err.message}`, 'search');
206
- return [];
207
- }
208
- }
209
- return [];
210
- }
211
- /**
212
- * Search Reddit with multiple queries (bounded concurrency)
213
- * NEVER throws - searchReddit never throws, pMap preserves order
214
- */
215
- async searchRedditMultiple(queries, dateAfter) {
216
- if (queries.length === 0) {
217
- return new Map();
218
- }
219
- const results = await pMap(queries, q => this.searchReddit(q, dateAfter), MAX_SEARCH_CONCURRENCY);
220
- return new Map(queries.map((q, i) => [q, results[i] || []]));
221
- }
222
- }
223
- //# sourceMappingURL=search.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"search.js","sourceRoot":"","sources":["../../src/clients/search.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,KAAK,EACL,SAAS,GAEV,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAC/C,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,kBAAkB;AAElB,MAAM,cAAc,GAAG,kCAA2C,CAAC;AACnE,MAAM,2BAA2B,GAAG,EAAW,CAAC;AAChD,MAAM,sBAAsB,GAAG,CAAU,CAAC;AAC1C,MAAM,WAAW,GAAG,CAAU,CAAC;AAkC/B,4BAA4B;AAE5B,MAAM,mBAAmB,GAAG;IAC1B,UAAU,EAAE,WAAW;IACvB,WAAW,EAAE,IAAI;IACjB,UAAU,EAAE,KAAK;IACjB,SAAS,EAAE,KAAK;CACR,CAAC;AAEX,MAAM,sBAAsB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAElE,gDAAgD;AAChD,MAAM,iBAAiB,GAAG,sBAAsB,CAAC;AACjD,MAAM,6BAA6B,GAAG,YAAY,CAAC;AACnD,MAAM,mBAAmB,GAAG,YAAY,CAAC;AAEzC,sEAAsE;AAEtE,SAAS,oBAAoB,CAC3B,SAAyC,EACzC,QAAkB;IAElB,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QACnC,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAmC,CAAC;YACvE,MAAM,OAAO,GAAmB,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;gBAC1D,KAAK,EAAG,IAAI,CAAC,KAAgB,IAAI,UAAU;gBAC3C,IAAI,EAAG,IAAI,CAAC,IAAe,IAAI,GAAG;gBAClC,OAAO,EAAG,IAAI,CAAC,OAAkB,IAAI,EAAE;gBACvC,IAAI,EAAE,IAAI,CAAC,IAA0B;gBACrC,QAAQ,EAAG,IAAI,CAAC,QAAmB,IAAI,GAAG,GAAG,CAAC;aAC/C,CAAC,CAAC,CAAC;YAEJ,MAAM,UAAU,GAAG,IAAI,CAAC,iBAAwD,CAAC;YACjF,MAAM,YAAY,GAAG,UAAU,EAAE,YAAY;gBAC3C,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;gBACjE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;YAEnB,MAAM,eAAe,GAAG,CAAC,IAAI,CAAC,eAAe,IAAI,EAAE,CAAmC,CAAC;YACvF,MAAM,OAAO,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAE,CAAC,CAAC,KAAgB,IAAI,EAAE,CAAC,CAAC;YAEtE,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC;QAC5E,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QACvF,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,mDAAmD;AAEnD,KAAK,UAAU,sBAAsB,CACnC,MAAc,EACd,IAAa,EACb,WAA0D;IAE1D,IAAI,SAAsC,CAAC;IAE3C,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,mBAAmB,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;QAC3E,IAAI,CAAC;YACH,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChB,MAAM,CAAC,SAAS,EAAE,iBAAiB,OAAO,IAAI,mBAAmB,CAAC,UAAU,EAAE,EAAE,QAAQ,CAAC,CAAC;YAC5F,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,cAAc,EAAE;gBACtD,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,WAAW,EAAE,MAAM;oBACnB,cAAc,EAAE,kBAAkB;iBACnC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;gBAC1B,SAAS,EAAE,mBAAmB,CAAC,SAAS;aACzC,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;gBACxD,SAAS,GAAG,aAAa,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;gBAE3E,IAAI,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,OAAO,GAAG,mBAAmB,CAAC,UAAU,EAAE,CAAC;oBAC7E,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,EAAE,mBAAmB,CAAC,WAAW,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;oBAC3G,MAAM,CAAC,SAAS,EAAE,gBAAgB,QAAQ,CAAC,MAAM,iBAAiB,OAAO,OAAO,EAAE,QAAQ,CAAC,CAAC;oBAC5F,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;oBACrB,SAAS;gBACX,CAAC;gBAED,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;YAC/C,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACnC,OAAO,EAAE,IAAI,EAAE,CAAC;YAClB,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO;oBACL,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC,WAAW,EAAE,OAAO,EAAE,iCAAiC,EAAE,SAAS,EAAE,KAAK,EAAE;iBACrG,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;YAEjC,IAAI,WAAW,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,OAAO,GAAG,mBAAmB,CAAC,UAAU,EAAE,CAAC;gBAC9E,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,EAAE,mBAAmB,CAAC,WAAW,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;gBAC3G,MAAM,CAAC,SAAS,EAAE,GAAG,SAAS,CAAC,IAAI,KAAK,SAAS,CAAC,OAAO,iBAAiB,OAAO,OAAO,EAAE,QAAQ,CAAC,CAAC;gBACpG,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;gBACrB,SAAS;YACX,CAAC;YAED,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;QAC/C,CAAC;IACH,CAAC;IAED,OAAO;QACL,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,SAAS,IAAI,EAAE,IAAI,EAAE,SAAS,CAAC,aAAa,EAAE,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,KAAK,EAAE;KAClG,CAAC;AACJ,CAAC;AAED,qBAAqB;AAErB,MAAM,OAAO,YAAY;IACf,MAAM,CAAS;IAEvB,YAAY,MAAe;QACzB,MAAM,GAAG,GAAG,QAAQ,EAAE,CAAC;QACvB,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,GAAG,CAAC,cAAc,IAAI,EAAE,CAAC;QAEjD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,sFAAsF,CAAC,CAAC;QAC1G,CAAC;IACH,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,MAAe,EAAE,KAAe;QAClD,IAAI,MAAM,IAAI,sBAAsB,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,OAAO,IAAI,CAAC;QAE9D,IAAI,KAAK,IAAI,IAAI;YAAE,OAAO,KAAK,CAAC;QAChC,MAAM,OAAO,GAAG,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,SAAS,IAAI,KAAK,IAAI,OAAQ,KAA8B,CAAC,OAAO,KAAK,QAAQ,CAAC;YAC9H,CAAC,CAAE,KAA6B,CAAC,OAAO,CAAC,WAAW,EAAE;YACtD,CAAC,CAAC,EAAE,CAAC;QACP,OAAO,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;IACzG,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc,CAAC,QAAkB;QACrC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO;gBACL,QAAQ,EAAE,EAAE;gBACZ,aAAa,EAAE,CAAC;gBAChB,aAAa,EAAE,CAAC;gBAChB,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC,aAAa,EAAE,OAAO,EAAE,sBAAsB,EAAE,SAAS,EAAE,KAAK,EAAE;aAC5F,CAAC;QACJ,CAAC;QAED,MAAM,aAAa,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;QAChE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,sBAAsB,CAClD,IAAI,CAAC,MAAM,EACX,aAAa,EACb,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,GAAG,CAAC,CAC/C,CAAC;QAEF,IAAI,KAAK,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YAChC,OAAO;gBACL,QAAQ,EAAE,EAAE;gBACZ,aAAa,EAAE,QAAQ,CAAC,MAAM;gBAC9B,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;gBACrC,KAAK;aACN,CAAC;QACJ,CAAC;QAED,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACtD,MAAM,QAAQ,GAAG,oBAAoB,CAAC,SAA2C,EAAE,QAAQ,CAAC,CAAC;QAE7F,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,QAAQ,CAAC,MAAM,EAAE,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE,CAAC;IAC7F,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,SAAkB;QAClD,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC;YACnB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,IAAI,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,kBAAkB,CAAC;QAEzE,IAAI,SAAS,EAAE,CAAC;YACd,CAAC,IAAI,UAAU,SAAS,EAAE,CAAC;QAC7B,CAAC;QAED,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,mBAAmB,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YAC3E,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,cAAc,EAAE;oBACjD,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE,EAAE,WAAW,EAAE,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,kBAAkB,EAAE;oBACzE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,2BAA2B,EAAE,CAAC;oBAC7D,SAAS,EAAE,mBAAmB,CAAC,SAAS;iBACzC,CAAC,CAAC;gBAEH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;oBACZ,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,OAAO,GAAG,mBAAmB,CAAC,UAAU,EAAE,CAAC;wBAC7E,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,EAAE,mBAAmB,CAAC,WAAW,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;wBAC3G,MAAM,CAAC,SAAS,EAAE,iBAAiB,GAAG,CAAC,MAAM,iBAAiB,OAAO,OAAO,EAAE,QAAQ,CAAC,CAAC;wBACxF,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;wBACrB,SAAS;oBACX,CAAC;oBACD,MAAM,CAAC,OAAO,EAAE,oCAAoC,GAAG,CAAC,MAAM,EAAE,EAAE,QAAQ,CAAC,CAAC;oBAC5E,OAAO,EAAE,CAAC;gBACZ,CAAC;gBAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAA0F,CAAC;gBACtH,OAAO,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBACtC,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,6BAA6B,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,mBAAmB,EAAE,EAAE,CAAC;oBAClG,GAAG,EAAE,CAAC,CAAC,IAAI,IAAI,EAAE;oBACjB,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,EAAE;oBACxB,IAAI,EAAE,CAAC,CAAC,IAAI;iBACb,CAAC,CAAC,CAAC;YAEN,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,GAAG,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;gBACjC,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,OAAO,GAAG,mBAAmB,CAAC,UAAU,EAAE,CAAC;oBACnF,MAAM,OAAO,GAAG,gBAAgB,CAAC,OAAO,EAAE,mBAAmB,CAAC,WAAW,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;oBAC3G,MAAM,CAAC,SAAS,EAAE,iBAAiB,GAAG,CAAC,IAAI,iBAAiB,OAAO,OAAO,EAAE,QAAQ,CAAC,CAAC;oBACtF,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;oBACrB,SAAS;gBACX,CAAC;gBACD,MAAM,CAAC,OAAO,EAAE,yBAAyB,GAAG,CAAC,OAAO,EAAE,EAAE,QAAQ,CAAC,CAAC;gBAClE,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC;QAED,OAAO,EAAE,CAAC;IACZ,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,oBAAoB,CAAC,OAAiB,EAAE,SAAkB;QAC9D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,IAAI,GAAG,EAAE,CAAC;QACnB,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CACxB,OAAO,EACP,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,EAAE,SAAS,CAAC,EACpC,sBAAsB,CACvB,CAAC;QAEF,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/D,CAAC;CACF"}