mcp-researchpowerpack-http 3.10.0

package/dist/src/clients/scraper.js

@@ -0,0 +1,327 @@
+import { parseEnv } from "../config/index.js";
+import {
+  classifyError,
+  fetchWithTimeout,
+  sleep,
+  ErrorCode
+} from "../utils/errors.js";
+import { calculateBackoff } from "../utils/retry.js";
+import { pMapSettled } from "../utils/concurrency.js";
+import { mcpLog } from "../utils/logger.js";
+const SCRAPE_MODES = ["basic", "javascript", "javascript_geo"];
+const CREDIT_COSTS = { basic: 1, javascript: 5, javascript_geo: 5 };
+const DEFAULT_SCRAPE_CONCURRENCY = 10;
+const SCRAPE_BATCH_SIZE = 30;
+const MAX_RETRIES = 1;
+const FALLBACK_OVERALL_TIMEOUT_MS = 3e4;
+const RETRYABLE_STATUS_CODES = /* @__PURE__ */ new Set([429, 502, 503, 504, 510]);
+const PERMANENT_FAILURE_CODES = /* @__PURE__ */ new Set([400, 401, 403]);
+const MIN_USEFUL_CONTENT_LENGTH = 200;
+const FALLBACK_ATTEMPTS = [
+  { mode: "basic", description: "basic mode" },
+  { mode: "javascript", description: "javascript rendering" },
+  { mode: "javascript", country: "us", description: "javascript + US geo-targeting" }
+];
+class ScraperClient {
+  apiKey;
+  baseURL = "https://api.scrape.do";
+  constructor(apiKey) {
+    const env = parseEnv();
+    this.apiKey = apiKey || env.SCRAPER_API_KEY;
+    if (!this.apiKey) {
+      throw new Error("Web scraping capability is not configured. Please set up the required API credentials.");
+    }
+  }
+  /**
+   * Scrape a single URL with retry logic
+   * NEVER throws - always returns a ScrapeResponse (possibly with error)
+   */
+  async scrape(request, maxRetries = MAX_RETRIES) {
+    const { url, mode = "basic", timeout = 15, country } = request;
+    const credits = CREDIT_COSTS[mode] ?? 1;
+    try {
+      new URL(url);
+    } catch {
+      return {
+        content: `Invalid URL: ${url}`,
+        statusCode: 400,
+        credits: 0,
+        error: { code: ErrorCode.INVALID_INPUT, message: `Invalid URL: ${url}`, retryable: false }
+      };
+    }
+    const params = new URLSearchParams({
+      url,
+      token: this.apiKey,
+      timeout: String(timeout * 1e3)
+    });
+    if (mode === "javascript") {
+      params.append("render", "true");
+    }
+    if (country) {
+      params.append("geoCode", country.toUpperCase());
+    }
+    const apiUrl = `${this.baseURL}?${params.toString()}`;
+    let lastError;
+    for (let attempt = 0; attempt < maxRetries; attempt++) {
+      try {
+        const timeoutMs = (timeout + 5) * 1e3;
+        const response = await fetchWithTimeout(apiUrl, {
+          method: "GET",
+          headers: { Accept: "text/html,application/json" },
+          timeoutMs
+        });
+        let content;
+        try {
+          content = await response.text();
+        } catch (readError) {
+          content = `Failed to read response: ${readError instanceof Error ? readError.message : String(readError)}`;
+        }
+        if (response.ok) {
+          return {
+            content,
+            statusCode: response.status,
+            credits,
+            headers: Object.fromEntries(response.headers.entries())
+          };
+        }
+        if (response.status === 404) {
+          return {
+            content: "404 - Page not found",
+            statusCode: 404,
+            credits
+          };
+        }
+        if (PERMANENT_FAILURE_CODES.has(response.status)) {
+          const errorMsg = response.status === 401 ? "No credits remaining or subscription suspended" : `Request failed with status ${response.status}`;
+          return {
+            content: `Error: ${errorMsg}`,
+            statusCode: response.status,
+            credits: 0,
+            error: {
+              code: response.status === 401 ? ErrorCode.AUTH_ERROR : ErrorCode.INVALID_INPUT,
+              message: errorMsg,
+              retryable: false,
+              statusCode: response.status
+            }
+          };
+        }
+        if (RETRYABLE_STATUS_CODES.has(response.status)) {
+          lastError = {
+            code: response.status === 429 ? ErrorCode.RATE_LIMITED : ErrorCode.SERVICE_UNAVAILABLE,
+            message: `Server returned ${response.status}`,
+            retryable: true,
+            statusCode: response.status
+          };
+          if (attempt < maxRetries - 1) {
+            const delayMs = calculateBackoff(attempt);
+            mcpLog("warning", `${response.status} on attempt ${attempt + 1}/${maxRetries}. Retrying in ${delayMs}ms`, "scraper");
+            await sleep(delayMs);
+            continue;
+          }
+        }
+        lastError = classifyError({ status: response.status, message: content });
+        if (attempt < maxRetries - 1 && lastError.retryable) {
+          const delayMs = calculateBackoff(attempt);
+          mcpLog("warning", `Status ${response.status}. Retrying in ${delayMs}ms`, "scraper");
+          await sleep(delayMs);
+          continue;
+        }
+        return {
+          content: `Error: ${lastError.message}`,
+          statusCode: response.status,
+          credits: 0,
+          error: lastError
+        };
+      } catch (error) {
+        lastError = classifyError(error);
+        if (!lastError.retryable) {
+          return {
+            content: `Error: ${lastError.message}`,
+            statusCode: lastError.statusCode || 500,
+            credits: 0,
+            error: lastError
+          };
+        }
+        if (attempt < maxRetries - 1) {
+          const delayMs = calculateBackoff(attempt);
+          mcpLog("warning", `${lastError.code}: ${lastError.message}. Retry ${attempt + 1}/${maxRetries} in ${delayMs}ms`, "scraper");
+          await sleep(delayMs);
+          continue;
+        }
+      }
+    }
+    return {
+      content: `Error: Failed after ${maxRetries} attempts. ${lastError?.message || "Unknown error"}`,
+      statusCode: lastError?.statusCode || 500,
+      credits: 0,
+      error: lastError || { code: ErrorCode.UNKNOWN_ERROR, message: "All retries exhausted", retryable: false }
+    };
+  }
+  /**
+   * Scrape with automatic fallback through different modes
+   * NEVER throws - always returns a ScrapeResponse
+   */
+  async scrapeWithFallback(url, options = {}) {
+    const attemptResults = [];
+    let lastResult = null;
+    const deadline = Date.now() + FALLBACK_OVERALL_TIMEOUT_MS;
+    for (const attempt of FALLBACK_ATTEMPTS) {
+      if (Date.now() >= deadline) {
+        mcpLog("warning", `Overall fallback timeout reached for ${url} after ${attemptResults.length} attempt(s)`, "scraper");
+        break;
+      }
+      const result = await this.tryFallbackAttempt(url, attempt, options);
+      if (result.done) {
+        if (attemptResults.length > 0) {
+          mcpLog("info", `Success with ${attempt.description} after ${attemptResults.length} fallback(s)`, "scraper");
+        }
+        return result.response;
+      }
+      lastResult = result.response;
+      attemptResults.push(`${attempt.description}: ${result.response.error?.message || result.response.statusCode}`);
+      mcpLog("warning", `Failed with ${attempt.description} (${result.response.statusCode}), trying next fallback...`, "scraper");
+    }
+    const errorMessage = `Failed after ${attemptResults.length} fallback attempt(s): ${attemptResults.join("; ")}`;
+    return {
+      content: `Error: ${errorMessage}`,
+      statusCode: lastResult?.statusCode || 500,
+      credits: 0,
+      error: {
+        code: ErrorCode.SERVICE_UNAVAILABLE,
+        message: errorMessage,
+        retryable: false
+      }
+    };
+  }
+  /**
+   * Execute a single fallback attempt and determine whether to continue.
+   * Returns { done: true } on success/terminal or { done: false } to try the next mode.
+   */
+  async tryFallbackAttempt(url, attempt, options) {
+    const result = await this.scrape({
+      url,
+      mode: attempt.mode,
+      timeout: options.timeout,
+      country: attempt.country
+    });
+    if (result.statusCode >= 200 && result.statusCode < 300 && !result.error) {
+      const strippedLength = result.content.replace(/<[^>]*>/g, "").trim().length;
+      if (strippedLength < MIN_USEFUL_CONTENT_LENGTH && attempt.mode === "basic") {
+        mcpLog("info", `Basic mode returned only ${strippedLength} chars of text for ${url} \u2014 trying JS rendering`, "scraper");
+        return { done: false, response: result };
+      }
+      return { done: true, response: result };
+    }
+    if (result.statusCode === 404) {
+      return { done: true, response: result };
+    }
+    if (result.statusCode === 502) {
+      mcpLog("warning", `502 Bad Gateway for ${url} \u2014 likely WAF/CDN block, skipping fallback modes`, "scraper");
+      return { done: true, response: {
+        ...result,
+        error: {
+          code: ErrorCode.SERVICE_UNAVAILABLE,
+          message: "Bad gateway \u2014 site is blocking automated access",
+          retryable: false
+        }
+      } };
+    }
+    if (result.error && !result.error.retryable) {
+      mcpLog("error", `Non-retryable error with ${attempt.description}: ${result.error.message}`, "scraper");
+      return { done: true, response: result };
+    }
+    return { done: false, response: result };
+  }
+  /**
+   * Scrape multiple URLs with batching
+   * NEVER throws - always returns results array
+   */
+  async scrapeMultiple(urls, options = {}) {
+    if (urls.length === 0) {
+      return [];
+    }
+    if (urls.length <= SCRAPE_BATCH_SIZE) {
+      return this.processBatch(urls, options);
+    }
+    const result = await this.batchScrape(urls, options);
+    return result.results;
+  }
+  /**
+   * Batch scrape with progress callback
+   * NEVER throws - uses Promise.allSettled internally
+   */
+  async batchScrape(urls, options = {}, onBatchComplete) {
+    const totalBatches = Math.ceil(urls.length / SCRAPE_BATCH_SIZE);
+    const allResults = [];
+    let rateLimitHits = 0;
+    mcpLog("info", `Starting batch processing: ${urls.length} URLs in ${totalBatches} batch(es)`, "scraper");
+    for (let batchNum = 0; batchNum < totalBatches; batchNum++) {
+      const startIdx = batchNum * SCRAPE_BATCH_SIZE;
+      const endIdx = Math.min(startIdx + SCRAPE_BATCH_SIZE, urls.length);
+      const batchUrls = urls.slice(startIdx, endIdx);
+      mcpLog("info", `Processing batch ${batchNum + 1}/${totalBatches} (${batchUrls.length} URLs)`, "scraper");
+      const batchResults = await pMapSettled(
+        batchUrls,
+        (url) => this.scrapeWithFallback(url, options),
+        DEFAULT_SCRAPE_CONCURRENCY
+      );
+      for (let i = 0; i < batchResults.length; i++) {
+        const result = batchResults[i];
+        if (!result) continue;
+        const url = batchUrls[i] ?? "";
+        if (result.status === "fulfilled") {
+          const scrapeResult = result.value;
+          allResults.push({ ...scrapeResult, url });
+          if (scrapeResult.error?.code === ErrorCode.RATE_LIMITED) {
+            rateLimitHits++;
+          }
+        } else {
+          const errorMsg = result.reason instanceof Error ? result.reason.message : String(result.reason);
+          mcpLog("error", `Unexpected rejection for ${url}: ${errorMsg}`, "scraper");
+          allResults.push({
+            url,
+            content: `Error: Unexpected failure - ${errorMsg}`,
+            statusCode: 500,
+            credits: 0,
+            error: classifyError(result.reason)
+          });
+        }
+      }
+      try {
+        onBatchComplete?.(batchNum + 1, totalBatches, allResults.length);
+      } catch (callbackError) {
+        mcpLog("error", `onBatchComplete callback error: ${callbackError}`, "scraper");
+      }
+      mcpLog("info", `Completed batch ${batchNum + 1}/${totalBatches} (${allResults.length}/${urls.length} total)`, "scraper");
+      if (batchNum < totalBatches - 1) {
+        const batchDelay = rateLimitHits > 0 ? 2e3 : 500;
+        await sleep(batchDelay);
+      }
+    }
+    return { results: allResults, batchesProcessed: totalBatches, totalAttempted: urls.length, rateLimitHits };
+  }
+  /**
+   * Process a single batch of URLs
+   * NEVER throws
+   */
+  async processBatch(urls, options) {
+    const results = await pMapSettled(urls, (url) => this.scrapeWithFallback(url, options), DEFAULT_SCRAPE_CONCURRENCY);
+    return results.map((result, index) => {
+      const url = urls[index] || "";
+      if (result.status === "fulfilled") {
+        return { ...result.value, url };
+      }
+      return {
+        url,
+        content: `Error: ${result.reason instanceof Error ? result.reason.message : String(result.reason)}`,
+        statusCode: 500,
+        credits: 0,
+        error: classifyError(result.reason)
+      };
+    });
+  }
+}
+export {
+  ScraperClient
+};
+//# sourceMappingURL=scraper.js.map

package/dist/src/clients/scraper.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../src/clients/scraper.ts"],
+  "sourcesContent": ["/**\n * Web Scraper Client\n * Generic interface for URL scraping with automatic fallback modes\n * Implements robust error handling that NEVER crashes\n */\n\nimport { parseEnv, SCRAPER } from '../config/index.js';\nimport {\n  classifyError,\n  fetchWithTimeout,\n  sleep,\n  ErrorCode,\n  type StructuredError,\n} from '../utils/errors.js';\nimport { calculateBackoff } from '../utils/retry.js';\nimport { pMapSettled } from '../utils/concurrency.js';\nimport { mcpLog } from '../utils/logger.js';\n\n// \u2500\u2500 Constants \u2500\u2500\n\nconst SCRAPE_MODES = ['basic', 'javascript', 'javascript_geo'] as const;\ntype ScrapeMode = typeof SCRAPE_MODES[number];\n\nconst CREDIT_COSTS: Record<string, number> = { basic: 1, javascript: 5, javascript_geo: 5 } as const;\nconst DEFAULT_SCRAPE_CONCURRENCY = 10 as const;\nconst SCRAPE_BATCH_SIZE = 30 as const;\nconst MAX_RETRIES = 1 as const;\n/** Overall timeout for all fallback attempts on a single URL */\nconst FALLBACK_OVERALL_TIMEOUT_MS = 30_000 as const;\n\n// \u2500\u2500 Interfaces \u2500\u2500\n\ninterface ScrapeRequest {\n  readonly url: string;\n  readonly mode?: 'basic' | 'javascript';\n  readonly timeout?: number;\n  readonly country?: string;\n}\n\ninterface ScrapeResponse {\n  readonly content: string;\n  readonly statusCode: number;\n  readonly credits: number;\n  readonly headers?: Record<string, string>;\n  readonly error?: StructuredError;\n}\n\ninterface BatchScrapeResult {\n  readonly results: ReadonlyArray<ScrapeResponse & { readonly url: string }>;\n  readonly batchesProcessed: number;\n  readonly totalAttempted: number;\n  readonly rateLimitHits: number;\n}\n\n// Status codes that indicate we should retry (no credit consumed)\nconst RETRYABLE_STATUS_CODES = new Set([429, 502, 503, 504, 510]);\n// Status codes that are permanent failures (don't retry)\nconst PERMANENT_FAILURE_CODES = new Set([400, 401, 403]);\n\n/** Minimum stripped-text length to consider a scrape successful (filters out empty SPA shells) */\nconst MIN_USEFUL_CONTENT_LENGTH = 200 as const;\n\n/** Fallback attempt descriptor used by scrapeWithFallback */\ninterface FallbackAttempt {\n  readonly mode: 'basic' | 'javascript';\n  readonly country?: string;\n  readonly description: string;\n}\n\nconst FALLBACK_ATTEMPTS: readonly FallbackAttempt[] = [\n  { mode: 'basic', description: 'basic mode' },\n  { mode: 'javascript', description: 'javascript rendering' },\n  { mode: 'javascript', country: 'us', description: 'javascript + US geo-targeting' },\n] as const;\n\nexport class ScraperClient {\n  private apiKey: string;\n  private baseURL = 'https://api.scrape.do';\n\n  constructor(apiKey?: string) {\n    const env = parseEnv();\n    this.apiKey = apiKey || env.SCRAPER_API_KEY;\n\n    if (!this.apiKey) {\n      throw new Error('Web scraping capability is not configured. Please set up the required API credentials.');\n    }\n  }\n\n  /**\n   * Scrape a single URL with retry logic\n   * NEVER throws - always returns a ScrapeResponse (possibly with error)\n   */\n  async scrape(request: ScrapeRequest, maxRetries = MAX_RETRIES): Promise<ScrapeResponse> {\n    const { url, mode = 'basic', timeout = 15, country } = request;\n    const credits = CREDIT_COSTS[mode] ?? 1;\n\n    // Validate URL first\n    try {\n      new URL(url);\n    } catch {\n      return {\n        content: `Invalid URL: ${url}`,\n        statusCode: 400,\n        credits: 0,\n        error: { code: ErrorCode.INVALID_INPUT, message: `Invalid URL: ${url}`, retryable: false },\n      };\n    }\n\n    const params = new URLSearchParams({\n      url: url,\n      token: this.apiKey,\n      timeout: String(timeout * 1000),\n    });\n\n    if (mode === 'javascript') {\n      params.append('render', 'true');\n    }\n\n    if (country) {\n      params.append('geoCode', country.toUpperCase());\n    }\n\n    const apiUrl = `${this.baseURL}?${params.toString()}`;\n    let lastError: StructuredError | undefined;\n\n    for (let attempt = 0; attempt < maxRetries; attempt++) {\n      try {\n        // Use AbortController for timeout\n        const timeoutMs = (timeout + 5) * 1000; // Add 5s buffer over scrape timeout\n        const response = await fetchWithTimeout(apiUrl, {\n          method: 'GET',\n          headers: { Accept: 'text/html,application/json' },\n          timeoutMs,\n        });\n\n        // Safely read response body\n        let content: string;\n        try {\n          content = await response.text();\n        } catch (readError) {\n          content = `Failed to read response: ${readError instanceof Error ? readError.message : String(readError)}`;\n        }\n\n        // SUCCESS: 2xx - Successful API call\n        if (response.ok) {\n          return {\n            content,\n            statusCode: response.status,\n            credits,\n            headers: Object.fromEntries(response.headers.entries()),\n          };\n        }\n\n        // 404 - Target not found (permanent, but not an error for our purposes)\n        if (response.status === 404) {\n          return {\n            content: '404 - Page not found',\n            statusCode: 404,\n            credits,\n          };\n        }\n\n        // Permanent failures - don't retry\n        if (PERMANENT_FAILURE_CODES.has(response.status)) {\n          const errorMsg = response.status === 401\n            ? 'No credits remaining or subscription suspended'\n            : `Request failed with status ${response.status}`;\n          return {\n            content: `Error: ${errorMsg}`,\n            statusCode: response.status,\n            credits: 0,\n            error: {\n              code: response.status === 401 ? ErrorCode.AUTH_ERROR : ErrorCode.INVALID_INPUT,\n              message: errorMsg,\n              retryable: false,\n              statusCode: response.status,\n            },\n          };\n        }\n\n        // Retryable status codes\n        if (RETRYABLE_STATUS_CODES.has(response.status)) {\n          lastError = {\n            code: response.status === 429 ? ErrorCode.RATE_LIMITED : ErrorCode.SERVICE_UNAVAILABLE,\n            message: `Server returned ${response.status}`,\n            retryable: true,\n            statusCode: response.status,\n          };\n\n          if (attempt < maxRetries - 1) {\n            const delayMs = calculateBackoff(attempt);\n            mcpLog('warning', `${response.status} on attempt ${attempt + 1}/${maxRetries}. Retrying in ${delayMs}ms`, 'scraper');\n            await sleep(delayMs);\n            continue;\n          }\n        }\n\n        // Other non-success status - treat as retryable\n        lastError = classifyError({ status: response.status, message: content });\n        if (attempt < maxRetries - 1 && lastError.retryable) {\n          const delayMs = calculateBackoff(attempt);\n          mcpLog('warning', `Status ${response.status}. Retrying in ${delayMs}ms`, 'scraper');\n          await sleep(delayMs);\n          continue;\n        }\n\n        // Final attempt failed\n        return {\n          content: `Error: ${lastError.message}`,\n          statusCode: response.status,\n          credits: 0,\n          error: lastError,\n        };\n\n      } catch (error) {\n        lastError = classifyError(error);\n\n        // Non-retryable errors - return immediately\n        if (!lastError.retryable) {\n          return {\n            content: `Error: ${lastError.message}`,\n            statusCode: lastError.statusCode || 500,\n            credits: 0,\n            error: lastError,\n          };\n        }\n\n        // Retryable error - continue if attempts remaining\n        if (attempt < maxRetries - 1) {\n          const delayMs = calculateBackoff(attempt);\n          mcpLog('warning', `${lastError.code}: ${lastError.message}. Retry ${attempt + 1}/${maxRetries} in ${delayMs}ms`, 'scraper');\n          await sleep(delayMs);\n          continue;\n        }\n      }\n    }\n\n    // All retries exhausted\n    return {\n      content: `Error: Failed after ${maxRetries} attempts. ${lastError?.message || 'Unknown error'}`,\n      statusCode: lastError?.statusCode || 500,\n      credits: 0,\n      error: lastError || { code: ErrorCode.UNKNOWN_ERROR, message: 'All retries exhausted', retryable: false },\n    };\n  }\n\n  /**\n   * Scrape with automatic fallback through different modes\n   * NEVER throws - always returns a ScrapeResponse\n   */\n  async scrapeWithFallback(url: string, options: { timeout?: number } = {}): Promise<ScrapeResponse> {\n    const attemptResults: string[] = [];\n    let lastResult: ScrapeResponse | null = null;\n    const deadline = Date.now() + FALLBACK_OVERALL_TIMEOUT_MS;\n\n    for (const attempt of FALLBACK_ATTEMPTS) {\n      // Check overall deadline before starting next fallback\n      if (Date.now() >= deadline) {\n        mcpLog('warning', `Overall fallback timeout reached for ${url} after ${attemptResults.length} attempt(s)`, 'scraper');\n        break;\n      }\n\n      const result = await this.tryFallbackAttempt(url, attempt, options);\n\n      if (result.done) {\n        if (attemptResults.length > 0) {\n          mcpLog('info', `Success with ${attempt.description} after ${attemptResults.length} fallback(s)`, 'scraper');\n        }\n        return result.response;\n      }\n\n      lastResult = result.response;\n      attemptResults.push(`${attempt.description}: ${result.response.error?.message || result.response.statusCode}`);\n      mcpLog('warning', `Failed with ${attempt.description} (${result.response.statusCode}), trying next fallback...`, 'scraper');\n    }\n\n    // All fallbacks exhausted or deadline reached\n    const errorMessage = `Failed after ${attemptResults.length} fallback attempt(s): ${attemptResults.join('; ')}`;\n    return {\n      content: `Error: ${errorMessage}`,\n      statusCode: lastResult?.statusCode || 500,\n      credits: 0,\n      error: {\n        code: ErrorCode.SERVICE_UNAVAILABLE,\n        message: errorMessage,\n        retryable: false,\n      },\n    };\n  }\n\n  /**\n   * Execute a single fallback attempt and determine whether to continue.\n   * Returns { done: true } on success/terminal or { done: false } to try the next mode.\n   */\n  private async tryFallbackAttempt(\n    url: string,\n    attempt: FallbackAttempt,\n    options: { timeout?: number },\n  ): Promise<{ done: boolean; response: ScrapeResponse }> {\n    const result = await this.scrape({\n      url,\n      mode: attempt.mode,\n      timeout: options.timeout,\n      country: attempt.country,\n    });\n\n    // Success \u2014 but verify content isn't an empty SPA shell\n    if (result.statusCode >= 200 && result.statusCode < 300 && !result.error) {\n      const strippedLength = result.content.replace(/<[^>]*>/g, '').trim().length;\n      if (strippedLength < MIN_USEFUL_CONTENT_LENGTH && attempt.mode === 'basic') {\n        mcpLog('info', `Basic mode returned only ${strippedLength} chars of text for ${url} \u2014 trying JS rendering`, 'scraper');\n        return { done: false, response: result };\n      }\n      return { done: true, response: result };\n    }\n\n    // 404 is a valid response, not an error\n    if (result.statusCode === 404) {\n      return { done: true, response: result };\n    }\n\n    // 502 Bad Gateway \u2014 almost always a WAF/CDN block, not a transient issue.\n    // Switching render mode won't bypass CDN protection, so fail fast.\n    if (result.statusCode === 502) {\n      mcpLog('warning', `502 Bad Gateway for ${url} \u2014 likely WAF/CDN block, skipping fallback modes`, 'scraper');\n      return { done: true, response: {\n        ...result,\n        error: {\n          code: ErrorCode.SERVICE_UNAVAILABLE,\n          message: 'Bad gateway \u2014 site is blocking automated access',\n          retryable: false,\n        },\n      }};\n    }\n\n    // Non-retryable errors - don't try other modes\n    if (result.error && !result.error.retryable) {\n      mcpLog('error', `Non-retryable error with ${attempt.description}: ${result.error.message}`, 'scraper');\n      return { done: true, response: result };\n    }\n\n    return { done: false, response: result };\n  }\n\n  /**\n   * Scrape multiple URLs with batching\n   * NEVER throws - always returns results array\n   */\n  async scrapeMultiple(urls: string[], options: { timeout?: number } = {}): Promise<Array<ScrapeResponse & { url: string }>> {\n    if (urls.length === 0) {\n      return [];\n    }\n\n    if (urls.length <= SCRAPE_BATCH_SIZE) {\n      return this.processBatch(urls, options);\n    }\n\n    const result = await this.batchScrape(urls, options);\n    return result.results as Array<ScrapeResponse & { url: string }>;\n  }\n\n  /**\n   * Batch scrape with progress callback\n   * NEVER throws - uses Promise.allSettled internally\n   */\n  async batchScrape(\n    urls: string[],\n    options: { timeout?: number } = {},\n    onBatchComplete?: (batchNum: number, totalBatches: number, processed: number) => void\n  ): Promise<BatchScrapeResult> {\n    const totalBatches = Math.ceil(urls.length / SCRAPE_BATCH_SIZE);\n    const allResults: Array<ScrapeResponse & { url: string }> = [];\n    let rateLimitHits = 0;\n\n    mcpLog('info', `Starting batch processing: ${urls.length} URLs in ${totalBatches} batch(es)`, 'scraper');\n\n    for (let batchNum = 0; batchNum < totalBatches; batchNum++) {\n      const startIdx = batchNum * SCRAPE_BATCH_SIZE;\n      const endIdx = Math.min(startIdx + SCRAPE_BATCH_SIZE, urls.length);\n      const batchUrls = urls.slice(startIdx, endIdx);\n\n      mcpLog('info', `Processing batch ${batchNum + 1}/${totalBatches} (${batchUrls.length} URLs)`, 'scraper');\n\n      const batchResults = await pMapSettled(\n        batchUrls,\n        url => this.scrapeWithFallback(url, options),\n        DEFAULT_SCRAPE_CONCURRENCY\n      );\n\n      for (let i = 0; i < batchResults.length; i++) {\n        const result = batchResults[i];\n        if (!result) continue;\n        const url = batchUrls[i] ?? '';\n\n        if (result.status === 'fulfilled') {\n          const scrapeResult = result.value;\n          allResults.push({ ...scrapeResult, url });\n\n          // Track rate limits\n          if (scrapeResult.error?.code === ErrorCode.RATE_LIMITED) {\n            rateLimitHits++;\n          }\n        } else {\n          // This shouldn't happen since scrapeWithFallback never throws,\n          // but handle it gracefully just in case\n          const errorMsg = result.reason instanceof Error ? result.reason.message : String(result.reason);\n          mcpLog('error', `Unexpected rejection for ${url}: ${errorMsg}`, 'scraper');\n\n          allResults.push({\n            url,\n            content: `Error: Unexpected failure - ${errorMsg}`,\n            statusCode: 500,\n            credits: 0,\n            error: classifyError(result.reason),\n          });\n        }\n      }\n\n      // Safe callback invocation\n      try {\n        onBatchComplete?.(batchNum + 1, totalBatches, allResults.length);\n      } catch (callbackError) {\n        mcpLog('error', `onBatchComplete callback error: ${callbackError}`, 'scraper');\n      }\n\n      mcpLog('info', `Completed batch ${batchNum + 1}/${totalBatches} (${allResults.length}/${urls.length} total)`, 'scraper');\n\n      // Adaptive delay between batches \u2014 back off harder under rate limiting\n      if (batchNum < totalBatches - 1) {\n        const batchDelay = rateLimitHits > 0 ? 2000 : 500;\n        await sleep(batchDelay);\n      }\n    }\n\n    return { results: allResults, batchesProcessed: totalBatches, totalAttempted: urls.length, rateLimitHits };\n  }\n\n  /**\n   * Process a single batch of URLs\n   * NEVER throws\n   */\n  private async processBatch(urls: string[], options: { timeout?: number }): Promise<Array<ScrapeResponse & { url: string }>> {\n    const results = await pMapSettled(urls, url => this.scrapeWithFallback(url, options), DEFAULT_SCRAPE_CONCURRENCY);\n\n    return results.map((result, index) => {\n      const url = urls[index] || '';\n\n      if (result.status === 'fulfilled') {\n        return { ...result.value, url };\n      }\n\n      // Shouldn't happen, but handle gracefully\n      return {\n        url,\n        content: `Error: ${result.reason instanceof Error ? result.reason.message : String(result.reason)}`,\n        statusCode: 500,\n        credits: 0,\n        error: classifyError(result.reason),\n      };\n    });\n  }\n}\n"],
+  "mappings": "AAMA,SAAS,gBAAyB;AAClC;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAEK;AACP,SAAS,wBAAwB;AACjC,SAAS,mBAAmB;AAC5B,SAAS,cAAc;AAIvB,MAAM,eAAe,CAAC,SAAS,cAAc,gBAAgB;AAG7D,MAAM,eAAuC,EAAE,OAAO,GAAG,YAAY,GAAG,gBAAgB,EAAE;AAC1F,MAAM,6BAA6B;AACnC,MAAM,oBAAoB;AAC1B,MAAM,cAAc;AAEpB,MAAM,8BAA8B;AA2BpC,MAAM,yBAAyB,oBAAI,IAAI,CAAC,KAAK,KAAK,KAAK,KAAK,GAAG,CAAC;AAEhE,MAAM,0BAA0B,oBAAI,IAAI,CAAC,KAAK,KAAK,GAAG,CAAC;AAGvD,MAAM,4BAA4B;AASlC,MAAM,oBAAgD;AAAA,EACpD,EAAE,MAAM,SAAS,aAAa,aAAa;AAAA,EAC3C,EAAE,MAAM,cAAc,aAAa,uBAAuB;AAAA,EAC1D,EAAE,MAAM,cAAc,SAAS,MAAM,aAAa,gCAAgC;AACpF;AAEO,MAAM,cAAc;AAAA,EACjB;AAAA,EACA,UAAU;AAAA,EAElB,YAAY,QAAiB;AAC3B,UAAM,MAAM,SAAS;AACrB,SAAK,SAAS,UAAU,IAAI;AAE5B,QAAI,CAAC,KAAK,QAAQ;AAChB,YAAM,IAAI,MAAM,wFAAwF;AAAA,IAC1G;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,OAAO,SAAwB,aAAa,aAAsC;AACtF,UAAM,EAAE,KAAK,OAAO,SAAS,UAAU,IAAI,QAAQ,IAAI;AACvD,UAAM,UAAU,aAAa,IAAI,KAAK;AAGtC,QAAI;AACF,UAAI,IAAI,GAAG;AAAA,IACb,QAAQ;AACN,aAAO;AAAA,QACL,SAAS,gBAAgB,GAAG;AAAA,QAC5B,YAAY;AAAA,QACZ,SAAS;AAAA,QACT,OAAO,EAAE,MAAM,UAAU,eAAe,SAAS,gBAAgB,GAAG,IAAI,WAAW,MAAM;AAAA,MAC3F;AAAA,IACF;AAEA,UAAM,SAAS,IAAI,gBAAgB;AAAA,MACjC;AAAA,MACA,OAAO,KAAK;AAAA,MACZ,SAAS,OAAO,UAAU,GAAI;AAAA,IAChC,CAAC;AAED,QAAI,SAAS,cAAc;AACzB,aAAO,OAAO,UAAU,MAAM;AAAA,IAChC;AAEA,QAAI,SAAS;AACX,aAAO,OAAO,WAAW,QAAQ,YAAY,CAAC;AAAA,IAChD;AAEA,UAAM,SAAS,GAAG,KAAK,OAAO,IAAI,OAAO,SAAS,CAAC;AACnD,QAAI;AAEJ,aAAS,UAAU,GAAG,UAAU,YAAY,WAAW;AACrD,UAAI;AAEF,cAAM,aAAa,UAAU,KAAK;AAClC,cAAM,WAAW,MAAM,iBAAiB,QAAQ;AAAA,UAC9C,QAAQ;AAAA,UACR,SAAS,EAAE,QAAQ,6BAA6B;AAAA,UAChD;AAAA,QACF,CAAC;AAGD,YAAI;AACJ,YAAI;AACF,oBAAU,MAAM,SAAS,KAAK;AAAA,QAChC,SAAS,WAAW;AAClB,oBAAU,4BAA4B,qBAAqB,QAAQ,UAAU,UAAU,OAAO,SAAS,CAAC;AAAA,QAC1G;AAGA,YAAI,SAAS,IAAI;AACf,iBAAO;AAAA,YACL;AAAA,YACA,YAAY,SAAS;AAAA,YACrB;AAAA,YACA,SAAS,OAAO,YAAY,SAAS,QAAQ,QAAQ,CAAC;AAAA,UACxD;AAAA,QACF;AAGA,YAAI,SAAS,WAAW,KAAK;AAC3B,iBAAO;AAAA,YACL,SAAS;AAAA,YACT,YAAY;AAAA,YACZ;AAAA,UACF;AAAA,QACF;AAGA,YAAI,wBAAwB,IAAI,SAAS,MAAM,GAAG;AAChD,gBAAM,WAAW,SAAS,WAAW,MACjC,mDACA,8BAA8B,SAAS,MAAM;AACjD,iBAAO;AAAA,YACL,SAAS,UAAU,QAAQ;AAAA,YAC3B,YAAY,SAAS;AAAA,YACrB,SAAS;AAAA,YACT,OAAO;AAAA,cACL,MAAM,SAAS,WAAW,MAAM,UAAU,aAAa,UAAU;AAAA,cACjE,SAAS;AAAA,cACT,WAAW;AAAA,cACX,YAAY,SAAS;AAAA,YACvB;AAAA,UACF;AAAA,QACF;AAGA,YAAI,uBAAuB,IAAI,SAAS,MAAM,GAAG;AAC/C,sBAAY;AAAA,YACV,MAAM,SAAS,WAAW,MAAM,UAAU,eAAe,UAAU;AAAA,YACnE,SAAS,mBAAmB,SAAS,MAAM;AAAA,YAC3C,WAAW;AAAA,YACX,YAAY,SAAS;AAAA,UACvB;AAEA,cAAI,UAAU,aAAa,GAAG;AAC5B,kBAAM,UAAU,iBAAiB,OAAO;AACxC,mBAAO,WAAW,GAAG,SAAS,MAAM,eAAe,UAAU,CAAC,IAAI,UAAU,iBAAiB,OAAO,MAAM,SAAS;AACnH,kBAAM,MAAM,OAAO;AACnB;AAAA,UACF;AAAA,QACF;AAGA,oBAAY,cAAc,EAAE,QAAQ,SAAS,QAAQ,SAAS,QAAQ,CAAC;AACvE,YAAI,UAAU,aAAa,KAAK,UAAU,WAAW;AACnD,gBAAM,UAAU,iBAAiB,OAAO;AACxC,iBAAO,WAAW,UAAU,SAAS,MAAM,iBAAiB,OAAO,MAAM,SAAS;AAClF,gBAAM,MAAM,OAAO;AACnB;AAAA,QACF;AAGA,eAAO;AAAA,UACL,SAAS,UAAU,UAAU,OAAO;AAAA,UACpC,YAAY,SAAS;AAAA,UACrB,SAAS;AAAA,UACT,OAAO;AAAA,QACT;AAAA,MAEF,SAAS,OAAO;AACd,oBAAY,cAAc,KAAK;AAG/B,YAAI,CAAC,UAAU,WAAW;AACxB,iBAAO;AAAA,YACL,SAAS,UAAU,UAAU,OAAO;AAAA,YACpC,YAAY,UAAU,cAAc;AAAA,YACpC,SAAS;AAAA,YACT,OAAO;AAAA,UACT;AAAA,QACF;AAGA,YAAI,UAAU,aAAa,GAAG;AAC5B,gBAAM,UAAU,iBAAiB,OAAO;AACxC,iBAAO,WAAW,GAAG,UAAU,IAAI,KAAK,UAAU,OAAO,WAAW,UAAU,CAAC,IAAI,UAAU,OAAO,OAAO,MAAM,SAAS;AAC1H,gBAAM,MAAM,OAAO;AACnB;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAGA,WAAO;AAAA,MACL,SAAS,uBAAuB,UAAU,cAAc,WAAW,WAAW,eAAe;AAAA,MAC7F,YAAY,WAAW,cAAc;AAAA,MACrC,SAAS;AAAA,MACT,OAAO,aAAa,EAAE,MAAM,UAAU,eAAe,SAAS,yBAAyB,WAAW,MAAM;AAAA,IAC1G;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,mBAAmB,KAAa,UAAgC,CAAC,GAA4B;AACjG,UAAM,iBAA2B,CAAC;AAClC,QAAI,aAAoC;AACxC,UAAM,WAAW,KAAK,IAAI,IAAI;AAE9B,eAAW,WAAW,mBAAmB;AAEvC,UAAI,KAAK,IAAI,KAAK,UAAU;AAC1B,eAAO,WAAW,wCAAwC,GAAG,UAAU,eAAe,MAAM,eAAe,SAAS;AACpH;AAAA,MACF;AAEA,YAAM,SAAS,MAAM,KAAK,mBAAmB,KAAK,SAAS,OAAO;AAElE,UAAI,OAAO,MAAM;AACf,YAAI,eAAe,SAAS,GAAG;AAC7B,iBAAO,QAAQ,gBAAgB,QAAQ,WAAW,UAAU,eAAe,MAAM,gBAAgB,SAAS;AAAA,QAC5G;AACA,eAAO,OAAO;AAAA,MAChB;AAEA,mBAAa,OAAO;AACpB,qBAAe,KAAK,GAAG,QAAQ,WAAW,KAAK,OAAO,SAAS,OAAO,WAAW,OAAO,SAAS,UAAU,EAAE;AAC7G,aAAO,WAAW,eAAe,QAAQ,WAAW,KAAK,OAAO,SAAS,UAAU,8BAA8B,SAAS;AAAA,IAC5H;AAGA,UAAM,eAAe,gBAAgB,eAAe,MAAM,yBAAyB,eAAe,KAAK,IAAI,CAAC;AAC5G,WAAO;AAAA,MACL,SAAS,UAAU,YAAY;AAAA,MAC/B,YAAY,YAAY,cAAc;AAAA,MACtC,SAAS;AAAA,MACT,OAAO;AAAA,QACL,MAAM,UAAU;AAAA,QAChB,SAAS;AAAA,QACT,WAAW;AAAA,MACb;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAc,mBACZ,KACA,SACA,SACsD;AACtD,UAAM,SAAS,MAAM,KAAK,OAAO;AAAA,MAC/B;AAAA,MACA,MAAM,QAAQ;AAAA,MACd,SAAS,QAAQ;AAAA,MACjB,SAAS,QAAQ;AAAA,IACnB,CAAC;AAGD,QAAI,OAAO,cAAc,OAAO,OAAO,aAAa,OAAO,CAAC,OAAO,OAAO;AACxE,YAAM,iBAAiB,OAAO,QAAQ,QAAQ,YAAY,EAAE,EAAE,KAAK,EAAE;AACrE,UAAI,iBAAiB,6BAA6B,QAAQ,SAAS,SAAS;AAC1E,eAAO,QAAQ,4BAA4B,cAAc,sBAAsB,GAAG,+BAA0B,SAAS;AACrH,eAAO,EAAE,MAAM,OAAO,UAAU,OAAO;AAAA,MACzC;AACA,aAAO,EAAE,MAAM,MAAM,UAAU,OAAO;AAAA,IACxC;AAGA,QAAI,OAAO,eAAe,KAAK;AAC7B,aAAO,EAAE,MAAM,MAAM,UAAU,OAAO;AAAA,IACxC;AAIA,QAAI,OAAO,eAAe,KAAK;AAC7B,aAAO,WAAW,uBAAuB,GAAG,yDAAoD,SAAS;AACzG,aAAO,EAAE,MAAM,MAAM,UAAU;AAAA,QAC7B,GAAG;AAAA,QACH,OAAO;AAAA,UACL,MAAM,UAAU;AAAA,UAChB,SAAS;AAAA,UACT,WAAW;AAAA,QACb;AAAA,MACF,EAAC;AAAA,IACH;AAGA,QAAI,OAAO,SAAS,CAAC,OAAO,MAAM,WAAW;AAC3C,aAAO,SAAS,4BAA4B,QAAQ,WAAW,KAAK,OAAO,MAAM,OAAO,IAAI,SAAS;AACrG,aAAO,EAAE,MAAM,MAAM,UAAU,OAAO;AAAA,IACxC;AAEA,WAAO,EAAE,MAAM,OAAO,UAAU,OAAO;AAAA,EACzC;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,eAAe,MAAgB,UAAgC,CAAC,GAAqD;AACzH,QAAI,KAAK,WAAW,GAAG;AACrB,aAAO,CAAC;AAAA,IACV;AAEA,QAAI,KAAK,UAAU,mBAAmB;AACpC,aAAO,KAAK,aAAa,MAAM,OAAO;AAAA,IACxC;AAEA,UAAM,SAAS,MAAM,KAAK,YAAY,MAAM,OAAO;AACnD,WAAO,OAAO;AAAA,EAChB;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAM,YACJ,MACA,UAAgC,CAAC,GACjC,iBAC4B;AAC5B,UAAM,eAAe,KAAK,KAAK,KAAK,SAAS,iBAAiB;AAC9D,UAAM,aAAsD,CAAC;AAC7D,QAAI,gBAAgB;AAEpB,WAAO,QAAQ,8BAA8B,KAAK,MAAM,YAAY,YAAY,cAAc,SAAS;AAEvG,aAAS,WAAW,GAAG,WAAW,cAAc,YAAY;AAC1D,YAAM,WAAW,WAAW;AAC5B,YAAM,SAAS,KAAK,IAAI,WAAW,mBAAmB,KAAK,MAAM;AACjE,YAAM,YAAY,KAAK,MAAM,UAAU,MAAM;AAE7C,aAAO,QAAQ,oBAAoB,WAAW,CAAC,IAAI,YAAY,KAAK,UAAU,MAAM,UAAU,SAAS;AAEvG,YAAM,eAAe,MAAM;AAAA,QACzB;AAAA,QACA,SAAO,KAAK,mBAAmB,KAAK,OAAO;AAAA,QAC3C;AAAA,MACF;AAEA,eAAS,IAAI,GAAG,IAAI,aAAa,QAAQ,KAAK;AAC5C,cAAM,SAAS,aAAa,CAAC;AAC7B,YAAI,CAAC,OAAQ;AACb,cAAM,MAAM,UAAU,CAAC,KAAK;AAE5B,YAAI,OAAO,WAAW,aAAa;AACjC,gBAAM,eAAe,OAAO;AAC5B,qBAAW,KAAK,EAAE,GAAG,cAAc,IAAI,CAAC;AAGxC,cAAI,aAAa,OAAO,SAAS,UAAU,cAAc;AACvD;AAAA,UACF;AAAA,QACF,OAAO;AAGL,gBAAM,WAAW,OAAO,kBAAkB,QAAQ,OAAO,OAAO,UAAU,OAAO,OAAO,MAAM;AAC9F,iBAAO,SAAS,4BAA4B,GAAG,KAAK,QAAQ,IAAI,SAAS;AAEzE,qBAAW,KAAK;AAAA,YACd;AAAA,YACA,SAAS,+BAA+B,QAAQ;AAAA,YAChD,YAAY;AAAA,YACZ,SAAS;AAAA,YACT,OAAO,cAAc,OAAO,MAAM;AAAA,UACpC,CAAC;AAAA,QACH;AAAA,MACF;AAGA,UAAI;AACF,0BAAkB,WAAW,GAAG,cAAc,WAAW,MAAM;AAAA,MACjE,SAAS,eAAe;AACtB,eAAO,SAAS,mCAAmC,aAAa,IAAI,SAAS;AAAA,MAC/E;AAEA,aAAO,QAAQ,mBAAmB,WAAW,CAAC,IAAI,YAAY,KAAK,WAAW,MAAM,IAAI,KAAK,MAAM,WAAW,SAAS;AAGvH,UAAI,WAAW,eAAe,GAAG;AAC/B,cAAM,aAAa,gBAAgB,IAAI,MAAO;AAC9C,cAAM,MAAM,UAAU;AAAA,MACxB;AAAA,IACF;AAEA,WAAO,EAAE,SAAS,YAAY,kBAAkB,cAAc,gBAAgB,KAAK,QAAQ,cAAc;AAAA,EAC3G;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,MAAc,aAAa,MAAgB,SAAiF;AAC1H,UAAM,UAAU,MAAM,YAAY,MAAM,SAAO,KAAK,mBAAmB,KAAK,OAAO,GAAG,0BAA0B;AAEhH,WAAO,QAAQ,IAAI,CAAC,QAAQ,UAAU;AACpC,YAAM,MAAM,KAAK,KAAK,KAAK;AAE3B,UAAI,OAAO,WAAW,aAAa;AACjC,eAAO,EAAE,GAAG,OAAO,OAAO,IAAI;AAAA,MAChC;AAGA,aAAO;AAAA,QACL;AAAA,QACA,SAAS,UAAU,OAAO,kBAAkB,QAAQ,OAAO,OAAO,UAAU,OAAO,OAAO,MAAM,CAAC;AAAA,QACjG,YAAY;AAAA,QACZ,SAAS;AAAA,QACT,OAAO,cAAc,OAAO,MAAM;AAAA,MACpC;AAAA,IACF,CAAC;AAAA,EACH;AACF;",
+  "names": []
+}

package/dist/src/clients/search.d.ts

@@ -0,0 +1,57 @@
+/**
+ * Web Search Client
+ * Generic interface for web search via Google (Serper implementation)
+ * Implements robust error handling that NEVER crashes
+ */
+import { type StructuredError } from '../utils/errors.js';
+interface SearchResult {
+    readonly title: string;
+    readonly link: string;
+    readonly snippet: string;
+    readonly date?: string;
+    readonly position: number;
+}
+export interface KeywordSearchResult {
+    readonly keyword: string;
+    readonly results: SearchResult[];
+    readonly totalResults: number;
+    readonly related: string[];
+    readonly error?: StructuredError;
+}
+interface MultipleSearchResponse {
+    readonly searches: KeywordSearchResult[];
+    readonly totalKeywords: number;
+    readonly executionTime: number;
+    readonly error?: StructuredError;
+}
+export interface RedditSearchResult {
+    readonly title: string;
+    readonly url: string;
+    readonly snippet: string;
+    readonly date?: string;
+}
+export declare class SearchClient {
+    private apiKey;
+    constructor(apiKey?: string);
+    /**
+     * Check if error is retryable
+     */
+    private isRetryable;
+    /**
+     * Search multiple keywords in parallel
+     * NEVER throws - always returns a valid response
+     */
+    searchMultiple(keywords: string[]): Promise<MultipleSearchResponse>;
+    /**
+     * Search Reddit via Google (adds site:reddit.com automatically)
+     * NEVER throws - returns empty array on failure
+     */
+    searchReddit(query: string, dateAfter?: string): Promise<RedditSearchResult[]>;
+    /**
+     * Search Reddit with multiple queries (bounded concurrency)
+     * NEVER throws - searchReddit never throws, pMap preserves order
+     */
+    searchRedditMultiple(queries: string[], dateAfter?: string): Promise<Map<string, RedditSearchResult[]>>;
+}
+export {};
+//# sourceMappingURL=search.d.ts.map

package/dist/src/clients/search.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../../src/clients/search.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,EAKL,KAAK,eAAe,EACrB,MAAM,oBAAoB,CAAC;AAc5B,UAAU,YAAY;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,YAAY,EAAE,CAAC;IACjC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,eAAe,CAAC;CAClC;AAED,UAAU,sBAAsB;IAC9B,QAAQ,CAAC,QAAQ,EAAE,mBAAmB,EAAE,CAAC;IACzC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,KAAK,CAAC,EAAE,eAAe,CAAC;CAClC;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;CACxB;AAwHD,qBAAa,YAAY;IACvB,OAAO,CAAC,MAAM,CAAS;gBAEX,MAAM,CAAC,EAAE,MAAM;IAS3B;;OAEG;IACH,OAAO,CAAC,WAAW;IAUnB;;;OAGG;IACG,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAkCzE;;;OAGG;IACG,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAuDpF;;;OAGG;IACG,oBAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,kBAAkB,EAAE,CAAC,CAAC;CAa9G"}

package/dist/src/clients/search.js

@@ -0,0 +1,218 @@
+import { parseEnv } from "../config/index.js";
+import {
+  classifyError,
+  fetchWithTimeout,
+  sleep,
+  ErrorCode
+} from "../utils/errors.js";
+import { calculateBackoff } from "../utils/retry.js";
+import { pMap } from "../utils/concurrency.js";
+import { mcpLog } from "../utils/logger.js";
+const SERPER_API_URL = "https://google.serper.dev/search";
+const DEFAULT_RESULTS_PER_KEYWORD = 10;
+const MAX_SEARCH_CONCURRENCY = 8;
+const MAX_RETRIES = 3;
+const SEARCH_RETRY_CONFIG = {
+  maxRetries: MAX_RETRIES,
+  baseDelayMs: 1e3,
+  maxDelayMs: 1e4,
+  timeoutMs: 3e4
+};
+const RETRYABLE_SEARCH_CODES = /* @__PURE__ */ new Set([429, 500, 502, 503, 504]);
+const REDDIT_SITE_REGEX = /site:\s*reddit\.com/i;
+const REDDIT_SUBREDDIT_SUFFIX_REGEX = / : r\/\w+$/;
+const REDDIT_SUFFIX_REGEX = / - Reddit$/;
+function parseSearchResponses(responses, keywords) {
+  return responses.map((resp, index) => {
+    try {
+      const organic = resp.organic || [];
+      const results = organic.map((item, idx) => ({
+        title: item.title || "No title",
+        link: item.link || "#",
+        snippet: item.snippet || "",
+        date: item.date,
+        position: item.position || idx + 1
+      }));
+      const searchInfo = resp.searchInformation;
+      const totalResults = searchInfo?.totalResults ? parseInt(String(searchInfo.totalResults).replace(/,/g, ""), 10) : results.length;
+      const relatedSearches = resp.relatedSearches || [];
+      const related = relatedSearches.map((r) => r.query || "");
+      return { keyword: keywords[index] || "", results, totalResults, related };
+    } catch {
+      return { keyword: keywords[index] || "", results: [], totalResults: 0, related: [] };
+    }
+  });
+}
+async function executeSearchWithRetry(apiKey, body, isRetryable) {
+  let lastError;
+  for (let attempt = 0; attempt <= SEARCH_RETRY_CONFIG.maxRetries; attempt++) {
+    try {
+      if (attempt > 0) {
+        mcpLog("warning", `Retry attempt ${attempt}/${SEARCH_RETRY_CONFIG.maxRetries}`, "search");
+      }
+      const response = await fetchWithTimeout(SERPER_API_URL, {
+        method: "POST",
+        headers: {
+          "X-API-KEY": apiKey,
+          "Content-Type": "application/json"
+        },
+        body: JSON.stringify(body),
+        timeoutMs: SEARCH_RETRY_CONFIG.timeoutMs
+      });
+      if (!response.ok) {
+        const errorText = await response.text().catch(() => "");
+        lastError = classifyError({ status: response.status, message: errorText });
+        if (isRetryable(response.status) && attempt < SEARCH_RETRY_CONFIG.maxRetries) {
+          const delayMs = calculateBackoff(attempt, SEARCH_RETRY_CONFIG.baseDelayMs, SEARCH_RETRY_CONFIG.maxDelayMs);
+          mcpLog("warning", `API returned ${response.status}, retrying in ${delayMs}ms...`, "search");
+          await sleep(delayMs);
+          continue;
+        }
+        return { data: void 0, error: lastError };
+      }
+      try {
+        const data = await response.json();
+        return { data };
+      } catch {
+        return {
+          data: void 0,
+          error: { code: ErrorCode.PARSE_ERROR, message: "Failed to parse search response", retryable: false }
+        };
+      }
+    } catch (error) {
+      lastError = classifyError(error);
+      if (isRetryable(void 0, error) && attempt < SEARCH_RETRY_CONFIG.maxRetries) {
+        const delayMs = calculateBackoff(attempt, SEARCH_RETRY_CONFIG.baseDelayMs, SEARCH_RETRY_CONFIG.maxDelayMs);
+        mcpLog("warning", `${lastError.code}: ${lastError.message}, retrying in ${delayMs}ms...`, "search");
+        await sleep(delayMs);
+        continue;
+      }
+      return { data: void 0, error: lastError };
+    }
+  }
+  return {
+    data: void 0,
+    error: lastError || { code: ErrorCode.UNKNOWN_ERROR, message: "Search failed", retryable: false }
+  };
+}
+class SearchClient {
+  apiKey;
+  constructor(apiKey) {
+    const env = parseEnv();
+    this.apiKey = apiKey || env.SEARCH_API_KEY || "";
+    if (!this.apiKey) {
+      throw new Error("Web search capability is not configured. Please set up the required API credentials.");
+    }
+  }
+  /**
+   * Check if error is retryable
+   */
+  isRetryable(status, error) {
+    if (status && RETRYABLE_SEARCH_CODES.has(status)) return true;
+    if (error == null) return false;
+    const message = typeof error === "object" && "message" in error && typeof error.message === "string" ? error.message.toLowerCase() : "";
+    return message.includes("timeout") || message.includes("rate limit") || message.includes("connection");
+  }
+  /**
+   * Search multiple keywords in parallel
+   * NEVER throws - always returns a valid response
+   */
+  async searchMultiple(keywords) {
+    const startTime = Date.now();
+    if (keywords.length === 0) {
+      return {
+        searches: [],
+        totalKeywords: 0,
+        executionTime: 0,
+        error: { code: ErrorCode.INVALID_INPUT, message: "No keywords provided", retryable: false }
+      };
+    }
+    const searchQueries = keywords.map((keyword) => ({ q: keyword }));
+    const { data, error } = await executeSearchWithRetry(
+      this.apiKey,
+      searchQueries,
+      (status, err) => this.isRetryable(status, err)
+    );
+    if (error || data === void 0) {
+      return {
+        searches: [],
+        totalKeywords: keywords.length,
+        executionTime: Date.now() - startTime,
+        error
+      };
+    }
+    const responses = Array.isArray(data) ? data : [data];
+    const searches = parseSearchResponses(responses, keywords);
+    return { searches, totalKeywords: keywords.length, executionTime: Date.now() - startTime };
+  }
+  /**
+   * Search Reddit via Google (adds site:reddit.com automatically)
+   * NEVER throws - returns empty array on failure
+   */
+  async searchReddit(query, dateAfter) {
+    if (!query?.trim()) {
+      return [];
+    }
+    let q = query.replace(REDDIT_SITE_REGEX, "").trim() + " site:reddit.com";
+    if (dateAfter) {
+      q += ` after:${dateAfter}`;
+    }
+    for (let attempt = 0; attempt <= SEARCH_RETRY_CONFIG.maxRetries; attempt++) {
+      try {
+        const res = await fetchWithTimeout(SERPER_API_URL, {
+          method: "POST",
+          headers: { "X-API-KEY": this.apiKey, "Content-Type": "application/json" },
+          body: JSON.stringify({ q, num: DEFAULT_RESULTS_PER_KEYWORD }),
+          timeoutMs: SEARCH_RETRY_CONFIG.timeoutMs
+        });
+        if (!res.ok) {
+          if (this.isRetryable(res.status) && attempt < SEARCH_RETRY_CONFIG.maxRetries) {
+            const delayMs = calculateBackoff(attempt, SEARCH_RETRY_CONFIG.baseDelayMs, SEARCH_RETRY_CONFIG.maxDelayMs);
+            mcpLog("warning", `Reddit search ${res.status}, retrying in ${delayMs}ms...`, "search");
+            await sleep(delayMs);
+            continue;
+          }
+          mcpLog("error", `Reddit search failed with status ${res.status}`, "search");
+          return [];
+        }
+        const data = await res.json();
+        return (data.organic || []).map((r) => ({
+          title: (r.title || "").replace(REDDIT_SUBREDDIT_SUFFIX_REGEX, "").replace(REDDIT_SUFFIX_REGEX, ""),
+          url: r.link || "",
+          snippet: r.snippet || "",
+          date: r.date
+        }));
+      } catch (error) {
+        const err = classifyError(error);
+        if (this.isRetryable(void 0, error) && attempt < SEARCH_RETRY_CONFIG.maxRetries) {
+          const delayMs = calculateBackoff(attempt, SEARCH_RETRY_CONFIG.baseDelayMs, SEARCH_RETRY_CONFIG.maxDelayMs);
+          mcpLog("warning", `Reddit search ${err.code}, retrying in ${delayMs}ms...`, "search");
+          await sleep(delayMs);
+          continue;
+        }
+        mcpLog("error", `Reddit search failed: ${err.message}`, "search");
+        return [];
+      }
+    }
+    return [];
+  }
+  /**
+   * Search Reddit with multiple queries (bounded concurrency)
+   * NEVER throws - searchReddit never throws, pMap preserves order
+   */
+  async searchRedditMultiple(queries, dateAfter) {
+    if (queries.length === 0) {
+      return /* @__PURE__ */ new Map();
+    }
+    const results = await pMap(
+      queries,
+      (q) => this.searchReddit(q, dateAfter),
+      MAX_SEARCH_CONCURRENCY
+    );
+    return new Map(queries.map((q, i) => [q, results[i] || []]));
+  }
+}
+export {
+  SearchClient
+};
+//# sourceMappingURL=search.js.map