@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/core/search-provider.d.ts

@@ -0,0 +1,243 @@
+/**
+ * Search provider abstraction
+ *
+ * WebPeel supports multiple web search backends. DuckDuckGo is the default
+ * (no API key required). The StealthSearchProvider uses WebPeel's own stealth
+ * browser to scrape multiple search engines in parallel — fully self-hosted,
+ * no external API keys required.
+ *
+ * Provider fallback chain (DDG):
+ *   DDG HTTP → DDG Lite → Brave (if key) → StealthSearchProvider (Bing + Ecosia)
+ *
+ * In production with no API keys configured, getBestSearchProvider() returns
+ * StealthSearchProvider since DDG HTTP is often blocked on datacenter IPs.
+ */
+export type SearchProviderId = 'duckduckgo' | 'brave' | 'stealth' | 'google' | 'baidu' | 'yandex' | 'naver' | 'yahoo_japan';
+export interface WebSearchResult {
+    title: string;
+    url: string;
+    snippet: string;
+    /** Relevance score (0–1) based on keyword overlap with query. Added by filterRelevantResults. */
+    relevanceScore?: number;
+    /** Thumbnail/image URL from SearXNG results (img_src or thumbnail field). */
+    imageUrl?: string;
+    /** Structured SERP data when structured=true is passed in WebSearchOptions. */
+    serp?: import('./google-serp-parser.js').GoogleSerpResult;
+}
+export interface WebSearchOptions {
+    /** Number of results (1-10) */
+    count: number;
+    /** Provider API key (required for some providers, e.g. Brave) */
+    apiKey?: string;
+    /** Time filter (DuckDuckGo: df param) */
+    tbs?: string;
+    /** Country code for geo-targeting */
+    country?: string;
+    /** Location/region for geo-targeting */
+    location?: string;
+    /**
+     * Locale for search results (e.g. 'jp-jp' for Japanese, 'fr-fr' for French).
+     * In DuckDuckGo, maps to the `kl` parameter.
+     * In SearXNG, maps to the `language` parameter.
+     * Falls back to 'en-US' when not provided.
+     */
+    locale?: string;
+    /** Optional AbortSignal */
+    signal?: AbortSignal;
+    /** Return structured SERP data (knowledge panel, PAA, featured snippets, etc.) */
+    structured?: boolean;
+}
+export interface SearchProvider {
+    readonly id: SearchProviderId;
+    readonly requiresApiKey: boolean;
+    searchWeb(query: string, options: WebSearchOptions): Promise<WebSearchResult[]>;
+}
+declare class ProviderStatsTracker {
+    private readonly history;
+    private readonly windowSize;
+    private readonly failThreshold;
+    private readonly minSamples;
+    private readonly decayMs;
+    constructor(windowSize?: number, failThreshold?: number, minSamples?: number, decayMs?: number);
+    /** Record the outcome of a single attempt for the given source. */
+    record(sourceId: string, success: boolean): void;
+    /**
+     * Returns the failure rate (0–1) for the given source based on
+     * the sliding window of recorded attempts.  Returns 0 if fewer
+     * than minSamples have been recorded, or if all samples are older
+     * than decayMs (failures expire so cold-start blips don't permanently
+     * lock out a provider).
+     */
+    getFailureRate(sourceId: string): number;
+    /**
+     * Returns true when the source should be skipped (failure rate >=
+     * failThreshold with at least minSamples recent recorded).
+     */
+    shouldSkip(sourceId: string): boolean;
+    /** Debug snapshot for a source. */
+    getStats(sourceId: string): {
+        attempts: number;
+        failures: number;
+        failureRate: number;
+        skipRecommended: boolean;
+    };
+    /** Clear history — useful in tests. */
+    reset(sourceId?: string): void;
+}
+/**
+ * Module-level singleton. Exported so callers can inspect or reset stats
+ * (e.g. in tests) and to log diagnostics.
+ */
+export declare const providerStats: ProviderStatsTracker;
+/**
+ * Merge results from multiple sources, deduplicating by normalized URL.
+ * Preserves original order (first occurrence wins) and limits to maxCount.
+ */
+export declare function mergeSearchResults(results: WebSearchResult[], maxCount: number): WebSearchResult[];
+/**
+ * Filter and rank results by relevance to the original query.
+ *
+ * 1. Extract meaningful keywords from the query (remove stop words).
+ * 2. Score each result by keyword overlap with title + URL + snippet.
+ * 3. Remove results with zero overlap (completely irrelevant).
+ * 4. Sort descending by score, keeping original index as tiebreaker.
+ * 5. Attach `relevanceScore` (0–1) to each surviving result.
+ *
+ * Results without any scores (query produced no keywords) are returned as-is.
+ */
+export declare function filterRelevantResults(results: WebSearchResult[], query: string): WebSearchResult[];
+/**
+ * StealthSearchProvider — self-hosted multi-engine search
+ *
+ * Uses WebPeel's own stealth browser (rebrowser-playwright with anti-detection)
+ * to scrape DuckDuckGo, Bing, and Ecosia in parallel. No external API keys
+ * required. Results are deduplicated by normalized URL before returning.
+ *
+ * Timeout: 15s per engine, 20s total.
+ */
+export declare class StealthSearchProvider implements SearchProvider {
+    readonly id: SearchProviderId;
+    readonly requiresApiKey = false;
+    /**
+     * Short-TTL in-memory cache for search results.
+     * Key: normalised "query::count", Value: { results, ts }.
+     * Entries expire after SEARCH_CACHE_TTL_MS.  Avoids redundant browser scrapes
+     * when the same query is issued within a short window (e.g. transit outbound +
+     * general enrichment both searching similar terms).
+     */
+    private static readonly SEARCH_CACHE_TTL_MS;
+    private static readonly SEARCH_CACHE_MAX;
+    private static searchCache;
+    private getCachedSearch;
+    private setCachedSearch;
+    /** Validate and normalize a URL; returns null if invalid/non-http or a DDG ad URL */
+    private validateUrl;
+    /**
+     * Scrape DuckDuckGo HTML endpoint with stealth browser.
+     * Uses the warm shared stealth browser (new context per call) for speed.
+     */
+    private scrapeDDG;
+    /**
+     * Scrape Bing web search with stealth browser.
+     * Selectors: li.b_algo for result containers.
+     */
+    private scrapeBing;
+    /**
+     * Scrape Ecosia web search with stealth browser.
+     * Uses the warm shared stealth browser (new context per call) for speed.
+     * Tries multiple selector patterns since Ecosia updates their HTML frequently.
+     */
+    private scrapeEcosia;
+    searchWeb(query: string, options: WebSearchOptions): Promise<WebSearchResult[]>;
+}
+export declare class DuckDuckGoProvider implements SearchProvider {
+    readonly id: SearchProviderId;
+    readonly requiresApiKey = false;
+    private buildQueryAttempts;
+    private buildSearchUrl;
+    private searchOnce;
+    /**
+     * Fallback: DuckDuckGo Lite endpoint. Different HTML structure, sometimes
+     * works when the main HTML endpoint is temporarily blocked on datacenter IPs.
+     */
+    private searchLite;
+    /**
+     * HTTP-only Bing scraping via undici + cheerio. No browser required.
+     * Routes through Webshare proxy (proxy first, direct fallback).
+     * Tracks stats via providerStats('bing-http').
+     */
+    private _searchBingHttp;
+    /**
+     * HTTP-only Google scraping via undici + cheerio. No browser required.
+     * Routes through Webshare proxy (proxy first, direct fallback).
+     * Sends CONSENT cookie to bypass Google consent page.
+     * Tracks stats via providerStats('google-http').
+     */
+    private _searchGoogleHttp;
+    searchWeb(query: string, options: WebSearchOptions): Promise<WebSearchResult[]>;
+    /**
+     * Exposed for testing: score and filter a pre-fetched result list against a query.
+     * Equivalent to calling filterRelevantResults() directly.
+     */
+    filterResults(results: WebSearchResult[], query: string): WebSearchResult[];
+}
+export declare class BraveSearchProvider implements SearchProvider {
+    readonly id: SearchProviderId;
+    readonly requiresApiKey = true;
+    searchWeb(query: string, options: WebSearchOptions): Promise<WebSearchResult[]>;
+}
+/**
+ * GoogleSearchProvider — Google Search via stealth browser or Custom Search JSON API
+ *
+ * Two modes:
+ *   1. Custom Search JSON API (BYOK): set GOOGLE_SEARCH_KEY + GOOGLE_SEARCH_CX env vars.
+ *      Reliable, structured, 100 free queries/day. Works from any IP.
+ *   2. Stealth browser scraping (no API key): uses playwright-extra stealth plugin to
+ *      scrape google.com/search directly. Works from datacenter IPs where DDG/Bing/Ecosia
+ *      are blocked. Gracefully returns [] if Playwright is unavailable.
+ *
+ * Docs: https://developers.google.com/custom-search/v1/overview
+ */
+export declare class GoogleSearchProvider implements SearchProvider {
+    readonly id: SearchProviderId;
+    /**
+     * requiresApiKey is false: works without API keys via stealth browser fallback.
+     */
+    readonly requiresApiKey = false;
+    /**
+     * Map standard freshness values to Google's dateRestrict format.
+     * Google dateRestrict: d[n]=past n days, w[n]=past n weeks,
+     *                      m[n]=past n months, y[n]=past n years.
+     */
+    private mapFreshnessToDateRestrict;
+    /** Validate URL; returns null if invalid/non-http or a DDG ad URL */
+    private validateUrl;
+    /**
+     * Stealth browser scrape of google.com/search.
+     * Used when no Custom Search API key is configured.
+     * Strategy A: peel() with stealth rendering (consistent with StealthSearchProvider).
+     * Strategy B: direct playwright-extra launch (if peel returns no results).
+     */
+    private scrapeGoogleStealth;
+    /** Parse Google search result HTML using cheerio. No DOM lib types required. */
+    private _parseGoogleHtml;
+    searchWeb(query: string, options: WebSearchOptions): Promise<WebSearchResult[]>;
+}
+export declare function getSearchProvider(id: SearchProviderId | undefined): SearchProvider;
+/**
+ /**
+ * Get the best available search provider based on configured API keys and
+ * available runtime dependencies.
+ *
+ * Priority:
+ *   1. Google Custom Search JSON API (if GOOGLE_SEARCH_KEY + GOOGLE_SEARCH_CX set)
+ *   2. Brave Search (if BRAVE_SEARCH_KEY is set)
+ *   3. Google stealth browser scraping (works from datacenter IPs; no API key needed)
+ *      — only when playwright-extra is available in node_modules
+ *   4. DuckDuckGo with full fallback chain (DDG HTTP → DDG Lite → stealth multi-engine (Bing + Ecosia))
+ */
+export declare function getBestSearchProvider(): {
+    provider: SearchProvider;
+    apiKey?: string;
+};
+export {};