@webcrawl/mcp 1.4.2 → 1.4.4

package/dist/core/search/providers.d.ts

@@ -0,0 +1,84 @@
+import { type CveRecord, type ProductOffer } from "../extract/index.js";
+import type { RawCandidateDocument, UserLocationContext } from "../ranker/index.js";
+import { type SearchRequest } from "./catalog.js";
+/**
+ * Wikipedia only helps for entity / definition queries ("retrieval augmented
+ * generation", "what is rag"). For intent queries ("best X under 500",
+ * "A vs B", "how to ...") it injects off-topic pages (songs, unrelated topics).
+ */
+export declare function isEntityQuery(q: string): boolean;
+/**
+ * Stage 1: Hedged Multi-Engine Native Federation with Early-Exit Quorum.
+ * Fires all native zero-key search engines concurrently and exits as soon as a fast
+ * quorum (at least 2 engines + >= limit results + 80ms grace window) finishes,
+ * preventing any single slow engine from inflating tail latency.
+ */
+export declare function fetchHedgedNativeFederation(rawQuery: string, localizedQuery: string, perEngineLimit: number, req: SearchRequest, userLocation: UserLocationContext, quorumTimeoutMs: number, targetSites: string[], onOffer: (offer: ProductOffer) => void, onCve?: (cve: CveRecord) => void): Promise<RawCandidateDocument[]>;
+/** Below this many unique URLs from the free engines, the keyed search fallback runs. */
+export declare const JINA_FALLBACK_MIN_URLS = 3;
+/** True when a scraped SERP is a bot-check / captcha rather than results. */
+export declare function isSerpChallengeHtml(html: string, status?: number): boolean;
+export declare const EMPTY_SEARCH_ERROR: string;
+/** Canonical URL for search dedupe (host lowercased, trailing slash / tracking params stripped). */
+export declare function canonicalSearchUrl(raw: string | undefined): string;
+/** Round-robin merge so one source (e.g. Wikipedia) cannot fill `-n` by itself. */
+export declare function mergeSearchCandidates(groups: RawCandidateDocument[][], limit: number): RawCandidateDocument[];
+export declare function withRetry<T>(fn: () => Promise<T>, opts?: {
+    retries?: number;
+    delayMs?: number;
+    isEmpty?: (v: T) => boolean;
+}): Promise<T>;
+/** Parse Bing's `format=rss` web-search feed into candidate docs. */
+export declare function parseBingWebRss(xml: string, limit: number): RawCandidateDocument[];
+/**
+ * Last-resort keyless engines used when DDG/Brave HTML SERPs come back empty
+ * or thinner than `limit`. Wikipedia OpenSearch, HN Algolia, Bing RSS, and
+ * DuckDuckGo Instant Answer do not require a JS challenge or API key.
+ */
+export declare function fetchKeylessSearchFallbacks(query: string, limit: number): Promise<RawCandidateDocument[]>;
+/** DuckDuckGo Instant Answer API — keyless JSON, unrelated to the HTML SERP challenge. */
+export declare function fetchDuckDuckGoInstantAnswer(query: string, limit: number): Promise<RawCandidateDocument[]>;
+export declare function fetchJinaSearch(query: string, limit: number, req?: {
+    country?: string;
+    lang?: string;
+}): Promise<RawCandidateDocument[]>;
+/**
+ * 0A. Bing Shopping Native Provider (`bing-shopping`) via Impit Chrome TLS impersonation.
+ * Parses embedded `CustomData` JSON + `<li class="br-item">` seller/rating cards with zero API keys and zero headless browsers (~400ms).
+ * When `targetSites` is set (e.g. `["jbhifi.com.au", "amazon.com.au"]`), queries per-site in parallel so even bot-gated retailers return live prices and direct product URLs.
+ */
+export declare function fetchBingShoppingNative(rawQuery: string, limit: number, userLocation?: UserLocationContext, targetSites?: string[], onOffer?: (offer: ProductOffer) => void): Promise<RawCandidateDocument[]>;
+/**
+ * 0B. Google Shopping Provider (`google-shopping`).
+ * Tier 1: Serper `/shopping` endpoint when `SERPER_API_KEY` is configured.
+ * Tier 2 (Zero-key fallback): Shopping-intent Brave query ("<query> buy price") with snippet price extraction.
+ */
+export declare function fetchGoogleShoppingProvider(rawQuery: string, localizedQuery: string, limit: number, userLocation?: UserLocationContext, targetSites?: string[], onOffer?: (offer: ProductOffer) => void): Promise<RawCandidateDocument[]>;
+/**
+ * 1. Brave Search Native HTML parser via Impit Chrome TLS impersonation (`search.brave.com/search`).
+ * Independent web index with zero API keys required.
+ */
+export declare function fetchBraveNativeSerp(query: string, limit: number, userLocation?: UserLocationContext): Promise<RawCandidateDocument[]>;
+/**
+ * 3. Google News Search RSS (`news.google.com/rss/search`) with dynamic edition localization (`gl`, `hl`, `ceid`).
+ */
+export declare function fetchGoogleNewsSearchRss(query: string, limit: number, userLocation?: UserLocationContext, lang?: string, timeoutMs?: number): Promise<RawCandidateDocument[]>;
+/**
+ * 6. ArXiv API (`export.arxiv.org/api/query`) + OpenAlex Scholarly Works API
+ * for scientific, ML/AI, biomedical, and peer-reviewed research paper search.
+ */
+export declare function buildArxivSearchQuery(rawQuery: string): string;
+export declare function fetchArxivApi(query: string, limit: number): Promise<RawCandidateDocument[]>;
+export declare function nvdKeywordFromQuery(query: string): string;
+/**
+ * Exact CVE ids, cache first: hits come from Postgres (stale ones refresh in the
+ * background), and only misses go to NVD, one request each, capped at `cap`.
+ * Used for query ids and page-mentioned ids. Write-back happens in the pipeline
+ * once NVD and OSV records are merged.
+ */
+export declare function fetchNvdByIds(ids: string[], cap?: number): Promise<CveRecord[]>;
+/** "lodash npm vulnerabilities" -> { name: "lodash", ecosystem: "npm" } */
+export declare function osvPackageFromQuery(query: string): {
+    name: string;
+    ecosystem: string;
+} | null;