@librechat/agents 3.2.32 → 3.2.34

package/dist/cjs/tools/search/search.cjs

@@ -1,551 +1,449 @@
-'use strict';
-
-var axios = require('axios');
-var textsplitters = require('@langchain/textsplitters');
-var utils = require('./utils.cjs');
-var tavilySearch = require('./tavily-search.cjs');
-
+const require_runtime = require("../../_virtual/_rolldown/runtime.cjs");
+const require_utils = require("./utils.cjs");
+const require_tavily_search = require("./tavily-search.cjs");
+let axios = require("axios");
+axios = require_runtime.__toESM(axios, 1);
+let _langchain_textsplitters = require("@langchain/textsplitters");
+//#region src/tools/search/search.ts
 const chunker = {
-    cleanText: (text) => {
-        if (!text)
-            return '';
-        /** Normalized all line endings to '\n' */
-        const normalizedText = text.replace(/\r\n/g, '\n').replace(/\r/g, '\n');
-        /** Handle multiple backslashes followed by newlines
-         * This replaces patterns like '\\\\\\n' with a single newline */
-        const fixedBackslashes = normalizedText.replace(/\\+\n/g, '\n');
-        /** Cleaned up consecutive newlines, tabs, and spaces around newlines */
-        const cleanedNewlines = fixedBackslashes.replace(/[\t ]*\n[\t \n]*/g, '\n');
-        /** Cleaned up excessive spaces and tabs */
-        const cleanedSpaces = cleanedNewlines.replace(/[ \t]+/g, ' ');
-        return cleanedSpaces.trim();
-    },
-    splitText: async (text, options) => {
-        const chunkSize = options?.chunkSize ?? 150;
-        const chunkOverlap = options?.chunkOverlap ?? 50;
-        const separators = options?.separators || ['\n\n', '\n'];
-        const splitter = new textsplitters.RecursiveCharacterTextSplitter({
-            separators,
-            chunkSize,
-            chunkOverlap,
-        });
-        return await splitter.splitText(text);
-    },
-    splitTexts: async (texts, options, logger) => {
-        // Split multiple texts
-        const logger_ = logger || utils.createDefaultLogger();
-        const promises = texts.map((text) => chunker.splitText(text, options).catch((error) => {
-            logger_.error('Error splitting text:', error);
-            return [text];
-        }));
-        return Promise.all(promises);
-    },
+	cleanText: (text) => {
+		if (!text) return "";
+		return text.replace(/\r\n/g, "\n").replace(/\r/g, "\n").replace(/\\+\n/g, "\n").replace(/[\t ]*\n[\t \n]*/g, "\n").replace(/[ \t]+/g, " ").trim();
+	},
+	splitText: async (text, options) => {
+		const chunkSize = options?.chunkSize ?? 150;
+		const chunkOverlap = options?.chunkOverlap ?? 50;
+		return await new _langchain_textsplitters.RecursiveCharacterTextSplitter({
+			separators: options?.separators || ["\n\n", "\n"],
+			chunkSize,
+			chunkOverlap
+		}).splitText(text);
+	},
+	splitTexts: async (texts, options, logger) => {
+		const logger_ = logger || require_utils.createDefaultLogger();
+		const promises = texts.map((text) => chunker.splitText(text, options).catch((error) => {
+			logger_.error("Error splitting text:", error);
+			return [text];
+		}));
+		return Promise.all(promises);
+	}
 };
+const DEFAULT_MAX_CONTENT_LENGTH = 5e4;
+/** Resolves the per-source scraped content cap from config, the
+* `SEARCH_MAX_CONTENT_LENGTH` env var, or the default (50,000 chars) */
+function resolveMaxContentLength(maxContentLength) {
+	if (maxContentLength != null && maxContentLength > 0) return maxContentLength;
+	const envValue = Number(process.env.SEARCH_MAX_CONTENT_LENGTH);
+	if (Number.isFinite(envValue) && envValue > 0) return envValue;
+	return DEFAULT_MAX_CONTENT_LENGTH;
+}
+function truncateContent(content, maxLength) {
+	return content.length > maxLength ? content.slice(0, maxLength) : content;
+}
 function createSourceUpdateCallback(sourceMap) {
-    return (link, update) => {
-        const source = sourceMap.get(link);
-        if (source) {
-            sourceMap.set(link, {
-                ...source,
-                ...update,
-            });
-        }
-    };
+	return (link, update) => {
+		const source = sourceMap.get(link);
+		if (source) sourceMap.set(link, {
+			...source,
+			...update
+		});
+	};
 }
-const getHighlights = async ({ query, content, reranker, topResults = 5, logger, }) => {
-    const logger_ = logger || utils.createDefaultLogger();
-    if (!content) {
-        logger_.warn('No content provided for highlights');
-        return;
-    }
-    if (!reranker) {
-        logger_.warn('No reranker provided for highlights');
-        return;
-    }
-    try {
-        const documents = await chunker.splitText(content);
-        if (Array.isArray(documents)) {
-            return await reranker.rerank(query, documents, topResults);
-        }
-        else {
-            logger_.error('Expected documents to be an array, got:', typeof documents);
-            return;
-        }
-    }
-    catch (error) {
-        logger_.error('Error in content processing:', error);
-        return;
-    }
+const getHighlights = async ({ query, content, reranker, topResults = 5, maxContentLength = DEFAULT_MAX_CONTENT_LENGTH, logger }) => {
+	const logger_ = logger || require_utils.createDefaultLogger();
+	if (!content) {
+		logger_.warn("No content provided for highlights");
+		return;
+	}
+	if (!reranker) {
+		logger_.warn("No reranker provided for highlights");
+		return;
+	}
+	try {
+		const documents = await chunker.splitText(truncateContent(content, maxContentLength));
+		if (Array.isArray(documents)) return await reranker.rerank(query, documents, topResults);
+		else {
+			logger_.error("Expected documents to be an array, got:", typeof documents);
+			return;
+		}
+	} catch (error) {
+		logger_.error("Error in content processing:", error);
+		return;
+	}
 };
 const createSerperAPI = (apiKey) => {
-    const config = {
-        apiKey: apiKey ?? process.env.SERPER_API_KEY,
-        apiUrl: 'https://google.serper.dev/search',
-        timeout: 10000,
-    };
-    if (config.apiKey == null || config.apiKey === '') {
-        throw new Error('SERPER_API_KEY is required for SerperAPI');
-    }
-    const getSources = async ({ query, date, country, safeSearch, numResults = 8, type, }) => {
-        if (!query.trim()) {
-            return { success: false, error: 'Query cannot be empty' };
-        }
-        try {
-            const safe = ['off', 'moderate', 'active'];
-            const payload = {
-                q: query,
-                safe: safe[safeSearch ?? 1],
-                num: Math.min(Math.max(1, numResults), 10),
-            };
-            // Set the search type if provided
-            if (type) {
-                payload.type = type;
-            }
-            if (date != null) {
-                payload.tbs = `qdr:${date}`;
-            }
-            if (country != null && country !== '') {
-                payload['gl'] = country.toLowerCase();
-            }
-            // Determine the API endpoint based on the search type
-            let apiEndpoint = config.apiUrl;
-            if (type === 'images') {
-                apiEndpoint = 'https://google.serper.dev/images';
-            }
-            else if (type === 'videos') {
-                apiEndpoint = 'https://google.serper.dev/videos';
-            }
-            else if (type === 'news') {
-                apiEndpoint = 'https://google.serper.dev/news';
-            }
-            const response = await axios.post(apiEndpoint, payload, {
-                headers: {
-                    'X-API-KEY': config.apiKey,
-                    'Content-Type': 'application/json',
-                },
-                timeout: config.timeout,
-            });
-            const data = response.data;
-            const results = {
-                organic: data.organic,
-                images: data.images ?? [],
-                answerBox: data.answerBox,
-                topStories: data.topStories ?? [],
-                peopleAlsoAsk: data.peopleAlsoAsk,
-                knowledgeGraph: data.knowledgeGraph,
-                relatedSearches: data.relatedSearches,
-                videos: data.videos ?? [],
-                news: data.news ?? [],
-            };
-            return { success: true, data: results };
-        }
-        catch (error) {
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            return { success: false, error: `API request failed: ${errorMessage}` };
-        }
-    };
-    return { getSources };
+	const config = {
+		apiKey: apiKey ?? process.env.SERPER_API_KEY,
+		apiUrl: "https://google.serper.dev/search",
+		timeout: 1e4
+	};
+	if (config.apiKey == null || config.apiKey === "") throw new Error("SERPER_API_KEY is required for SerperAPI");
+	const getSources = async ({ query, date, country, safeSearch, numResults = 8, type }) => {
+		if (!query.trim()) return {
+			success: false,
+			error: "Query cannot be empty"
+		};
+		try {
+			const payload = {
+				q: query,
+				safe: [
+					"off",
+					"moderate",
+					"active"
+				][safeSearch ?? 1],
+				num: Math.min(Math.max(1, numResults), 10)
+			};
+			if (type) payload.type = type;
+			if (date != null) payload.tbs = `qdr:${date}`;
+			if (country != null && country !== "") payload["gl"] = country.toLowerCase();
+			let apiEndpoint = config.apiUrl;
+			if (type === "images") apiEndpoint = "https://google.serper.dev/images";
+			else if (type === "videos") apiEndpoint = "https://google.serper.dev/videos";
+			else if (type === "news") apiEndpoint = "https://google.serper.dev/news";
+			const data = (await axios.default.post(apiEndpoint, payload, {
+				headers: {
+					"X-API-KEY": config.apiKey,
+					"Content-Type": "application/json"
+				},
+				timeout: config.timeout
+			})).data;
+			return {
+				success: true,
+				data: {
+					organic: data.organic,
+					images: data.images ?? [],
+					answerBox: data.answerBox,
+					topStories: data.topStories ?? [],
+					peopleAlsoAsk: data.peopleAlsoAsk,
+					knowledgeGraph: data.knowledgeGraph,
+					relatedSearches: data.relatedSearches,
+					videos: data.videos ?? [],
+					news: data.news ?? []
+				}
+			};
+		} catch (error) {
+			return {
+				success: false,
+				error: `API request failed: ${error instanceof Error ? error.message : String(error)}`
+			};
+		}
+	};
+	return { getSources };
 };
 const createSearXNGAPI = (instanceUrl, apiKey) => {
-    const config = {
-        instanceUrl: instanceUrl ?? process.env.SEARXNG_INSTANCE_URL,
-        apiKey: apiKey ?? process.env.SEARXNG_API_KEY,
-        defaultLocation: 'all',
-        timeout: 10000,
-    };
-    if (config.instanceUrl == null || config.instanceUrl === '') {
-        throw new Error('SEARXNG_INSTANCE_URL is required for SearXNG API');
-    }
-    const getSources = async ({ query, numResults = 8, safeSearch, type, }) => {
-        if (!query.trim()) {
-            return { success: false, error: 'Query cannot be empty' };
-        }
-        try {
-            // Ensure the instance URL ends with /search
-            if (config.instanceUrl == null || config.instanceUrl === '') {
-                return { success: false, error: 'Instance URL is not defined' };
-            }
-            let searchUrl = config.instanceUrl;
-            if (!searchUrl.endsWith('/search')) {
-                searchUrl = searchUrl.replace(/\/$/, '') + '/search';
-            }
-            // Determine the search category based on the type
-            let category = 'general';
-            if (type === 'images') {
-                category = 'images';
-            }
-            else if (type === 'videos') {
-                category = 'videos';
-            }
-            else if (type === 'news') {
-                category = 'news';
-            }
-            // Prepare parameters for SearXNG
-            const params = {
-                q: query,
-                format: 'json',
-                pageno: 1,
-                categories: category,
-                language: 'all',
-                safesearch: safeSearch,
-                engines: 'google,bing,duckduckgo',
-            };
-            const headers = {
-                'Content-Type': 'application/json',
-            };
-            if (config.apiKey != null && config.apiKey !== '') {
-                headers['X-API-Key'] = config.apiKey;
-            }
-            const response = await axios.get(searchUrl, {
-                headers,
-                params,
-                timeout: config.timeout,
-            });
-            const data = response.data;
-            // Helper function to identify news results since SearXNG doesn't provide that classification by default
-            const isNewsResult = (result) => {
-                const url = result.url?.toLowerCase() ?? '';
-                const title = result.title?.toLowerCase() ?? '';
-                // News-related keywords in title/content
-                const newsKeywords = [
-                    'breaking news',
-                    'latest news',
-                    'top stories',
-                    'news today',
-                    'developing story',
-                    'trending news',
-                    'news',
-                ];
-                // Check if title/content contains news keywords
-                const hasNewsKeywords = newsKeywords.some((keyword) => title.toLowerCase().includes(keyword) // just title probably fine, content parsing is overkill for what we need: || content.includes(keyword)
-                );
-                // Check if URL contains news-related paths
-                const hasNewsPath = url.includes('/news/') ||
-                    url.includes('/world/') ||
-                    url.includes('/politics/') ||
-                    url.includes('/breaking/');
-                return hasNewsKeywords || hasNewsPath;
-            };
-            // Transform SearXNG results to match SerperAPI format
-            const organicResults = (data.results ?? [])
-                .slice(0, numResults)
-                .map((result, index) => {
-                let attribution = '';
-                try {
-                    attribution = new URL(result.url ?? '').hostname;
-                }
-                catch {
-                    attribution = '';
-                }
-                return {
-                    position: index + 1,
-                    title: result.title ?? '',
-                    link: result.url ?? '',
-                    snippet: result.content ?? '',
-                    date: result.publishedDate ?? '',
-                    attribution,
-                };
-            });
-            const imageResults = (data.results ?? [])
-                .filter((result) => result.img_src)
-                .slice(0, 6)
-                .map((result, index) => ({
-                title: result.title ?? '',
-                imageUrl: result.img_src ?? '',
-                position: index + 1,
-                source: new URL(result.url ?? '').hostname,
-                domain: new URL(result.url ?? '').hostname,
-                link: result.url ?? '',
-            }));
-            // Extract news results from organic results
-            const newsResults = (data.results ?? [])
-                .filter(isNewsResult)
-                .map((result, index) => {
-                let attribution = '';
-                try {
-                    attribution = new URL(result.url ?? '').hostname;
-                }
-                catch {
-                    attribution = '';
-                }
-                return {
-                    title: result.title ?? '',
-                    link: result.url ?? '',
-                    snippet: result.content ?? '',
-                    date: result.publishedDate ?? '',
-                    source: attribution,
-                    imageUrl: result.img_src ?? '',
-                    position: index + 1,
-                };
-            });
-            const topStories = newsResults.slice(0, 5);
-            const relatedSearches = Array.isArray(data.suggestions)
-                ? data.suggestions.map((suggestion) => ({ query: suggestion }))
-                : [];
-            const results = {
-                organic: organicResults,
-                images: imageResults,
-                topStories: topStories, // Use first 5 extracted news as top stories
-                relatedSearches,
-                videos: [],
-                news: newsResults,
-                // Add empty arrays for other Serper fields to maintain parity
-                places: [],
-                shopping: [],
-                peopleAlsoAsk: [],
-                knowledgeGraph: undefined,
-                answerBox: undefined,
-            };
-            return { success: true, data: results };
-        }
-        catch (error) {
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            return {
-                success: false,
-                error: `SearXNG API request failed: ${errorMessage}`,
-            };
-        }
-    };
-    return { getSources };
+	const config = {
+		instanceUrl: instanceUrl ?? process.env.SEARXNG_INSTANCE_URL,
+		apiKey: apiKey ?? process.env.SEARXNG_API_KEY,
+		defaultLocation: "all",
+		timeout: 1e4
+	};
+	if (config.instanceUrl == null || config.instanceUrl === "") throw new Error("SEARXNG_INSTANCE_URL is required for SearXNG API");
+	const getSources = async ({ query, numResults = 8, safeSearch, type }) => {
+		if (!query.trim()) return {
+			success: false,
+			error: "Query cannot be empty"
+		};
+		try {
+			if (config.instanceUrl == null || config.instanceUrl === "") return {
+				success: false,
+				error: "Instance URL is not defined"
+			};
+			let searchUrl = config.instanceUrl;
+			if (!searchUrl.endsWith("/search")) searchUrl = searchUrl.replace(/\/$/, "") + "/search";
+			let category = "general";
+			if (type === "images") category = "images";
+			else if (type === "videos") category = "videos";
+			else if (type === "news") category = "news";
+			const params = {
+				q: query,
+				format: "json",
+				pageno: 1,
+				categories: category,
+				language: "all",
+				safesearch: safeSearch,
+				engines: "google,bing,duckduckgo"
+			};
+			const headers = { "Content-Type": "application/json" };
+			if (config.apiKey != null && config.apiKey !== "") headers["X-API-Key"] = config.apiKey;
+			const data = (await axios.default.get(searchUrl, {
+				headers,
+				params,
+				timeout: config.timeout
+			})).data;
+			const isNewsResult = (result) => {
+				const url = result.url?.toLowerCase() ?? "";
+				const title = result.title?.toLowerCase() ?? "";
+				const hasNewsKeywords = [
+					"breaking news",
+					"latest news",
+					"top stories",
+					"news today",
+					"developing story",
+					"trending news",
+					"news"
+				].some((keyword) => title.toLowerCase().includes(keyword));
+				const hasNewsPath = url.includes("/news/") || url.includes("/world/") || url.includes("/politics/") || url.includes("/breaking/");
+				return hasNewsKeywords || hasNewsPath;
+			};
+			const organicResults = (data.results ?? []).slice(0, numResults).map((result, index) => {
+				let attribution = "";
+				try {
+					attribution = new URL(result.url ?? "").hostname;
+				} catch {
+					attribution = "";
+				}
+				return {
+					position: index + 1,
+					title: result.title ?? "",
+					link: result.url ?? "",
+					snippet: result.content ?? "",
+					date: result.publishedDate ?? "",
+					attribution
+				};
+			});
+			const imageResults = (data.results ?? []).filter((result) => result.img_src).slice(0, 6).map((result, index) => ({
+				title: result.title ?? "",
+				imageUrl: result.img_src ?? "",
+				position: index + 1,
+				source: new URL(result.url ?? "").hostname,
+				domain: new URL(result.url ?? "").hostname,
+				link: result.url ?? ""
+			}));
+			const newsResults = (data.results ?? []).filter(isNewsResult).map((result, index) => {
+				let attribution = "";
+				try {
+					attribution = new URL(result.url ?? "").hostname;
+				} catch {
+					attribution = "";
+				}
+				return {
+					title: result.title ?? "",
+					link: result.url ?? "",
+					snippet: result.content ?? "",
+					date: result.publishedDate ?? "",
+					source: attribution,
+					imageUrl: result.img_src ?? "",
+					position: index + 1
+				};
+			});
+			return {
+				success: true,
+				data: {
+					organic: organicResults,
+					images: imageResults,
+					topStories: newsResults.slice(0, 5),
+					relatedSearches: Array.isArray(data.suggestions) ? data.suggestions.map((suggestion) => ({ query: suggestion })) : [],
+					videos: [],
+					news: newsResults,
+					places: [],
+					shopping: [],
+					peopleAlsoAsk: [],
+					knowledgeGraph: void 0,
+					answerBox: void 0
+				}
+			};
+		} catch (error) {
+			return {
+				success: false,
+				error: `SearXNG API request failed: ${error instanceof Error ? error.message : String(error)}`
+			};
+		}
+	};
+	return { getSources };
 };
 const createSearchAPI = (config) => {
-    const { searchProvider = 'serper', serperApiKey, searxngInstanceUrl, searxngApiKey, tavilyApiKey, tavilySearchUrl, tavilySearchOptions, } = config;
-    if (searchProvider.toLowerCase() === 'serper') {
-        return createSerperAPI(serperApiKey);
-    }
-    else if (searchProvider.toLowerCase() === 'searxng') {
-        return createSearXNGAPI(searxngInstanceUrl, searxngApiKey);
-    }
-    else if (searchProvider.toLowerCase() === 'tavily') {
-        return tavilySearch.createTavilyAPI(tavilyApiKey, tavilySearchUrl, tavilySearchOptions);
-    }
-    else {
-        throw new Error(`Invalid search provider: ${searchProvider}. Must be 'serper', 'searxng', or 'tavily'`);
-    }
+	const { searchProvider = "serper", serperApiKey, searxngInstanceUrl, searxngApiKey, tavilyApiKey, tavilySearchUrl, tavilySearchOptions } = config;
+	if (searchProvider.toLowerCase() === "serper") return createSerperAPI(serperApiKey);
+	else if (searchProvider.toLowerCase() === "searxng") return createSearXNGAPI(searxngInstanceUrl, searxngApiKey);
+	else if (searchProvider.toLowerCase() === "tavily") return require_tavily_search.createTavilyAPI(tavilyApiKey, tavilySearchUrl, tavilySearchOptions);
+	else throw new Error(`Invalid search provider: ${searchProvider}. Must be 'serper', 'searxng', or 'tavily'`);
 };
 const createSourceProcessor = (config = {}, scraperInstance) => {
-    if (!scraperInstance) {
-        throw new Error('Scraper instance is required');
-    }
-    const { topResults = 5, 
-    // strategies = ['no_extraction'],
-    // filterContent = true,
-    reranker, logger, } = config;
-    const logger_ = logger || utils.createDefaultLogger();
-    const scraper = scraperInstance;
-    const processResponse = (url, response) => {
-        const rawMetadata = scraper.extractMetadata(response);
-        const metadata = Object.keys(rawMetadata).length > 0 ? rawMetadata : undefined;
-        const attribution = utils.getAttribution(url, metadata, logger_);
-        if (response.success && response.data) {
-            const [content, references] = scraper.extractContent(response);
-            return {
-                url,
-                references,
-                attribution,
-                content: chunker.cleanText(content),
-            };
-        }
-        logger_.error(`Error scraping ${url}: ${response.error ?? 'Unknown error'}`);
-        return { url, attribution, error: true, content: '' };
-    };
-    const addHighlights = async (result, query, onGetHighlights) => {
-        if (result.error != null) {
-            return result;
-        }
-        try {
-            const highlights = await getHighlights({
-                query,
-                reranker,
-                content: result.content,
-                logger: logger_,
-            });
-            if (onGetHighlights) {
-                onGetHighlights(result.url);
-            }
-            return { ...result, highlights };
-        }
-        catch (error) {
-            logger_.error('Error processing scraped content:', error);
-            return result;
-        }
-    };
-    const webScraper = {
-        scrapeMany: async ({ query, links, onGetHighlights, }) => {
-            logger_.debug(`Scraping ${links.length} links`);
-            try {
-                let responses;
-                if (scraper.scrapeUrls) {
-                    responses = await scraper.scrapeUrls(links);
-                }
-                else {
-                    responses = await Promise.all(links.map((link) => scraper
-                        .scrapeUrl(link, {})
-                        .catch((error) => {
-                        logger_.error(`Error scraping ${link}:`, error);
-                        return [link, { success: false, error: String(error) }];
-                    })));
-                }
-                const withHighlights = await Promise.all(responses.map(([url, response]) => addHighlights(processResponse(url, response), query, onGetHighlights)));
-                return withHighlights;
-            }
-            catch (error) {
-                logger_.error('Error in scrapeMany:', error);
-                return [];
-            }
-        },
-    };
-    const fetchContents = async ({ links, query, target, onGetHighlights, onContentScraped, }) => {
-        const initialLinks = links.slice(0, target);
-        // const remainingLinks = links.slice(target).reverse();
-        const results = await webScraper.scrapeMany({
-            query,
-            links: initialLinks,
-            onGetHighlights,
-        });
-        for (const result of results) {
-            if (result.error === true) {
-                continue;
-            }
-            const { url, content, attribution, references, highlights } = result;
-            onContentScraped?.(url, {
-                content,
-                attribution,
-                references,
-                highlights,
-            });
-        }
-    };
-    const processSources = async ({ result, numElements, query, news, proMode = true, onGetHighlights, }) => {
-        try {
-            if (!result.data) {
-                return {
-                    organic: [],
-                    topStories: [],
-                    images: [],
-                    relatedSearches: [],
-                };
-            }
-            else if (!result.data.organic) {
-                return result.data;
-            }
-            if (!proMode) {
-                const wikiSources = result.data.organic.filter((source) => source.link.includes('wikipedia.org'));
-                if (!wikiSources.length) {
-                    return result.data;
-                }
-                const wikiSourceMap = new Map();
-                wikiSourceMap.set(wikiSources[0].link, wikiSources[0]);
-                const onContentScraped = createSourceUpdateCallback(wikiSourceMap);
-                await fetchContents({
-                    query,
-                    target: 1,
-                    onGetHighlights,
-                    onContentScraped,
-                    links: [wikiSources[0].link],
-                });
-                for (let i = 0; i < result.data.organic.length; i++) {
-                    const source = result.data.organic[i];
-                    const updatedSource = wikiSourceMap.get(source.link);
-                    if (updatedSource) {
-                        result.data.organic[i] = {
-                            ...source,
-                            ...updatedSource,
-                        };
-                    }
-                }
-                return result.data;
-            }
-            const sourceMap = new Map();
-            const organicLinksSet = new Set();
-            // Collect organic links
-            const organicLinks = collectLinks(result.data.organic, sourceMap, organicLinksSet);
-            // Collect top story links, excluding any that are already in organic links
-            const topStories = result.data.topStories ?? [];
-            const topStoryLinks = collectLinks(topStories, sourceMap, organicLinksSet);
-            if (organicLinks.length === 0 && (topStoryLinks.length === 0 || !news)) {
-                return result.data;
-            }
-            const onContentScraped = createSourceUpdateCallback(sourceMap);
-            const promises = [];
-            // Process organic links
-            if (organicLinks.length > 0) {
-                promises.push(fetchContents({
-                    query,
-                    onGetHighlights,
-                    onContentScraped,
-                    links: organicLinks,
-                    target: numElements,
-                }));
-            }
-            // Process top story links
-            if (news && topStoryLinks.length > 0) {
-                promises.push(fetchContents({
-                    query,
-                    onGetHighlights,
-                    onContentScraped,
-                    links: topStoryLinks,
-                    target: numElements,
-                }));
-            }
-            await Promise.all(promises);
-            if (result.data.organic.length > 0) {
-                updateSourcesWithContent(result.data.organic, sourceMap);
-            }
-            if (news && topStories.length > 0) {
-                updateSourcesWithContent(topStories, sourceMap);
-            }
-            return result.data;
-        }
-        catch (error) {
-            logger_.error('Error in processSources:', error);
-            return {
-                organic: [],
-                topStories: [],
-                images: [],
-                relatedSearches: [],
-                ...result.data,
-                error: error instanceof Error ? error.message : String(error),
-            };
-        }
-    };
-    return {
-        processSources,
-        topResults,
-    };
+	if (!scraperInstance) throw new Error("Scraper instance is required");
+	const { topResults = 5, reranker, logger } = config;
+	const maxContentLength = resolveMaxContentLength(config.maxContentLength);
+	const logger_ = logger || require_utils.createDefaultLogger();
+	const scraper = scraperInstance;
+	const processResponse = (url, response) => {
+		const rawMetadata = scraper.extractMetadata(response);
+		const attribution = require_utils.getAttribution(url, Object.keys(rawMetadata).length > 0 ? rawMetadata : void 0, logger_);
+		if (response.success && response.data) {
+			const [content, references] = scraper.extractContent(response);
+			return {
+				url,
+				references,
+				attribution,
+				content: truncateContent(chunker.cleanText(content), maxContentLength)
+			};
+		}
+		logger_.error(`Error scraping ${url}: ${response.error ?? "Unknown error"}`);
+		return {
+			url,
+			attribution,
+			error: true,
+			content: ""
+		};
+	};
+	const addHighlights = async (result, query, onGetHighlights) => {
+		if (result.error != null) return result;
+		try {
+			const highlights = await getHighlights({
+				query,
+				reranker,
+				content: result.content,
+				maxContentLength,
+				logger: logger_
+			});
+			if (onGetHighlights) onGetHighlights(result.url);
+			return {
+				...result,
+				highlights
+			};
+		} catch (error) {
+			logger_.error("Error processing scraped content:", error);
+			return result;
+		}
+	};
+	const webScraper = { scrapeMany: async ({ query, links, onGetHighlights }) => {
+		logger_.debug(`Scraping ${links.length} links`);
+		try {
+			let responses;
+			if (scraper.scrapeUrls) responses = await scraper.scrapeUrls(links);
+			else responses = await Promise.all(links.map((link) => scraper.scrapeUrl(link, {}).catch((error) => {
+				logger_.error(`Error scraping ${link}:`, error);
+				return [link, {
+					success: false,
+					error: String(error)
+				}];
+			})));
+			return await Promise.all(responses.map(([url, response]) => addHighlights(processResponse(url, response), query, onGetHighlights)));
+		} catch (error) {
+			logger_.error("Error in scrapeMany:", error);
+			return [];
+		}
+	} };
+	const fetchContents = async ({ links, query, target, onGetHighlights, onContentScraped }) => {
+		const initialLinks = links.slice(0, target);
+		const results = await webScraper.scrapeMany({
+			query,
+			links: initialLinks,
+			onGetHighlights
+		});
+		for (const result of results) {
+			if (result.error === true) continue;
+			const { url, content, attribution, references, highlights } = result;
+			onContentScraped?.(url, {
+				content,
+				attribution,
+				references,
+				highlights
+			});
+		}
+	};
+	const processSources = async ({ result, numElements, query, news, proMode = true, onGetHighlights }) => {
+		try {
+			if (!result.data) return {
+				organic: [],
+				topStories: [],
+				images: [],
+				relatedSearches: []
+			};
+			if (result.data.topStories != null && result.data.topStories.length > numElements)
+ /** Merged news results can far exceed the requested source count;
+			* every entry is formatted into the LLM output, so cap them up
+			* front — before any early return below and before scraping
+			* entries the cap would discard */
+			result.data.topStories = result.data.topStories.slice(0, numElements);
+			if (!result.data.organic) return result.data;
+			if (!proMode) {
+				const wikiSources = result.data.organic.filter((source) => source.link.includes("wikipedia.org"));
+				if (!wikiSources.length) return result.data;
+				const wikiSourceMap = /* @__PURE__ */ new Map();
+				wikiSourceMap.set(wikiSources[0].link, wikiSources[0]);
+				await fetchContents({
+					query,
+					target: 1,
+					onGetHighlights,
+					onContentScraped: createSourceUpdateCallback(wikiSourceMap),
+					links: [wikiSources[0].link]
+				});
+				for (let i = 0; i < result.data.organic.length; i++) {
+					const source = result.data.organic[i];
+					const updatedSource = wikiSourceMap.get(source.link);
+					if (updatedSource) result.data.organic[i] = {
+						...source,
+						...updatedSource
+					};
+				}
+				return result.data;
+			}
+			const sourceMap = /* @__PURE__ */ new Map();
+			const organicLinksSet = /* @__PURE__ */ new Set();
+			const organicLinks = collectLinks(result.data.organic, sourceMap, organicLinksSet);
+			const topStories = result.data.topStories ?? [];
+			const topStoryLinks = collectLinks(topStories, sourceMap, organicLinksSet);
+			if (organicLinks.length === 0 && (topStoryLinks.length === 0 || !news)) return result.data;
+			const onContentScraped = createSourceUpdateCallback(sourceMap);
+			const promises = [];
+			if (organicLinks.length > 0) promises.push(fetchContents({
+				query,
+				onGetHighlights,
+				onContentScraped,
+				links: organicLinks,
+				target: numElements
+			}));
+			if (news && topStoryLinks.length > 0) promises.push(fetchContents({
+				query,
+				onGetHighlights,
+				onContentScraped,
+				links: topStoryLinks,
+				target: numElements
+			}));
+			await Promise.all(promises);
+			if (result.data.organic.length > 0) updateSourcesWithContent(result.data.organic, sourceMap);
+			if (news && topStories.length > 0) updateSourcesWithContent(topStories, sourceMap);
+			return result.data;
+		} catch (error) {
+			logger_.error("Error in processSources:", error);
+			return {
+				organic: [],
+				topStories: [],
+				images: [],
+				relatedSearches: [],
+				...result.data,
+				error: error instanceof Error ? error.message : String(error)
+			};
+		}
+	};
+	return {
+		processSources,
+		topResults
+	};
 };
 /** Helper function to collect links and update sourceMap */
 function collectLinks(sources, sourceMap, existingLinksSet) {
-    const links = [];
-    for (const source of sources) {
-        if (source.link) {
-            // For topStories, only add if not already in organic links
-            if (existingLinksSet && existingLinksSet.has(source.link)) {
-                continue;
-            }
-            links.push(source.link);
-            if (existingLinksSet) {
-                existingLinksSet.add(source.link);
-            }
-            sourceMap.set(source.link, source);
-        }
-    }
-    return links;
+	const links = [];
+	for (const source of sources) if (source.link) {
+		if (existingLinksSet && existingLinksSet.has(source.link)) continue;
+		links.push(source.link);
+		if (existingLinksSet) existingLinksSet.add(source.link);
+		sourceMap.set(source.link, source);
+	}
+	return links;
 }
 /** Helper function to update sources with scraped content */
 function updateSourcesWithContent(sources, sourceMap) {
-    for (let i = 0; i < sources.length; i++) {
-        const source = sources[i];
-        const updatedSource = sourceMap.get(source.link);
-        if (updatedSource) {
-            sources[i] = {
-                ...source,
-                ...updatedSource,
-            };
-        }
-    }
+	for (let i = 0; i < sources.length; i++) {
+		const source = sources[i];
+		const updatedSource = sourceMap.get(source.link);
+		if (updatedSource) sources[i] = {
+			...source,
+			...updatedSource
+		};
+	}
 }
-
+//#endregion
 exports.createSearchAPI = createSearchAPI;
 exports.createSourceProcessor = createSourceProcessor;
-//# sourceMappingURL=search.cjs.map
+
+//# sourceMappingURL=search.cjs.map