@librechat/agents 3.2.32 → 3.2.33

package/dist/esm/tools/search/tool.mjs

@@ -1,305 +1,267 @@
-import { tool } from '@langchain/core/tools';
-import { WebSearchToolDescription, WebSearchToolName, countrySchema, newsSchema, videosSchema, imagesSchema, dateSchema, querySchema } from './schema.mjs';
-import { createSearchAPI, createSourceProcessor } from './search.mjs';
-import { createSerperScraper } from './serper-scraper.mjs';
-import { createTavilyScraper } from './tavily-scraper.mjs';
-import { createFirecrawlScraper } from './firecrawl.mjs';
-import { expandHighlights } from './highlights.mjs';
-import { formatResultsForLLM } from './format.mjs';
-import { createDefaultLogger } from './utils.mjs';
-import { createReranker } from './rerankers.mjs';
-import { Constants } from '../../common/enum.mjs';
-
+import "../../common/enum.mjs";
+import "../../common/index.mjs";
+import { createDefaultLogger } from "./utils.mjs";
+import { formatResultsForLLM } from "./format.mjs";
+import { WebSearchToolDescription, WebSearchToolName, countrySchema, dateSchema, imagesSchema, newsSchema, querySchema, videosSchema } from "./schema.mjs";
+import { createSearchAPI, createSourceProcessor } from "./search.mjs";
+import { createSerperScraper } from "./serper-scraper.mjs";
+import { createTavilyScraper } from "./tavily-scraper.mjs";
+import { createFirecrawlScraper } from "./firecrawl.mjs";
+import { expandHighlights } from "./highlights.mjs";
+import { createReranker } from "./rerankers.mjs";
+import { tool } from "@langchain/core/tools";
+//#region src/tools/search/tool.ts
 /**
- * Executes parallel searches and merges the results
- */
-async function executeParallelSearches({ searchAPI, query, date, country, safeSearch, images, videos, news, logger, }) {
-    // Prepare all search tasks to run in parallel
-    const searchTasks = [
-        // Main search
-        searchAPI.getSources({
-            query,
-            date,
-            country,
-            safeSearch,
-        }),
-    ];
-    if (images) {
-        searchTasks.push(searchAPI
-            .getSources({
-            query,
-            date,
-            country,
-            safeSearch,
-            type: 'images',
-        })
-            .catch((error) => {
-            logger.error('Error fetching images:', error);
-            return {
-                success: false,
-                error: `Images search failed: ${error instanceof Error ? error.message : String(error)}`,
-            };
-        }));
-    }
-    if (videos) {
-        searchTasks.push(searchAPI
-            .getSources({
-            query,
-            date,
-            country,
-            safeSearch,
-            type: 'videos',
-        })
-            .catch((error) => {
-            logger.error('Error fetching videos:', error);
-            return {
-                success: false,
-                error: `Videos search failed: ${error instanceof Error ? error.message : String(error)}`,
-            };
-        }));
-    }
-    if (news) {
-        searchTasks.push(searchAPI
-            .getSources({
-            query,
-            date,
-            country,
-            safeSearch,
-            type: 'news',
-        })
-            .catch((error) => {
-            logger.error('Error fetching news:', error);
-            return {
-                success: false,
-                error: `News search failed: ${error instanceof Error ? error.message : String(error)}`,
-            };
-        }));
-    }
-    // Run all searches in parallel
-    const results = await Promise.all(searchTasks);
-    // Get the main search result (first result)
-    const mainResult = results[0];
-    if (!mainResult.success) {
-        throw new Error(mainResult.error ?? 'Search failed');
-    }
-    // Merge additional results with the main results
-    const mergedResults = { ...mainResult.data };
-    // Convert existing news to topStories if present
-    if (mergedResults.news !== undefined && mergedResults.news.length > 0) {
-        const existingNewsAsTopStories = mergedResults.news
-            .filter((newsItem) => newsItem.link !== undefined && newsItem.link !== '')
-            .map((newsItem) => ({
-            title: newsItem.title ?? '',
-            link: newsItem.link ?? '',
-            source: newsItem.source ?? '',
-            date: newsItem.date ?? '',
-            imageUrl: newsItem.imageUrl ?? '',
-            processed: false,
-        }));
-        mergedResults.topStories = [
-            ...(mergedResults.topStories ?? []),
-            ...existingNewsAsTopStories,
-        ];
-        delete mergedResults.news;
-    }
-    results.slice(1).forEach((result) => {
-        if (result.success && result.data !== undefined) {
-            if (result.data.images !== undefined && result.data.images.length > 0) {
-                mergedResults.images = [
-                    ...(mergedResults.images ?? []),
-                    ...result.data.images,
-                ];
-            }
-            if (result.data.videos !== undefined && result.data.videos.length > 0) {
-                mergedResults.videos = [
-                    ...(mergedResults.videos ?? []),
-                    ...result.data.videos,
-                ];
-            }
-            if (result.data.news !== undefined && result.data.news.length > 0) {
-                const newsAsTopStories = result.data.news.map((newsItem) => ({
-                    ...newsItem,
-                    link: newsItem.link ?? '',
-                }));
-                mergedResults.topStories = [
-                    ...(mergedResults.topStories ?? []),
-                    ...newsAsTopStories,
-                ];
-            }
-        }
-    });
-    return { success: true, data: mergedResults };
+* Executes parallel searches and merges the results,
+* deduplicating top stories by link
+*/
+async function executeParallelSearches({ searchAPI, query, date, country, safeSearch, images, videos, news, logger }) {
+	const searchTasks = [searchAPI.getSources({
+		query,
+		date,
+		country,
+		safeSearch
+	})];
+	if (images) searchTasks.push(searchAPI.getSources({
+		query,
+		date,
+		country,
+		safeSearch,
+		type: "images"
+	}).catch((error) => {
+		logger.error("Error fetching images:", error);
+		return {
+			success: false,
+			error: `Images search failed: ${error instanceof Error ? error.message : String(error)}`
+		};
+	}));
+	if (videos) searchTasks.push(searchAPI.getSources({
+		query,
+		date,
+		country,
+		safeSearch,
+		type: "videos"
+	}).catch((error) => {
+		logger.error("Error fetching videos:", error);
+		return {
+			success: false,
+			error: `Videos search failed: ${error instanceof Error ? error.message : String(error)}`
+		};
+	}));
+	if (news) searchTasks.push(searchAPI.getSources({
+		query,
+		date,
+		country,
+		safeSearch,
+		type: "news"
+	}).catch((error) => {
+		logger.error("Error fetching news:", error);
+		return {
+			success: false,
+			error: `News search failed: ${error instanceof Error ? error.message : String(error)}`
+		};
+	}));
+	const results = await Promise.all(searchTasks);
+	const mainResult = results[0];
+	if (!mainResult.success) throw new Error(mainResult.error ?? "Search failed");
+	const mergedResults = { ...mainResult.data };
+	if (mergedResults.news !== void 0 && mergedResults.news.length > 0) {
+		const existingNewsAsTopStories = mergedResults.news.filter((newsItem) => newsItem.link !== void 0 && newsItem.link !== "").map((newsItem) => ({
+			title: newsItem.title ?? "",
+			link: newsItem.link ?? "",
+			source: newsItem.source ?? "",
+			date: newsItem.date ?? "",
+			imageUrl: newsItem.imageUrl ?? "",
+			processed: false
+		}));
+		mergedResults.topStories = [...mergedResults.topStories ?? [], ...existingNewsAsTopStories];
+		delete mergedResults.news;
+	}
+	results.slice(1).forEach((result) => {
+		if (result.success && result.data !== void 0) {
+			if (result.data.images !== void 0 && result.data.images.length > 0) mergedResults.images = [...mergedResults.images ?? [], ...result.data.images];
+			if (result.data.videos !== void 0 && result.data.videos.length > 0) mergedResults.videos = [...mergedResults.videos ?? [], ...result.data.videos];
+			if (result.data.news !== void 0 && result.data.news.length > 0) {
+				const newsAsTopStories = result.data.news.map((newsItem) => ({
+					...newsItem,
+					link: newsItem.link ?? ""
+				}));
+				mergedResults.topStories = [...mergedResults.topStories ?? [], ...newsAsTopStories];
+			}
+		}
+	});
+	if (mergedResults.topStories !== void 0 && mergedResults.topStories.length > 1) {
+		/** The main search's own news results and the parallel news sub-search
+		* frequently return the same stories — keep the first occurrence of each
+		* link so duplicates aren't scraped, reranked, and formatted repeatedly */
+		const seenLinks = /* @__PURE__ */ new Set();
+		mergedResults.topStories = mergedResults.topStories.filter((story) => {
+			if (!story.link || seenLinks.has(story.link)) return false;
+			seenLinks.add(story.link);
+			return true;
+		});
+	}
+	return {
+		success: true,
+		data: mergedResults
+	};
 }
-function createSearchProcessor({ searchAPI, safeSearch, supportsVideos, sourceProcessor, onGetHighlights, logger, }) {
-    return async function ({ query, date, country, proMode = true, maxSources = 5, onSearchResults, images = false, videos = false, news = false, }) {
-        try {
-            // Execute parallel searches and merge results
-            const searchResult = await executeParallelSearches({
-                searchAPI,
-                query,
-                date,
-                country,
-                safeSearch,
-                images,
-                videos: supportsVideos && videos,
-                news,
-                logger,
-            });
-            onSearchResults?.(searchResult);
-            const processedSources = await sourceProcessor.processSources({
-                query,
-                news,
-                result: searchResult,
-                proMode,
-                onGetHighlights,
-                numElements: maxSources,
-            });
-            return expandHighlights(processedSources);
-        }
-        catch (error) {
-            logger.error('Error in search:', error);
-            return {
-                organic: [],
-                topStories: [],
-                images: [],
-                videos: [],
-                news: [],
-                relatedSearches: [],
-                error: error instanceof Error ? error.message : String(error),
-            };
-        }
-    };
+function createSearchProcessor({ searchAPI, safeSearch, supportsVideos, sourceProcessor, onGetHighlights, logger }) {
+	return async function({ query, date, country, proMode = true, maxSources = 5, onSearchResults, images = false, videos = false, news = false }) {
+		try {
+			const searchResult = await executeParallelSearches({
+				searchAPI,
+				query,
+				date,
+				country,
+				safeSearch,
+				images,
+				videos: supportsVideos && videos,
+				news,
+				logger
+			});
+			onSearchResults?.(searchResult);
+			return expandHighlights(await sourceProcessor.processSources({
+				query,
+				news,
+				result: searchResult,
+				proMode,
+				onGetHighlights,
+				numElements: maxSources
+			}));
+		} catch (error) {
+			logger.error("Error in search:", error);
+			return {
+				organic: [],
+				topStories: [],
+				images: [],
+				videos: [],
+				news: [],
+				relatedSearches: [],
+				error: error instanceof Error ? error.message : String(error)
+			};
+		}
+	};
 }
-function createOnSearchResults({ runnableConfig, onSearchResults, }) {
-    return function (results) {
-        if (!onSearchResults) {
-            return;
-        }
-        onSearchResults(results, runnableConfig);
-    };
+function createOnSearchResults({ runnableConfig, onSearchResults }) {
+	return function(results) {
+		if (!onSearchResults) return;
+		onSearchResults(results, runnableConfig);
+	};
 }
-function createTool({ schema, search, onSearchResults: _onSearchResults, }) {
-    return tool(async (rawParams, runnableConfig) => {
-        const params = rawParams;
-        const { query, date, country: _c, images, videos, news } = params;
-        const country = typeof _c === 'string' && _c ? _c : undefined;
-        const searchResult = await search({
-            query,
-            date,
-            country,
-            images,
-            videos,
-            news,
-            onSearchResults: createOnSearchResults({
-                runnableConfig,
-                onSearchResults: _onSearchResults,
-            }),
-        });
-        const turn = runnableConfig.toolCall?.turn ?? 0;
-        const { output, references } = formatResultsForLLM(turn, searchResult);
-        const data = { turn, ...searchResult, references };
-        return [output, { [Constants.WEB_SEARCH]: data }];
-    }, {
-        name: WebSearchToolName,
-        description: WebSearchToolDescription,
-        schema: schema,
-        responseFormat: Constants.CONTENT_AND_ARTIFACT,
-    });
+function createTool({ schema, search, onSearchResults: _onSearchResults }) {
+	return tool(async (rawParams, runnableConfig) => {
+		const { query, date, country: _c, images, videos, news } = rawParams;
+		const searchResult = await search({
+			query,
+			date,
+			country: typeof _c === "string" && _c ? _c : void 0,
+			images,
+			videos,
+			news,
+			onSearchResults: createOnSearchResults({
+				runnableConfig,
+				onSearchResults: _onSearchResults
+			})
+		});
+		const turn = runnableConfig.toolCall?.turn ?? 0;
+		const { output, references } = formatResultsForLLM(turn, searchResult);
+		const data = {
+			turn,
+			...searchResult,
+			references
+		};
+		return [output, { ["web_search"]: data }];
+	}, {
+		name: WebSearchToolName,
+		description: WebSearchToolDescription,
+		schema,
+		responseFormat: "content_and_artifact"
+	});
 }
 const createSearchTool = (config = {}) => {
-    const { searchProvider = 'serper', serperApiKey, searxngInstanceUrl, searxngApiKey, tavilyApiKey, tavilySearchUrl, tavilyExtractUrl, tavilySearchOptions, rerankerType = 'cohere', topResults = 5, strategies = ['no_extraction'], filterContent = true, safeSearch = 1, scraperProvider = 'firecrawl', firecrawlApiKey, firecrawlApiUrl, firecrawlVersion, firecrawlOptions, serperScraperOptions, tavilyScraperOptions, scraperTimeout, jinaApiKey, jinaApiUrl, cohereApiKey, onSearchResults: _onSearchResults, onGetHighlights, } = config;
-    const logger = config.logger || createDefaultLogger();
-    const effectiveTavilySearchOptions = searchProvider === 'tavily' && config.safeSearch != null
-        ? {
-            ...tavilySearchOptions,
-            safeSearch: config.safeSearch !== 0,
-        }
-        : tavilySearchOptions;
-    const schemaProperties = {
-        query: querySchema,
-        date: dateSchema,
-        images: imagesSchema,
-        videos: videosSchema,
-        news: newsSchema,
-    };
-    if (searchProvider === 'serper' || searchProvider === 'tavily') {
-        schemaProperties.country = countrySchema;
-    }
-    const toolSchema = {
-        type: 'object',
-        properties: schemaProperties,
-        required: ['query'],
-    };
-    const searchAPI = createSearchAPI({
-        searchProvider,
-        serperApiKey,
-        searxngInstanceUrl,
-        searxngApiKey,
-        tavilyApiKey,
-        tavilySearchUrl,
-        tavilySearchOptions: effectiveTavilySearchOptions,
-    });
-    /** Create scraper based on scraperProvider */
-    let scraperInstance;
-    if (scraperProvider === 'serper') {
-        scraperInstance = createSerperScraper({
-            ...serperScraperOptions,
-            apiKey: serperApiKey,
-            timeout: scraperTimeout ?? serperScraperOptions?.timeout,
-            logger,
-        });
-    }
-    else if (scraperProvider === 'tavily') {
-        scraperInstance = createTavilyScraper({
-            ...tavilyScraperOptions,
-            apiKey: tavilyScraperOptions?.apiKey ??
-                tavilyApiKey ??
-                process.env.TAVILY_API_KEY,
-            apiUrl: tavilyScraperOptions?.apiUrl ?? tavilyExtractUrl,
-            timeout: scraperTimeout ?? tavilyScraperOptions?.timeout,
-            logger,
-        });
-    }
-    else {
-        scraperInstance = createFirecrawlScraper({
-            ...firecrawlOptions,
-            apiKey: firecrawlApiKey ?? process.env.FIRECRAWL_API_KEY,
-            apiUrl: firecrawlApiUrl,
-            version: firecrawlVersion,
-            timeout: scraperTimeout ?? firecrawlOptions?.timeout,
-            formats: firecrawlOptions?.formats ?? ['markdown', 'rawHtml'],
-            logger,
-        });
-    }
-    const selectedReranker = createReranker({
-        rerankerType,
-        jinaApiKey,
-        jinaApiUrl,
-        cohereApiKey,
-        logger,
-    });
-    if (!selectedReranker) {
-        logger.warn('No reranker selected. Using default ranking.');
-    }
-    const sourceProcessor = createSourceProcessor({
-        reranker: selectedReranker,
-        topResults,
-        logger,
-    }, scraperInstance);
-    const search = createSearchProcessor({
-        searchAPI,
-        safeSearch,
-        supportsVideos: searchProvider !== 'tavily',
-        sourceProcessor,
-        onGetHighlights,
-        logger,
-    });
-    return createTool({
-        search,
-        schema: toolSchema,
-        onSearchResults: _onSearchResults,
-    });
+	const { searchProvider = "serper", serperApiKey, searxngInstanceUrl, searxngApiKey, tavilyApiKey, tavilySearchUrl, tavilyExtractUrl, tavilySearchOptions, rerankerType = "cohere", topResults = 5, maxContentLength, strategies = ["no_extraction"], filterContent = true, safeSearch = 1, scraperProvider = "firecrawl", firecrawlApiKey, firecrawlApiUrl, firecrawlVersion, firecrawlOptions, serperScraperOptions, tavilyScraperOptions, scraperTimeout, jinaApiKey, jinaApiUrl, cohereApiKey, onSearchResults: _onSearchResults, onGetHighlights } = config;
+	const logger = config.logger || createDefaultLogger();
+	const effectiveTavilySearchOptions = searchProvider === "tavily" && config.safeSearch != null ? {
+		...tavilySearchOptions,
+		safeSearch: config.safeSearch !== 0
+	} : tavilySearchOptions;
+	const schemaProperties = {
+		query: querySchema,
+		date: dateSchema,
+		images: imagesSchema,
+		videos: videosSchema,
+		news: newsSchema
+	};
+	if (searchProvider === "serper" || searchProvider === "tavily") schemaProperties.country = countrySchema;
+	const toolSchema = {
+		type: "object",
+		properties: schemaProperties,
+		required: ["query"]
+	};
+	const searchAPI = createSearchAPI({
+		searchProvider,
+		serperApiKey,
+		searxngInstanceUrl,
+		searxngApiKey,
+		tavilyApiKey,
+		tavilySearchUrl,
+		tavilySearchOptions: effectiveTavilySearchOptions
+	});
+	/** Create scraper based on scraperProvider */
+	let scraperInstance;
+	if (scraperProvider === "serper") scraperInstance = createSerperScraper({
+		...serperScraperOptions,
+		apiKey: serperApiKey,
+		timeout: scraperTimeout ?? serperScraperOptions?.timeout,
+		logger
+	});
+	else if (scraperProvider === "tavily") scraperInstance = createTavilyScraper({
+		...tavilyScraperOptions,
+		apiKey: tavilyScraperOptions?.apiKey ?? tavilyApiKey ?? process.env.TAVILY_API_KEY,
+		apiUrl: tavilyScraperOptions?.apiUrl ?? tavilyExtractUrl,
+		timeout: scraperTimeout ?? tavilyScraperOptions?.timeout,
+		logger
+	});
+	else scraperInstance = createFirecrawlScraper({
+		...firecrawlOptions,
+		apiKey: firecrawlApiKey ?? process.env.FIRECRAWL_API_KEY,
+		apiUrl: firecrawlApiUrl,
+		version: firecrawlVersion,
+		timeout: scraperTimeout ?? firecrawlOptions?.timeout,
+		formats: firecrawlOptions?.formats ?? ["markdown", "rawHtml"],
+		logger
+	});
+	const selectedReranker = createReranker({
+		rerankerType,
+		jinaApiKey,
+		jinaApiUrl,
+		cohereApiKey,
+		logger
+	});
+	if (!selectedReranker) logger.warn("No reranker selected. Using default ranking.");
+	const sourceProcessor = createSourceProcessor({
+		reranker: selectedReranker,
+		topResults,
+		maxContentLength,
+		strategies,
+		filterContent,
+		logger
+	}, scraperInstance);
+	return createTool({
+		search: createSearchProcessor({
+			searchAPI,
+			safeSearch,
+			supportsVideos: searchProvider !== "tavily",
+			sourceProcessor,
+			onGetHighlights,
+			logger
+		}),
+		schema: toolSchema,
+		onSearchResults: _onSearchResults
+	});
 };
+//#endregion
+export { createSearchTool, executeParallelSearches };
 
-export { createSearchTool };
-//# sourceMappingURL=tool.mjs.map
+//# sourceMappingURL=tool.mjs.map