@librechat/agents 2.4.321 → 3.0.0-rc1

package/src/tools/search/search.ts

@@ -1,10 +1,9 @@
-/* eslint-disable no-console */
 import axios from 'axios';
 import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
 import type * as t from './types';
+import { getAttribution, createDefaultLogger } from './utils';
 import { FirecrawlScraper } from './firecrawl';
 import { BaseReranker } from './rerankers';
-import { getAttribution } from './utils';
 
 const chunker = {
   cleanText: (text: string): string => {
@@ -52,12 +51,14 @@ const chunker = {
       chunkSize?: number;
       chunkOverlap?: number;
       separators?: string[];
-    }
+    },
+    logger?: t.Logger
   ): Promise<string[][]> => {
     // Split multiple texts
+    const logger_ = logger || createDefaultLogger();
     const promises = texts.map((text) =>
       chunker.splitText(text, options).catch((error) => {
-        console.error('Error splitting text:', error);
+        logger_.error('Error splitting text:', error);
         return [text];
       })
     );
@@ -82,18 +83,22 @@ const getHighlights = async ({
   content,
   reranker,
   topResults = 5,
+  logger,
 }: {
   content: string;
   query: string;
   reranker?: BaseReranker;
   topResults?: number;
+  logger?: t.Logger;
 }): Promise<t.Highlight[] | undefined> => {
+  const logger_ = logger || createDefaultLogger();
+
   if (!content) {
-    console.warn('No content provided for highlights');
+    logger_.warn('No content provided for highlights');
     return;
   }
   if (!reranker) {
-    console.warn('No reranker provided for highlights');
+    logger_.warn('No reranker provided for highlights');
     return;
   }
 
@@ -102,14 +107,14 @@ const getHighlights = async ({
     if (Array.isArray(documents)) {
       return await reranker.rerank(query, documents, topResults);
     } else {
-      console.error(
+      logger_.error(
         'Expected documents to be an array, got:',
         typeof documents
       );
       return;
     }
   } catch (error) {
-    console.error('Error in content processing:', error);
+    logger_.error('Error in content processing:', error);
     return;
   }
 };
@@ -131,25 +136,49 @@ const createSerperAPI = (
 
   const getSources = async ({
     query,
+    date,
     country,
+    safeSearch,
     numResults = 8,
+    type,
   }: t.GetSourcesParams): Promise<t.SearchResult> => {
     if (!query.trim()) {
       return { success: false, error: 'Query cannot be empty' };
     }
 
     try {
+      const safe = ['off', 'moderate', 'active'] as const;
       const payload: t.SerperSearchPayload = {
         q: query,
+        safe: safe[safeSearch ?? 1],
         num: Math.min(Math.max(1, numResults), 10),
       };
 
+      // Set the search type if provided
+      if (type) {
+        payload.type = type;
+      }
+
+      if (date != null) {
+        payload.tbs = `qdr:${date}`;
+      }
+
       if (country != null && country !== '') {
         payload['gl'] = country.toLowerCase();
       }
 
+      // Determine the API endpoint based on the search type
+      let apiEndpoint = config.apiUrl;
+      if (type === 'images') {
+        apiEndpoint = 'https://google.serper.dev/images';
+      } else if (type === 'videos') {
+        apiEndpoint = 'https://google.serper.dev/videos';
+      } else if (type === 'news') {
+        apiEndpoint = 'https://google.serper.dev/news';
+      }
+
       const response = await axios.post<t.SerperResultData>(
-        config.apiUrl,
+        apiEndpoint,
         payload,
         {
           headers: {
@@ -169,6 +198,8 @@ const createSerperAPI = (
         peopleAlsoAsk: data.peopleAlsoAsk,
         knowledgeGraph: data.knowledgeGraph,
         relatedSearches: data.relatedSearches,
+        videos: data.videos ?? [],
+        news: data.news ?? [],
       };
 
       return { success: true, data: results };
@@ -202,6 +233,8 @@ const createSearXNGAPI = (
   const getSources = async ({
     query,
     numResults = 8,
+    safeSearch,
+    type,
   }: t.GetSourcesParams): Promise<t.SearchResult> => {
     if (!query.trim()) {
       return { success: false, error: 'Query cannot be empty' };
@@ -218,14 +251,24 @@ const createSearXNGAPI = (
         searchUrl = searchUrl.replace(/\/$/, '') + '/search';
       }
 
+      // Determine the search category based on the type
+      let category = 'general';
+      if (type === 'images') {
+        category = 'images';
+      } else if (type === 'videos') {
+        category = 'videos';
+      } else if (type === 'news') {
+        category = 'news';
+      }
+
       // Prepare parameters for SearXNG
       const params: t.SearxNGSearchPayload = {
         q: query,
         format: 'json',
         pageno: 1,
-        categories: 'general',
+        categories: category,
         language: 'all',
-        safesearch: 0,
+        safesearch: safeSearch,
         engines: 'google,bing,duckduckgo',
       };
 
@@ -245,32 +288,111 @@ const createSearXNGAPI = (
 
       const data = response.data;
 
+      // Helper function to identify news results since SearXNG doesn't provide that classification by default
+      const isNewsResult = (result: t.SearXNGResult): boolean => {
+        const url = result.url?.toLowerCase() ?? '';
+        const title = result.title?.toLowerCase() ?? '';
+
+        // News-related keywords in title/content
+        const newsKeywords = [
+          'breaking news',
+          'latest news',
+          'top stories',
+          'news today',
+          'developing story',
+          'trending news',
+          'news',
+        ];
+
+        // Check if title/content contains news keywords
+        const hasNewsKeywords = newsKeywords.some(
+          (keyword) => title.toLowerCase().includes(keyword) // just title probably fine, content parsing is overkill for what we need: || content.includes(keyword)
+        );
+
+        // Check if URL contains news-related paths
+        const hasNewsPath =
+          url.includes('/news/') ||
+          url.includes('/world/') ||
+          url.includes('/politics/') ||
+          url.includes('/breaking/');
+
+        return hasNewsKeywords || hasNewsPath;
+      };
+
       // Transform SearXNG results to match SerperAPI format
       const organicResults = (data.results ?? [])
         .slice(0, numResults)
-        .map((result: t.SearXNGResult) => ({
-          title: result.title ?? '',
-          link: result.url ?? '',
-          snippet: result.content ?? '',
-          date: result.publishedDate ?? '',
-        }));
+        .map((result: t.SearXNGResult, index: number) => {
+          let attribution = '';
+          try {
+            attribution = new URL(result.url ?? '').hostname;
+          } catch {
+            attribution = '';
+          }
+
+          return {
+            position: index + 1,
+            title: result.title ?? '',
+            link: result.url ?? '',
+            snippet: result.content ?? '',
+            date: result.publishedDate ?? '',
+            attribution,
+          };
+        });
 
-      // Extract image results if available
       const imageResults = (data.results ?? [])
         .filter((result: t.SearXNGResult) => result.img_src)
         .slice(0, 6)
-        .map((result: t.SearXNGResult) => ({
+        .map((result: t.SearXNGResult, index: number) => ({
           title: result.title ?? '',
           imageUrl: result.img_src ?? '',
+          position: index + 1,
+          source: new URL(result.url ?? '').hostname,
+          domain: new URL(result.url ?? '').hostname,
+          link: result.url ?? '',
         }));
 
-      // Format results to match SerperAPI structure
+      // Extract news results from organic results
+      const newsResults = (data.results ?? [])
+        .filter(isNewsResult)
+        .map((result: t.SearXNGResult, index: number) => {
+          let attribution = '';
+          try {
+            attribution = new URL(result.url ?? '').hostname;
+          } catch {
+            attribution = '';
+          }
+
+          return {
+            title: result.title ?? '',
+            link: result.url ?? '',
+            snippet: result.content ?? '',
+            date: result.publishedDate ?? '',
+            source: attribution,
+            imageUrl: result.img_src ?? '',
+            position: index + 1,
+          };
+        });
+
+      const topStories = newsResults.slice(0, 5);
+
+      const relatedSearches = Array.isArray(data.suggestions)
+        ? data.suggestions.map((suggestion: string) => ({ query: suggestion }))
+        : [];
+
       const results: t.SearchResultData = {
         organic: organicResults,
         images: imageResults,
-        topStories: [],
-        // Use undefined instead of null for optional properties
-        relatedSearches: data.suggestions ?? [],
+        topStories: topStories, // Use first 5 extracted news as top stories
+        relatedSearches,
+        videos: [],
+        news: newsResults,
+        // Add empty arrays for other Serper fields to maintain parity
+        places: [],
+        shopping: [],
+        peopleAlsoAsk: [],
+        knowledgeGraph: undefined,
+        answerBox: undefined,
       };
 
       return { success: true, data: results };
@@ -327,8 +449,10 @@ export const createSourceProcessor = (
     // strategies = ['no_extraction'],
     // filterContent = true,
     reranker,
+    logger,
   } = config;
 
+  const logger_ = logger || createDefaultLogger();
   const firecrawlScraper = scraperInstance;
 
   const webScraper = {
@@ -341,7 +465,7 @@ export const createSourceProcessor = (
       links: string[];
       onGetHighlights: t.SearchToolConfig['onGetHighlights'];
     }): Promise<Array<t.ScrapeResult>> => {
-      console.log(`Scraping ${links.length} links with Firecrawl`);
+      logger_.debug(`Scraping ${links.length} links with Firecrawl`);
       const promises: Array<Promise<t.ScrapeResult>> = [];
       try {
         for (let i = 0; i < links.length; i++) {
@@ -349,7 +473,11 @@ export const createSourceProcessor = (
           const promise: Promise<t.ScrapeResult> = firecrawlScraper
             .scrapeUrl(currentLink, {})
             .then(([url, response]) => {
-              const attribution = getAttribution(url, response.data?.metadata);
+              const attribution = getAttribution(
+                url,
+                response.data?.metadata,
+                logger_
+              );
               if (response.success && response.data) {
                 const [content, references] =
                   firecrawlScraper.extractContent(response);
@@ -359,6 +487,10 @@ export const createSourceProcessor = (
                   attribution,
                   content: chunker.cleanText(content),
                 } as t.ScrapeResult;
+              } else {
+                logger_.error(
+                  `Error scraping ${url}: ${response.error ?? 'Unknown error'}`
+                );
               }
 
               return {
@@ -371,7 +503,7 @@ export const createSourceProcessor = (
             .then(async (result) => {
               try {
                 if (result.error != null) {
-                  console.error(
+                  logger_.error(
                     `Error scraping ${result.url}: ${result.content}`
                   );
                   return {
@@ -382,6 +514,7 @@ export const createSourceProcessor = (
                   query,
                   reranker,
                   content: result.content,
+                  logger: logger_,
                 });
                 if (onGetHighlights) {
                   onGetHighlights(result.url);
@@ -391,14 +524,14 @@ export const createSourceProcessor = (
                   highlights,
                 };
               } catch (error) {
-                console.error('Error processing scraped content:', error);
+                logger_.error('Error processing scraped content:', error);
                 return {
                   ...result,
                 };
               }
             })
             .catch((error) => {
-              console.error(`Error scraping ${currentLink}:`, error);
+              logger_.error(`Error scraping ${currentLink}:`, error);
               return {
                 url: currentLink,
                 error: true,
@@ -409,7 +542,7 @@ export const createSourceProcessor = (
         }
         return await Promise.all(promises);
       } catch (error) {
-        console.error('Error in scrapeMany:', error);
+        logger_.error('Error in scrapeMany:', error);
         return [];
       }
     },
@@ -453,6 +586,7 @@ export const createSourceProcessor = (
     result,
     numElements,
     query,
+    news,
     proMode = true,
     onGetHighlights,
   }: t.ProcessSourcesFields): Promise<t.SearchResultData> => {
@@ -520,7 +654,7 @@ export const createSourceProcessor = (
         organicLinksSet
       );
 
-      if (organicLinks.length === 0 && topStoryLinks.length === 0) {
+      if (organicLinks.length === 0 && (topStoryLinks.length === 0 || !news)) {
         return result.data;
       }
 
@@ -541,7 +675,7 @@ export const createSourceProcessor = (
       }
 
       // Process top story links
-      if (topStoryLinks.length > 0) {
+      if (news && topStoryLinks.length > 0) {
         promises.push(
           fetchContents({
             query,
@@ -555,18 +689,17 @@ export const createSourceProcessor = (
 
       await Promise.all(promises);
 
-      // Update sources with scraped content
       if (result.data.organic.length > 0) {
         updateSourcesWithContent(result.data.organic, sourceMap);
       }
 
-      if (topStories.length > 0) {
+      if (news && topStories.length > 0) {
         updateSourcesWithContent(topStories, sourceMap);
       }
 
       return result.data;
     } catch (error) {
-      console.error('Error in processSources:', error);
+      logger_.error('Error in processSources:', error);
       return {
         organic: [],
         topStories: [],