@librechat/agents 2.4.320 → 2.4.322

package/src/tools/search/search.ts

@@ -1,10 +1,9 @@
-/* eslint-disable no-console */
 import axios from 'axios';
 import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
 import type * as t from './types';
+import { getAttribution, createDefaultLogger } from './utils';
 import { FirecrawlScraper } from './firecrawl';
 import { BaseReranker } from './rerankers';
-import { getAttribution } from './utils';
 
 const chunker = {
   cleanText: (text: string): string => {
@@ -52,12 +51,14 @@ const chunker = {
       chunkSize?: number;
       chunkOverlap?: number;
       separators?: string[];
-    }
+    },
+    logger?: t.Logger
   ): Promise<string[][]> => {
     // Split multiple texts
+    const logger_ = logger || createDefaultLogger();
     const promises = texts.map((text) =>
       chunker.splitText(text, options).catch((error) => {
-        console.error('Error splitting text:', error);
+        logger_.error('Error splitting text:', error);
         return [text];
       })
     );
@@ -82,18 +83,22 @@ const getHighlights = async ({
   content,
   reranker,
   topResults = 5,
+  logger,
 }: {
   content: string;
   query: string;
   reranker?: BaseReranker;
   topResults?: number;
+  logger?: t.Logger;
 }): Promise<t.Highlight[] | undefined> => {
+  const logger_ = logger || createDefaultLogger();
+
   if (!content) {
-    console.warn('No content provided for highlights');
+    logger_.warn('No content provided for highlights');
     return;
   }
   if (!reranker) {
-    console.warn('No reranker provided for highlights');
+    logger_.warn('No reranker provided for highlights');
     return;
   }
 
@@ -102,14 +107,14 @@ const getHighlights = async ({
     if (Array.isArray(documents)) {
       return await reranker.rerank(query, documents, topResults);
     } else {
-      console.error(
+      logger_.error(
         'Expected documents to be an array, got:',
         typeof documents
       );
       return;
     }
   } catch (error) {
-    console.error('Error in content processing:', error);
+    logger_.error('Error in content processing:', error);
     return;
   }
 };
@@ -131,25 +136,49 @@ const createSerperAPI = (
 
   const getSources = async ({
     query,
+    date,
     country,
+    safeSearch,
     numResults = 8,
+    type,
   }: t.GetSourcesParams): Promise<t.SearchResult> => {
     if (!query.trim()) {
       return { success: false, error: 'Query cannot be empty' };
     }
 
     try {
+      const safe = ['off', 'moderate', 'active'] as const;
       const payload: t.SerperSearchPayload = {
         q: query,
+        safe: safe[safeSearch ?? 1],
         num: Math.min(Math.max(1, numResults), 10),
       };
 
+      // Set the search type if provided
+      if (type) {
+        payload.type = type;
+      }
+
+      if (date != null) {
+        payload.tbs = `qdr:${date}`;
+      }
+
       if (country != null && country !== '') {
         payload['gl'] = country.toLowerCase();
       }
 
+      // Determine the API endpoint based on the search type
+      let apiEndpoint = config.apiUrl;
+      if (type === 'images') {
+        apiEndpoint = 'https://google.serper.dev/images';
+      } else if (type === 'videos') {
+        apiEndpoint = 'https://google.serper.dev/videos';
+      } else if (type === 'news') {
+        apiEndpoint = 'https://google.serper.dev/news';
+      }
+
       const response = await axios.post<t.SerperResultData>(
-        config.apiUrl,
+        apiEndpoint,
         payload,
         {
           headers: {
@@ -169,6 +198,8 @@ const createSerperAPI = (
         peopleAlsoAsk: data.peopleAlsoAsk,
         knowledgeGraph: data.knowledgeGraph,
         relatedSearches: data.relatedSearches,
+        videos: data.videos ?? [],
+        news: data.news ?? [],
       };
 
       return { success: true, data: results };
@@ -202,6 +233,7 @@ const createSearXNGAPI = (
   const getSources = async ({
     query,
     numResults = 8,
+    type,
   }: t.GetSourcesParams): Promise<t.SearchResult> => {
     if (!query.trim()) {
       return { success: false, error: 'Query cannot be empty' };
@@ -218,12 +250,22 @@ const createSearXNGAPI = (
         searchUrl = searchUrl.replace(/\/$/, '') + '/search';
       }
 
+      // Determine the search category based on the type
+      let category = 'general';
+      if (type === 'images') {
+        category = 'images';
+      } else if (type === 'videos') {
+        category = 'videos';
+      } else if (type === 'news') {
+        category = 'news';
+      }
+
       // Prepare parameters for SearXNG
       const params: t.SearxNGSearchPayload = {
         q: query,
         format: 'json',
         pageno: 1,
-        categories: 'general',
+        categories: category,
         language: 'all',
         safesearch: 0,
         engines: 'google,bing,duckduckgo',
@@ -271,6 +313,8 @@ const createSearXNGAPI = (
         topStories: [],
         // Use undefined instead of null for optional properties
         relatedSearches: data.suggestions ?? [],
+        videos: [],
+        news: [],
       };
 
       return { success: true, data: results };
@@ -327,8 +371,10 @@ export const createSourceProcessor = (
     // strategies = ['no_extraction'],
     // filterContent = true,
     reranker,
+    logger,
   } = config;
 
+  const logger_ = logger || createDefaultLogger();
   const firecrawlScraper = scraperInstance;
 
   const webScraper = {
@@ -341,7 +387,7 @@ export const createSourceProcessor = (
       links: string[];
       onGetHighlights: t.SearchToolConfig['onGetHighlights'];
     }): Promise<Array<t.ScrapeResult>> => {
-      console.log(`Scraping ${links.length} links with Firecrawl`);
+      logger_.debug(`Scraping ${links.length} links with Firecrawl`);
       const promises: Array<Promise<t.ScrapeResult>> = [];
       try {
         for (let i = 0; i < links.length; i++) {
@@ -349,7 +395,11 @@ export const createSourceProcessor = (
           const promise: Promise<t.ScrapeResult> = firecrawlScraper
             .scrapeUrl(currentLink, {})
             .then(([url, response]) => {
-              const attribution = getAttribution(url, response.data?.metadata);
+              const attribution = getAttribution(
+                url,
+                response.data?.metadata,
+                logger_
+              );
               if (response.success && response.data) {
                 const [content, references] =
                   firecrawlScraper.extractContent(response);
@@ -365,14 +415,15 @@ export const createSourceProcessor = (
                 url,
                 attribution,
                 error: true,
-                content: `Failed to scrape ${url}: ${response.error ?? 'Unknown error'}`,
+                content: '',
               } as t.ScrapeResult;
             })
             .then(async (result) => {
               try {
                 if (result.error != null) {
-                  console.error(
-                    `Error scraping ${result.url}: ${result.content}`
+                  logger_.error(
+                    `Error scraping ${result.url}: ${result.content}`,
+                    result.error
                   );
                   return {
                     ...result,
@@ -382,6 +433,7 @@ export const createSourceProcessor = (
                   query,
                   reranker,
                   content: result.content,
+                  logger: logger_,
                 });
                 if (onGetHighlights) {
                   onGetHighlights(result.url);
@@ -391,25 +443,25 @@ export const createSourceProcessor = (
                   highlights,
                 };
               } catch (error) {
-                console.error('Error processing scraped content:', error);
+                logger_.error('Error processing scraped content:', error);
                 return {
                   ...result,
                 };
               }
             })
             .catch((error) => {
-              console.error(`Error scraping ${currentLink}:`, error);
+              logger_.error(`Error scraping ${currentLink}:`, error);
               return {
                 url: currentLink,
                 error: true,
-                content: `Failed to scrape ${currentLink}: ${error.message ?? 'Unknown error'}`,
+                content: '',
               };
             });
           promises.push(promise);
         }
         return await Promise.all(promises);
       } catch (error) {
-        console.error('Error in scrapeMany:', error);
+        logger_.error('Error in scrapeMany:', error);
         return [];
       }
     },
@@ -453,6 +505,7 @@ export const createSourceProcessor = (
     result,
     numElements,
     query,
+    news,
     proMode = true,
     onGetHighlights,
   }: t.ProcessSourcesFields): Promise<t.SearchResultData> => {
@@ -503,52 +556,69 @@ export const createSourceProcessor = (
       }
 
       const sourceMap = new Map<string, t.ValidSource>();
-      const allLinks: string[] = [];
+      const organicLinksSet = new Set<string>();
 
-      for (const source of result.data.organic) {
-        if (source.link) {
-          allLinks.push(source.link);
-          sourceMap.set(source.link, source);
-        }
-      }
+      // Collect organic links
+      const organicLinks = collectLinks(
+        result.data.organic,
+        sourceMap,
+        organicLinksSet
+      );
 
-      if (allLinks.length === 0) {
+      // Collect top story links, excluding any that are already in organic links
+      const topStories = result.data.topStories ?? [];
+      const topStoryLinks = collectLinks(
+        topStories,
+        sourceMap,
+        organicLinksSet
+      );
+
+      if (organicLinks.length === 0 && (topStoryLinks.length === 0 || !news)) {
         return result.data;
       }
 
       const onContentScraped = createSourceUpdateCallback(sourceMap);
-      await fetchContents({
-        query,
-        links: allLinks,
-        onGetHighlights,
-        onContentScraped,
-        target: numElements,
-      });
+      const promises: Promise<void>[] = [];
+
+      // Process organic links
+      if (organicLinks.length > 0) {
+        promises.push(
+          fetchContents({
+            query,
+            onGetHighlights,
+            onContentScraped,
+            links: organicLinks,
+            target: numElements,
+          })
+        );
+      }
 
-      for (let i = 0; i < result.data.organic.length; i++) {
-        const source = result.data.organic[i];
-        const updatedSource = sourceMap.get(source.link);
-        if (updatedSource) {
-          result.data.organic[i] = {
-            ...source,
-            ...updatedSource,
-          };
-        }
+      // Process top story links
+      if (news && topStoryLinks.length > 0) {
+        promises.push(
+          fetchContents({
+            query,
+            onGetHighlights,
+            onContentScraped,
+            links: topStoryLinks,
+            target: numElements,
+          })
+        );
       }
 
-      const successfulSources = result.data.organic
-        .filter(
-          (source) =>
-            source.content != null && !source.content.startsWith('Failed')
-        )
-        .slice(0, numElements);
+      await Promise.all(promises);
 
-      if (successfulSources.length > 0) {
-        result.data.organic = successfulSources;
+      if (result.data.organic.length > 0) {
+        updateSourcesWithContent(result.data.organic, sourceMap);
       }
+
+      if (news && topStories.length > 0) {
+        updateSourcesWithContent(topStories, sourceMap);
+      }
+
       return result.data;
     } catch (error) {
-      console.error('Error in processSources:', error);
+      logger_.error('Error in processSources:', error);
       return {
         organic: [],
         topStories: [],
@@ -565,3 +635,46 @@ export const createSourceProcessor = (
     topResults,
   };
 };
+
+/** Helper function to collect links and update sourceMap */
+function collectLinks(
+  sources: Array<t.OrganicResult | t.TopStoryResult>,
+  sourceMap: Map<string, t.ValidSource>,
+  existingLinksSet?: Set<string>
+): string[] {
+  const links: string[] = [];
+
+  for (const source of sources) {
+    if (source.link) {
+      // For topStories, only add if not already in organic links
+      if (existingLinksSet && existingLinksSet.has(source.link)) {
+        continue;
+      }
+
+      links.push(source.link);
+      if (existingLinksSet) {
+        existingLinksSet.add(source.link);
+      }
+      sourceMap.set(source.link, source as t.ValidSource);
+    }
+  }
+
+  return links;
+}
+
+/** Helper function to update sources with scraped content */
+function updateSourcesWithContent<T extends t.ValidSource>(
+  sources: T[],
+  sourceMap: Map<string, t.ValidSource>
+): void {
+  for (let i = 0; i < sources.length; i++) {
+    const source = sources[i];
+    const updatedSource = sourceMap.get(source.link);
+    if (updatedSource) {
+      sources[i] = {
+        ...source,
+        ...updatedSource,
+      } as T;
+    }
+  }
+}