@librechat/agents 2.4.31 → 2.4.33

package/src/tools/search/search.ts

@@ -1,8 +1,8 @@
-/* eslint-disable no-console */
 import axios from 'axios';
 import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
 import type * as t from './types';
-import { getAttribution, FirecrawlScraper } from './firecrawl';
+import { getAttribution, createDefaultLogger } from './utils';
+import { FirecrawlScraper } from './firecrawl';
 import { BaseReranker } from './rerankers';
 
 const chunker = {
@@ -51,12 +51,14 @@ const chunker = {
       chunkSize?: number;
       chunkOverlap?: number;
       separators?: string[];
-    }
+    },
+    logger?: t.Logger
   ): Promise<string[][]> => {
     // Split multiple texts
+    const logger_ = logger || createDefaultLogger();
     const promises = texts.map((text) =>
       chunker.splitText(text, options).catch((error) => {
-        console.error('Error splitting text:', error);
+        logger_.error('Error splitting text:', error);
         return [text];
       })
     );
@@ -64,7 +66,7 @@ const chunker = {
   },
 };
 
-const createSourceUpdateCallback = (sourceMap: Map<string, t.ValidSource>) => {
+function createSourceUpdateCallback(sourceMap: Map<string, t.ValidSource>) {
   return (link: string, update?: Partial<t.ValidSource>): void => {
     const source = sourceMap.get(link);
     if (source) {
@@ -74,25 +76,29 @@ const createSourceUpdateCallback = (sourceMap: Map<string, t.ValidSource>) => {
       });
     }
   };
-};
+}
 
 const getHighlights = async ({
   query,
   content,
   reranker,
   topResults = 5,
+  logger,
 }: {
   content: string;
   query: string;
   reranker?: BaseReranker;
   topResults?: number;
+  logger?: t.Logger;
 }): Promise<t.Highlight[] | undefined> => {
+  const logger_ = logger || createDefaultLogger();
+
   if (!content) {
-    console.warn('No content provided for highlights');
+    logger_.warn('No content provided for highlights');
     return;
   }
   if (!reranker) {
-    console.warn('No reranker provided for highlights');
+    logger_.warn('No reranker provided for highlights');
     return;
   }
 
@@ -101,14 +107,14 @@ const getHighlights = async ({
     if (Array.isArray(documents)) {
       return await reranker.rerank(query, documents, topResults);
     } else {
-      console.error(
+      logger_.error(
         'Expected documents to be an array, got:',
         typeof documents
       );
       return;
     }
   } catch (error) {
-    console.error('Error in content processing:', error);
+    logger_.error('Error in content processing:', error);
     return;
   }
 };
@@ -116,16 +122,11 @@ const getHighlights = async ({
 const createSerperAPI = (
   apiKey?: string
 ): {
-  getSources: (
-    query: string,
-    numResults?: number,
-    storedLocation?: string
-  ) => Promise<t.SearchResult>;
+  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;
 } => {
   const config = {
     apiKey: apiKey ?? process.env.SERPER_API_KEY,
     apiUrl: 'https://google.serper.dev/search',
-    defaultLocation: 'us',
     timeout: 10000,
   };
 
@@ -133,43 +134,72 @@ const createSerperAPI = (
     throw new Error('SERPER_API_KEY is required for SerperAPI');
   }
 
-  const getSources = async (
-    query: string,
-    numResults: number = 8,
-    storedLocation?: string
-  ): Promise<t.SearchResult> => {
+  const getSources = async ({
+    query,
+    date,
+    country,
+    safeSearch,
+    numResults = 8,
+    type,
+  }: t.GetSourcesParams): Promise<t.SearchResult> => {
     if (!query.trim()) {
       return { success: false, error: 'Query cannot be empty' };
     }
 
     try {
-      const searchLocation = (
-        storedLocation ?? config.defaultLocation
-      ).toLowerCase();
-
-      const payload = {
+      const safe = ['off', 'moderate', 'active'] as const;
+      const payload: t.SerperSearchPayload = {
         q: query,
+        safe: safe[safeSearch ?? 1],
         num: Math.min(Math.max(1, numResults), 10),
-        gl: searchLocation,
       };
 
-      const response = await axios.post(config.apiUrl, payload, {
-        headers: {
-          'X-API-KEY': config.apiKey,
-          'Content-Type': 'application/json',
-        },
-        timeout: config.timeout,
-      });
+      // Set the search type if provided
+      if (type) {
+        payload.type = type;
+      }
+
+      if (date != null) {
+        payload.tbs = `qdr:${date}`;
+      }
+
+      if (country != null && country !== '') {
+        payload['gl'] = country.toLowerCase();
+      }
+
+      // Determine the API endpoint based on the search type
+      let apiEndpoint = config.apiUrl;
+      if (type === 'images') {
+        apiEndpoint = 'https://google.serper.dev/images';
+      } else if (type === 'videos') {
+        apiEndpoint = 'https://google.serper.dev/videos';
+      } else if (type === 'news') {
+        apiEndpoint = 'https://google.serper.dev/news';
+      }
+
+      const response = await axios.post<t.SerperResultData>(
+        apiEndpoint,
+        payload,
+        {
+          headers: {
+            'X-API-KEY': config.apiKey,
+            'Content-Type': 'application/json',
+          },
+          timeout: config.timeout,
+        }
+      );
 
       const data = response.data;
       const results: t.SearchResultData = {
         organic: data.organic,
         images: data.images ?? [],
+        answerBox: data.answerBox,
         topStories: data.topStories ?? [],
-        knowledgeGraph: data.knowledgeGraph as t.KnowledgeGraphResult,
-        answerBox: data.answerBox as t.AnswerBoxResult,
-        peopleAlsoAsk: data.peopleAlsoAsk as t.PeopleAlsoAskResult[],
-        relatedSearches: data.relatedSearches as string[],
+        peopleAlsoAsk: data.peopleAlsoAsk,
+        knowledgeGraph: data.knowledgeGraph,
+        relatedSearches: data.relatedSearches,
+        videos: data.videos ?? [],
+        news: data.news ?? [],
       };
 
       return { success: true, data: results };
@@ -187,11 +217,7 @@ const createSearXNGAPI = (
   instanceUrl?: string,
   apiKey?: string
 ): {
-  getSources: (
-    query: string,
-    numResults?: number,
-    storedLocation?: string
-  ) => Promise<t.SearchResult>;
+  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;
 } => {
   const config = {
     instanceUrl: instanceUrl ?? process.env.SEARXNG_INSTANCE_URL,
@@ -204,11 +230,11 @@ const createSearXNGAPI = (
     throw new Error('SEARXNG_INSTANCE_URL is required for SearXNG API');
   }
 
-  const getSources = async (
-    query: string,
-    numResults: number = 8,
-    storedLocation?: string
-  ): Promise<t.SearchResult> => {
+  const getSources = async ({
+    query,
+    numResults = 8,
+    type,
+  }: t.GetSourcesParams): Promise<t.SearchResult> => {
     if (!query.trim()) {
       return { success: false, error: 'Query cannot be empty' };
     }
@@ -224,22 +250,27 @@ const createSearXNGAPI = (
         searchUrl = searchUrl.replace(/\/$/, '') + '/search';
       }
 
+      // Determine the search category based on the type
+      let category = 'general';
+      if (type === 'images') {
+        category = 'images';
+      } else if (type === 'videos') {
+        category = 'videos';
+      } else if (type === 'news') {
+        category = 'news';
+      }
+
       // Prepare parameters for SearXNG
-      const params: Record<string, string | number> = {
+      const params: t.SearxNGSearchPayload = {
         q: query,
         format: 'json',
         pageno: 1,
-        categories: 'general',
+        categories: category,
         language: 'all',
         safesearch: 0,
         engines: 'google,bing,duckduckgo',
-        max_results: Math.min(Math.max(1, numResults), 20),
       };
 
-      if (storedLocation != null && storedLocation !== 'all') {
-        params.language = storedLocation;
-      }
-
       const headers: Record<string, string> = {
         'Content-Type': 'application/json',
       };
@@ -282,6 +313,8 @@ const createSearXNGAPI = (
         topStories: [],
         // Use undefined instead of null for optional properties
         relatedSearches: data.suggestions ?? [],
+        videos: [],
+        news: [],
       };
 
       return { success: true, data: results };
@@ -301,11 +334,7 @@ const createSearXNGAPI = (
 export const createSearchAPI = (
   config: t.SearchConfig
 ): {
-  getSources: (
-    query: string,
-    numResults?: number,
-    storedLocation?: string
-  ) => Promise<t.SearchResult>;
+  getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;
 } => {
   const {
     searchProvider = 'serper',
@@ -330,10 +359,7 @@ export const createSourceProcessor = (
   scraperInstance?: FirecrawlScraper
 ): {
   processSources: (
-    result: t.SearchResult,
-    numElements: number,
-    query: string,
-    proMode?: boolean
+    fields: t.ProcessSourcesFields
   ) => Promise<t.SearchResultData>;
   topResults: number;
 } => {
@@ -345,47 +371,59 @@ export const createSourceProcessor = (
     // strategies = ['no_extraction'],
     // filterContent = true,
     reranker,
+    logger,
   } = config;
 
+  const logger_ = logger || createDefaultLogger();
   const firecrawlScraper = scraperInstance;
 
   const webScraper = {
     scrapeMany: async ({
       query,
       links,
+      onGetHighlights,
     }: {
       query: string;
       links: string[];
+      onGetHighlights: t.SearchToolConfig['onGetHighlights'];
     }): Promise<Array<t.ScrapeResult>> => {
-      console.log(`Scraping ${links.length} links with Firecrawl`);
+      logger_.debug(`Scraping ${links.length} links with Firecrawl`);
       const promises: Array<Promise<t.ScrapeResult>> = [];
       try {
-        for (const currentLink of links) {
+        for (let i = 0; i < links.length; i++) {
+          const currentLink = links[i];
           const promise: Promise<t.ScrapeResult> = firecrawlScraper
             .scrapeUrl(currentLink, {})
             .then(([url, response]) => {
-              const attribution = getAttribution(url, response.data?.metadata);
+              const attribution = getAttribution(
+                url,
+                response.data?.metadata,
+                logger_
+              );
               if (response.success && response.data) {
-                const content = firecrawlScraper.extractContent(response);
+                const [content, references] =
+                  firecrawlScraper.extractContent(response);
                 return {
                   url,
+                  references,
                   attribution,
                   content: chunker.cleanText(content),
-                };
+                } as t.ScrapeResult;
               }
 
               return {
                 url,
                 attribution,
                 error: true,
-                content: `Failed to scrape ${url}: ${response.error ?? 'Unknown error'}`,
-              };
+                content: '',
+              } as t.ScrapeResult;
             })
             .then(async (result) => {
               try {
                 if (result.error != null) {
-                  console.error(
-                    `Error scraping ${result.url}: ${result.content}`
+                  logger_.error(
+                    `Error scraping ${result.url}: ${result.content}`,
+                    result.error
                   );
                   return {
                     ...result,
@@ -395,31 +433,35 @@ export const createSourceProcessor = (
                   query,
                   reranker,
                   content: result.content,
+                  logger: logger_,
                 });
+                if (onGetHighlights) {
+                  onGetHighlights(result.url);
+                }
                 return {
                   ...result,
                   highlights,
                 };
               } catch (error) {
-                console.error('Error processing scraped content:', error);
+                logger_.error('Error processing scraped content:', error);
                 return {
                   ...result,
                 };
               }
             })
             .catch((error) => {
-              console.error(`Error scraping ${currentLink}:`, error);
+              logger_.error(`Error scraping ${currentLink}:`, error);
               return {
                 url: currentLink,
                 error: true,
-                content: `Failed to scrape ${currentLink}: ${error.message ?? 'Unknown error'}`,
+                content: '',
               };
             });
           promises.push(promise);
         }
         return await Promise.all(promises);
       } catch (error) {
-        console.error('Error in scrapeMany:', error);
+        logger_.error('Error in scrapeMany:', error);
         return [];
       }
     },
@@ -429,35 +471,44 @@ export const createSourceProcessor = (
     links,
     query,
     target,
+    onGetHighlights,
     onContentScraped,
   }: {
     links: string[];
     query: string;
     target: number;
+    onGetHighlights: t.SearchToolConfig['onGetHighlights'];
     onContentScraped?: (link: string, update?: Partial<t.ValidSource>) => void;
   }): Promise<void> => {
     const initialLinks = links.slice(0, target);
     // const remainingLinks = links.slice(target).reverse();
-    const results = await webScraper.scrapeMany({ query, links: initialLinks });
+    const results = await webScraper.scrapeMany({
+      query,
+      links: initialLinks,
+      onGetHighlights,
+    });
     for (const result of results) {
       if (result.error === true) {
         continue;
       }
-      const { url, content, attribution, highlights } = result;
+      const { url, content, attribution, references, highlights } = result;
       onContentScraped?.(url, {
         content,
         attribution,
+        references,
         highlights,
       });
     }
   };
 
-  const processSources = async (
-    result: t.SearchResult,
-    numElements: number,
-    query: string,
-    proMode: boolean = false
-  ): Promise<t.SearchResultData> => {
+  const processSources = async ({
+    result,
+    numElements,
+    query,
+    news,
+    proMode = true,
+    onGetHighlights,
+  }: t.ProcessSourcesFields): Promise<t.SearchResultData> => {
     try {
       if (!result.data) {
         return {
@@ -485,6 +536,7 @@ export const createSourceProcessor = (
         await fetchContents({
           query,
           target: 1,
+          onGetHighlights,
           onContentScraped,
           links: [wikiSources[0].link],
         });
@@ -504,51 +556,69 @@ export const createSourceProcessor = (
       }
 
       const sourceMap = new Map<string, t.ValidSource>();
-      const allLinks: string[] = [];
+      const organicLinksSet = new Set<string>();
 
-      for (const source of result.data.organic) {
-        if (source.link) {
-          allLinks.push(source.link);
-          sourceMap.set(source.link, source);
-        }
-      }
+      // Collect organic links
+      const organicLinks = collectLinks(
+        result.data.organic,
+        sourceMap,
+        organicLinksSet
+      );
+
+      // Collect top story links, excluding any that are already in organic links
+      const topStories = result.data.topStories ?? [];
+      const topStoryLinks = collectLinks(
+        topStories,
+        sourceMap,
+        organicLinksSet
+      );
 
-      if (allLinks.length === 0) {
+      if (organicLinks.length === 0 && (topStoryLinks.length === 0 || !news)) {
         return result.data;
       }
 
       const onContentScraped = createSourceUpdateCallback(sourceMap);
-      await fetchContents({
-        links: allLinks,
-        query,
-        onContentScraped,
-        target: numElements,
-      });
+      const promises: Promise<void>[] = [];
+
+      // Process organic links
+      if (organicLinks.length > 0) {
+        promises.push(
+          fetchContents({
+            query,
+            onGetHighlights,
+            onContentScraped,
+            links: organicLinks,
+            target: numElements,
+          })
+        );
+      }
 
-      for (let i = 0; i < result.data.organic.length; i++) {
-        const source = result.data.organic[i];
-        const updatedSource = sourceMap.get(source.link);
-        if (updatedSource) {
-          result.data.organic[i] = {
-            ...source,
-            ...updatedSource,
-          };
-        }
+      // Process top story links
+      if (news && topStoryLinks.length > 0) {
+        promises.push(
+          fetchContents({
+            query,
+            onGetHighlights,
+            onContentScraped,
+            links: topStoryLinks,
+            target: numElements,
+          })
+        );
       }
 
-      const successfulSources = result.data.organic
-        .filter(
-          (source) =>
-            source.content != null && !source.content.startsWith('Failed')
-        )
-        .slice(0, numElements);
+      await Promise.all(promises);
 
-      if (successfulSources.length > 0) {
-        result.data.organic = successfulSources;
+      if (result.data.organic.length > 0) {
+        updateSourcesWithContent(result.data.organic, sourceMap);
       }
+
+      if (news && topStories.length > 0) {
+        updateSourcesWithContent(topStories, sourceMap);
+      }
+
       return result.data;
     } catch (error) {
-      console.error('Error in processSources:', error);
+      logger_.error('Error in processSources:', error);
       return {
         organic: [],
         topStories: [],
@@ -565,3 +635,46 @@ export const createSourceProcessor = (
     topResults,
   };
 };
+
+/** Helper function to collect links and update sourceMap */
+function collectLinks(
+  sources: Array<t.OrganicResult | t.TopStoryResult>,
+  sourceMap: Map<string, t.ValidSource>,
+  existingLinksSet?: Set<string>
+): string[] {
+  const links: string[] = [];
+
+  for (const source of sources) {
+    if (source.link) {
+      // For topStories, only add if not already in organic links
+      if (existingLinksSet && existingLinksSet.has(source.link)) {
+        continue;
+      }
+
+      links.push(source.link);
+      if (existingLinksSet) {
+        existingLinksSet.add(source.link);
+      }
+      sourceMap.set(source.link, source as t.ValidSource);
+    }
+  }
+
+  return links;
+}
+
+/** Helper function to update sources with scraped content */
+function updateSourcesWithContent<T extends t.ValidSource>(
+  sources: T[],
+  sourceMap: Map<string, t.ValidSource>
+): void {
+  for (let i = 0; i < sources.length; i++) {
+    const source = sources[i];
+    const updatedSource = sourceMap.get(source.link);
+    if (updatedSource) {
+      sources[i] = {
+        ...source,
+        ...updatedSource,
+      } as T;
+    }
+  }
+}