@librechat/agents 2.4.322 → 3.0.0-rc1

package/src/tools/search/search.ts

@@ -233,6 +233,7 @@ const createSearXNGAPI = (
   const getSources = async ({
     query,
     numResults = 8,
+    safeSearch,
     type,
   }: t.GetSourcesParams): Promise<t.SearchResult> => {
     if (!query.trim()) {
@@ -267,7 +268,7 @@ const createSearXNGAPI = (
         pageno: 1,
         categories: category,
         language: 'all',
-        safesearch: 0,
+        safesearch: safeSearch,
         engines: 'google,bing,duckduckgo',
       };
 
@@ -287,34 +288,111 @@ const createSearXNGAPI = (
 
       const data = response.data;
 
+      // Helper function to identify news results since SearXNG doesn't provide that classification by default
+      const isNewsResult = (result: t.SearXNGResult): boolean => {
+        const url = result.url?.toLowerCase() ?? '';
+        const title = result.title?.toLowerCase() ?? '';
+
+        // News-related keywords in title/content
+        const newsKeywords = [
+          'breaking news',
+          'latest news',
+          'top stories',
+          'news today',
+          'developing story',
+          'trending news',
+          'news',
+        ];
+
+        // Check if title/content contains news keywords
+        const hasNewsKeywords = newsKeywords.some(
+          (keyword) => title.toLowerCase().includes(keyword) // just title probably fine, content parsing is overkill for what we need: || content.includes(keyword)
+        );
+
+        // Check if URL contains news-related paths
+        const hasNewsPath =
+          url.includes('/news/') ||
+          url.includes('/world/') ||
+          url.includes('/politics/') ||
+          url.includes('/breaking/');
+
+        return hasNewsKeywords || hasNewsPath;
+      };
+
       // Transform SearXNG results to match SerperAPI format
       const organicResults = (data.results ?? [])
         .slice(0, numResults)
-        .map((result: t.SearXNGResult) => ({
-          title: result.title ?? '',
-          link: result.url ?? '',
-          snippet: result.content ?? '',
-          date: result.publishedDate ?? '',
-        }));
+        .map((result: t.SearXNGResult, index: number) => {
+          let attribution = '';
+          try {
+            attribution = new URL(result.url ?? '').hostname;
+          } catch {
+            attribution = '';
+          }
+
+          return {
+            position: index + 1,
+            title: result.title ?? '',
+            link: result.url ?? '',
+            snippet: result.content ?? '',
+            date: result.publishedDate ?? '',
+            attribution,
+          };
+        });
 
-      // Extract image results if available
       const imageResults = (data.results ?? [])
         .filter((result: t.SearXNGResult) => result.img_src)
         .slice(0, 6)
-        .map((result: t.SearXNGResult) => ({
+        .map((result: t.SearXNGResult, index: number) => ({
           title: result.title ?? '',
           imageUrl: result.img_src ?? '',
+          position: index + 1,
+          source: new URL(result.url ?? '').hostname,
+          domain: new URL(result.url ?? '').hostname,
+          link: result.url ?? '',
         }));
 
-      // Format results to match SerperAPI structure
+      // Extract news results from organic results
+      const newsResults = (data.results ?? [])
+        .filter(isNewsResult)
+        .map((result: t.SearXNGResult, index: number) => {
+          let attribution = '';
+          try {
+            attribution = new URL(result.url ?? '').hostname;
+          } catch {
+            attribution = '';
+          }
+
+          return {
+            title: result.title ?? '',
+            link: result.url ?? '',
+            snippet: result.content ?? '',
+            date: result.publishedDate ?? '',
+            source: attribution,
+            imageUrl: result.img_src ?? '',
+            position: index + 1,
+          };
+        });
+
+      const topStories = newsResults.slice(0, 5);
+
+      const relatedSearches = Array.isArray(data.suggestions)
+        ? data.suggestions.map((suggestion: string) => ({ query: suggestion }))
+        : [];
+
       const results: t.SearchResultData = {
         organic: organicResults,
         images: imageResults,
-        topStories: [],
-        // Use undefined instead of null for optional properties
-        relatedSearches: data.suggestions ?? [],
+        topStories: topStories, // Use first 5 extracted news as top stories
+        relatedSearches,
         videos: [],
-        news: [],
+        news: newsResults,
+        // Add empty arrays for other Serper fields to maintain parity
+        places: [],
+        shopping: [],
+        peopleAlsoAsk: [],
+        knowledgeGraph: undefined,
+        answerBox: undefined,
       };
 
       return { success: true, data: results };
@@ -409,6 +487,10 @@ export const createSourceProcessor = (
                   attribution,
                   content: chunker.cleanText(content),
                 } as t.ScrapeResult;
+              } else {
+                logger_.error(
+                  `Error scraping ${url}: ${response.error ?? 'Unknown error'}`
+                );
               }
 
               return {
@@ -422,8 +504,7 @@ export const createSourceProcessor = (
               try {
                 if (result.error != null) {
                   logger_.error(
-                    `Error scraping ${result.url}: ${result.content}`,
-                    result.error
+                    `Error scraping ${result.url}: ${result.content}`
                   );
                   return {
                     ...result,

package/src/tools/search/tool.ts

@@ -346,7 +346,8 @@ export const createSearchTool = (
     safeSearch = 1,
     firecrawlApiKey,
     firecrawlApiUrl,
-    firecrawlFormats = ['markdown', 'html'],
+    firecrawlOptions,
+    scraperTimeout,
     jinaApiKey,
     cohereApiKey,
     onSearchResults: _onSearchResults,
@@ -384,9 +385,11 @@ export const createSearchTool = (
   });
 
   const firecrawlScraper = createFirecrawlScraper({
+    ...firecrawlOptions,
     apiKey: firecrawlApiKey ?? process.env.FIRECRAWL_API_KEY,
     apiUrl: firecrawlApiUrl,
-    formats: firecrawlFormats,
+    timeout: scraperTimeout ?? firecrawlOptions?.timeout,
+    formats: firecrawlOptions?.formats ?? ['markdown', 'rawHtml'],
   });
 
   const selectedReranker = createReranker({

package/src/tools/search/types.ts

@@ -94,7 +94,7 @@ export interface ProcessSourcesConfig {
 export interface FirecrawlConfig {
   firecrawlApiKey?: string;
   firecrawlApiUrl?: string;
-  firecrawlFormats?: string[];
+  firecrawlOptions?: FirecrawlScraperConfig;
 }
 
 export interface ScraperContentResult {
@@ -150,6 +150,7 @@ export interface SearchToolConfig
   jinaApiKey?: string;
   cohereApiKey?: string;
   rerankerType?: RerankerType;
+  scraperTimeout?: number;
   onSearchResults?: (
     results: SearchResult,
     runnableConfig?: RunnableConfig
@@ -169,15 +170,10 @@ export type UsedReferences = {
 }[];
 
 /** Firecrawl */
-
-export interface FirecrawlScrapeOptions {
-  formats?: string[];
-  includeTags?: string[];
-  excludeTags?: string[];
-  headers?: Record<string, string>;
-  waitFor?: number;
-  timeout?: number;
-}
+export type FirecrawlScrapeOptions = Omit<
+  FirecrawlScraperConfig,
+  'apiKey' | 'apiUrl' | 'logger'
+>;
 
 export interface ScrapeMetadata {
   // Core source information
@@ -260,6 +256,21 @@ export interface FirecrawlScraperConfig {
   formats?: string[];
   timeout?: number;
   logger?: Logger;
+  includeTags?: string[];
+  excludeTags?: string[];
+  waitFor?: number;
+  maxAge?: number;
+  mobile?: boolean;
+  skipTlsVerification?: boolean;
+  blockAds?: boolean;
+  removeBase64Images?: boolean;
+  parsePDF?: boolean;
+  storeInCache?: boolean;
+  zeroDataRetention?: boolean;
+  headers?: Record<string, string>;
+  location?: { country?: string; languages?: string[] };
+  onlyMainContent?: boolean;
+  changeTrackingOptions?: object;
 }
 
 export type GetSourcesParams = {
@@ -580,6 +591,15 @@ export interface SearXNGResult {
   content?: string;
   publishedDate?: string;
   img_src?: string;
+  score?: number;
+  engine?: string;
+  category?: string;
+  thumbnail?: string;
+  priority?: string;
+  engines?: string[];
+  positions?: number[];
+  template?: string;
+  parsed_url?: string[];
 }
 
 export type ProcessSourcesFields = {

package/src/tools/search/utils.ts

@@ -25,7 +25,7 @@ export const createDefaultLogger = (): t.Logger => {
 };
 
 export const fileExtRegex =
-  /\.(pdf|jpe?g|png|gif|svg|webp|bmp|ico|tiff?|avif|heic|doc[xm]?|xls[xm]?|ppt[xm]?|zip|rar|mp[34]|mov|avi|wav)$/i;
+  /\.(pdf|jpe?g|png|gif|svg|webp|bmp|ico|tiff?|avif|heic|doc[xm]?|xls[xm]?|ppt[xm]?|zip|rar|mp[34]|mov|avi|wav)(?:\?.*)?$/i;
 
 export const getDomainName = (
   link: string,