@librechat/agents 2.4.319 → 2.4.321

package/src/tools/search/format.ts

@@ -1,107 +1,165 @@
 import type * as t from './types';
 import { getDomainName } from './utils';
 
+function addHighlightSection(): string[] {
+  return ['\n## Highlights', ''];
+}
+
+// Helper function to format a source (organic or top story)
+function formatSource(
+  source: t.ValidSource,
+  index: number,
+  turn: number,
+  sourceType: 'search' | 'news',
+  references: t.ResultReference[]
+): string {
+  /** Array of all lines to include in the output */
+  const outputLines: string[] = [];
+
+  // Add the title
+  outputLines.push(
+    `# ${sourceType.charAt(0).toUpperCase() + sourceType.slice(1)} ${index}: ${source.title != null && source.title ? `"${source.title}"` : '(no title)'}`
+  );
+  outputLines.push(`\nAnchor: \\ue202turn${turn}${sourceType}${index}`);
+  outputLines.push(`URL: ${source.link}`);
+
+  // Add optional fields
+  if ('snippet' in source && source.snippet != null) {
+    outputLines.push(`Summary: ${source.snippet}`);
+  }
+
+  if (source.date != null) {
+    outputLines.push(`Date: ${source.date}`);
+  }
+
+  if (source.attribution != null) {
+    outputLines.push(`Source: ${source.attribution}`);
+  }
+
+  // Add highlight section or empty line
+  if ((source.highlights?.length ?? 0) > 0) {
+    outputLines.push(...addHighlightSection());
+  } else {
+    outputLines.push('');
+  }
+
+  // Process highlights if they exist
+  (source.highlights ?? [])
+    .filter((h) => h.text.trim().length > 0)
+    .forEach((h, hIndex) => {
+      outputLines.push(
+        `### Highlight ${hIndex + 1} [Relevance: ${h.score.toFixed(2)}]`
+      );
+      outputLines.push('');
+      outputLines.push('```text');
+      outputLines.push(h.text.trim());
+      outputLines.push('```');
+      outputLines.push('');
+
+      if (h.references != null && h.references.length) {
+        let hasHeader = false;
+        const refLines: string[] = [];
+
+        for (let j = 0; j < h.references.length; j++) {
+          const ref = h.references[j];
+          references.push({
+            type: ref.type,
+            link: ref.reference.originalUrl,
+            attribution: getDomainName(ref.reference.originalUrl),
+            title: (
+              ((ref.reference.title ?? '') || ref.reference.text) ??
+              ''
+            ).split('\n')[0],
+          });
+
+          if (ref.type !== 'link') {
+            continue;
+          }
+
+          if (!hasHeader) {
+            refLines.push('Core References:');
+            hasHeader = true;
+          }
+
+          refLines.push(
+            `- ${ref.type}#${ref.originalIndex + 1}: ${ref.reference.originalUrl}`
+          );
+          refLines.push(
+            `\t- Anchor: \\ue202turn${turn}ref${references.length - 1}`
+          );
+        }
+
+        if (hasHeader) {
+          outputLines.push(...refLines);
+          outputLines.push('');
+        }
+      }
+
+      if (hIndex < (source.highlights?.length ?? 0) - 1) {
+        outputLines.push('---');
+        outputLines.push('');
+      }
+    });
+
+  outputLines.push('');
+  return outputLines.join('\n');
+}
+
 export function formatResultsForLLM(
   turn: number,
   results: t.SearchResultData
 ): { output: string; references: t.ResultReference[] } {
-  let output = '';
+  /** Array to collect all output lines */
+  const outputLines: string[] = [];
 
   const addSection = (title: string): void => {
-    output += `\n=== ${title} ===\n`;
+    outputLines.push('');
+    outputLines.push(`=== ${title} ===`);
+    outputLines.push('');
   };
 
   const references: t.ResultReference[] = [];
+
   // Organic (web) results
   if (results.organic?.length != null && results.organic.length > 0) {
     addSection(`Web Results, Turn ${turn}`);
     for (let i = 0; i < results.organic.length; i++) {
       const r = results.organic[i];
-      output += [
-        `# Source ${i}: "${r.title ?? '(no title)'}"`,
-        `Anchor: \\ue202turn${turn}search${i}`,
-        `URL: ${r.link}`,
-        r.snippet != null ? `Summary: ${r.snippet}` : '',
-        r.date != null ? `Date: ${r.date}` : '',
-        r.attribution != null ? `Source: ${r.attribution}` : '',
-        '',
-        '\n## Highlights\n\n',
-        '',
-        '',
-      ]
-        .filter(Boolean)
-        .join('\n');
-
-      (r.highlights ?? [])
-        .filter((h) => h.text.trim().length > 0)
-        .forEach((h, hIndex) => {
-          output += `### Highlight ${hIndex + 1} [Relevance: ${h.score.toFixed(2)}]\n\n`;
-          output += '```text\n' + h.text.trim() + '\n```\n\n';
-
-          if (h.references != null && h.references.length) {
-            output += 'Core References:\n';
-            output += h.references
-              .map((ref) => {
-                references.push({
-                  link: ref.reference.originalUrl,
-                  attribution: getDomainName(ref.reference.originalUrl),
-                  title: (
-                    ((ref.reference.title ?? '') || ref.reference.text) ??
-                    ''
-                  ).split('\n')[0],
-                });
-                return `- ${ref.type}#${ref.originalIndex + 1}: ${ref.reference.originalUrl}\n\t- Anchor: \\ue202turn${turn}ref${references.length - 1}`;
-              })
-              .join('\n');
-            output += '\n\n';
-          }
-
-          if (hIndex < (r.highlights?.length ?? 0) - 1) {
-            output += '---\n\n';
-          }
-        });
-
+      outputLines.push(formatSource(r, i, turn, 'search', references));
       delete results.organic[i].highlights;
-      output += '\n';
     }
   }
 
-  // Ignoring these sections for now
-  // // Top stories (news)
-  // const topStores = results.topStories ?? [];
-  // if (topStores.length) {
-  //   addSection('News Results');
-  //   topStores.forEach((r, i) => {
-  //     output += [
-  //       `Anchor: \ue202turn0news${i}`,
-  //       `Title: ${r.title ?? '(no title)'}`,
-  //       `URL: ${r.link}`,
-  //       r.snippet != null ? `Snippet: ${r.snippet}` : '',
-  //       r.date != null ? `Date: ${r.date}` : '',
-  //       r.attribution != null ? `Source: ${r.attribution}` : '',
-  //       ''
-  //     ].filter(Boolean).join('\n');
-  //   });
-  // }
+  // Top stories (news)
+  const topStories = results.topStories ?? [];
+  if (topStories.length) {
+    addSection('News Results');
+    for (let i = 0; i < topStories.length; i++) {
+      const r = topStories[i];
+      outputLines.push(formatSource(r, i, turn, 'news', references));
+      if (results.topStories?.[i]?.highlights) {
+        delete results.topStories[i].highlights;
+      }
+    }
+  }
 
   // // Images
   // const images = results.images ?? [];
   // if (images.length) {
   //   addSection('Image Results');
-  //   images.forEach((img, i) => {
-  //     output += [
-  //       `Anchor: \ue202turn0image${i}`,
-  //       `Title: ${img.title ?? '(no title)'}`,
-  //       `Image URL: ${img.imageUrl}`,
-  //       ''
-  //     ].join('\n');
-  //   });
+  //   const imageLines = images.map((img, i) => [
+  //     `Anchor: \ue202turn0image${i}`,
+  //     `Title: ${img.title ?? '(no title)'}`,
+  //     `Image URL: ${img.imageUrl}`,
+  //     ''
+  //   ].join('\n'));
+  //   outputLines.push(imageLines.join('\n'));
   // }
 
   // Knowledge Graph
   if (results.knowledgeGraph != null) {
     addSection('Knowledge Graph');
-    output += [
+    const kgLines = [
       `**Title:** ${results.knowledgeGraph.title ?? '(no title)'}`,
       results.knowledgeGraph.type != null
         ? `**Type:** ${results.knowledgeGraph.type}`
@@ -129,15 +187,15 @@ export function formatResultsForLLM(
         )}\n\`\`\``
         : '',
       '',
-    ]
-      .filter(Boolean)
-      .join('\n\n');
+    ].filter(Boolean);
+
+    outputLines.push(kgLines.join('\n\n'));
   }
 
   // Answer Box
   if (results.answerBox != null) {
     addSection('Answer Box');
-    output += [
+    const abLines = [
       results.answerBox.title != null
         ? `**Title:** ${results.answerBox.title}`
         : '',
@@ -153,30 +211,35 @@ export function formatResultsForLLM(
         ? `**Link:** ${results.answerBox.link}`
         : '',
       '',
-    ]
-      .filter(Boolean)
-      .join('\n\n');
+    ].filter(Boolean);
+
+    outputLines.push(abLines.join('\n\n'));
   }
 
   // People also ask
   const peopleAlsoAsk = results.peopleAlsoAsk ?? [];
   if (peopleAlsoAsk.length) {
     addSection('People Also Ask');
+
+    const paaLines: string[] = [];
     peopleAlsoAsk.forEach((p, i) => {
-      output += [
+      const questionLines = [
         `### Question ${i + 1}:`,
         `"${p.question}"`,
-        `${p.snippet != null && p.snippet ? `Snippet: ${p.snippet}}` : ''}`,
+        `${p.snippet != null && p.snippet ? `Snippet: ${p.snippet}` : ''}`,
         `${p.title != null && p.title ? `Title: ${p.title}` : ''}`,
         `${p.link != null && p.link ? `Link: ${p.link}` : ''}`,
         '',
-      ]
-        .filter(Boolean)
-        .join('\n\n');
+      ].filter(Boolean);
+
+      paaLines.push(questionLines.join('\n\n'));
     });
+
+    outputLines.push(paaLines.join(''));
   }
+
   return {
-    output: output.trim(),
+    output: outputLines.join('\n').trim(),
     references,
   };
 }

package/src/tools/search/search.ts

@@ -65,7 +65,7 @@ const chunker = {
   },
 };
 
-const createSourceUpdateCallback = (sourceMap: Map<string, t.ValidSource>) => {
+function createSourceUpdateCallback(sourceMap: Map<string, t.ValidSource>) {
   return (link: string, update?: Partial<t.ValidSource>): void => {
     const source = sourceMap.get(link);
     if (source) {
@@ -75,7 +75,7 @@ const createSourceUpdateCallback = (sourceMap: Map<string, t.ValidSource>) => {
       });
     }
   };
-};
+}
 
 const getHighlights = async ({
   query,
@@ -315,10 +315,7 @@ export const createSourceProcessor = (
   scraperInstance?: FirecrawlScraper
 ): {
   processSources: (
-    result: t.SearchResult,
-    numElements: number,
-    query: string,
-    proMode?: boolean
+    fields: t.ProcessSourcesFields
   ) => Promise<t.SearchResultData>;
   topResults: number;
 } => {
@@ -338,14 +335,17 @@ export const createSourceProcessor = (
     scrapeMany: async ({
       query,
       links,
+      onGetHighlights,
     }: {
       query: string;
       links: string[];
+      onGetHighlights: t.SearchToolConfig['onGetHighlights'];
     }): Promise<Array<t.ScrapeResult>> => {
       console.log(`Scraping ${links.length} links with Firecrawl`);
       const promises: Array<Promise<t.ScrapeResult>> = [];
       try {
-        for (const currentLink of links) {
+        for (let i = 0; i < links.length; i++) {
+          const currentLink = links[i];
           const promise: Promise<t.ScrapeResult> = firecrawlScraper
             .scrapeUrl(currentLink, {})
             .then(([url, response]) => {
@@ -365,7 +365,7 @@ export const createSourceProcessor = (
                 url,
                 attribution,
                 error: true,
-                content: `Failed to scrape ${url}: ${response.error ?? 'Unknown error'}`,
+                content: '',
               } as t.ScrapeResult;
             })
             .then(async (result) => {
@@ -383,6 +383,9 @@ export const createSourceProcessor = (
                   reranker,
                   content: result.content,
                 });
+                if (onGetHighlights) {
+                  onGetHighlights(result.url);
+                }
                 return {
                   ...result,
                   highlights,
@@ -399,7 +402,7 @@ export const createSourceProcessor = (
               return {
                 url: currentLink,
                 error: true,
-                content: `Failed to scrape ${currentLink}: ${error.message ?? 'Unknown error'}`,
+                content: '',
               };
             });
           promises.push(promise);
@@ -416,16 +419,22 @@ export const createSourceProcessor = (
     links,
     query,
     target,
+    onGetHighlights,
     onContentScraped,
   }: {
     links: string[];
     query: string;
     target: number;
+    onGetHighlights: t.SearchToolConfig['onGetHighlights'];
     onContentScraped?: (link: string, update?: Partial<t.ValidSource>) => void;
   }): Promise<void> => {
     const initialLinks = links.slice(0, target);
     // const remainingLinks = links.slice(target).reverse();
-    const results = await webScraper.scrapeMany({ query, links: initialLinks });
+    const results = await webScraper.scrapeMany({
+      query,
+      links: initialLinks,
+      onGetHighlights,
+    });
     for (const result of results) {
       if (result.error === true) {
         continue;
@@ -440,12 +449,13 @@ export const createSourceProcessor = (
     }
   };
 
-  const processSources = async (
-    result: t.SearchResult,
-    numElements: number,
-    query: string,
-    proMode: boolean = false
-  ): Promise<t.SearchResultData> => {
+  const processSources = async ({
+    result,
+    numElements,
+    query,
+    proMode = true,
+    onGetHighlights,
+  }: t.ProcessSourcesFields): Promise<t.SearchResultData> => {
     try {
       if (!result.data) {
         return {
@@ -473,6 +483,7 @@ export const createSourceProcessor = (
         await fetchContents({
           query,
           target: 1,
+          onGetHighlights,
           onContentScraped,
           links: [wikiSources[0].link],
         });
@@ -492,48 +503,67 @@ export const createSourceProcessor = (
       }
 
       const sourceMap = new Map<string, t.ValidSource>();
-      const allLinks: string[] = [];
+      const organicLinksSet = new Set<string>();
 
-      for (const source of result.data.organic) {
-        if (source.link) {
-          allLinks.push(source.link);
-          sourceMap.set(source.link, source);
-        }
-      }
+      // Collect organic links
+      const organicLinks = collectLinks(
+        result.data.organic,
+        sourceMap,
+        organicLinksSet
+      );
 
-      if (allLinks.length === 0) {
+      // Collect top story links, excluding any that are already in organic links
+      const topStories = result.data.topStories ?? [];
+      const topStoryLinks = collectLinks(
+        topStories,
+        sourceMap,
+        organicLinksSet
+      );
+
+      if (organicLinks.length === 0 && topStoryLinks.length === 0) {
         return result.data;
       }
 
       const onContentScraped = createSourceUpdateCallback(sourceMap);
-      await fetchContents({
-        links: allLinks,
-        query,
-        onContentScraped,
-        target: numElements,
-      });
+      const promises: Promise<void>[] = [];
+
+      // Process organic links
+      if (organicLinks.length > 0) {
+        promises.push(
+          fetchContents({
+            query,
+            onGetHighlights,
+            onContentScraped,
+            links: organicLinks,
+            target: numElements,
+          })
+        );
+      }
 
-      for (let i = 0; i < result.data.organic.length; i++) {
-        const source = result.data.organic[i];
-        const updatedSource = sourceMap.get(source.link);
-        if (updatedSource) {
-          result.data.organic[i] = {
-            ...source,
-            ...updatedSource,
-          };
-        }
+      // Process top story links
+      if (topStoryLinks.length > 0) {
+        promises.push(
+          fetchContents({
+            query,
+            onGetHighlights,
+            onContentScraped,
+            links: topStoryLinks,
+            target: numElements,
+          })
+        );
       }
 
-      const successfulSources = result.data.organic
-        .filter(
-          (source) =>
-            source.content != null && !source.content.startsWith('Failed')
-        )
-        .slice(0, numElements);
+      await Promise.all(promises);
 
-      if (successfulSources.length > 0) {
-        result.data.organic = successfulSources;
+      // Update sources with scraped content
+      if (result.data.organic.length > 0) {
+        updateSourcesWithContent(result.data.organic, sourceMap);
       }
+
+      if (topStories.length > 0) {
+        updateSourcesWithContent(topStories, sourceMap);
+      }
+
       return result.data;
     } catch (error) {
       console.error('Error in processSources:', error);
@@ -553,3 +583,46 @@ export const createSourceProcessor = (
     topResults,
   };
 };
+
+/** Helper function to collect links and update sourceMap */
+function collectLinks(
+  sources: Array<t.OrganicResult | t.TopStoryResult>,
+  sourceMap: Map<string, t.ValidSource>,
+  existingLinksSet?: Set<string>
+): string[] {
+  const links: string[] = [];
+
+  for (const source of sources) {
+    if (source.link) {
+      // For topStories, only add if not already in organic links
+      if (existingLinksSet && existingLinksSet.has(source.link)) {
+        continue;
+      }
+
+      links.push(source.link);
+      if (existingLinksSet) {
+        existingLinksSet.add(source.link);
+      }
+      sourceMap.set(source.link, source as t.ValidSource);
+    }
+  }
+
+  return links;
+}
+
+/** Helper function to update sources with scraped content */
+function updateSourcesWithContent<T extends t.ValidSource>(
+  sources: T[],
+  sourceMap: Map<string, t.ValidSource>
+): void {
+  for (let i = 0; i < sources.length; i++) {
+    const source = sources[i];
+    const updatedSource = sourceMap.get(source.link);
+    if (updatedSource) {
+      sources[i] = {
+        ...source,
+        ...updatedSource,
+      } as T;
+    }
+  }
+}