@librechat/agents 2.4.31 → 2.4.34

package/src/tools/search/format.ts

@@ -1,121 +1,252 @@
 import type * as t from './types';
+import { getDomainName, fileExtRegex } from './utils';
 
-export function formatResultsForLLM(results: t.SearchResultData): string {
-  let output = '';
+function addHighlightSection(): string[] {
+  return ['\n## Highlights', ''];
+}
+
+// Helper function to format a source (organic or top story)
+function formatSource(
+  source: t.ValidSource,
+  index: number,
+  turn: number,
+  sourceType: 'search' | 'news',
+  references: t.ResultReference[]
+): string {
+  /** Array of all lines to include in the output */
+  const outputLines: string[] = [];
+
+  // Add the title
+  outputLines.push(
+    `# ${sourceType.charAt(0).toUpperCase() + sourceType.slice(1)} ${index}: ${source.title != null && source.title ? `"${source.title}"` : '(no title)'}`
+  );
+  outputLines.push(`\nAnchor: \\ue202turn${turn}${sourceType}${index}`);
+  outputLines.push(`URL: ${source.link}`);
+
+  // Add optional fields
+  if ('snippet' in source && source.snippet != null) {
+    outputLines.push(`Summary: ${source.snippet}`);
+  }
+
+  if (source.date != null) {
+    outputLines.push(`Date: ${source.date}`);
+  }
+
+  if (source.attribution != null) {
+    outputLines.push(`Source: ${source.attribution}`);
+  }
+
+  // Add highlight section or empty line
+  if ((source.highlights?.length ?? 0) > 0) {
+    outputLines.push(...addHighlightSection());
+  } else {
+    outputLines.push('');
+  }
+
+  // Process highlights if they exist
+  (source.highlights ?? [])
+    .filter((h) => h.text.trim().length > 0)
+    .forEach((h, hIndex) => {
+      outputLines.push(
+        `### Highlight ${hIndex + 1} [Relevance: ${h.score.toFixed(2)}]`
+      );
+      outputLines.push('');
+      outputLines.push('```text');
+      outputLines.push(h.text.trim());
+      outputLines.push('```');
+      outputLines.push('');
+
+      if (h.references != null && h.references.length) {
+        let hasHeader = false;
+        const refLines: string[] = [];
+
+        for (let j = 0; j < h.references.length; j++) {
+          const ref = h.references[j];
+          if (ref.reference.originalUrl.includes('mailto:')) {
+            continue;
+          }
+          references.push({
+            type: ref.type,
+            link: ref.reference.originalUrl,
+            attribution: getDomainName(ref.reference.originalUrl),
+            title: (
+              ((ref.reference.title ?? '') || ref.reference.text) ??
+              ''
+            ).split('\n')[0],
+          });
+
+          if (ref.type !== 'link') {
+            continue;
+          }
+
+          if (fileExtRegex.test(ref.reference.originalUrl)) {
+            continue;
+          }
+
+          if (!hasHeader) {
+            refLines.push('Core References:');
+            hasHeader = true;
+          }
+
+          refLines.push(
+            `- ${ref.type}#${ref.originalIndex + 1}: ${ref.reference.originalUrl}`
+          );
+          refLines.push(
+            `\t- Anchor: \\ue202turn${turn}ref${references.length - 1}`
+          );
+        }
+
+        if (hasHeader) {
+          outputLines.push(...refLines);
+          outputLines.push('');
+        }
+      }
+
+      if (hIndex < (source.highlights?.length ?? 0) - 1) {
+        outputLines.push('---');
+        outputLines.push('');
+      }
+    });
+
+  outputLines.push('');
+  return outputLines.join('\n');
+}
+
+export function formatResultsForLLM(
+  turn: number,
+  results: t.SearchResultData
+): { output: string; references: t.ResultReference[] } {
+  /** Array to collect all output lines */
+  const outputLines: string[] = [];
 
   const addSection = (title: string): void => {
-    output += `\n=== ${title} ===\n`;
+    outputLines.push('');
+    outputLines.push(`=== ${title} ===`);
+    outputLines.push('');
   };
 
+  const references: t.ResultReference[] = [];
+
   // Organic (web) results
-  const organic = results.organic ?? [];
-  if (organic.length) {
-    addSection('Web Results');
-    organic.forEach((r, i) => {
-      output += [
-        `Source ${i + 1}: ${r.title ?? '(no title)'}`,
-        `Citation Anchor: \\ue202turn0search${i + 1}`,
-        `URL: ${r.link}`,
-        r.snippet != null ? `Summary: ${r.snippet}` : '',
-        r.date != null ? `Date: ${r.date}` : '',
-        r.attribution != null ? `Source: ${r.attribution}` : '',
-        '',
-        '--- Content Highlights ---',
-        ...(r.highlights ?? [])
-          .filter((h) => h.text.trim().length > 0)
-          .map((h) => `[Relevance: ${h.score.toFixed(2)}]\n${h.text.trim()}`),
-        '',
-      ]
-        .filter(Boolean)
-        .join('\n');
-    });
+  if (results.organic?.length != null && results.organic.length > 0) {
+    addSection(`Web Results, Turn ${turn}`);
+    for (let i = 0; i < results.organic.length; i++) {
+      const r = results.organic[i];
+      outputLines.push(formatSource(r, i, turn, 'search', references));
+      delete results.organic[i].highlights;
+    }
   }
 
-  // Ignoring these sections for now
-  // // Top stories (news)
-  // const topStores = results.topStories ?? [];
-  // if (topStores.length) {
-  //   addSection('News Results');
-  //   topStores.forEach((r, i) => {
-  //     output += [
-  //       `Anchor: \ue202turn0news${i + 1}`,
-  //       `Title: ${r.title ?? '(no title)'}`,
-  //       `URL: ${r.link}`,
-  //       r.snippet != null ? `Snippet: ${r.snippet}` : '',
-  //       r.date != null ? `Date: ${r.date}` : '',
-  //       r.attribution != null ? `Source: ${r.attribution}` : '',
-  //       ''
-  //     ].filter(Boolean).join('\n');
-  //   });
-  // }
+  // Top stories (news)
+  const topStories = results.topStories ?? [];
+  if (topStories.length) {
+    addSection('News Results');
+    for (let i = 0; i < topStories.length; i++) {
+      const r = topStories[i];
+      outputLines.push(formatSource(r, i, turn, 'news', references));
+      if (results.topStories?.[i]?.highlights) {
+        delete results.topStories[i].highlights;
+      }
+    }
+  }
 
   // // Images
   // const images = results.images ?? [];
   // if (images.length) {
   //   addSection('Image Results');
-  //   images.forEach((img, i) => {
-  //     output += [
-  //       `Anchor: \ue202turn0image${i + 1}`,
-  //       `Title: ${img.title ?? '(no title)'}`,
-  //       `Image URL: ${img.imageUrl}`,
-  //       ''
-  //     ].join('\n');
-  //   });
+  //   const imageLines = images.map((img, i) => [
+  //     `Anchor: \ue202turn0image${i}`,
+  //     `Title: ${img.title ?? '(no title)'}`,
+  //     `Image URL: ${img.imageUrl}`,
+  //     ''
+  //   ].join('\n'));
+  //   outputLines.push(imageLines.join('\n'));
   // }
 
   // Knowledge Graph
   if (results.knowledgeGraph != null) {
     addSection('Knowledge Graph');
-    output += [
-      `Title: ${results.knowledgeGraph.title ?? '(no title)'}`,
+    const kgLines = [
+      `**Title:** ${results.knowledgeGraph.title ?? '(no title)'}`,
+      results.knowledgeGraph.type != null
+        ? `**Type:** ${results.knowledgeGraph.type}`
+        : '',
       results.knowledgeGraph.description != null
-        ? `Description: ${results.knowledgeGraph.description}`
+        ? `**Description:** ${results.knowledgeGraph.description}`
         : '',
-      results.knowledgeGraph.type != null
-        ? `Type: ${results.knowledgeGraph.type}`
+      results.knowledgeGraph.descriptionSource != null
+        ? `**Description Source:** ${results.knowledgeGraph.descriptionSource}`
+        : '',
+      results.knowledgeGraph.descriptionLink != null
+        ? `**Description Link:** ${results.knowledgeGraph.descriptionLink}`
         : '',
       results.knowledgeGraph.imageUrl != null
-        ? `Image URL: ${results.knowledgeGraph.imageUrl}`
+        ? `**Image URL:** ${results.knowledgeGraph.imageUrl}`
+        : '',
+      results.knowledgeGraph.website != null
+        ? `**Website:** ${results.knowledgeGraph.website}`
         : '',
       results.knowledgeGraph.attributes != null
-        ? `Attributes: ${JSON.stringify(results.knowledgeGraph.attributes, null, 2)}`
+        ? `**Attributes:**\n\`\`\`json\n${JSON.stringify(
+          results.knowledgeGraph.attributes,
+          null,
+          2
+        )}\n\`\`\``
         : '',
       '',
-    ]
-      .filter(Boolean)
-      .join('\n');
+    ].filter(Boolean);
+
+    outputLines.push(kgLines.join('\n\n'));
   }
 
   // Answer Box
   if (results.answerBox != null) {
     addSection('Answer Box');
-    output += [
+    const abLines = [
       results.answerBox.title != null
-        ? `Title: ${results.answerBox.title}`
-        : '',
-      results.answerBox.answer != null
-        ? `Answer: ${results.answerBox.answer}`
+        ? `**Title:** ${results.answerBox.title}`
         : '',
       results.answerBox.snippet != null
-        ? `Snippet: ${results.answerBox.snippet}`
+        ? `**Snippet:** ${results.answerBox.snippet}`
+        : '',
+      results.answerBox.snippetHighlighted != null
+        ? `**Snippet Highlighted:** ${results.answerBox.snippetHighlighted
+          .map((s) => `\`${s}\``)
+          .join(' ')}`
+        : '',
+      results.answerBox.link != null
+        ? `**Link:** ${results.answerBox.link}`
         : '',
-      results.answerBox.date != null ? `Date: ${results.answerBox.date}` : '',
       '',
-    ]
-      .filter(Boolean)
-      .join('\n');
+    ].filter(Boolean);
+
+    outputLines.push(abLines.join('\n\n'));
   }
 
   // People also ask
   const peopleAlsoAsk = results.peopleAlsoAsk ?? [];
   if (peopleAlsoAsk.length) {
     addSection('People Also Ask');
-    peopleAlsoAsk.forEach((p, _i) => {
-      output += [`Q: ${p.question}`, `A: ${p.answer}`, '']
-        .filter(Boolean)
-        .join('\n');
+
+    const paaLines: string[] = [];
+    peopleAlsoAsk.forEach((p, i) => {
+      const questionLines = [
+        `### Question ${i + 1}:`,
+        `"${p.question}"`,
+        `${p.snippet != null && p.snippet ? `Snippet: ${p.snippet}` : ''}`,
+        `${p.title != null && p.title ? `Title: ${p.title}` : ''}`,
+        `${p.link != null && p.link ? `Link: ${p.link}` : ''}`,
+        '',
+      ].filter(Boolean);
+
+      paaLines.push(questionLines.join('\n\n'));
     });
+
+    outputLines.push(paaLines.join(''));
   }
 
-  return output.trim();
+  return {
+    output: outputLines.join('\n').trim(),
+    references,
+  };
 }

package/src/tools/search/highlights.ts

@@ -122,6 +122,89 @@ function findBestBoundary(text: string, direction = 'backward'): number {
   return direction === 'backward' ? text.length : 0;
 }
 
+/**
+ * Tracks references used in a highlight without changing their numbers
+ */
+function trackReferencesInHighlight(
+  text: string,
+  sourceResult: t.ValidSource // Source containing the original references
+): {
+  references: {
+    type: 'link' | 'image' | 'video';
+    originalIndex: number;
+    reference: t.MediaReference; // Original reference object
+  }[];
+} {
+  // Track used references
+  const references: {
+    type: 'link' | 'image' | 'video';
+    originalIndex: number;
+    reference: t.MediaReference;
+  }[] = [];
+
+  if (!text || text.length === 0 || !text.includes('#')) {
+    return { references }; // Early return
+  }
+
+  // Quick check for reference markers
+  if (
+    !text.includes('link#') &&
+    !text.includes('image#') &&
+    !text.includes('video#')
+  ) {
+    return { references };
+  }
+
+  // Get references from the source if available
+  const sourceRefs = sourceResult.references || {
+    links: [],
+    images: [],
+    videos: [],
+  };
+
+  // Find references but don't modify text
+  const refRegex = /\((link|image|video)#(\d+)(?:\s+"([^"]*)")?\)/g;
+  let match;
+
+  while ((match = refRegex.exec(text)) !== null) {
+    const [, type, indexStr] = match;
+    const originalIndex = parseInt(indexStr, 10) - 1; // Convert to 0-based
+
+    // Get the source array for this type
+    const refType = type as 'link' | 'image' | 'video';
+    const sourceArray = sourceRefs[`${refType}s`] as
+      | t.MediaReference[]
+      | undefined;
+
+    // Skip if invalid reference
+    if (
+      !sourceArray ||
+      originalIndex < 0 ||
+      originalIndex >= sourceArray.length
+    ) {
+      continue; // Skip invalid references
+    }
+
+    // Get original reference
+    const reference = sourceArray[originalIndex];
+
+    // Track if not already tracked
+    const alreadyTracked = references.some(
+      (ref) => ref.type === refType && ref.originalIndex === originalIndex
+    );
+
+    if (!alreadyTracked) {
+      references.push({
+        type: refType,
+        originalIndex,
+        reference,
+      });
+    }
+  }
+
+  return { references };
+}
+
 /**
  * Expand highlights in search results using smart boundary detection.
  *
@@ -131,25 +214,19 @@ function findBestBoundary(text: string, direction = 'backward'): number {
  * @param searchResults - Search results object
  * @param mainExpandBy - Primary expansion size on each side (default: 300)
  * @param separatorExpandBy - Additional range to look for separators (default: 150)
- * @returns Copy of search results with expanded highlights
+ * @returns Copy of search results with expanded highlights and tracked references
  */
 export function expandHighlights(
   searchResults: t.SearchResultData,
   mainExpandBy = 300,
   separatorExpandBy = 150
 ): t.SearchResultData {
-  // 1. Avoid full deep copy - only copy what we modify
+  // Avoid deep copy - only copy what we modify
   const resultCopy = { ...searchResults };
+  if (resultCopy.organic) resultCopy.organic = [...resultCopy.organic];
+  if (resultCopy.topStories) resultCopy.topStories = [...resultCopy.topStories];
 
-  // Only deep copy the relevant arrays
-  if (resultCopy.organic) {
-    resultCopy.organic = [...resultCopy.organic];
-  }
-  if (resultCopy.topStories) {
-    resultCopy.topStories = [...resultCopy.topStories];
-  }
-
-  // 5. Process the results efficiently
+  // Process the results efficiently
   const processResultTypes = ['organic', 'topStories'] as const;
 
   for (const resultType of processResultTypes) {
@@ -170,23 +247,26 @@ export function expandHighlights(
       const resultCopy = { ...result };
       const content = result.content;
       const highlights = [];
-
       // Process each highlight
       for (const highlight of result.highlights) {
-        const highlightText = highlight.text;
+        const { references } = trackReferencesInHighlight(
+          highlight.text,
+          result
+        );
 
-        let startPos = content.indexOf(highlightText);
-        let highlightLen = highlightText.length;
+        let startPos = content.indexOf(highlight.text);
+        let highlightLen = highlight.text.length;
 
         if (startPos === -1) {
           // Try with stripped whitespace
-          const strippedHighlight = highlightText.trim();
+          const strippedHighlight = highlight.text.trim();
           startPos = content.indexOf(strippedHighlight);
 
           if (startPos === -1) {
             highlights.push({
               text: highlight.text,
               score: highlight.score,
+              references,
             });
             continue;
           }
@@ -225,10 +305,13 @@ export function expandHighlights(
         highlights.push({
           text: expandedHighlightText,
           score: highlight.score,
+          references,
         });
       }
 
       resultCopy.highlights = highlights;
+      delete resultCopy.content;
+      delete resultCopy.references;
       return resultCopy;
     });
   }