@librechat/agents 2.4.316 → 2.4.318

package/src/tools/search/content.ts

@@ -0,0 +1,147 @@
+import * as cheerio from 'cheerio';
+import type { References, MediaReference } from './types';
+
+export function processContent(
+  html: string,
+  markdown: string
+): {
+  markdown: string;
+} & References {
+  const linkMap = new Map<string, MediaReference>();
+  const imageMap = new Map<string, MediaReference>();
+  const videoMap = new Map<string, MediaReference>();
+  const iframeMap = new Map<string, MediaReference>();
+
+  const $ = cheerio.load(html, {
+    xmlMode: false,
+  });
+
+  // Extract all media references
+  $('a[href]').each((_, el) => {
+    const href = $(el).attr('href');
+    if (href != null && href) {
+      linkMap.set(href, {
+        originalUrl: href,
+        title: $(el).attr('title'),
+        text: $(el).text().trim(),
+      });
+    }
+  });
+
+  $('img[src]').each((_, el) => {
+    const src = $(el).attr('src');
+    if (src != null && src) {
+      imageMap.set(src, {
+        originalUrl: src,
+        title: $(el).attr('alt') ?? $(el).attr('title'),
+      });
+    }
+  });
+
+  // Handle videos (dedicated video elements and video platforms in iframes)
+  $('video[src], iframe[src*="youtube"], iframe[src*="vimeo"]').each(
+    (_, el) => {
+      const src = $(el).attr('src');
+      if (src != null && src) {
+        videoMap.set(src, {
+          originalUrl: src,
+          title: $(el).attr('title'),
+        });
+      }
+    }
+  );
+
+  // Handle all other generic iframes that aren't already captured as videos
+  $('iframe').each((_, el) => {
+    const src = $(el).attr('src');
+    if (
+      src != null &&
+      src &&
+      !src.includes('youtube') &&
+      !src.includes('vimeo')
+    ) {
+      iframeMap.set(src, {
+        originalUrl: src,
+        title: $(el).attr('title'),
+      });
+    }
+  });
+
+  // Create lookup maps with indices
+  const linkIndexMap = new Map<string, number>();
+  const imageIndexMap = new Map<string, number>();
+  const videoIndexMap = new Map<string, number>();
+  const iframeIndexMap = new Map<string, number>();
+
+  Array.from(linkMap.keys()).forEach((url, i) => linkIndexMap.set(url, i + 1));
+  Array.from(imageMap.keys()).forEach((url, i) =>
+    imageIndexMap.set(url, i + 1)
+  );
+  Array.from(videoMap.keys()).forEach((url, i) =>
+    videoIndexMap.set(url, i + 1)
+  );
+  Array.from(iframeMap.keys()).forEach((url, i) =>
+    iframeIndexMap.set(url, i + 1)
+  );
+
+  // Process the markdown
+  let result = markdown;
+
+  // Replace each URL one by one, starting with the longest URLs first to avoid partial matches
+  const allUrls = [
+    ...Array.from(imageMap.keys()).map((url) => ({
+      url,
+      type: 'image',
+      idx: imageIndexMap.get(url),
+    })),
+    ...Array.from(videoMap.keys()).map((url) => ({
+      url,
+      type: 'video',
+      idx: videoIndexMap.get(url),
+    })),
+    ...Array.from(iframeMap.keys()).map((url) => ({
+      url,
+      type: 'iframe',
+      idx: iframeIndexMap.get(url),
+    })),
+    ...Array.from(linkMap.keys()).map((url) => ({
+      url,
+      type: 'link',
+      idx: linkIndexMap.get(url),
+    })),
+  ].sort((a, b) => b.url.length - a.url.length);
+
+  // Create a function to escape special characters in URLs for regex
+  function escapeRegex(string: string): string {
+    return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+  }
+
+  // Replace each URL in the markdown
+  for (const { url, type, idx } of allUrls) {
+    // Create a regex that captures URLs in markdown links
+    const regex = new RegExp(`\\(${escapeRegex(url)}(?:\\s+"[^"]*")?\\)`, 'g');
+
+    result = result.replace(regex, (match) => {
+      // Keep any title attribute that might exist
+      const titleMatch = match.match(/\s+"([^"]*)"/);
+      const titlePart = titleMatch ? ` "${titleMatch[1]}"` : '';
+
+      return `(${type}#${idx}${titlePart})`;
+    });
+  }
+
+  iframeMap.clear();
+  const links = Array.from(linkMap.values());
+  linkMap.clear();
+  const images = Array.from(imageMap.values());
+  imageMap.clear();
+  const videos = Array.from(videoMap.values());
+  videoMap.clear();
+
+  return {
+    markdown: result,
+    links,
+    images,
+    videos,
+  };
+}

package/src/tools/search/firecrawl.ts

@@ -1,136 +1,7 @@
 /* eslint-disable no-console */
 import axios from 'axios';
-
-export interface FirecrawlScrapeOptions {
-  formats?: string[];
-  includeTags?: string[];
-  excludeTags?: string[];
-  headers?: Record<string, string>;
-  waitFor?: number;
-  timeout?: number;
-}
-
-interface ScrapeMetadata {
-  // Core source information
-  sourceURL?: string;
-  url?: string;
-  scrapeId?: string;
-  statusCode?: number;
-  // Basic metadata
-  title?: string;
-  description?: string;
-  language?: string;
-  favicon?: string;
-  viewport?: string;
-  robots?: string;
-  'theme-color'?: string;
-  // Open Graph metadata
-  'og:url'?: string;
-  'og:title'?: string;
-  'og:description'?: string;
-  'og:type'?: string;
-  'og:image'?: string;
-  'og:image:width'?: string;
-  'og:image:height'?: string;
-  'og:site_name'?: string;
-  ogUrl?: string;
-  ogTitle?: string;
-  ogDescription?: string;
-  ogImage?: string;
-  ogSiteName?: string;
-  // Article metadata
-  'article:author'?: string;
-  'article:published_time'?: string;
-  'article:modified_time'?: string;
-  'article:section'?: string;
-  'article:tag'?: string;
-  'article:publisher'?: string;
-  publishedTime?: string;
-  modifiedTime?: string;
-  // Twitter metadata
-  'twitter:site'?: string;
-  'twitter:creator'?: string;
-  'twitter:card'?: string;
-  'twitter:image'?: string;
-  'twitter:dnt'?: string;
-  'twitter:app:name:iphone'?: string;
-  'twitter:app:id:iphone'?: string;
-  'twitter:app:url:iphone'?: string;
-  'twitter:app:name:ipad'?: string;
-  'twitter:app:id:ipad'?: string;
-  'twitter:app:url:ipad'?: string;
-  'twitter:app:name:googleplay'?: string;
-  'twitter:app:id:googleplay'?: string;
-  'twitter:app:url:googleplay'?: string;
-  // Facebook metadata
-  'fb:app_id'?: string;
-  // App links
-  'al:ios:url'?: string;
-  'al:ios:app_name'?: string;
-  'al:ios:app_store_id'?: string;
-  // Allow for additional properties that might be present
-  [key: string]: string | number | boolean | null | undefined;
-}
-
-export interface FirecrawlScrapeResponse {
-  success: boolean;
-  data?: {
-    markdown?: string;
-    html?: string;
-    rawHtml?: string;
-    screenshot?: string;
-    links?: string[];
-    metadata?: ScrapeMetadata;
-  };
-  error?: string;
-}
-
-export interface FirecrawlScraperConfig {
-  apiKey?: string;
-  apiUrl?: string;
-  formats?: string[];
-  timeout?: number;
-}
-const getDomainName = (
-  link: string,
-  metadata?: ScrapeMetadata
-): string | undefined => {
-  try {
-    const url = metadata?.sourceURL ?? metadata?.url ?? (link || '');
-    const domain = new URL(url).hostname.replace(/^www\./, '');
-    if (domain) {
-      return domain;
-    }
-  } catch (e) {
-    // URL parsing failed
-    console.error('Error parsing URL:', e);
-  }
-
-  return;
-};
-
-export function getAttribution(
-  link: string,
-  metadata?: ScrapeMetadata
-): string | undefined {
-  if (!metadata) return getDomainName(link, metadata);
-
-  const possibleAttributions = [
-    metadata.ogSiteName,
-    metadata['og:site_name'],
-    metadata.title?.split('|').pop()?.trim(),
-    metadata['twitter:site']?.replace(/^@/, ''),
-  ];
-
-  const attribution = possibleAttributions.find(
-    (attr) => attr != null && typeof attr === 'string' && attr.trim() !== ''
-  );
-  if (attribution != null) {
-    return attribution;
-  }
-
-  return getDomainName(link, metadata);
-}
+import { processContent } from './content';
+import type * as t from './types';
 
 /**
  * Firecrawl scraper implementation
@@ -142,7 +13,7 @@ export class FirecrawlScraper {
   private defaultFormats: string[];
   private timeout: number;
 
-  constructor(config: FirecrawlScraperConfig = {}) {
+  constructor(config: t.FirecrawlScraperConfig = {}) {
     this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';
 
     const baseUrl =
@@ -169,8 +40,8 @@ export class FirecrawlScraper {
    */
   async scrapeUrl(
     url: string,
-    options: FirecrawlScrapeOptions = {}
-  ): Promise<[string, FirecrawlScrapeResponse]> {
+    options: t.FirecrawlScrapeOptions = {}
+  ): Promise<[string, t.FirecrawlScrapeResponse]> {
     if (!this.apiKey) {
       return [
         url,
@@ -221,27 +92,39 @@ export class FirecrawlScraper {
    * @param response Scrape response
    * @returns Extracted content or empty string if not available
    */
-  extractContent(response: FirecrawlScrapeResponse): string {
+  extractContent(
+    response: t.FirecrawlScrapeResponse
+  ): [string, undefined | t.References] {
     if (!response.success || !response.data) {
-      return '';
+      return ['', undefined];
     }
 
-    // Prefer markdown content if available
-    if (response.data.markdown != null) {
-      return response.data.markdown;
+    if (response.data.markdown != null && response.data.html != null) {
+      try {
+        const { markdown, ...rest } = processContent(
+          response.data.html,
+          response.data.markdown
+        );
+        return [markdown, rest];
+      } catch (error) {
+        console.error('Error processing content:', error);
+        return [response.data.markdown, undefined];
+      }
+    } else if (response.data.markdown != null) {
+      return [response.data.markdown, undefined];
     }
 
     // Fall back to HTML content
     if (response.data.html != null) {
-      return response.data.html;
+      return [response.data.html, undefined];
     }
 
     // Fall back to raw HTML content
     if (response.data.rawHtml != null) {
-      return response.data.rawHtml;
+      return [response.data.rawHtml, undefined];
     }
 
-    return '';
+    return ['', undefined];
   }
 
   /**
@@ -249,7 +132,7 @@ export class FirecrawlScraper {
    * @param response Scrape response
    * @returns Metadata object
    */
-  extractMetadata(response: FirecrawlScrapeResponse): ScrapeMetadata {
+  extractMetadata(response: t.FirecrawlScrapeResponse): t.ScrapeMetadata {
     if (!response.success || !response.data || !response.data.metadata) {
       return {};
     }
@@ -264,7 +147,7 @@ export class FirecrawlScraper {
  * @returns Firecrawl scraper instance
  */
 export const createFirecrawlScraper = (
-  config: FirecrawlScraperConfig = {}
+  config: t.FirecrawlScraperConfig = {}
 ): FirecrawlScraper => {
   return new FirecrawlScraper(config);
 };

package/src/tools/search/format.ts

@@ -1,37 +1,69 @@
 import type * as t from './types';
+import { getDomainName } from './utils';
 
 export function formatResultsForLLM(
   turn: number,
   results: t.SearchResultData
-): string {
+): { output: string; references: t.ResultReference[] } {
   let output = '';
 
   const addSection = (title: string): void => {
     output += `\n=== ${title} ===\n`;
   };
 
+  const references: t.ResultReference[] = [];
   // Organic (web) results
-  const organic = results.organic ?? [];
-  if (organic.length) {
-    addSection('Web Results, Turn ' + turn);
-    organic.forEach((r, i) => {
+  if (results.organic?.length != null && results.organic.length > 0) {
+    addSection(`Web Results, Turn ${turn}`);
+    for (let i = 0; i < results.organic.length; i++) {
+      const r = results.organic[i];
       output += [
-        `Source ${i}: ${r.title ?? '(no title)'}`,
-        `Citation Anchor: \\ue202turn${turn}search${i}`,
+        `# Source ${i}: "${r.title ?? '(no title)'}"`,
+        `Anchor: \\ue202turn${turn}search${i}`,
         `URL: ${r.link}`,
         r.snippet != null ? `Summary: ${r.snippet}` : '',
         r.date != null ? `Date: ${r.date}` : '',
         r.attribution != null ? `Source: ${r.attribution}` : '',
         '',
-        '--- Content Highlights ---',
-        ...(r.highlights ?? [])
-          .filter((h) => h.text.trim().length > 0)
-          .map((h) => `[Relevance: ${h.score.toFixed(2)}]\n${h.text.trim()}`),
+        '\n## Highlights\n\n',
+        '',
         '',
       ]
         .filter(Boolean)
         .join('\n');
-    });
+
+      (r.highlights ?? [])
+        .filter((h) => h.text.trim().length > 0)
+        .forEach((h, hIndex) => {
+          output += `### Highlight ${hIndex + 1} [Relevance: ${h.score.toFixed(2)}]\n\n`;
+          output += '```text\n' + h.text.trim() + '\n```\n\n';
+
+          if (h.references != null && h.references.length) {
+            output += 'Core References:\n';
+            output += h.references
+              .map((ref) => {
+                references.push({
+                  link: ref.reference.originalUrl,
+                  attribution: getDomainName(ref.reference.originalUrl),
+                  title: (
+                    ((ref.reference.title ?? '') || ref.reference.text) ??
+                    ''
+                  ).split('\n')[0],
+                });
+                return `- ${ref.type}#${ref.originalIndex + 1}: ${ref.reference.originalUrl}\n\t- Anchor: \\ue202turn${turn}ref${references.length - 1}`;
+              })
+              .join('\n');
+            output += '\n\n';
+          }
+
+          if (hIndex < (r.highlights?.length ?? 0) - 1) {
+            output += '---\n\n';
+          }
+        });
+
+      delete results.organic[i].highlights;
+      output += '\n';
+    }
   }
 
   // Ignoring these sections for now
@@ -70,23 +102,36 @@ export function formatResultsForLLM(
   if (results.knowledgeGraph != null) {
     addSection('Knowledge Graph');
     output += [
-      `Title: ${results.knowledgeGraph.title ?? '(no title)'}`,
+      `**Title:** ${results.knowledgeGraph.title ?? '(no title)'}`,
+      results.knowledgeGraph.type != null
+        ? `**Type:** ${results.knowledgeGraph.type}`
+        : '',
       results.knowledgeGraph.description != null
-        ? `Description: ${results.knowledgeGraph.description}`
+        ? `**Description:** ${results.knowledgeGraph.description}`
         : '',
-      results.knowledgeGraph.type != null
-        ? `Type: ${results.knowledgeGraph.type}`
+      results.knowledgeGraph.descriptionSource != null
+        ? `**Description Source:** ${results.knowledgeGraph.descriptionSource}`
+        : '',
+      results.knowledgeGraph.descriptionLink != null
+        ? `**Description Link:** ${results.knowledgeGraph.descriptionLink}`
         : '',
       results.knowledgeGraph.imageUrl != null
-        ? `Image URL: ${results.knowledgeGraph.imageUrl}`
+        ? `**Image URL:** ${results.knowledgeGraph.imageUrl}`
+        : '',
+      results.knowledgeGraph.website != null
+        ? `**Website:** ${results.knowledgeGraph.website}`
         : '',
       results.knowledgeGraph.attributes != null
-        ? `Attributes: ${JSON.stringify(results.knowledgeGraph.attributes, null, 2)}`
+        ? `**Attributes:**\n\`\`\`json\n${JSON.stringify(
+          results.knowledgeGraph.attributes,
+          null,
+          2
+        )}\n\`\`\``
         : '',
       '',
     ]
       .filter(Boolean)
-      .join('\n');
+      .join('\n\n');
   }
 
   // Answer Box
@@ -94,31 +139,44 @@ export function formatResultsForLLM(
     addSection('Answer Box');
     output += [
       results.answerBox.title != null
-        ? `Title: ${results.answerBox.title}`
-        : '',
-      results.answerBox.answer != null
-        ? `Answer: ${results.answerBox.answer}`
+        ? `**Title:** ${results.answerBox.title}`
         : '',
       results.answerBox.snippet != null
-        ? `Snippet: ${results.answerBox.snippet}`
+        ? `**Snippet:** ${results.answerBox.snippet}`
+        : '',
+      results.answerBox.snippetHighlighted != null
+        ? `**Snippet Highlighted:** ${results.answerBox.snippetHighlighted
+          .map((s) => `\`${s}\``)
+          .join(' ')}`
+        : '',
+      results.answerBox.link != null
+        ? `**Link:** ${results.answerBox.link}`
         : '',
-      results.answerBox.date != null ? `Date: ${results.answerBox.date}` : '',
       '',
     ]
       .filter(Boolean)
-      .join('\n');
+      .join('\n\n');
   }
 
   // People also ask
   const peopleAlsoAsk = results.peopleAlsoAsk ?? [];
   if (peopleAlsoAsk.length) {
     addSection('People Also Ask');
-    peopleAlsoAsk.forEach((p, _i) => {
-      output += [`Q: ${p.question}`, `A: ${p.answer}`, '']
+    peopleAlsoAsk.forEach((p, i) => {
+      output += [
+        `### Question ${i + 1}:`,
+        `"${p.question}"`,
+        `${p.snippet != null && p.snippet ? `Snippet: ${p.snippet}}` : ''}`,
+        `${p.title != null && p.title ? `Title: ${p.title}` : ''}`,
+        `${p.link != null && p.link ? `Link: ${p.link}` : ''}`,
+        '',
+      ]
         .filter(Boolean)
-        .join('\n');
+        .join('\n\n');
     });
   }
-
-  return output.trim();
+  return {
+    output: output.trim(),
+    references,
+  };
 }