@librechat/agents 2.4.30 → 2.4.33

package/src/tools/search/content.test.ts

@@ -0,0 +1,173 @@
+/* eslint-disable @typescript-eslint/no-unused-vars */
+/* eslint-disable no-console */
+// content.test.ts
+import * as fs from 'fs';
+import { processContent } from './content';
+
+describe('Link Processor', () => {
+  afterAll(() => {
+    if (fs.existsSync('./temp.html')) {
+      fs.unlinkSync('./temp.html');
+    }
+    if (fs.existsSync('./temp.md')) {
+      fs.unlinkSync('./temp.md');
+    }
+  });
+  // Basic functionality tests
+  test('should replace basic links with references', () => {
+    const html = `
+      <p>Test with <a href="https://example.com/link" title="Example">a link</a></p>
+      <p>And an <img src="https://example.com/img.jpg" alt="image"></p>
+      <p>Plus a <video src="https://example.com/video.mp4"></video></p>
+    `;
+
+    const markdown = `
+      Test with [a link](https://example.com/link "Example")
+      And an ![image](https://example.com/img.jpg)
+      Plus a [video](https://example.com/video.mp4)
+    `;
+
+    const result = processContent(html, markdown);
+
+    expect(result.links.length).toBe(1);
+    expect(result.images.length).toBe(1);
+    expect(result.videos.length).toBe(1);
+    expect(result.markdown).toContain('link#1');
+    expect(result.markdown).toContain('image#1');
+    expect(result.markdown).toContain('video#1');
+  });
+
+  // Edge case tests
+  test('should handle links with parentheses and special characters', () => {
+    const html = `
+      <a href="https://example.com/page(1).html" title="Parens">Link with parens</a>
+      <a href="https://example.com/path?query=test&param=value">Link with query</a>
+    `;
+
+    const markdown = `
+      [Link with parens](https://example.com/page(1).html "Parens")
+      [Link with query](https://example.com/path?query=test&param=value)
+    `;
+
+    const result = processContent(html, markdown);
+
+    expect(result.links.length).toBe(2);
+    expect(result.markdown).toContain('link#1');
+    expect(result.markdown).toContain('link#2');
+  });
+
+  // Performance test with large files
+  test('should process large files efficiently', () => {
+    const html = fs.readFileSync('src/tools/search/test.html', 'utf-8');
+    const markdown = fs.readFileSync('src/tools/search/test.md', 'utf-8');
+
+    // const largeHtml = generateLargeHtml(1000); // 1000 links
+    // fs.writeFileSync('./temp.html', largeHtml);
+
+    // const largeMd = generateLargeMarkdown(1000); // 1000 links
+    // fs.writeFileSync('./temp.md', largeMd);
+
+    // const html = fs.readFileSync('./temp.html', 'utf-8');
+    // const markdown = fs.readFileSync('./temp.md', 'utf-8');
+
+    // Measure time taken to process
+    const startTime = process.hrtime();
+    const result = processContent(html, markdown);
+    const elapsed = process.hrtime(startTime);
+    const timeInMs = elapsed[0] * 1000 + elapsed[1] / 1000000;
+
+    console.log(
+      `Processed ${result.links.length} links, ${result.images.length} images, and ${result.videos.length} videos in ${timeInMs.toFixed(2)}ms`
+    );
+
+    // Basic validations for large file processing
+    expect(result.links.length).toBeGreaterThan(0);
+    expect(result.markdown).toContain('link#');
+
+    // Check if all links were replaced (sample check)
+    expect(result.markdown).not.toContain('https://example.com/link');
+  });
+
+  // Memory usage test
+  test('should have reasonable memory usage', () => {
+    const html = fs.readFileSync('src/tools/search/test.html', 'utf-8');
+    const markdown = fs.readFileSync('src/tools/search/test.md', 'utf-8');
+
+    const beforeMem = process.memoryUsage();
+    processContent(html, markdown);
+    const afterMem = process.memoryUsage();
+
+    const heapUsed = (afterMem.heapUsed - beforeMem.heapUsed) / 1024 / 1024; // MB
+
+    console.log(`Memory used: ${heapUsed.toFixed(2)} MB`);
+
+    // This is a loose check - actual thresholds depend on your environment
+    expect(heapUsed).toBeLessThan(100); // Should use less than 100MB additional heap
+  });
+
+  // Real-world file test (if available)
+  test('should process real-world Wikipedia content', () => {
+    // Try to find real-world test files if they exist
+    const wikiHtml = 'src/tools/search/test.html';
+    const wikiMd = 'src/tools/search/test.md';
+
+    if (fs.existsSync(wikiHtml) && fs.existsSync(wikiMd)) {
+      const html = fs.readFileSync(wikiHtml, 'utf-8');
+      const markdown = fs.readFileSync(wikiMd, 'utf-8');
+
+      const result = processContent(html, markdown);
+
+      console.log(
+        `Processed ${result.links.length} Wikipedia links, ${result.images.length} images, and ${result.videos.length} videos`
+      );
+
+      expect(result.links.length).toBeGreaterThan(10); // Wikipedia articles typically have many links
+      expect(result.markdown).not.toMatch(/\]\(https?:\/\/[^\s")]+\)/); // No regular URLs should remain
+    } else {
+      console.log('Wikipedia test files not found, skipping this test');
+    }
+  });
+});
+
+// Helper function to generate large HTML test data
+function generateLargeHtml(linkCount: number): string {
+  let html = '<html><body>';
+
+  for (let i = 1; i <= linkCount; i++) {
+    html += `<p>Paragraph ${i} with <a href="https://example.com/link${i}" title="Link ${i}">link ${i}</a>`;
+
+    if (i % 10 === 0) {
+      html += ` and <img src="https://example.com/image${i / 10}.jpg" alt="Image ${i / 10}">`;
+    }
+
+    if (i % 50 === 0) {
+      html += ` and <video src="https://example.com/video${i / 50}.mp4" title="Video ${i / 50}"></video>`;
+    }
+
+    html += '</p>';
+  }
+
+  html += '</body></html>';
+  return html;
+}
+
+/** Helper function to generate large Markdown test data  */
+function generateLargeMarkdown(linkCount: number): string {
+  let markdown = '# Test Document\n\n';
+
+  for (let i = 1; i <= linkCount; i++) {
+    markdown += `Paragraph ${i} with [link ${i}](https://example.com/link${i} "Link ${i}")`;
+
+    if (i % 10 === 0) {
+      markdown += ` and ![Image ${i / 10}](https://example.com/image${i / 10}.jpg)`;
+    }
+
+    if (i % 50 === 0) {
+      markdown += ` and [Video ${i / 50}](https://example.com/video${i / 50}.mp4 "Video ${i / 50}")`;
+    }
+
+    markdown += '\n\n';
+  }
+
+  return markdown;
+}

package/src/tools/search/content.ts

@@ -0,0 +1,147 @@
+import * as cheerio from 'cheerio';
+import type { References, MediaReference } from './types';
+
+export function processContent(
+  html: string,
+  markdown: string
+): {
+  markdown: string;
+} & References {
+  const linkMap = new Map<string, MediaReference>();
+  const imageMap = new Map<string, MediaReference>();
+  const videoMap = new Map<string, MediaReference>();
+  const iframeMap = new Map<string, MediaReference>();
+
+  const $ = cheerio.load(html, {
+    xmlMode: false,
+  });
+
+  // Extract all media references
+  $('a[href]').each((_, el) => {
+    const href = $(el).attr('href');
+    if (href != null && href) {
+      linkMap.set(href, {
+        originalUrl: href,
+        title: $(el).attr('title'),
+        text: $(el).text().trim(),
+      });
+    }
+  });
+
+  $('img[src]').each((_, el) => {
+    const src = $(el).attr('src');
+    if (src != null && src) {
+      imageMap.set(src, {
+        originalUrl: src,
+        title: $(el).attr('alt') ?? $(el).attr('title'),
+      });
+    }
+  });
+
+  // Handle videos (dedicated video elements and video platforms in iframes)
+  $('video[src], iframe[src*="youtube"], iframe[src*="vimeo"]').each(
+    (_, el) => {
+      const src = $(el).attr('src');
+      if (src != null && src) {
+        videoMap.set(src, {
+          originalUrl: src,
+          title: $(el).attr('title'),
+        });
+      }
+    }
+  );
+
+  // Handle all other generic iframes that aren't already captured as videos
+  $('iframe').each((_, el) => {
+    const src = $(el).attr('src');
+    if (
+      src != null &&
+      src &&
+      !src.includes('youtube') &&
+      !src.includes('vimeo')
+    ) {
+      iframeMap.set(src, {
+        originalUrl: src,
+        title: $(el).attr('title'),
+      });
+    }
+  });
+
+  // Create lookup maps with indices
+  const linkIndexMap = new Map<string, number>();
+  const imageIndexMap = new Map<string, number>();
+  const videoIndexMap = new Map<string, number>();
+  const iframeIndexMap = new Map<string, number>();
+
+  Array.from(linkMap.keys()).forEach((url, i) => linkIndexMap.set(url, i + 1));
+  Array.from(imageMap.keys()).forEach((url, i) =>
+    imageIndexMap.set(url, i + 1)
+  );
+  Array.from(videoMap.keys()).forEach((url, i) =>
+    videoIndexMap.set(url, i + 1)
+  );
+  Array.from(iframeMap.keys()).forEach((url, i) =>
+    iframeIndexMap.set(url, i + 1)
+  );
+
+  // Process the markdown
+  let result = markdown;
+
+  // Replace each URL one by one, starting with the longest URLs first to avoid partial matches
+  const allUrls = [
+    ...Array.from(imageMap.keys()).map((url) => ({
+      url,
+      type: 'image',
+      idx: imageIndexMap.get(url),
+    })),
+    ...Array.from(videoMap.keys()).map((url) => ({
+      url,
+      type: 'video',
+      idx: videoIndexMap.get(url),
+    })),
+    ...Array.from(iframeMap.keys()).map((url) => ({
+      url,
+      type: 'iframe',
+      idx: iframeIndexMap.get(url),
+    })),
+    ...Array.from(linkMap.keys()).map((url) => ({
+      url,
+      type: 'link',
+      idx: linkIndexMap.get(url),
+    })),
+  ].sort((a, b) => b.url.length - a.url.length);
+
+  // Create a function to escape special characters in URLs for regex
+  function escapeRegex(string: string): string {
+    return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+  }
+
+  // Replace each URL in the markdown
+  for (const { url, type, idx } of allUrls) {
+    // Create a regex that captures URLs in markdown links
+    const regex = new RegExp(`\\(${escapeRegex(url)}(?:\\s+"[^"]*")?\\)`, 'g');
+
+    result = result.replace(regex, (match) => {
+      // Keep any title attribute that might exist
+      const titleMatch = match.match(/\s+"([^"]*)"/);
+      const titlePart = titleMatch ? ` "${titleMatch[1]}"` : '';
+
+      return `(${type}#${idx}${titlePart})`;
+    });
+  }
+
+  iframeMap.clear();
+  const links = Array.from(linkMap.values());
+  linkMap.clear();
+  const images = Array.from(imageMap.values());
+  imageMap.clear();
+  const videos = Array.from(videoMap.values());
+  videoMap.clear();
+
+  return {
+    markdown: result,
+    links,
+    images,
+    videos,
+  };
+}

package/src/tools/search/firecrawl.ts

@@ -0,0 +1,158 @@
+import axios from 'axios';
+import { processContent } from './content';
+import type * as t from './types';
+import { createDefaultLogger } from './utils';
+
+/**
+ * Firecrawl scraper implementation
+ * Uses the Firecrawl API to scrape web pages
+ */
+export class FirecrawlScraper {
+  private apiKey: string;
+  private apiUrl: string;
+  private defaultFormats: string[];
+  private timeout: number;
+  private logger: t.Logger;
+
+  constructor(config: t.FirecrawlScraperConfig = {}) {
+    this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';
+
+    const baseUrl =
+      config.apiUrl ??
+      process.env.FIRECRAWL_BASE_URL ??
+      'https://api.firecrawl.dev';
+    this.apiUrl = `${baseUrl.replace(/\/+$/, '')}/v1/scrape`;
+
+    this.defaultFormats = config.formats ?? ['markdown', 'html'];
+    this.timeout = config.timeout ?? 15000;
+
+    this.logger = config.logger || createDefaultLogger();
+
+    if (!this.apiKey) {
+      this.logger.warn('FIRECRAWL_API_KEY is not set. Scraping will not work.');
+    }
+
+    this.logger.debug(
+      `Firecrawl scraper initialized with API URL: ${this.apiUrl}`
+    );
+  }
+
+  /**
+   * Scrape a single URL
+   * @param url URL to scrape
+   * @param options Scrape options
+   * @returns Scrape response
+   */
+  async scrapeUrl(
+    url: string,
+    options: t.FirecrawlScrapeOptions = {}
+  ): Promise<[string, t.FirecrawlScrapeResponse]> {
+    if (!this.apiKey) {
+      return [
+        url,
+        {
+          success: false,
+          error: 'FIRECRAWL_API_KEY is not set',
+        },
+      ];
+    }
+
+    try {
+      const response = await axios.post(
+        this.apiUrl,
+        {
+          url,
+          formats: options.formats || this.defaultFormats,
+          includeTags: options.includeTags,
+          excludeTags: options.excludeTags,
+          headers: options.headers,
+          waitFor: options.waitFor,
+          timeout: options.timeout ?? this.timeout,
+        },
+        {
+          headers: {
+            'Content-Type': 'application/json',
+            Authorization: `Bearer ${this.apiKey}`,
+          },
+          timeout: this.timeout,
+        }
+      );
+
+      return [url, response.data];
+    } catch (error) {
+      const errorMessage =
+        error instanceof Error ? error.message : String(error);
+      return [
+        url,
+        {
+          success: false,
+          error: `Firecrawl API request failed: ${errorMessage}`,
+        },
+      ];
+    }
+  }
+
+  /**
+   * Extract content from scrape response
+   * @param response Scrape response
+   * @returns Extracted content or empty string if not available
+   */
+  extractContent(
+    response: t.FirecrawlScrapeResponse
+  ): [string, undefined | t.References] {
+    if (!response.success || !response.data) {
+      return ['', undefined];
+    }
+
+    if (response.data.markdown != null && response.data.html != null) {
+      try {
+        const { markdown, ...rest } = processContent(
+          response.data.html,
+          response.data.markdown
+        );
+        return [markdown, rest];
+      } catch (error) {
+        this.logger.error('Error processing content:', error);
+        return [response.data.markdown, undefined];
+      }
+    } else if (response.data.markdown != null) {
+      return [response.data.markdown, undefined];
+    }
+
+    // Fall back to HTML content
+    if (response.data.html != null) {
+      return [response.data.html, undefined];
+    }
+
+    // Fall back to raw HTML content
+    if (response.data.rawHtml != null) {
+      return [response.data.rawHtml, undefined];
+    }
+
+    return ['', undefined];
+  }
+
+  /**
+   * Extract metadata from scrape response
+   * @param response Scrape response
+   * @returns Metadata object
+   */
+  extractMetadata(response: t.FirecrawlScrapeResponse): t.ScrapeMetadata {
+    if (!response.success || !response.data || !response.data.metadata) {
+      return {};
+    }
+
+    return response.data.metadata;
+  }
+}
+
+/**
+ * Create a Firecrawl scraper instance
+ * @param config Scraper configuration
+ * @returns Firecrawl scraper instance
+ */
+export const createFirecrawlScraper = (
+  config: t.FirecrawlScraperConfig = {}
+): FirecrawlScraper => {
+  return new FirecrawlScraper(config);
+};