@librechat/agents 2.4.83 → 2.4.85

package/src/messages/formatMessage.test.ts

@@ -1,8 +1,43 @@
-import { HumanMessage, AIMessage, SystemMessage } from '@langchain/core/messages';
-import { formatMessage, formatLangChainMessages, formatFromLangChain } from './format';
+import {
+  HumanMessage,
+  AIMessage,
+  SystemMessage,
+} from '@langchain/core/messages';
+import type { MessageContentComplex } from '@/types';
+import {
+  formatMessage,
+  formatLangChainMessages,
+  formatFromLangChain,
+  formatMediaMessage,
+} from './format';
+import { Providers } from '@/common';
 
 const NO_PARENT = '00000000-0000-0000-0000-000000000000';
 
+/**
+ * Type for formatted message results with media content
+ */
+interface FormattedMediaMessage {
+  role: string;
+  content: MessageContentComplex[];
+  name?: string;
+}
+
+/**
+ * Type guard to check if result is a FormattedMediaMessage
+ */
+function isFormattedMediaMessage(
+  result: unknown
+): result is FormattedMediaMessage {
+  return (
+    typeof result === 'object' &&
+    result !== null &&
+    'role' in result &&
+    'content' in result &&
+    Array.isArray((result as FormattedMediaMessage).content)
+  );
+}
+
 describe('formatMessage', () => {
   it('formats user message', () => {
     const input = {
@@ -187,6 +222,387 @@ describe('formatMessage', () => {
   });
 });
 
+describe('formatMediaMessage', () => {
+  it('formats message with images for default provider', () => {
+    const message = {
+      role: 'user',
+      content: 'Check out this image',
+      name: 'John',
+    };
+    const mediaParts = [
+      {
+        type: 'image_url',
+        image_url: { url: 'https://example.com/image1.jpg' },
+      },
+      {
+        type: 'image_url',
+        image_url: { url: 'https://example.com/image2.jpg' },
+      },
+    ];
+
+    const result = formatMediaMessage({ message, mediaParts });
+
+    expect(result.role).toBe('user');
+    expect(result.name).toBe('John');
+    expect(Array.isArray(result.content)).toBe(true);
+    expect(result.content).toHaveLength(3);
+    expect(result.content[0]).toEqual({
+      type: 'text',
+      text: 'Check out this image',
+    });
+    expect(result.content[1]).toEqual(mediaParts[0]);
+    expect(result.content[2]).toEqual(mediaParts[1]);
+  });
+
+  it('formats message with images for Anthropic (media first)', () => {
+    const message = {
+      role: 'user',
+      content: 'Check out this image',
+    };
+    const mediaParts = [
+      {
+        type: 'image_url',
+        image_url: { url: 'https://example.com/image.jpg' },
+      },
+    ];
+
+    const result = formatMediaMessage({
+      message,
+      mediaParts,
+      endpoint: Providers.ANTHROPIC,
+    });
+
+    expect(result.content).toHaveLength(2);
+    expect(result.content[0]).toEqual(mediaParts[0]);
+    expect(result.content[1]).toEqual({
+      type: 'text',
+      text: 'Check out this image',
+    });
+  });
+
+  it('formats message with multiple media types', () => {
+    const message = {
+      role: 'user',
+      content: 'Check out these files',
+    };
+    const mediaParts = [
+      { type: 'document', document: { url: 'https://example.com/doc.pdf' } },
+      { type: 'video', video: { url: 'https://example.com/video.mp4' } },
+      { type: 'audio', audio: { url: 'https://example.com/audio.mp3' } },
+      {
+        type: 'image_url',
+        image_url: { url: 'https://example.com/image.jpg' },
+      },
+    ];
+
+    const result = formatMediaMessage({ message, mediaParts });
+
+    expect(result.content).toHaveLength(5);
+    expect(result.content[0]).toEqual({
+      type: 'text',
+      text: 'Check out these files',
+    });
+    expect(result.content[1]).toEqual(mediaParts[0]);
+    expect(result.content[2]).toEqual(mediaParts[1]);
+    expect(result.content[3]).toEqual(mediaParts[2]);
+    expect(result.content[4]).toEqual(mediaParts[3]);
+  });
+});
+
+describe('formatMessage with media', () => {
+  it('formats user message with image_urls (backward compatibility)', () => {
+    const input = {
+      message: {
+        sender: 'user',
+        text: 'Check out this image',
+        image_urls: [
+          {
+            type: 'image_url' as const,
+            image_url: { url: 'https://example.com/image.jpg' },
+          },
+        ],
+      },
+      userName: 'John',
+    };
+
+    const result = formatMessage(input);
+
+    expect(isFormattedMediaMessage(result)).toBe(true);
+    if (isFormattedMediaMessage(result)) {
+      expect(result.role).toBe('user');
+      expect(result.name).toBe('John');
+      expect(Array.isArray(result.content)).toBe(true);
+      expect(result.content).toHaveLength(2);
+      expect(result.content[0]).toEqual({
+        type: 'text',
+        text: 'Check out this image',
+      });
+      expect(result.content[1]).toEqual(input.message.image_urls[0]);
+    }
+  });
+
+  it('formats user message with documents', () => {
+    const input = {
+      message: {
+        role: 'user',
+        content: 'Review this document',
+        documents: [
+          {
+            type: 'document',
+            document: { url: 'https://example.com/report.pdf' },
+          },
+        ],
+      },
+    };
+
+    const result = formatMessage(input);
+
+    expect(isFormattedMediaMessage(result)).toBe(true);
+    if (isFormattedMediaMessage(result)) {
+      expect(result.role).toBe('user');
+      expect(Array.isArray(result.content)).toBe(true);
+      expect(result.content).toHaveLength(2);
+      expect(result.content[0]).toEqual({
+        type: 'text',
+        text: 'Review this document',
+      });
+      expect(result.content[1]).toEqual(input.message.documents[0]);
+    }
+  });
+
+  it('formats user message with videos', () => {
+    const input = {
+      message: {
+        role: 'user',
+        content: 'Watch this video',
+        videos: [
+          { type: 'video', video: { url: 'https://example.com/demo.mp4' } },
+        ],
+      },
+    };
+
+    const result = formatMessage(input);
+
+    expect(isFormattedMediaMessage(result)).toBe(true);
+    if (isFormattedMediaMessage(result)) {
+      expect(result.role).toBe('user');
+      expect(Array.isArray(result.content)).toBe(true);
+      expect(result.content).toHaveLength(2);
+      expect(result.content[0]).toEqual({
+        type: 'text',
+        text: 'Watch this video',
+      });
+      expect(result.content[1]).toEqual(input.message.videos[0]);
+    }
+  });
+
+  it('formats user message with audios', () => {
+    const input = {
+      message: {
+        role: 'user',
+        content: 'Listen to this',
+        audios: [
+          { type: 'audio', audio: { url: 'https://example.com/podcast.mp3' } },
+        ],
+      },
+    };
+
+    const result = formatMessage(input);
+
+    expect(isFormattedMediaMessage(result)).toBe(true);
+    if (isFormattedMediaMessage(result)) {
+      expect(result.role).toBe('user');
+      expect(Array.isArray(result.content)).toBe(true);
+      expect(result.content).toHaveLength(2);
+      expect(result.content[0]).toEqual({
+        type: 'text',
+        text: 'Listen to this',
+      });
+      expect(result.content[1]).toEqual(input.message.audios[0]);
+    }
+  });
+
+  it('formats user message with all media types in correct order', () => {
+    const input = {
+      message: {
+        role: 'user',
+        content: 'Check out all these files',
+        documents: [
+          {
+            type: 'document',
+            document: { url: 'https://example.com/doc.pdf' },
+          },
+        ],
+        videos: [
+          { type: 'video', video: { url: 'https://example.com/video.mp4' } },
+        ],
+        audios: [
+          { type: 'audio', audio: { url: 'https://example.com/audio.mp3' } },
+        ],
+        image_urls: [
+          {
+            type: 'image_url' as const,
+            image_url: { url: 'https://example.com/image.jpg' },
+          },
+        ],
+      },
+    };
+
+    const result = formatMessage(input);
+
+    expect(isFormattedMediaMessage(result)).toBe(true);
+    if (isFormattedMediaMessage(result)) {
+      expect(result.role).toBe('user');
+      expect(Array.isArray(result.content)).toBe(true);
+      expect(result.content).toHaveLength(5);
+      // Text first
+      expect(result.content[0]).toEqual({
+        type: 'text',
+        text: 'Check out all these files',
+      });
+      // Then documents, videos, audios, images
+      expect(result.content[1]).toEqual(input.message.documents[0]);
+      expect(result.content[2]).toEqual(input.message.videos[0]);
+      expect(result.content[3]).toEqual(input.message.audios[0]);
+      expect(result.content[4]).toEqual(input.message.image_urls[0]);
+    }
+  });
+
+  it('formats user message with multiple files of the same type', () => {
+    const input = {
+      message: {
+        role: 'user',
+        content: 'Review these documents',
+        documents: [
+          {
+            type: 'document',
+            document: { url: 'https://example.com/doc1.pdf' },
+          },
+          {
+            type: 'document',
+            document: { url: 'https://example.com/doc2.pdf' },
+          },
+          {
+            type: 'document',
+            document: { url: 'https://example.com/doc3.pdf' },
+          },
+        ],
+      },
+    };
+
+    const result = formatMessage(input);
+
+    expect(isFormattedMediaMessage(result)).toBe(true);
+    if (isFormattedMediaMessage(result)) {
+      expect(result.content).toHaveLength(4);
+      expect(result.content[0].type).toBe('text');
+      expect(result.content[1]).toEqual(input.message.documents[0]);
+      expect(result.content[2]).toEqual(input.message.documents[1]);
+      expect(result.content[3]).toEqual(input.message.documents[2]);
+    }
+  });
+
+  it('respects Anthropic provider ordering (media before text)', () => {
+    const input = {
+      message: {
+        role: 'user',
+        content: 'Check this out',
+        documents: [
+          {
+            type: 'document',
+            document: { url: 'https://example.com/doc.pdf' },
+          },
+        ],
+        image_urls: [
+          {
+            type: 'image_url' as const,
+            image_url: { url: 'https://example.com/image.jpg' },
+          },
+        ],
+      },
+      endpoint: Providers.ANTHROPIC,
+    };
+
+    const result = formatMessage(input);
+
+    expect(isFormattedMediaMessage(result)).toBe(true);
+    if (isFormattedMediaMessage(result)) {
+      expect(result.content).toHaveLength(3);
+      // Media first for Anthropic
+      expect(result.content[0]).toEqual(input.message.documents[0]);
+      expect(result.content[1]).toEqual(input.message.image_urls[0]);
+      expect(result.content[2]).toEqual({
+        type: 'text',
+        text: 'Check this out',
+      });
+    }
+  });
+
+  it('does not format media for assistant messages', () => {
+    const input = {
+      message: {
+        role: 'assistant',
+        content: 'Here is a response',
+        documents: [
+          {
+            type: 'document',
+            document: { url: 'https://example.com/doc.pdf' },
+          },
+        ],
+      },
+    };
+
+    const result = formatMessage(input);
+
+    expect(result).toMatchObject({
+      role: 'assistant',
+      content: 'Here is a response',
+    });
+  });
+
+  it('handles empty media arrays gracefully', () => {
+    const input = {
+      message: {
+        role: 'user',
+        content: 'Just text',
+        documents: [],
+        videos: [],
+        audios: [],
+        image_urls: [],
+      },
+    };
+
+    const result = formatMessage(input);
+
+    expect(result).toMatchObject({
+      role: 'user',
+      content: 'Just text',
+    });
+  });
+
+  it('formats media with langChain flag', () => {
+    const input = {
+      message: {
+        role: 'user',
+        content: 'Check this image',
+        image_urls: [
+          {
+            type: 'image_url' as const,
+            image_url: { url: 'https://example.com/image.jpg' },
+          },
+        ],
+      },
+      langChain: true,
+    };
+
+    const result = formatMessage(input);
+
+    expect(result).toBeInstanceOf(HumanMessage);
+    expect(Array.isArray(result.lc_kwargs.content)).toBe(true);
+    expect(result.lc_kwargs.content).toHaveLength(2);
+  });
+});
+
 describe('formatLangChainMessages', () => {
   it('formats an array of messages for LangChain', () => {
     const messages = [

package/src/scripts/search.ts

@@ -83,7 +83,11 @@ async function testStandardStreaming(): Promise<void> {
     graphConfig: {
       type: 'standard',
       llmConfig,
-      tools: [createSearchTool()],
+      tools: [
+        createSearchTool({
+          scraperProvider: 'serper',
+        }),
+      ],
       instructions:
         'You are a friendly AI assistant. Always address the user by their name.',
       // additional_instructions: `The user's name is ${userName} and they are located in ${location}.`,

package/src/tools/search/firecrawl.ts

@@ -7,9 +7,10 @@ import { createDefaultLogger } from './utils';
  * Firecrawl scraper implementation
  * Uses the Firecrawl API to scrape web pages
  */
-export class FirecrawlScraper {
+export class FirecrawlScraper implements t.BaseScraper {
   private apiKey: string;
   private apiUrl: string;
+  private version: string;
   private defaultFormats: string[];
   private timeout: number;
   private logger: t.Logger;
@@ -32,11 +33,13 @@ export class FirecrawlScraper {
   constructor(config: t.FirecrawlScraperConfig = {}) {
     this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';
 
+    this.version = config.version ?? 'v2';
+
     const baseUrl =
       config.apiUrl ??
       process.env.FIRECRAWL_BASE_URL ??
       'https://api.firecrawl.dev';
-    this.apiUrl = `${baseUrl.replace(/\/+$/, '')}/v1/scrape`;
+    this.apiUrl = `${baseUrl.replace(/\/+$/, '')}/${this.version}/scrape`;
 
     this.defaultFormats = config.formats ?? ['markdown', 'rawHtml'];
     this.timeout = config.timeout ?? 7500;

package/src/tools/search/search.ts

@@ -2,7 +2,6 @@ import axios from 'axios';
 import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters';
 import type * as t from './types';
 import { getAttribution, createDefaultLogger } from './utils';
-import { FirecrawlScraper } from './firecrawl';
 import { BaseReranker } from './rerankers';
 
 const chunker = {
@@ -434,7 +433,7 @@ export const createSearchAPI = (
 
 export const createSourceProcessor = (
   config: t.ProcessSourcesConfig = {},
-  scraperInstance?: FirecrawlScraper
+  scraperInstance?: t.BaseScraper
 ): {
   processSources: (
     fields: t.ProcessSourcesFields
@@ -442,7 +441,7 @@ export const createSourceProcessor = (
   topResults: number;
 } => {
   if (!scraperInstance) {
-    throw new Error('Firecrawl scraper instance is required');
+    throw new Error('Scraper instance is required');
   }
   const {
     topResults = 5,
@@ -453,7 +452,7 @@ export const createSourceProcessor = (
   } = config;
 
   const logger_ = logger || createDefaultLogger();
-  const firecrawlScraper = scraperInstance;
+  const scraper = scraperInstance;
 
   const webScraper = {
     scrapeMany: async ({
@@ -465,12 +464,12 @@ export const createSourceProcessor = (
       links: string[];
       onGetHighlights: t.SearchToolConfig['onGetHighlights'];
     }): Promise<Array<t.ScrapeResult>> => {
-      logger_.debug(`Scraping ${links.length} links with Firecrawl`);
+      logger_.debug(`Scraping ${links.length} links`);
       const promises: Array<Promise<t.ScrapeResult>> = [];
       try {
         for (let i = 0; i < links.length; i++) {
           const currentLink = links[i];
-          const promise: Promise<t.ScrapeResult> = firecrawlScraper
+          const promise: Promise<t.ScrapeResult> = scraper
             .scrapeUrl(currentLink, {})
             .then(([url, response]) => {
               const attribution = getAttribution(
@@ -479,8 +478,7 @@ export const createSourceProcessor = (
                 logger_
               );
               if (response.success && response.data) {
-                const [content, references] =
-                  firecrawlScraper.extractContent(response);
+                const [content, references] = scraper.extractContent(response);
                 return {
                   url,
                   references,

package/src/tools/search/serper-scraper.ts

@@ -0,0 +1,155 @@
+import axios from 'axios';
+import type * as t from './types';
+import { createDefaultLogger } from './utils';
+
+/**
+ * Serper scraper implementation
+ * Uses the Serper Scrape API (https://scrape.serper.dev) to scrape web pages
+ *
+ * Features:
+ * - Simple API with single endpoint
+ * - Returns both text and markdown content
+ * - Includes metadata from scraped pages
+ * - Credits-based pricing model
+ *
+ * @example
+ * ```typescript
+ * const scraper = createSerperScraper({
+ *   apiKey: 'your-serper-api-key',
+ *   includeMarkdown: true,
+ *   timeout: 10000
+ * });
+ *
+ * const [url, response] = await scraper.scrapeUrl('https://example.com');
+ * if (response.success) {
+ *   const [content] = scraper.extractContent(response);
+ *   console.log(content);
+ * }
+ * ```
+ */
+export class SerperScraper implements t.BaseScraper {
+  private apiKey: string;
+  private apiUrl: string;
+  private timeout: number;
+  private logger: t.Logger;
+  private includeMarkdown: boolean;
+
+  constructor(config: t.SerperScraperConfig = {}) {
+    this.apiKey = config.apiKey ?? process.env.SERPER_API_KEY ?? '';
+
+    this.apiUrl =
+      config.apiUrl ??
+      process.env.SERPER_SCRAPE_URL ??
+      'https://scrape.serper.dev';
+
+    this.timeout = config.timeout ?? 7500;
+    this.includeMarkdown = config.includeMarkdown ?? true;
+
+    this.logger = config.logger || createDefaultLogger();
+
+    if (!this.apiKey) {
+      this.logger.warn('SERPER_API_KEY is not set. Scraping will not work.');
+    }
+
+    this.logger.debug(
+      `Serper scraper initialized with API URL: ${this.apiUrl}`
+    );
+  }
+
+  /**
+   * Scrape a single URL
+   * @param url URL to scrape
+   * @param options Scrape options
+   * @returns Scrape response
+   */
+  async scrapeUrl(
+    url: string,
+    options: t.SerperScrapeOptions = {}
+  ): Promise<[string, t.SerperScrapeResponse]> {
+    if (!this.apiKey) {
+      return [
+        url,
+        {
+          success: false,
+          error: 'SERPER_API_KEY is not set',
+        },
+      ];
+    }
+
+    try {
+      const payload = {
+        url,
+        includeMarkdown: options.includeMarkdown ?? this.includeMarkdown,
+      };
+
+      const response = await axios.post(this.apiUrl, payload, {
+        headers: {
+          'X-API-KEY': this.apiKey,
+          'Content-Type': 'application/json',
+        },
+        timeout: options.timeout ?? this.timeout,
+      });
+
+      return [url, { success: true, data: response.data }];
+    } catch (error) {
+      const errorMessage =
+        error instanceof Error ? error.message : String(error);
+      return [
+        url,
+        {
+          success: false,
+          error: `Serper Scrape API request failed: ${errorMessage}`,
+        },
+      ];
+    }
+  }
+
+  /**
+   * Extract content from scrape response
+   * @param response Scrape response
+   * @returns Extracted content or empty string if not available
+   */
+  extractContent(
+    response: t.SerperScrapeResponse
+  ): [string, undefined | t.References] {
+    if (!response.success || !response.data) {
+      return ['', undefined];
+    }
+
+    if (response.data.markdown != null) {
+      return [response.data.markdown, undefined];
+    }
+
+    if (response.data.text != null) {
+      return [response.data.text, undefined];
+    }
+
+    return ['', undefined];
+  }
+
+  /**
+   * Extract metadata from scrape response
+   * @param response Scrape response
+   * @returns Metadata object
+   */
+  extractMetadata(
+    response: t.SerperScrapeResponse
+  ): Record<string, string | number | boolean | null | undefined> {
+    if (!response.success || !response.data || !response.data.metadata) {
+      return {};
+    }
+
+    return response.data.metadata;
+  }
+}
+
+/**
+ * Create a Serper scraper instance
+ * @param config Scraper configuration
+ * @returns Serper scraper instance
+ */
+export const createSerperScraper = (
+  config: t.SerperScraperConfig = {}
+): SerperScraper => {
+  return new SerperScraper(config);
+};