@librechat/agents 2.4.83 → 2.4.85

package/dist/types/tools/search/search.d.ts

@@ -1,9 +1,8 @@
 import type * as t from './types';
-import { FirecrawlScraper } from './firecrawl';
 export declare const createSearchAPI: (config: t.SearchConfig) => {
     getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;
 };
-export declare const createSourceProcessor: (config?: t.ProcessSourcesConfig, scraperInstance?: FirecrawlScraper) => {
+export declare const createSourceProcessor: (config?: t.ProcessSourcesConfig, scraperInstance?: t.BaseScraper) => {
     processSources: (fields: t.ProcessSourcesFields) => Promise<t.SearchResultData>;
     topResults: number;
 };

package/dist/types/tools/search/serper-scraper.d.ts

@@ -0,0 +1,59 @@
+import type * as t from './types';
+/**
+ * Serper scraper implementation
+ * Uses the Serper Scrape API (https://scrape.serper.dev) to scrape web pages
+ *
+ * Features:
+ * - Simple API with single endpoint
+ * - Returns both text and markdown content
+ * - Includes metadata from scraped pages
+ * - Credits-based pricing model
+ *
+ * @example
+ * ```typescript
+ * const scraper = createSerperScraper({
+ *   apiKey: 'your-serper-api-key',
+ *   includeMarkdown: true,
+ *   timeout: 10000
+ * });
+ *
+ * const [url, response] = await scraper.scrapeUrl('https://example.com');
+ * if (response.success) {
+ *   const [content] = scraper.extractContent(response);
+ *   console.log(content);
+ * }
+ * ```
+ */
+export declare class SerperScraper implements t.BaseScraper {
+    private apiKey;
+    private apiUrl;
+    private timeout;
+    private logger;
+    private includeMarkdown;
+    constructor(config?: t.SerperScraperConfig);
+    /**
+     * Scrape a single URL
+     * @param url URL to scrape
+     * @param options Scrape options
+     * @returns Scrape response
+     */
+    scrapeUrl(url: string, options?: t.SerperScrapeOptions): Promise<[string, t.SerperScrapeResponse]>;
+    /**
+     * Extract content from scrape response
+     * @param response Scrape response
+     * @returns Extracted content or empty string if not available
+     */
+    extractContent(response: t.SerperScrapeResponse): [string, undefined | t.References];
+    /**
+     * Extract metadata from scrape response
+     * @param response Scrape response
+     * @returns Metadata object
+     */
+    extractMetadata(response: t.SerperScrapeResponse): Record<string, string | number | boolean | null | undefined>;
+}
+/**
+ * Create a Serper scraper instance
+ * @param config Scraper configuration
+ * @returns Serper scraper instance
+ */
+export declare const createSerperScraper: (config?: t.SerperScraperConfig) => SerperScraper;

package/dist/types/tools/search/tool.d.ts

@@ -6,6 +6,27 @@ import { DATE_RANGE } from './schema';
  * Creates a search tool with a schema that dynamically includes the country field
  * only when the searchProvider is 'serper'.
  *
+ * Supports multiple scraper providers:
+ * - Firecrawl (default): Full-featured web scraping with multiple formats
+ * - Serper: Lightweight scraping using Serper's scrape API
+ *
+ * @example
+ * ```typescript
+ * // Using Firecrawl scraper (default)
+ * const searchTool = createSearchTool({
+ *   searchProvider: 'serper',
+ *   scraperProvider: 'firecrawl',
+ *   firecrawlApiKey: 'your-firecrawl-key'
+ * });
+ *
+ * // Using Serper scraper
+ * const searchTool = createSearchTool({
+ *   searchProvider: 'serper',
+ *   scraperProvider: 'serper',
+ *   serperApiKey: 'your-serper-key'
+ * });
+ * ```
+ *
  * @param config - The search tool configuration
  * @returns A DynamicStructuredTool with a schema that depends on the searchProvider
  */

package/dist/types/tools/search/types.d.ts

@@ -4,6 +4,7 @@ import type { RunnableConfig } from '@langchain/core/runnables';
 import type { BaseReranker } from './rerankers';
 import { DATE_RANGE } from './schema';
 export type SearchProvider = 'serper' | 'searxng';
+export type ScraperProvider = 'firecrawl' | 'serper';
 export type RerankerType = 'infinity' | 'jina' | 'cohere' | 'none';
 export interface Highlight {
     score: number;
@@ -85,8 +86,16 @@ export interface ProcessSourcesConfig {
 export interface FirecrawlConfig {
     firecrawlApiKey?: string;
     firecrawlApiUrl?: string;
+    firecrawlVersion?: string;
     firecrawlOptions?: FirecrawlScraperConfig;
 }
+export interface SerperScraperConfig {
+    apiKey?: string;
+    apiUrl?: string;
+    timeout?: number;
+    logger?: Logger;
+    includeMarkdown?: boolean;
+}
 export interface ScraperContentResult {
     content: string;
 }
@@ -133,7 +142,9 @@ export interface SearchToolConfig extends SearchConfig, ProcessSourcesConfig, Fi
     jinaApiUrl?: string;
     cohereApiKey?: string;
     rerankerType?: RerankerType;
+    scraperProvider?: ScraperProvider;
     scraperTimeout?: number;
+    serperScraperOptions?: SerperScraperConfig;
     onSearchResults?: (results: SearchResult, runnableConfig?: RunnableConfig) => void;
     onGetHighlights?: (link: string) => void;
 }
@@ -147,8 +158,15 @@ export type UsedReferences = {
     originalIndex: number;
     reference: MediaReference;
 }[];
+/** Base Scraper Interface */
+export interface BaseScraper {
+    scrapeUrl(url: string, options?: unknown): Promise<[string, FirecrawlScrapeResponse | SerperScrapeResponse]>;
+    extractContent(response: FirecrawlScrapeResponse | SerperScrapeResponse): [string, undefined | References];
+    extractMetadata(response: FirecrawlScrapeResponse | SerperScrapeResponse): ScrapeMetadata | Record<string, string | number | boolean | null | undefined>;
+}
 /** Firecrawl */
-export type FirecrawlScrapeOptions = Omit<FirecrawlScraperConfig, 'apiKey' | 'apiUrl' | 'logger'>;
+export type FirecrawlScrapeOptions = Omit<FirecrawlScraperConfig, 'apiKey' | 'apiUrl' | 'version' | 'logger'>;
+export type SerperScrapeOptions = Omit<SerperScraperConfig, 'apiKey' | 'apiUrl' | 'logger'>;
 export interface ScrapeMetadata {
     sourceURL?: string;
     url?: string;
@@ -214,9 +232,20 @@ export interface FirecrawlScrapeResponse {
     };
     error?: string;
 }
+export interface SerperScrapeResponse {
+    success: boolean;
+    data?: {
+        text?: string;
+        markdown?: string;
+        metadata?: Record<string, string | number | boolean | null | undefined>;
+        credits?: number;
+    };
+    error?: string;
+}
 export interface FirecrawlScraperConfig {
     apiKey?: string;
     apiUrl?: string;
+    version?: string;
     formats?: string[];
     timeout?: number;
     logger?: Logger;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@librechat/agents",
-  "version": "2.4.83",
+  "version": "2.4.85",
   "main": "./dist/cjs/main.cjs",
   "module": "./dist/esm/main.mjs",
   "types": "./dist/types/index.d.ts",

package/src/llm/google/utils/common.ts

@@ -301,6 +301,20 @@ function _convertLangChainContentToPart(
         mimeType,
       },
     };
+  } else if (
+    content.type === 'document' ||
+    content.type === 'audio' ||
+    content.type === 'video'
+  ) {
+    if (!isMultimodalModel) {
+      throw new Error(`This model does not support ${content.type}s`);
+    }
+    return {
+      inlineData: {
+        data: content.data,
+        mimeType: content.mimeType,
+      },
+    };
   } else if (content.type === 'media') {
     return messageContentMedia(content);
   } else if (content.type === 'tool_use') {

package/src/messages/format.ts

@@ -17,28 +17,28 @@ import type {
 } from '@/types';
 import { Providers, ContentTypes } from '@/common';
 
-interface VisionMessageParams {
+interface MediaMessageParams {
   message: {
     role: string;
     content: string;
     name?: string;
     [key: string]: any;
   };
-  image_urls: MessageContentImageUrl[];
+  mediaParts: MessageContentComplex[];
   endpoint?: Providers;
 }
 
 /**
- * Formats a message to OpenAI Vision API payload format.
+ * Formats a message with media content (images, documents, videos, audios) to API payload format.
  *
- * @param {VisionMessageParams} params - The parameters for formatting.
- * @returns {Object} - The formatted message.
+ * @param params - The parameters for formatting.
+ * @returns - The formatted message.
  */
-export const formatVisionMessage = ({
+export const formatMediaMessage = ({
   message,
-  image_urls,
   endpoint,
-}: VisionMessageParams): {
+  mediaParts,
+}: MediaMessageParams): {
   role: string;
   content: MessageContentComplex[];
   name?: string;
@@ -57,7 +57,7 @@ export const formatVisionMessage = ({
 
   if (endpoint === Providers.ANTHROPIC) {
     result.content = [
-      ...image_urls,
+      ...mediaParts,
       { type: ContentTypes.TEXT, text: message.content },
     ] as MessageContentComplex[];
     return result;
@@ -65,7 +65,7 @@ export const formatVisionMessage = ({
 
   result.content = [
     { type: ContentTypes.TEXT, text: message.content },
-    ...image_urls,
+    ...mediaParts,
   ] as MessageContentComplex[];
 
   return result;
@@ -78,6 +78,9 @@ interface MessageInput {
   text?: string;
   content?: string | MessageContentComplex[];
   image_urls?: MessageContentImageUrl[];
+  documents?: MessageContentComplex[];
+  videos?: MessageContentComplex[];
+  audios?: MessageContentComplex[];
   lc_id?: string[];
   [key: string]: any;
 }
@@ -100,14 +103,14 @@ interface FormattedMessage {
 /**
  * Formats a message to OpenAI payload format based on the provided options.
  *
- * @param {FormatMessageParams} params - The parameters for formatting.
- * @returns {FormattedMessage | HumanMessage | AIMessage | SystemMessage} - The formatted message.
+ * @param params - The parameters for formatting.
+ * @returns - The formatted message.
  */
 export const formatMessage = ({
   message,
   userName,
-  assistantName,
   endpoint,
+  assistantName,
   langChain = false,
 }: FormatMessageParams):
   | FormattedMessage
@@ -135,21 +138,7 @@ export const formatMessage = ({
     content,
   };
 
-  const { image_urls } = message;
-  if (Array.isArray(image_urls) && image_urls.length > 0 && role === 'user') {
-    return formatVisionMessage({
-      message: {
-        ...formattedMessage,
-        content:
-          typeof formattedMessage.content === 'string'
-            ? formattedMessage.content
-            : '',
-      },
-      image_urls,
-      endpoint,
-    });
-  }
-
+  // Set name fields first
   if (_name != null && _name) {
     formattedMessage.name = _name;
   }
@@ -179,6 +168,45 @@ export const formatMessage = ({
     }
   }
 
+  const { image_urls, documents, videos, audios } = message;
+  const mediaParts: MessageContentComplex[] = [];
+
+  if (Array.isArray(documents) && documents.length > 0) {
+    mediaParts.push(...documents);
+  }
+
+  if (Array.isArray(videos) && videos.length > 0) {
+    mediaParts.push(...videos);
+  }
+
+  if (Array.isArray(audios) && audios.length > 0) {
+    mediaParts.push(...audios);
+  }
+
+  if (Array.isArray(image_urls) && image_urls.length > 0) {
+    mediaParts.push(...image_urls);
+  }
+
+  if (mediaParts.length > 0 && role === 'user') {
+    const mediaMessage = formatMediaMessage({
+      message: {
+        ...formattedMessage,
+        content:
+          typeof formattedMessage.content === 'string'
+            ? formattedMessage.content
+            : '',
+      },
+      mediaParts,
+      endpoint,
+    });
+
+    if (!langChain) {
+      return mediaMessage;
+    }
+
+    return new HumanMessage(mediaMessage);
+  }
+
   if (!langChain) {
     return formattedMessage;
   }
@@ -195,9 +223,9 @@ export const formatMessage = ({
 /**
  * Formats an array of messages for LangChain.
  *
- * @param {Array<MessageInput>} messages - The array of messages to format.
- * @param {Omit<FormatMessageParams, 'message' | 'langChain'>} formatOptions - The options for formatting each message.
- * @returns {Array<HumanMessage | AIMessage | SystemMessage>} - The array of formatted LangChain messages.
+ * @param messages - The array of messages to format.
+ * @param formatOptions - The options for formatting each message.
+ * @returns - The array of formatted LangChain messages.
  */
 export const formatLangChainMessages = (
   messages: Array<MessageInput>,
@@ -228,8 +256,8 @@ interface LangChainMessage {
 /**
  * Formats a LangChain message object by merging properties from `lc_kwargs` or `kwargs` and `additional_kwargs`.
  *
- * @param {LangChainMessage} message - The message object to format.
- * @returns {Record<string, any>} The formatted LangChain message.
+ * @param message - The message object to format.
+ * @returns - The formatted LangChain message.
  */
 export const formatFromLangChain = (
   message: LangChainMessage
@@ -357,10 +385,10 @@ function formatAssistantMessage(
 /**
  * Formats an array of messages for LangChain, handling tool calls and creating ToolMessage instances.
  *
- * @param {TPayload} payload - The array of messages to format.
- * @param {Record<number, number>} [indexTokenCountMap] - Optional map of message indices to token counts.
- * @param {Set<string>} [tools] - Optional set of tool names that are allowed in the request.
- * @returns {Object} - Object containing formatted messages and updated indexTokenCountMap if provided.
+ * @param payload - The array of messages to format.
+ * @param indexTokenCountMap - Optional map of message indices to token counts.
+ * @param tools - Optional set of tool names that are allowed in the request.
+ * @returns - Object containing formatted messages and updated indexTokenCountMap if provided.
  */
 export const formatAgentMessages = (
   payload: TPayload,
@@ -539,8 +567,8 @@ export const formatAgentMessages = (
 
 /**
  * Formats an array of messages for LangChain, making sure all content fields are strings
- * @param {Array<HumanMessage | AIMessage | SystemMessage | ToolMessage>} payload - The array of messages to format.
- * @returns {Array<HumanMessage | AIMessage | SystemMessage | ToolMessage>} - The array of formatted LangChain messages, including ToolMessages for tool calls.
+ * @param payload - The array of messages to format.
+ * @returns - The array of formatted LangChain messages, including ToolMessages for tool calls.
  */
 export const formatContentStrings = (
   payload: Array<BaseMessage>