@librechat/agents 3.0.0-rc10 → 3.0.0-rc11

package/src/messages/formatMessage.test.ts

@@ -1,8 +1,43 @@
-import { HumanMessage, AIMessage, SystemMessage } from '@langchain/core/messages';
-import { formatMessage, formatLangChainMessages, formatFromLangChain } from './format';
+import {
+  HumanMessage,
+  AIMessage,
+  SystemMessage,
+} from '@langchain/core/messages';
+import type { MessageContentComplex } from '@/types';
+import {
+  formatMessage,
+  formatLangChainMessages,
+  formatFromLangChain,
+  formatMediaMessage,
+} from './format';
+import { Providers } from '@/common';
 
 const NO_PARENT = '00000000-0000-0000-0000-000000000000';
 
+/**
+ * Type for formatted message results with media content
+ */
+interface FormattedMediaMessage {
+  role: string;
+  content: MessageContentComplex[];
+  name?: string;
+}
+
+/**
+ * Type guard to check if result is a FormattedMediaMessage
+ */
+function isFormattedMediaMessage(
+  result: unknown
+): result is FormattedMediaMessage {
+  return (
+    typeof result === 'object' &&
+    result !== null &&
+    'role' in result &&
+    'content' in result &&
+    Array.isArray((result as FormattedMediaMessage).content)
+  );
+}
+
 describe('formatMessage', () => {
   it('formats user message', () => {
     const input = {
@@ -187,6 +222,387 @@ describe('formatMessage', () => {
   });
 });
 
+describe('formatMediaMessage', () => {
+  it('formats message with images for default provider', () => {
+    const message = {
+      role: 'user',
+      content: 'Check out this image',
+      name: 'John',
+    };
+    const mediaParts = [
+      {
+        type: 'image_url',
+        image_url: { url: 'https://example.com/image1.jpg' },
+      },
+      {
+        type: 'image_url',
+        image_url: { url: 'https://example.com/image2.jpg' },
+      },
+    ];
+
+    const result = formatMediaMessage({ message, mediaParts });
+
+    expect(result.role).toBe('user');
+    expect(result.name).toBe('John');
+    expect(Array.isArray(result.content)).toBe(true);
+    expect(result.content).toHaveLength(3);
+    expect(result.content[0]).toEqual({
+      type: 'text',
+      text: 'Check out this image',
+    });
+    expect(result.content[1]).toEqual(mediaParts[0]);
+    expect(result.content[2]).toEqual(mediaParts[1]);
+  });
+
+  it('formats message with images for Anthropic (media first)', () => {
+    const message = {
+      role: 'user',
+      content: 'Check out this image',
+    };
+    const mediaParts = [
+      {
+        type: 'image_url',
+        image_url: { url: 'https://example.com/image.jpg' },
+      },
+    ];
+
+    const result = formatMediaMessage({
+      message,
+      mediaParts,
+      endpoint: Providers.ANTHROPIC,
+    });
+
+    expect(result.content).toHaveLength(2);
+    expect(result.content[0]).toEqual(mediaParts[0]);
+    expect(result.content[1]).toEqual({
+      type: 'text',
+      text: 'Check out this image',
+    });
+  });
+
+  it('formats message with multiple media types', () => {
+    const message = {
+      role: 'user',
+      content: 'Check out these files',
+    };
+    const mediaParts = [
+      { type: 'document', document: { url: 'https://example.com/doc.pdf' } },
+      { type: 'video', video: { url: 'https://example.com/video.mp4' } },
+      { type: 'audio', audio: { url: 'https://example.com/audio.mp3' } },
+      {
+        type: 'image_url',
+        image_url: { url: 'https://example.com/image.jpg' },
+      },
+    ];
+
+    const result = formatMediaMessage({ message, mediaParts });
+
+    expect(result.content).toHaveLength(5);
+    expect(result.content[0]).toEqual({
+      type: 'text',
+      text: 'Check out these files',
+    });
+    expect(result.content[1]).toEqual(mediaParts[0]);
+    expect(result.content[2]).toEqual(mediaParts[1]);
+    expect(result.content[3]).toEqual(mediaParts[2]);
+    expect(result.content[4]).toEqual(mediaParts[3]);
+  });
+});
+
+describe('formatMessage with media', () => {
+  it('formats user message with image_urls (backward compatibility)', () => {
+    const input = {
+      message: {
+        sender: 'user',
+        text: 'Check out this image',
+        image_urls: [
+          {
+            type: 'image_url' as const,
+            image_url: { url: 'https://example.com/image.jpg' },
+          },
+        ],
+      },
+      userName: 'John',
+    };
+
+    const result = formatMessage(input);
+
+    expect(isFormattedMediaMessage(result)).toBe(true);
+    if (isFormattedMediaMessage(result)) {
+      expect(result.role).toBe('user');
+      expect(result.name).toBe('John');
+      expect(Array.isArray(result.content)).toBe(true);
+      expect(result.content).toHaveLength(2);
+      expect(result.content[0]).toEqual({
+        type: 'text',
+        text: 'Check out this image',
+      });
+      expect(result.content[1]).toEqual(input.message.image_urls[0]);
+    }
+  });
+
+  it('formats user message with documents', () => {
+    const input = {
+      message: {
+        role: 'user',
+        content: 'Review this document',
+        documents: [
+          {
+            type: 'document',
+            document: { url: 'https://example.com/report.pdf' },
+          },
+        ],
+      },
+    };
+
+    const result = formatMessage(input);
+
+    expect(isFormattedMediaMessage(result)).toBe(true);
+    if (isFormattedMediaMessage(result)) {
+      expect(result.role).toBe('user');
+      expect(Array.isArray(result.content)).toBe(true);
+      expect(result.content).toHaveLength(2);
+      expect(result.content[0]).toEqual({
+        type: 'text',
+        text: 'Review this document',
+      });
+      expect(result.content[1]).toEqual(input.message.documents[0]);
+    }
+  });
+
+  it('formats user message with videos', () => {
+    const input = {
+      message: {
+        role: 'user',
+        content: 'Watch this video',
+        videos: [
+          { type: 'video', video: { url: 'https://example.com/demo.mp4' } },
+        ],
+      },
+    };
+
+    const result = formatMessage(input);
+
+    expect(isFormattedMediaMessage(result)).toBe(true);
+    if (isFormattedMediaMessage(result)) {
+      expect(result.role).toBe('user');
+      expect(Array.isArray(result.content)).toBe(true);
+      expect(result.content).toHaveLength(2);
+      expect(result.content[0]).toEqual({
+        type: 'text',
+        text: 'Watch this video',
+      });
+      expect(result.content[1]).toEqual(input.message.videos[0]);
+    }
+  });
+
+  it('formats user message with audios', () => {
+    const input = {
+      message: {
+        role: 'user',
+        content: 'Listen to this',
+        audios: [
+          { type: 'audio', audio: { url: 'https://example.com/podcast.mp3' } },
+        ],
+      },
+    };
+
+    const result = formatMessage(input);
+
+    expect(isFormattedMediaMessage(result)).toBe(true);
+    if (isFormattedMediaMessage(result)) {
+      expect(result.role).toBe('user');
+      expect(Array.isArray(result.content)).toBe(true);
+      expect(result.content).toHaveLength(2);
+      expect(result.content[0]).toEqual({
+        type: 'text',
+        text: 'Listen to this',
+      });
+      expect(result.content[1]).toEqual(input.message.audios[0]);
+    }
+  });
+
+  it('formats user message with all media types in correct order', () => {
+    const input = {
+      message: {
+        role: 'user',
+        content: 'Check out all these files',
+        documents: [
+          {
+            type: 'document',
+            document: { url: 'https://example.com/doc.pdf' },
+          },
+        ],
+        videos: [
+          { type: 'video', video: { url: 'https://example.com/video.mp4' } },
+        ],
+        audios: [
+          { type: 'audio', audio: { url: 'https://example.com/audio.mp3' } },
+        ],
+        image_urls: [
+          {
+            type: 'image_url' as const,
+            image_url: { url: 'https://example.com/image.jpg' },
+          },
+        ],
+      },
+    };
+
+    const result = formatMessage(input);
+
+    expect(isFormattedMediaMessage(result)).toBe(true);
+    if (isFormattedMediaMessage(result)) {
+      expect(result.role).toBe('user');
+      expect(Array.isArray(result.content)).toBe(true);
+      expect(result.content).toHaveLength(5);
+      // Text first
+      expect(result.content[0]).toEqual({
+        type: 'text',
+        text: 'Check out all these files',
+      });
+      // Then documents, videos, audios, images
+      expect(result.content[1]).toEqual(input.message.documents[0]);
+      expect(result.content[2]).toEqual(input.message.videos[0]);
+      expect(result.content[3]).toEqual(input.message.audios[0]);
+      expect(result.content[4]).toEqual(input.message.image_urls[0]);
+    }
+  });
+
+  it('formats user message with multiple files of the same type', () => {
+    const input = {
+      message: {
+        role: 'user',
+        content: 'Review these documents',
+        documents: [
+          {
+            type: 'document',
+            document: { url: 'https://example.com/doc1.pdf' },
+          },
+          {
+            type: 'document',
+            document: { url: 'https://example.com/doc2.pdf' },
+          },
+          {
+            type: 'document',
+            document: { url: 'https://example.com/doc3.pdf' },
+          },
+        ],
+      },
+    };
+
+    const result = formatMessage(input);
+
+    expect(isFormattedMediaMessage(result)).toBe(true);
+    if (isFormattedMediaMessage(result)) {
+      expect(result.content).toHaveLength(4);
+      expect(result.content[0].type).toBe('text');
+      expect(result.content[1]).toEqual(input.message.documents[0]);
+      expect(result.content[2]).toEqual(input.message.documents[1]);
+      expect(result.content[3]).toEqual(input.message.documents[2]);
+    }
+  });
+
+  it('respects Anthropic provider ordering (media before text)', () => {
+    const input = {
+      message: {
+        role: 'user',
+        content: 'Check this out',
+        documents: [
+          {
+            type: 'document',
+            document: { url: 'https://example.com/doc.pdf' },
+          },
+        ],
+        image_urls: [
+          {
+            type: 'image_url' as const,
+            image_url: { url: 'https://example.com/image.jpg' },
+          },
+        ],
+      },
+      endpoint: Providers.ANTHROPIC,
+    };
+
+    const result = formatMessage(input);
+
+    expect(isFormattedMediaMessage(result)).toBe(true);
+    if (isFormattedMediaMessage(result)) {
+      expect(result.content).toHaveLength(3);
+      // Media first for Anthropic
+      expect(result.content[0]).toEqual(input.message.documents[0]);
+      expect(result.content[1]).toEqual(input.message.image_urls[0]);
+      expect(result.content[2]).toEqual({
+        type: 'text',
+        text: 'Check this out',
+      });
+    }
+  });
+
+  it('does not format media for assistant messages', () => {
+    const input = {
+      message: {
+        role: 'assistant',
+        content: 'Here is a response',
+        documents: [
+          {
+            type: 'document',
+            document: { url: 'https://example.com/doc.pdf' },
+          },
+        ],
+      },
+    };
+
+    const result = formatMessage(input);
+
+    expect(result).toMatchObject({
+      role: 'assistant',
+      content: 'Here is a response',
+    });
+  });
+
+  it('handles empty media arrays gracefully', () => {
+    const input = {
+      message: {
+        role: 'user',
+        content: 'Just text',
+        documents: [],
+        videos: [],
+        audios: [],
+        image_urls: [],
+      },
+    };
+
+    const result = formatMessage(input);
+
+    expect(result).toMatchObject({
+      role: 'user',
+      content: 'Just text',
+    });
+  });
+
+  it('formats media with langChain flag', () => {
+    const input = {
+      message: {
+        role: 'user',
+        content: 'Check this image',
+        image_urls: [
+          {
+            type: 'image_url' as const,
+            image_url: { url: 'https://example.com/image.jpg' },
+          },
+        ],
+      },
+      langChain: true,
+    };
+
+    const result = formatMessage(input);
+
+    expect(result).toBeInstanceOf(HumanMessage);
+    expect(Array.isArray(result.lc_kwargs.content)).toBe(true);
+    expect(result.lc_kwargs.content).toHaveLength(2);
+  });
+});
+
 describe('formatLangChainMessages', () => {
   it('formats an array of messages for LangChain', () => {
     const messages = [

package/src/messages/prune.ts

@@ -389,6 +389,14 @@ export function checkValidNumber(value: unknown): value is number {
   return typeof value === 'number' && !isNaN(value) && value > 0;
 }
 
+type ThinkingBlocks = {
+  thinking_blocks?: Array<{
+    type: 'thinking';
+    thinking: string;
+    signature: string;
+  }>;
+};
+
 export function createPruneMessages(factoryParams: PruneMessagesFactoryParams) {
   const indexTokenCountMap = { ...factoryParams.indexTokenCountMap };
   let lastTurnStartIndex = factoryParams.startIndex;
@@ -402,6 +410,49 @@ export function createPruneMessages(factoryParams: PruneMessagesFactoryParams) {
     context: BaseMessage[];
     indexTokenCountMap: Record<string, number | undefined>;
   } {
+    if (
+      factoryParams.provider === Providers.OPENAI &&
+      factoryParams.thinkingEnabled === true
+    ) {
+      for (let i = lastTurnStartIndex; i < params.messages.length; i++) {
+        const m = params.messages[i];
+        if (
+          m.getType() === 'ai' &&
+          typeof m.additional_kwargs.reasoning_content === 'string' &&
+          Array.isArray(
+            (
+              m.additional_kwargs.provider_specific_fields as
+                | ThinkingBlocks
+                | undefined
+            )?.thinking_blocks
+          ) &&
+          (m as AIMessage).tool_calls &&
+          ((m as AIMessage).tool_calls?.length ?? 0) > 0
+        ) {
+          const message = m as AIMessage;
+          const thinkingBlocks = (
+            message.additional_kwargs.provider_specific_fields as ThinkingBlocks
+          ).thinking_blocks;
+          const signature =
+            thinkingBlocks?.[thinkingBlocks.length - 1].signature;
+          const thinkingBlock: ThinkingContentText = {
+            signature,
+            type: ContentTypes.THINKING,
+            thinking: message.additional_kwargs.reasoning_content as string,
+          };
+
+          params.messages[i] = new AIMessage({
+            ...message,
+            content: [thinkingBlock],
+            additional_kwargs: {
+              ...message.additional_kwargs,
+              reasoning_content: undefined,
+            },
+          });
+        }
+      }
+    }
+
     let currentUsage: UsageMetadata | undefined;
     if (
       params.usageMetadata &&

package/src/scripts/search.ts

@@ -77,7 +77,11 @@ async function testStandardStreaming(): Promise<void> {
     graphConfig: {
       type: 'standard',
       llmConfig,
-      tools: [createSearchTool()],
+      tools: [
+        createSearchTool({
+          scraperProvider: 'serper',
+        }),
+      ],
       instructions:
         'You are a friendly AI assistant. Always address the user by their name.',
       // additional_instructions: `The user's name is ${userName} and they are located in ${location}.`,

package/src/scripts/tools.ts

@@ -126,7 +126,10 @@ async function testStandardStreaming(): Promise<void> {
   const inputs = {
     messages: conversationHistory,
   };
-  const finalContentParts = await run.processStream(inputs, config);
+  const finalContentParts = await run.processStream(inputs, config, {
+    indexTokenCountMap: { 0: 35 },
+    maxContextTokens: 89000,
+  });
   const finalMessages = run.getRunMessages();
   if (finalMessages) {
     conversationHistory.push(...finalMessages);

package/src/tools/search/firecrawl.ts

@@ -7,9 +7,10 @@ import { createDefaultLogger } from './utils';
  * Firecrawl scraper implementation
  * Uses the Firecrawl API to scrape web pages
  */
-export class FirecrawlScraper {
+export class FirecrawlScraper implements t.BaseScraper {
   private apiKey: string;
   private apiUrl: string;
+  private version: string;
   private defaultFormats: string[];
   private timeout: number;
   private logger: t.Logger;
@@ -32,11 +33,13 @@ export class FirecrawlScraper {
   constructor(config: t.FirecrawlScraperConfig = {}) {
     this.apiKey = config.apiKey ?? process.env.FIRECRAWL_API_KEY ?? '';
 
+    this.version = config.version ?? 'v2';
+
     const baseUrl =
       config.apiUrl ??
       process.env.FIRECRAWL_BASE_URL ??
       'https://api.firecrawl.dev';
-    this.apiUrl = `${baseUrl.replace(/\/+$/, '')}/v1/scrape`;
+    this.apiUrl = `${baseUrl.replace(/\/+$/, '')}/${this.version}/scrape`;
 
     this.defaultFormats = config.formats ?? ['markdown', 'rawHtml'];
     this.timeout = config.timeout ?? 7500;

package/src/tools/search/jina-reranker.test.ts

@@ -0,0 +1,126 @@
+import { JinaReranker } from './rerankers';
+import { createDefaultLogger } from './utils';
+
+describe('JinaReranker', () => {
+  const mockLogger = createDefaultLogger();
+  
+  describe('constructor', () => {
+    it('should use default API URL when no apiUrl is provided', () => {
+      const reranker = new JinaReranker({
+        apiKey: 'test-key',
+        logger: mockLogger,
+      });
+      
+      // Access private property for testing
+      const apiUrl = (reranker as any).apiUrl;
+      expect(apiUrl).toBe('https://api.jina.ai/v1/rerank');
+    });
+
+    it('should use custom API URL when provided', () => {
+      const customUrl = 'https://custom-jina-endpoint.com/v1/rerank';
+      const reranker = new JinaReranker({
+        apiKey: 'test-key',
+        apiUrl: customUrl,
+        logger: mockLogger,
+      });
+      
+      const apiUrl = (reranker as any).apiUrl;
+      expect(apiUrl).toBe(customUrl);
+    });
+
+    it('should use environment variable JINA_API_URL when available', () => {
+      const originalEnv = process.env.JINA_API_URL;
+      process.env.JINA_API_URL = 'https://env-jina-endpoint.com/v1/rerank';
+      
+      const reranker = new JinaReranker({
+        apiKey: 'test-key',
+        logger: mockLogger,
+      });
+      
+      const apiUrl = (reranker as any).apiUrl;
+      expect(apiUrl).toBe('https://env-jina-endpoint.com/v1/rerank');
+      
+      // Restore original environment
+      if (originalEnv) {
+        process.env.JINA_API_URL = originalEnv;
+      } else {
+        delete process.env.JINA_API_URL;
+      }
+    });
+
+    it('should prioritize explicit apiUrl over environment variable', () => {
+      const originalEnv = process.env.JINA_API_URL;
+      process.env.JINA_API_URL = 'https://env-jina-endpoint.com/v1/rerank';
+      
+      const customUrl = 'https://explicit-jina-endpoint.com/v1/rerank';
+      const reranker = new JinaReranker({
+        apiKey: 'test-key',
+        apiUrl: customUrl,
+        logger: mockLogger,
+      });
+      
+      const apiUrl = (reranker as any).apiUrl;
+      expect(apiUrl).toBe(customUrl);
+      
+      // Restore original environment
+      if (originalEnv) {
+        process.env.JINA_API_URL = originalEnv;
+      } else {
+        delete process.env.JINA_API_URL;
+      }
+    });
+  });
+
+  describe('rerank method', () => {
+    it('should log the API URL being used', async () => {
+      const customUrl = 'https://test-jina-endpoint.com/v1/rerank';
+      const reranker = new JinaReranker({
+        apiKey: 'test-key',
+        apiUrl: customUrl,
+        logger: mockLogger,
+      });
+      
+      const logSpy = jest.spyOn(mockLogger, 'debug');
+      
+      try {
+        await reranker.rerank('test query', ['document1', 'document2'], 2);
+      } catch (error) {
+        // Expected to fail due to missing API key, but we can check the log
+      }
+      
+      expect(logSpy).toHaveBeenCalledWith(
+        expect.stringContaining(`Reranking 2 chunks with Jina using API URL: ${customUrl}`)
+      );
+      
+      logSpy.mockRestore();
+    });
+  });
+});
+
+describe('createReranker', () => {
+  const { createReranker } = require('./rerankers');
+  
+  it('should create JinaReranker with jinaApiUrl when provided', () => {
+    const customUrl = 'https://custom-jina-endpoint.com/v1/rerank';
+    const reranker = createReranker({
+      rerankerType: 'jina',
+      jinaApiKey: 'test-key',
+      jinaApiUrl: customUrl,
+    });
+    
+    expect(reranker).toBeInstanceOf(JinaReranker);
+    const apiUrl = (reranker as any).apiUrl;
+    expect(apiUrl).toBe(customUrl);
+  });
+
+  it('should create JinaReranker with default URL when jinaApiUrl is not provided', () => {
+    const reranker = createReranker({
+      rerankerType: 'jina',
+      jinaApiKey: 'test-key',
+    });
+    
+    expect(reranker).toBeInstanceOf(JinaReranker);
+    const apiUrl = (reranker as any).apiUrl;
+    expect(apiUrl).toBe('https://api.jina.ai/v1/rerank');
+  });
+});