@librechat/agents 2.4.21 → 2.4.30

package/src/llm/anthropic/types.ts

@@ -1,6 +1,4 @@
-
 import Anthropic from '@anthropic-ai/sdk';
-import type { Tool as AnthropicTool } from '@anthropic-ai/sdk/resources';
 import { BindToolsInput } from '@langchain/core/language_models/chat_models';
 
 export type AnthropicToolResponse = {
@@ -11,9 +9,10 @@ export type AnthropicToolResponse = {
   input: Record<string, any>;
 };
 
+export type AnthropicStreamUsage = Anthropic.Usage;
 export type AnthropicMessageParam = Anthropic.MessageParam;
-export type AnthropicMessageDeltaEvent= Anthropic.MessageDeltaEvent;
-export type AnthropicMessageStartEvent= Anthropic.MessageStartEvent;
+export type AnthropicMessageDeltaEvent = Anthropic.MessageDeltaEvent;
+export type AnthropicMessageStartEvent = Anthropic.MessageStartEvent;
 export type AnthropicMessageResponse =
   | Anthropic.ContentBlock
   | AnthropicToolResponse;
@@ -21,6 +20,7 @@ export type AnthropicMessageCreateParams =
   Anthropic.MessageCreateParamsNonStreaming;
 export type AnthropicStreamingMessageCreateParams =
   Anthropic.MessageCreateParamsStreaming;
+export type AnthropicThinkingConfigParam = Anthropic.ThinkingConfigParam;
 export type AnthropicMessageStreamEvent = Anthropic.MessageStreamEvent;
 export type AnthropicRequestOptions = Anthropic.RequestOptions;
 export type AnthropicToolChoice =
@@ -32,7 +32,7 @@ export type AnthropicToolChoice =
   | 'auto'
   | 'none'
   | string;
-export type ChatAnthropicToolType = AnthropicTool | BindToolsInput;
+export type ChatAnthropicToolType = Anthropic.Messages.Tool | BindToolsInput;
 export type AnthropicTextBlockParam = Anthropic.Messages.TextBlockParam;
 export type AnthropicImageBlockParam = Anthropic.Messages.ImageBlockParam;
 export type AnthropicToolUseBlockParam = Anthropic.Messages.ToolUseBlockParam;
@@ -41,39 +41,64 @@ export type AnthropicToolResultBlockParam =
 export type AnthropicDocumentBlockParam = Anthropic.Messages.DocumentBlockParam;
 export type AnthropicThinkingBlockParam = Anthropic.Messages.ThinkingBlockParam;
 export type AnthropicRedactedThinkingBlockParam =
-    Anthropic.Messages.RedactedThinkingBlockParam;
+  Anthropic.Messages.RedactedThinkingBlockParam;
+
+export function isAnthropicImageBlockParam(
+  block: unknown
+): block is AnthropicImageBlockParam {
+  if (block == null) {
+    return false;
+  }
+  if (typeof block !== 'object') {
+    return false;
+  }
+  if (!('type' in block) || block.type !== 'image') {
+    return false;
+  }
+
+  if (!('source' in block) || typeof block.source !== 'object') {
+    return false;
+  }
+
+  if (block.source == null) {
+    return false;
+  }
+
+  if (!('type' in block.source)) {
+    return false;
+  }
+
+  if (block.source.type === 'base64') {
+    if (!('media_type' in block.source)) {
+      return false;
+    }
+
+    if (typeof block.source.media_type !== 'string') {
+      return false;
+    }
+
+    if (!('data' in block.source)) {
+      return false;
+    }
+
+    if (typeof block.source.data !== 'string') {
+      return false;
+    }
+
+    return true;
+  }
+
+  if (block.source.type === 'url') {
+    if (!('url' in block.source)) {
+      return false;
+    }
+
+    if (typeof block.source.url !== 'string') {
+      return false;
+    }
 
-/**
- * Stream usage information for Anthropic API calls
- * @see https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching#pricing
- */
-export interface AnthropicStreamUsage {
-  /**
-   * The number of input tokens used in the request
-   */
-  input_tokens: number;
+    return true;
+  }
 
-  /**
-   * The number of cache creation input tokens used (write operations)
-   */
-  cache_creation_input_tokens?: number;
-  /**
-   * The number of cache input tokens used (read operations)
-   */
-  cache_read_input_tokens?: number;
-  /**
-   * The number of output tokens generated in the response
-   */
-  output_tokens: number;
-  /**
-   * The total number of tokens generated in the response
-   */
-  total_tokens: number;
-  /**
-   * Details about input token usage
-   */
-  input_token_details?: {
-    cache_creation: number;
-    cache_read: number;
-  };
-}
+  return false;
+}

package/src/llm/anthropic/utils/message_inputs.ts

@@ -1,47 +1,80 @@
+/* eslint-disable @typescript-eslint/explicit-function-return-type */
 /* eslint-disable no-console */
 /**
  * This util file contains functions for converting LangChain messages to Anthropic messages.
  */
 import {
-  AIMessage,
-  BaseMessage,
-  ToolMessage,
-  isAIMessage,
+  type BaseMessage,
+  type SystemMessage,
   HumanMessage,
-  SystemMessage,
-  MessageContent,
+  type AIMessage,
+  type ToolMessage,
+  type MessageContent,
+  isAIMessage,
+  type StandardContentBlockConverter,
+  type StandardTextBlock,
+  type StandardImageBlock,
+  type StandardFileBlock,
+  MessageContentComplex,
+  isDataContentBlock,
+  convertToProviderContentBlock,
+  parseBase64DataUrl,
 } from '@langchain/core/messages';
 import { ToolCall } from '@langchain/core/messages/tool';
 import type {
-  AnthropicToolResponse,
-  AnthropicMessageParam,
-  AnthropicTextBlockParam,
   AnthropicImageBlockParam,
-  AnthropicToolUseBlockParam,
   AnthropicMessageCreateParams,
+  AnthropicTextBlockParam,
+  AnthropicToolResponse,
   AnthropicToolResultBlockParam,
+  AnthropicToolUseBlockParam,
   AnthropicDocumentBlockParam,
   AnthropicThinkingBlockParam,
   AnthropicRedactedThinkingBlockParam,
 } from '@/llm/anthropic/types';
+import { isAnthropicImageBlockParam } from '@/llm/anthropic/types';
+
+function _formatImage(imageUrl: string) {
+  const parsed = parseBase64DataUrl({ dataUrl: imageUrl });
+  if (parsed) {
+    return {
+      type: 'base64',
+      media_type: parsed.mime_type,
+      data: parsed.data,
+    };
+  }
+  let parsedUrl: URL;
 
-function _formatImage(imageUrl: string): { type: string; media_type: string; data: string } {
-  const regex = /^data:(image\/.+);base64,(.+)$/;
-  const match = imageUrl.match(regex);
-  if (match === null) {
+  try {
+    parsedUrl = new URL(imageUrl);
+  } catch {
     throw new Error(
       [
-        'Anthropic only supports base64-encoded images currently.',
+        `Malformed image URL: ${JSON.stringify(
+          imageUrl
+        )}. Content blocks of type 'image_url' must be a valid http, https, or base64-encoded data URL.`,
         'Example: data:image/png;base64,/9j/4AAQSk...',
+        'Example: https://example.com/image.jpg',
       ].join('\n\n')
     );
   }
-  return {
-    type: 'base64',
-    media_type: match[1] ?? '',
-    data: match[2] ?? '',
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  } as any;
+
+  if (parsedUrl.protocol === 'http:' || parsedUrl.protocol === 'https:') {
+    return {
+      type: 'url',
+      url: imageUrl,
+    };
+  }
+
+  throw new Error(
+    [
+      `Invalid image URL protocol: ${JSON.stringify(
+        parsedUrl.protocol
+      )}. Anthropic only supports images as http, https, or base64-encoded data URLs on 'image_url' content blocks.`,
+      'Example: data:image/png;base64,/9j/4AAQSk...',
+      'Example: https://example.com/image.jpg',
+    ].join('\n\n')
+  );
 }
 
 function _ensureMessageContents(
@@ -52,16 +85,15 @@ function _ensureMessageContents(
   for (const message of messages) {
     if (message._getType() === 'tool') {
       if (typeof message.content === 'string') {
-        const previousMessage = updatedMsgs[updatedMsgs.length - 1] as BaseMessage | undefined;
+        const previousMessage = updatedMsgs[updatedMsgs.length - 1];
         if (
-          previousMessage &&
           previousMessage._getType() === 'human' &&
           Array.isArray(previousMessage.content) &&
           'type' in previousMessage.content[0] &&
           previousMessage.content[0].type === 'tool_result'
         ) {
           // If the previous message was a tool result, we merge this tool message into it.
-          previousMessage.content.push({
+          (previousMessage.content as MessageContentComplex[]).push({
             type: 'tool_result',
             content: message.content,
             tool_use_id: (message as ToolMessage).tool_call_id,
@@ -114,8 +146,203 @@ export function _convertLangChainToolCallToAnthropic(
   };
 }
 
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-function _formatContent(content: MessageContent): string | Record<string, any>[] {
+const standardContentBlockConverter: StandardContentBlockConverter<{
+  text: AnthropicTextBlockParam;
+  image: AnthropicImageBlockParam;
+  file: AnthropicDocumentBlockParam;
+}> = {
+  providerName: 'anthropic',
+
+  fromStandardTextBlock(block: StandardTextBlock): AnthropicTextBlockParam {
+    return {
+      type: 'text',
+      text: block.text,
+      ...('citations' in (block.metadata ?? {})
+        ? { citations: block.metadata!.citations }
+        : {}),
+      ...('cache_control' in (block.metadata ?? {})
+        ? { cache_control: block.metadata!.cache_control }
+        : {}),
+    } as AnthropicTextBlockParam;
+  },
+
+  fromStandardImageBlock(block: StandardImageBlock): AnthropicImageBlockParam {
+    if (block.source_type === 'url') {
+      const data = parseBase64DataUrl({
+        dataUrl: block.url,
+        asTypedArray: false,
+      });
+      if (data) {
+        return {
+          type: 'image',
+          source: {
+            type: 'base64',
+            data: data.data,
+            media_type: data.mime_type,
+          },
+          ...('cache_control' in (block.metadata ?? {})
+            ? { cache_control: block.metadata!.cache_control }
+            : {}),
+        } as AnthropicImageBlockParam;
+      } else {
+        return {
+          type: 'image',
+          source: {
+            type: 'url',
+            url: block.url,
+            media_type: block.mime_type ?? '',
+          },
+          ...('cache_control' in (block.metadata ?? {})
+            ? { cache_control: block.metadata!.cache_control }
+            : {}),
+        } as AnthropicImageBlockParam;
+      }
+    } else {
+      if (block.source_type === 'base64') {
+        return {
+          type: 'image',
+          source: {
+            type: 'base64',
+            data: block.data,
+            media_type: block.mime_type ?? '',
+          },
+          ...('cache_control' in (block.metadata ?? {})
+            ? { cache_control: block.metadata!.cache_control }
+            : {}),
+        } as AnthropicImageBlockParam;
+      } else {
+        throw new Error(`Unsupported image source type: ${block.source_type}`);
+      }
+    }
+  },
+
+  fromStandardFileBlock(block: StandardFileBlock): AnthropicDocumentBlockParam {
+    const mime_type = (block.mime_type ?? '').split(';')[0];
+
+    if (block.source_type === 'url') {
+      if (mime_type === 'application/pdf' || mime_type === '') {
+        return {
+          type: 'document',
+          source: {
+            type: 'url',
+            url: block.url,
+            media_type: block.mime_type ?? '',
+          },
+          ...('cache_control' in (block.metadata ?? {})
+            ? { cache_control: block.metadata!.cache_control }
+            : {}),
+          ...('citations' in (block.metadata ?? {})
+            ? { citations: block.metadata!.citations }
+            : {}),
+          ...('context' in (block.metadata ?? {})
+            ? { context: block.metadata!.context }
+            : {}),
+          ...('title' in (block.metadata ?? {})
+            ? { title: block.metadata!.title }
+            : {}),
+        } as AnthropicDocumentBlockParam;
+      }
+      throw new Error(
+        `Unsupported file mime type for file url source: ${block.mime_type}`
+      );
+    } else if (block.source_type === 'text') {
+      if (mime_type === 'text/plain' || mime_type === '') {
+        return {
+          type: 'document',
+          source: {
+            type: 'text',
+            data: block.text,
+            media_type: block.mime_type ?? '',
+          },
+          ...('cache_control' in (block.metadata ?? {})
+            ? { cache_control: block.metadata!.cache_control }
+            : {}),
+          ...('citations' in (block.metadata ?? {})
+            ? { citations: block.metadata!.citations }
+            : {}),
+          ...('context' in (block.metadata ?? {})
+            ? { context: block.metadata!.context }
+            : {}),
+          ...('title' in (block.metadata ?? {})
+            ? { title: block.metadata!.title }
+            : {}),
+        } as AnthropicDocumentBlockParam;
+      } else {
+        throw new Error(
+          `Unsupported file mime type for file text source: ${block.mime_type}`
+        );
+      }
+    } else if (block.source_type === 'base64') {
+      if (mime_type === 'application/pdf' || mime_type === '') {
+        return {
+          type: 'document',
+          source: {
+            type: 'base64',
+            data: block.data,
+            media_type: 'application/pdf',
+          },
+          ...('cache_control' in (block.metadata ?? {})
+            ? { cache_control: block.metadata!.cache_control }
+            : {}),
+          ...('citations' in (block.metadata ?? {})
+            ? { citations: block.metadata!.citations }
+            : {}),
+          ...('context' in (block.metadata ?? {})
+            ? { context: block.metadata!.context }
+            : {}),
+          ...('title' in (block.metadata ?? {})
+            ? { title: block.metadata!.title }
+            : {}),
+        } as AnthropicDocumentBlockParam;
+      } else if (
+        ['image/jpeg', 'image/png', 'image/gif', 'image/webp'].includes(
+          mime_type
+        )
+      ) {
+        return {
+          type: 'document',
+          source: {
+            type: 'content',
+            content: [
+              {
+                type: 'image',
+                source: {
+                  type: 'base64',
+                  data: block.data,
+                  media_type: mime_type as
+                    | 'image/jpeg'
+                    | 'image/png'
+                    | 'image/gif'
+                    | 'image/webp',
+                },
+              },
+            ],
+          },
+          ...('cache_control' in (block.metadata ?? {})
+            ? { cache_control: block.metadata!.cache_control }
+            : {}),
+          ...('citations' in (block.metadata ?? {})
+            ? { citations: block.metadata!.citations }
+            : {}),
+          ...('context' in (block.metadata ?? {})
+            ? { context: block.metadata!.context }
+            : {}),
+          ...('title' in (block.metadata ?? {})
+            ? { title: block.metadata!.title }
+            : {}),
+        } as AnthropicDocumentBlockParam;
+      } else {
+        throw new Error(
+          `Unsupported file mime type for file base64 source: ${block.mime_type}`
+        );
+      }
+    } else {
+      throw new Error(`Unsupported file source type: ${block.source_type}`);
+    }
+  },
+};
+
+function _formatContent(content: MessageContent) {
   const toolTypes = ['tool_use', 'tool_result', 'input_json_delta'];
   const textTypes = ['text', 'text_delta'];
 
@@ -123,6 +350,13 @@ function _formatContent(content: MessageContent): string | Record<string, any>[]
     return content;
   } else {
     const contentBlocks = content.map((contentPart) => {
+      if (isDataContentBlock(contentPart)) {
+        return convertToProviderContentBlock(
+          contentPart,
+          standardContentBlockConverter
+        );
+      }
+
       const cacheControl =
         'cache_control' in contentPart ? contentPart.cache_control : undefined;
 
@@ -138,6 +372,8 @@ function _formatContent(content: MessageContent): string | Record<string, any>[]
           source,
           ...(cacheControl ? { cache_control: cacheControl } : {}),
         };
+      } else if (isAnthropicImageBlockParam(contentPart)) {
+        return contentPart;
       } else if (contentPart.type === 'document') {
         // PDF
         return {
@@ -160,7 +396,7 @@ function _formatContent(content: MessageContent): string | Record<string, any>[]
         };
         return block;
       } else if (
-        textTypes.find((t) => t === contentPart.type) != null &&
+        textTypes.find((t) => t === contentPart.type) &&
         'text' in contentPart
       ) {
         // Assuming contentPart is of type MessageContentText here
@@ -169,7 +405,7 @@ function _formatContent(content: MessageContent): string | Record<string, any>[]
           text: contentPart.text,
           ...(cacheControl ? { cache_control: cacheControl } : {}),
         };
-      } else if (toolTypes.find((t) => t === contentPart.type) != null) {
+      } else if (toolTypes.find((t) => t === contentPart.type)) {
         const contentPartCopy = { ...contentPart };
         if ('index' in contentPartCopy) {
           // Anthropic does not support passing the index field here, so we remove it.
@@ -184,10 +420,12 @@ function _formatContent(content: MessageContent): string | Record<string, any>[]
 
         if ('input' in contentPartCopy) {
           // Anthropic tool use inputs should be valid objects, when applicable.
-          try {
-            contentPartCopy.input = JSON.parse(contentPartCopy.input);
-          } catch {
-            contentPartCopy.input = {};
+          if (typeof contentPartCopy.input === 'string') {
+            try {
+              contentPartCopy.input = JSON.parse(contentPartCopy.input);
+            } catch {
+              contentPartCopy.input = {};
+            }
           }
         }
 
@@ -282,14 +520,14 @@ export function _convertMessagesToAnthropicPayload(
     }
   });
   return {
-    messages: mergeMessages(formattedMessages as AnthropicMessageCreateParams['messages']),
+    messages: mergeMessages(formattedMessages),
     system,
   } as AnthropicMessageCreateParams;
 }
 
-function mergeMessages(messages?: AnthropicMessageCreateParams['messages']): AnthropicMessageParam[] {
+function mergeMessages(messages: AnthropicMessageCreateParams['messages']) {
   if (!messages || messages.length <= 1) {
-    return messages ?? [];
+    return messages;
   }
 
   const result: AnthropicMessageCreateParams['messages'] = [];
@@ -327,7 +565,7 @@ function mergeMessages(messages?: AnthropicMessageCreateParams['messages']): Ant
     return content;
   };
 
-  const isToolResultMessage = (msg: (typeof messages)[0]): boolean => {
+  const isToolResultMessage = (msg: (typeof messages)[0]) => {
     if (msg.role !== 'user') return false;
 
     if (typeof msg.content === 'string') {
@@ -363,4 +601,4 @@ function mergeMessages(messages?: AnthropicMessageCreateParams['messages']): Ant
 
   result.push(currentMessage);
   return result;
-}
+}

package/src/llm/anthropic/utils/message_outputs.ts

@@ -4,29 +4,13 @@
 import Anthropic from '@anthropic-ai/sdk';
 import {
   AIMessage,
-  UsageMetadata,
   AIMessageChunk,
+  UsageMetadata,
 } from '@langchain/core/messages';
 import type { ToolCallChunk } from '@langchain/core/messages/tool';
-import { ToolCall } from '@langchain/core/messages/tool';
 import { ChatGeneration } from '@langchain/core/outputs';
-import { AnthropicMessageResponse } from '../types.js';
-
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-export function extractToolCalls(content: Record<string, any>[]): ToolCall[] {
-  const toolCalls: ToolCall[] = [];
-  for (const block of content) {
-    if (block.type === 'tool_use') {
-      toolCalls.push({
-        name: block.name,
-        args: block.input,
-        id: block.id,
-        type: 'tool_call',
-      });
-    }
-  }
-  return toolCalls;
-}
+import { extractToolCalls } from './output_parsers';
+import { AnthropicMessageResponse } from '../types';
 
 export function _makeMessageChunkFromAnthropicEvent(
   data: Anthropic.Messages.RawMessageStreamEvent,
@@ -231,7 +215,6 @@ export function _makeMessageChunkFromAnthropicEvent(
       }),
     };
   }
-
   return null;
 }
 
@@ -284,4 +267,4 @@ export function anthropicResponseToChatMessages(
     ];
     return generations;
   }
-}
+}