@lobehub/chat 1.15.29 → 1.15.30

package/CHANGELOG.md

@@ -2,6 +2,31 @@
 
 # Changelog
 
+### [Version 1.15.30](https://github.com/lobehub/lobe-chat/compare/v1.15.29...v1.15.30)
+
+<sup>Released on **2024-09-09**</sup>
+
+#### 🐛 Bug Fixes
+
+- **misc**: Fix claude 3.5 image with s3 url.
+
+<br/>
+
+<details>
+<summary><kbd>Improvements and Fixes</kbd></summary>
+
+#### What's fixed
+
+- **misc**: Fix claude 3.5 image with s3 url, closes [#3870](https://github.com/lobehub/lobe-chat/issues/3870) ([89c8dd4](https://github.com/lobehub/lobe-chat/commit/89c8dd4))
+
+</details>
+
+<div align="right">
+
+[![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top)
+
+</div>
+
 ### [Version 1.15.29](https://github.com/lobehub/lobe-chat/compare/v1.15.28...v1.15.29)
 
 <sup>Released on **2024-09-09**</sup>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lobehub/chat",
-  "version": "1.15.29",
+  "version": "1.15.30",
   "description": "Lobe Chat - an open-source, high-performance chatbot framework that supports speech synthesis, multimodal, and extensible Function Call plugin system. Supports one-click free deployment of your private ChatGPT/LLM web application.",
   "keywords": [
     "framework",

package/src/libs/agent-runtime/anthropic/index.ts

@@ -29,7 +29,8 @@ export class LobeAnthropicAI implements LobeRuntimeAI {
 
   async chat(payload: ChatStreamPayload, options?: ChatCompetitionOptions) {
     try {
-      const anthropicPayload = this.buildAnthropicPayload(payload);
+      const anthropicPayload = await this.buildAnthropicPayload(payload);
+
       const response = await this.client.messages.create(
         { ...anthropicPayload, stream: true },
         {
@@ -86,20 +87,17 @@ export class LobeAnthropicAI implements LobeRuntimeAI {
     }
   }
 
-  private buildAnthropicPayload(payload: ChatStreamPayload) {
+  private async buildAnthropicPayload(payload: ChatStreamPayload) {
     const { messages, model, max_tokens = 4096, temperature, top_p, tools } = payload;
     const system_message = messages.find((m) => m.role === 'system');
     const user_messages = messages.filter((m) => m.role !== 'system');
 
     return {
       max_tokens,
-      messages: buildAnthropicMessages(user_messages),
+      messages: await buildAnthropicMessages(user_messages),
       model,
       system: system_message?.content as string,
-      temperature: 
-        payload.temperature !== undefined 
-        ? temperature / 2
-        : undefined,
+      temperature: payload.temperature !== undefined ? temperature / 2 : undefined,
       tools: buildAnthropicTools(tools),
       top_p,
     } satisfies Anthropic.MessageCreateParams;

package/src/libs/agent-runtime/bedrock/index.ts

@@ -62,7 +62,7 @@ export class LobeBedrockAI implements LobeRuntimeAI {
       body: JSON.stringify({
         anthropic_version: 'bedrock-2023-05-31',
         max_tokens: max_tokens || 4096,
-        messages: buildAnthropicMessages(user_messages),
+        messages: await buildAnthropicMessages(user_messages),
         system: system_message?.content as string,
         temperature: temperature / 2,
         tools: buildAnthropicTools(tools),

package/src/libs/agent-runtime/utils/anthropicHelpers.test.ts

@@ -1,6 +1,8 @@
 import { OpenAI } from 'openai';
 import { describe, expect, it } from 'vitest';
 
+import { imageUrlToBase64 } from '@/utils/imageToBase64';
+
 import { OpenAIChatMessage, UserMessageContentPart } from '../types/chat';
 import {
   buildAnthropicBlock,
@@ -10,28 +12,30 @@ import {
 } from './anthropicHelpers';
 import { parseDataUri } from './uriParser';
 
-describe('anthropicHelpers', () => {
-  // Mock the parseDataUri function since it's an implementation detail
-  vi.mock('./uriParser', () => ({
-    parseDataUri: vi.fn().mockReturnValue({
-      mimeType: 'image/jpeg',
-      base64: 'base64EncodedString',
-    }),
-  }));
+// Mock the parseDataUri function since it's an implementation detail
+vi.mock('./uriParser', () => ({
+  parseDataUri: vi.fn().mockReturnValue({
+    mimeType: 'image/jpeg',
+    base64: 'base64EncodedString',
+    type: 'base64',
+  }),
+}));
+vi.mock('@/utils/imageToBase64');
 
+describe('anthropicHelpers', () => {
   describe('buildAnthropicBlock', () => {
-    it('should return the content as is for text type', () => {
+    it('should return the content as is for text type', async () => {
       const content: UserMessageContentPart = { type: 'text', text: 'Hello!' };
-      const result = buildAnthropicBlock(content);
+      const result = await buildAnthropicBlock(content);
       expect(result).toEqual(content);
     });
 
-    it('should transform an image URL into an Anthropic.ImageBlockParam', () => {
+    it('should transform an image URL into an Anthropic.ImageBlockParam', async () => {
       const content: UserMessageContentPart = {
         type: 'image_url',
         image_url: { url: 'data:image/jpeg;base64,base64EncodedString' },
       };
-      const result = buildAnthropicBlock(content);
+      const result = await buildAnthropicBlock(content);
       expect(parseDataUri).toHaveBeenCalledWith(content.image_url.url);
       expect(result).toEqual({
         source: {
@@ -42,28 +46,96 @@ describe('anthropicHelpers', () => {
         type: 'image',
       });
     });
+
+    it('should transform a regular image URL into an Anthropic.ImageBlockParam', async () => {
+      vi.mocked(parseDataUri).mockReturnValueOnce({
+        mimeType: 'image/png',
+        base64: null,
+        type: 'url',
+      });
+      vi.mocked(imageUrlToBase64).mockResolvedValue('convertedBase64String');
+
+      const content = {
+        type: 'image_url',
+        image_url: { url: 'https://example.com/image.png' },
+      } as const;
+
+      const result = await buildAnthropicBlock(content);
+
+      expect(parseDataUri).toHaveBeenCalledWith(content.image_url.url);
+      expect(imageUrlToBase64).toHaveBeenCalledWith(content.image_url.url);
+      expect(result).toEqual({
+        source: {
+          data: 'convertedBase64String',
+          media_type: 'image/png',
+          type: 'base64',
+        },
+        type: 'image',
+      });
+    });
+
+    it('should use default media_type for URL images when mimeType is not provided', async () => {
+      vi.mocked(parseDataUri).mockReturnValueOnce({
+        mimeType: null,
+        base64: null,
+        type: 'url',
+      });
+      vi.mocked(imageUrlToBase64).mockResolvedValue('convertedBase64String');
+
+      const content = {
+        type: 'image_url',
+        image_url: { url: 'https://example.com/image' },
+      } as const;
+
+      const result = await buildAnthropicBlock(content);
+
+      expect(result).toEqual({
+        source: {
+          data: 'convertedBase64String',
+          media_type: 'image/png',
+          type: 'base64',
+        },
+        type: 'image',
+      });
+    });
+
+    it('should throw an error for invalid image URLs', async () => {
+      vi.mocked(parseDataUri).mockReturnValueOnce({
+        mimeType: null,
+        base64: null,
+        // @ts-ignore
+        type: 'invalid',
+      });
+
+      const content = {
+        type: 'image_url',
+        image_url: { url: 'invalid-url' },
+      } as const;
+
+      await expect(buildAnthropicBlock(content)).rejects.toThrow('Invalid image URL: invalid-url');
+    });
   });
 
   describe('buildAnthropicMessage', () => {
-    it('should correctly convert system message to assistant message', () => {
+    it('should correctly convert system message to assistant message', async () => {
       const message: OpenAIChatMessage = {
         content: [{ type: 'text', text: 'Hello!' }],
         role: 'system',
       };
-      const result = buildAnthropicMessage(message);
+      const result = await buildAnthropicMessage(message);
       expect(result).toEqual({ content: [{ type: 'text', text: 'Hello!' }], role: 'user' });
     });
 
-    it('should correctly convert user message with string content', () => {
+    it('should correctly convert user message with string content', async () => {
       const message: OpenAIChatMessage = {
         content: 'Hello!',
         role: 'user',
       };
-      const result = buildAnthropicMessage(message);
+      const result = await buildAnthropicMessage(message);
       expect(result).toEqual({ content: 'Hello!', role: 'user' });
     });
 
-    it('should correctly convert user message with content parts', () => {
+    it('should correctly convert user message with content parts', async () => {
       const message: OpenAIChatMessage = {
         content: [
           { type: 'text', text: 'Check out this image:' },
@@ -71,19 +143,19 @@ describe('anthropicHelpers', () => {
         ],
         role: 'user',
       };
-      const result = buildAnthropicMessage(message);
+      const result = await buildAnthropicMessage(message);
       expect(result.role).toBe('user');
       expect(result.content).toHaveLength(2);
       expect((result.content[1] as any).type).toBe('image');
     });
 
-    it('should correctly convert tool message', () => {
+    it('should correctly convert tool message', async () => {
       const message: OpenAIChatMessage = {
         content: 'Tool result content',
         role: 'tool',
         tool_call_id: 'tool123',
       };
-      const result = buildAnthropicMessage(message);
+      const result = await buildAnthropicMessage(message);
       expect(result.role).toBe('user');
       expect(result.content).toEqual([
         {
@@ -94,7 +166,7 @@ describe('anthropicHelpers', () => {
       ]);
     });
 
-    it('should correctly convert assistant message with tool calls', () => {
+    it('should correctly convert assistant message with tool calls', async () => {
       const message: OpenAIChatMessage = {
         content: 'Here is the result:',
         role: 'assistant',
@@ -109,7 +181,7 @@ describe('anthropicHelpers', () => {
           },
         ],
       };
-      const result = buildAnthropicMessage(message);
+      const result = await buildAnthropicMessage(message);
       expect(result.role).toBe('assistant');
       expect(result.content).toEqual([
         { text: 'Here is the result:', type: 'text' },
@@ -122,12 +194,12 @@ describe('anthropicHelpers', () => {
       ]);
     });
 
-    it('should correctly convert function message', () => {
+    it('should correctly convert function message', async () => {
       const message: OpenAIChatMessage = {
         content: 'def hello(name):\n  return f"Hello {name}"',
         role: 'function',
       };
-      const result = buildAnthropicMessage(message);
+      const result = await buildAnthropicMessage(message);
       expect(result).toEqual({
         content: 'def hello(name):\n  return f"Hello {name}"',
         role: 'assistant',
@@ -136,13 +208,13 @@ describe('anthropicHelpers', () => {
   });
 
   describe('buildAnthropicMessages', () => {
-    it('should correctly convert OpenAI Messages to Anthropic Messages', () => {
+    it('should correctly convert OpenAI Messages to Anthropic Messages', async () => {
       const messages: OpenAIChatMessage[] = [
         { content: 'Hello', role: 'user' },
         { content: 'Hi', role: 'assistant' },
       ];
 
-      const result = buildAnthropicMessages(messages);
+      const result = await buildAnthropicMessages(messages);
       expect(result).toHaveLength(2);
       expect(result).toEqual([
         { content: 'Hello', role: 'user' },
@@ -150,14 +222,14 @@ describe('anthropicHelpers', () => {
       ]);
     });
 
-    it('messages should end with user', () => {
+    it('messages should end with user', async () => {
       const messages: OpenAIChatMessage[] = [
         { content: 'Hello', role: 'user' },
         { content: 'Hello', role: 'user' },
         { content: 'Hi', role: 'assistant' },
       ];
 
-      const contents = buildAnthropicMessages(messages);
+      const contents = await buildAnthropicMessages(messages);
 
       expect(contents).toHaveLength(4);
       expect(contents).toEqual([
@@ -168,7 +240,7 @@ describe('anthropicHelpers', () => {
       ]);
     });
 
-    it('messages should pair', () => {
+    it('messages should pair', async () => {
       const messages: OpenAIChatMessage[] = [
         { content: 'a', role: 'assistant' },
         { content: 'b', role: 'assistant' },
@@ -177,7 +249,7 @@ describe('anthropicHelpers', () => {
         { content: '你好', role: 'user' },
       ];
 
-      const contents = buildAnthropicMessages(messages);
+      const contents = await buildAnthropicMessages(messages);
 
       expect(contents).toHaveLength(9);
       expect(contents).toEqual([
@@ -193,7 +265,7 @@ describe('anthropicHelpers', () => {
       ]);
     });
 
-    it('should correctly convert OpenAI tool message to Anthropic format', () => {
+    it('should correctly convert OpenAI tool message to Anthropic format', async () => {
       const messages: OpenAIChatMessage[] = [
         {
           content: '告诉我杭州和北京的天气，先回答我好的',
@@ -242,7 +314,7 @@ describe('anthropicHelpers', () => {
         },
       ];
 
-      const contents = buildAnthropicMessages(messages);
+      const contents = await buildAnthropicMessages(messages);
 
       expect(contents).toEqual([
         { content: '告诉我杭州和北京的天气，先回答我好的', role: 'user' },

package/src/libs/agent-runtime/utils/anthropicHelpers.ts

@@ -1,35 +1,52 @@
 import Anthropic from '@anthropic-ai/sdk';
 import OpenAI from 'openai';
 
+import { imageUrlToBase64 } from '@/utils/imageToBase64';
+
 import { OpenAIChatMessage, UserMessageContentPart } from '../types';
 import { parseDataUri } from './uriParser';
 
-export const buildAnthropicBlock = (
+export const buildAnthropicBlock = async (
   content: UserMessageContentPart,
-): Anthropic.ContentBlock | Anthropic.ImageBlockParam => {
+): Promise<Anthropic.ContentBlock | Anthropic.ImageBlockParam> => {
   switch (content.type) {
     case 'text': {
       return content;
     }
 
     case 'image_url': {
-      const { mimeType, base64 } = parseDataUri(content.image_url.url);
+      const { mimeType, base64, type } = parseDataUri(content.image_url.url);
 
-      return {
-        source: {
-          data: base64 as string,
-          media_type: mimeType as Anthropic.ImageBlockParam.Source['media_type'],
-          type: 'base64',
-        },
-        type: 'image',
-      };
+      if (type === 'base64')
+        return {
+          source: {
+            data: base64 as string,
+            media_type: mimeType as Anthropic.ImageBlockParam.Source['media_type'],
+            type: 'base64',
+          },
+          type: 'image',
+        };
+
+      if (type === 'url') {
+        const base64 = await imageUrlToBase64(content.image_url.url);
+        return {
+          source: {
+            data: base64 as string,
+            media_type: (mimeType as Anthropic.ImageBlockParam.Source['media_type']) || 'image/png',
+            type: 'base64',
+          },
+          type: 'image',
+        };
+      }
+
+      throw new Error(`Invalid image URL: ${content.image_url.url}`);
     }
   }
 };
 
-export const buildAnthropicMessage = (
+export const buildAnthropicMessage = async (
   message: OpenAIChatMessage,
-): Anthropic.Messages.MessageParam => {
+): Promise<Anthropic.Messages.MessageParam> => {
   const content = message.content as string | UserMessageContentPart[];
 
   switch (message.role) {
@@ -39,7 +56,10 @@ export const buildAnthropicMessage = (
 
     case 'user': {
       return {
-        content: typeof content === 'string' ? content : content.map((c) => buildAnthropicBlock(c)),
+        content:
+          typeof content === 'string'
+            ? content
+            : await Promise.all(content.map(async (c) => await buildAnthropicBlock(c))),
         role: 'user',
       };
     }
@@ -90,14 +110,15 @@ export const buildAnthropicMessage = (
   }
 };
 
-export const buildAnthropicMessages = (
+export const buildAnthropicMessages = async (
   oaiMessages: OpenAIChatMessage[],
-): Anthropic.Messages.MessageParam[] => {
+): Promise<Anthropic.Messages.MessageParam[]> => {
   const messages: Anthropic.Messages.MessageParam[] = [];
   let lastRole = 'assistant';
   let pendingToolResults: Anthropic.ToolResultBlockParam[] = [];
 
-  oaiMessages.forEach((message, index) => {
+  for (const message of oaiMessages) {
+    const index = oaiMessages.indexOf(message);
     // refs: https://docs.anthropic.com/claude/docs/tool-use#tool-use-and-tool-result-content-blocks
     if (message.role === 'tool') {
       pendingToolResults.push({
@@ -117,7 +138,7 @@ export const buildAnthropicMessages = (
         lastRole = 'user';
       }
     } else {
-      const anthropicMessage = buildAnthropicMessage(message);
+      const anthropicMessage = await buildAnthropicMessage(message);
 
       if (lastRole === anthropicMessage.role) {
         messages.push({ content: '_', role: lastRole === 'user' ? 'assistant' : 'user' });
@@ -126,7 +147,7 @@ export const buildAnthropicMessages = (
       lastRole = anthropicMessage.role;
       messages.push(anthropicMessage);
     }
-  });
+  }
 
   return messages;
 };

package/src/utils/imageToBase64.test.ts

@@ -0,0 +1,90 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { imageToBase64, imageUrlToBase64 } from './imageToBase64';
+
+describe('imageToBase64', () => {
+  let mockImage: HTMLImageElement;
+  let mockCanvas: HTMLCanvasElement;
+  let mockContext: CanvasRenderingContext2D;
+
+  beforeEach(() => {
+    mockImage = {
+      width: 200,
+      height: 100,
+    } as HTMLImageElement;
+
+    mockContext = {
+      drawImage: vi.fn(),
+    } as unknown as CanvasRenderingContext2D;
+
+    mockCanvas = {
+      width: 0,
+      height: 0,
+      getContext: vi.fn().mockReturnValue(mockContext),
+      toDataURL: vi.fn().mockReturnValue('data:image/webp;base64,mockBase64Data'),
+    } as unknown as HTMLCanvasElement;
+
+    vi.spyOn(document, 'createElement').mockReturnValue(mockCanvas);
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it('should convert image to base64 with correct size and type', () => {
+    const result = imageToBase64({ img: mockImage, size: 100, type: 'image/jpeg' });
+
+    expect(document.createElement).toHaveBeenCalledWith('canvas');
+    expect(mockCanvas.width).toBe(100);
+    expect(mockCanvas.height).toBe(100);
+    expect(mockCanvas.getContext).toHaveBeenCalledWith('2d');
+    expect(mockContext.drawImage).toHaveBeenCalledWith(mockImage, 50, 0, 100, 100, 0, 0, 100, 100);
+    expect(mockCanvas.toDataURL).toHaveBeenCalledWith('image/jpeg');
+    expect(result).toBe('data:image/webp;base64,mockBase64Data');
+  });
+
+  it('should use default type when not specified', () => {
+    imageToBase64({ img: mockImage, size: 100 });
+    expect(mockCanvas.toDataURL).toHaveBeenCalledWith('image/webp');
+  });
+
+  it('should handle taller images correctly', () => {
+    mockImage.width = 100;
+    mockImage.height = 200;
+    imageToBase64({ img: mockImage, size: 100 });
+    expect(mockContext.drawImage).toHaveBeenCalledWith(mockImage, 0, 50, 100, 100, 0, 0, 100, 100);
+  });
+});
+
+describe('imageUrlToBase64', () => {
+  const mockFetch = vi.fn();
+  const mockArrayBuffer = new ArrayBuffer(8);
+
+  beforeEach(() => {
+    global.fetch = mockFetch;
+    global.btoa = vi.fn().mockReturnValue('mockBase64String');
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it('should convert image URL to base64 string', async () => {
+    mockFetch.mockResolvedValue({
+      arrayBuffer: () => Promise.resolve(mockArrayBuffer),
+    });
+
+    const result = await imageUrlToBase64('https://example.com/image.jpg');
+
+    expect(mockFetch).toHaveBeenCalledWith('https://example.com/image.jpg');
+    expect(global.btoa).toHaveBeenCalled();
+    expect(result).toBe('mockBase64String');
+  });
+
+  it('should throw an error when fetch fails', async () => {
+    const mockError = new Error('Fetch failed');
+    mockFetch.mockRejectedValue(mockError);
+
+    await expect(imageUrlToBase64('https://example.com/image.jpg')).rejects.toThrow('Fetch failed');
+  });
+});