@lobehub/chat 1.16.2 → 1.16.3

package/CHANGELOG.md

@@ -2,6 +2,31 @@
 
 # Changelog
 
+### [Version 1.16.3](https://github.com/lobehub/lobe-chat/compare/v1.16.2...v1.16.3)
+
+<sup>Released on **2024-09-11**</sup>
+
+#### 🐛 Bug Fixes
+
+- **misc**: Add `LLM_VISION_IMAGE_USE_BASE64` to support local s3 in vision model.
+
+<br/>
+
+<details>
+<summary><kbd>Improvements and Fixes</kbd></summary>
+
+#### What's fixed
+
+- **misc**: Add `LLM_VISION_IMAGE_USE_BASE64` to support local s3 in vision model, closes [#3887](https://github.com/lobehub/lobe-chat/issues/3887) ([16e57ed](https://github.com/lobehub/lobe-chat/commit/16e57ed))
+
+</details>
+
+<div align="right">
+
+[![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top)
+
+</div>
+
 ### [Version 1.16.2](https://github.com/lobehub/lobe-chat/compare/v1.16.1...v1.16.2)
 
 <sup>Released on **2024-09-11**</sup>

package/README.md

@@ -285,14 +285,14 @@ Our marketplace is not just a showcase platform but also a collaborative space.
 
 <!-- AGENT LIST -->
 
-| Recent Submits                                                                                                                                                                                 | Description                                                                                                                                                                                                                                                                                                                                                                                                     |
-| ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| [AI Agent Generator](https://chat-preview.lobehub.com/market?agent=ai-agent-generator)<br/><sup>By **[xyftw](https://github.com/xyftw)** on **2024-09-10**</sup>                               | Skilled at creating AI Agent character descriptions that meet the needs.<br/>`ai-agent` `character-creation`                                                                                                                                                                                                                                                                                                    |
-| [HTML to React](https://chat-preview.lobehub.com/market?agent=web-react)<br/><sup>By **[xingwang02](https://github.com/xingwang02)** on **2024-09-10**</sup>                                   | Input HTML snippets and convert them into React components<br/>`react` `html`                                                                                                                                                                                                                                                                                                                                   |
-| [FiveM & QBCore Framework Expert](https://chat-preview.lobehub.com/market?agent=lua-development)<br/><sup>By **[heartsiddharth1](https://github.com/heartsiddharth1)** on **2024-09-08**</sup> | Expertise in FiveM development, QBCore framework, Lua programming, JavaScript, database management, server administration, version control, full-stack web development, DevOps, and community engagement with a focus on performance, security, and best practices.<br/>`five-m` `qb-core` `lua` `java-script` `my-sql` `server-management` `git` `full-stack-web-development` `dev-ops` `community-engagement` |
-| [Nuxt 3/Vue.js Master Developer](https://chat-preview.lobehub.com/market?agent=nuxt-vue-developer)<br/><sup>By **[Kadreev](https://github.com/Kadreev)** on **2024-09-03**</sup>               | Specialized in full-stack development with Nuxt 3 expertise.<br/>`nuxt-3` `vue-js` `full-stack-development` `java-script` `web-applications`                                                                                                                                                                                                                                                                    |
-
-> 📊 Total agents: [<kbd>**325**</kbd> ](https://github.com/lobehub/lobe-chat-agents)
+| Recent Submits                                                                                                                                                   | Description                                                                                                                                                                           |
+| ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| [AI Agent Generator](https://chat-preview.lobehub.com/market?agent=ai-agent-generator)<br/><sup>By **[xyftw](https://github.com/xyftw)** on **2024-09-10**</sup> | Skilled at creating AI Agent character descriptions that meet the needs.<br/>`ai-agent` `character-creation`                                                                          |
+| [Desolate Friend](https://chat-preview.lobehub.com/market?agent=meu)<br/><sup>By **[adminewacc](https://github.com/adminewacc)** on **2024-09-10**</sup>         | Skilled at comforting and supporting friends<br/>`friendship` `sadness` `support`                                                                                                     |
+| [NetMaster](https://chat-preview.lobehub.com/market?agent=net-master)<br/><sup>By **[erhuoyan](https://github.com/erhuoyan)** on **2024-09-10**</sup>            | Network Engineer: Professional network topology construction and management<br/>`network-engineer` `network-configuration` `network-management` `network-topology` `network-security` |
+| [HTML to React](https://chat-preview.lobehub.com/market?agent=web-react)<br/><sup>By **[xingwang02](https://github.com/xingwang02)** on **2024-09-10**</sup>     | Input HTML snippets and convert them into React components<br/>`react` `html`                                                                                                         |
+
+> 📊 Total agents: [<kbd>**327**</kbd> ](https://github.com/lobehub/lobe-chat-agents)
 
  <!-- AGENT LIST -->
 

package/README.zh-CN.md

@@ -273,14 +273,14 @@ LobeChat 的插件生态系统是其核心功能的重要扩展，它极大地
 
 <!-- AGENT LIST -->
 
-| 最近新增                                                                                                                                                                                | 助手说明                                                                                                                                                                                                                                                                                                           |
-| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
-| [AI 代理生成器](https://chat-preview.lobehub.com/market?agent=ai-agent-generator)<br/><sup>By **[xyftw](https://github.com/xyftw)** on **2024-09-10**</sup>                             | 擅长创建满足需求的 AI 代理角色描述。<br/>`ai-agent` `角色创建`                                                                                                                                                                                                                                                     |
-| [HTML to React](https://chat-preview.lobehub.com/market?agent=web-react)<br/><sup>By **[xingwang02](https://github.com/xingwang02)** on **2024-09-10**</sup>                            | 输入 HTML 片段，转化为 React 组件<br/>`react、-html`                                                                                                                                                                                                                                                               |
-| [FiveM 和 QBCore 框架专家](https://chat-preview.lobehub.com/market?agent=lua-development)<br/><sup>By **[heartsiddharth1](https://github.com/heartsiddharth1)** on **2024-09-08**</sup> | 在 FiveM 开发、QBCore 框架、Lua 编程、JavaScript、数据库管理、服务器管理、版本控制、全栈 Web 开发、DevOps 和社区参与方面具有专业知识，重点关注性能、安全性和最佳实践。<br/>`five-m` `qb-core` `lua` `java-script` `my-sql` `server-management` `git` `full-stack-web-development` `dev-ops` `community-engagement` |
-| [Nuxt 3/Vue.js 大师开发者](https://chat-preview.lobehub.com/market?agent=nuxt-vue-developer)<br/><sup>By **[Kadreev](https://github.com/Kadreev)** on **2024-09-03**</sup>              | 专注于全栈开发，拥有 Nuxt 3 专业知识。<br/>`nuxt-3` `vue-js` `全栈开发` `java-script` `网络应用`                                                                                                                                                                                                                   |
-
-> 📊 Total agents: [<kbd>**325**</kbd> ](https://github.com/lobehub/lobe-chat-agents)
+| 最近新增                                                                                                                                                     | 助手说明                                                                                        |
+| ------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------- |
+| [AI 代理生成器](https://chat-preview.lobehub.com/market?agent=ai-agent-generator)<br/><sup>By **[xyftw](https://github.com/xyftw)** on **2024-09-10**</sup>  | 擅长创建满足需求的 AI 代理角色描述。<br/>`ai-agent` `角色创建`                                  |
+| [孤独的朋友](https://chat-preview.lobehub.com/market?agent=meu)<br/><sup>By **[adminewacc](https://github.com/adminewacc)** on **2024-09-10**</sup>          | 擅长安慰和支持朋友<br/>`友谊` `悲伤` `支持`                                                     |
+| [NetMaster](https://chat-preview.lobehub.com/market?agent=net-master)<br/><sup>By **[erhuoyan](https://github.com/erhuoyan)** on **2024-09-10**</sup>        | 网络工程师：专业网络拓扑搭建与管理<br/>`网络工程师` `网络配置` `网络管理` `网络拓扑` `网络安全` |
+| [HTML to React](https://chat-preview.lobehub.com/market?agent=web-react)<br/><sup>By **[xingwang02](https://github.com/xingwang02)** on **2024-09-10**</sup> | 输入 HTML 片段，转化为 React 组件<br/>`react、-html`                                            |
+
+> 📊 Total agents: [<kbd>**327**</kbd> ](https://github.com/lobehub/lobe-chat-agents)
 
  <!-- AGENT LIST -->
 

package/docs/self-hosting/advanced/auth/next-auth/logto.zh-CN.mdx

@@ -51,7 +51,7 @@ tags:
   src="https://github.com/user-attachments/assets/15af6d94-af4f-4aa9-bbab-7a46e9f9e837"
 />
 
-将获取到的 `Client ID` 和 `Client Secret`，设为 LobeChat 环境变量中的 `LOGTO_CLIENT_ID` 和 `LOGTO_CLIENT_SECRETT`。
+将获取到的 `Client ID` 和 `Client Secret`，设为 LobeChat 环境变量中的 `LOGTO_CLIENT_ID` 和 `LOGTO_CLIENT_SECRET`。
 
 配置 LobeChat 环境变量中 `LOGTO_ISSUER` 为：
 
@@ -75,4 +75,4 @@ tags:
 </Callout>
 </Steps>
 
-<Callout type={'info'}>部署成功后，用户将可以通过 Logto 身份认证并使用 LobeChat。</Callout>
+<Callout type={'info'}>部署成功后，用户将可以通过 Logto 身份认证并使用 LobeChat。</Callout>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lobehub/chat",
-  "version": "1.16.2",
+  "version": "1.16.3",
   "description": "Lobe Chat - an open-source, high-performance chatbot framework that supports speech synthesis, multimodal, and extensible Function Call plugin system. Supports one-click free deployment of your private ChatGPT/LLM web application.",
   "keywords": [
     "framework",

package/src/config/modelProviders/stepfun.ts

@@ -61,6 +61,10 @@ const Stepfun: ModelProviderCard = {
   id: 'stepfun',
   modelList: { showModelFetcher: true },
   name: '阶跃星辰',
+  smoothing: {
+    speed: 2,
+    text: true,
+  },
 };
 
 export default Stepfun;

package/src/libs/agent-runtime/google/index.test.ts

@@ -309,7 +309,10 @@ describe('LobeGoogleAI', () => {
         const mockBase64 = 'mockBase64Data';
 
         // Mock the imageUrlToBase64 function
-        vi.spyOn(imageToBase64Module, 'imageUrlToBase64').mockResolvedValueOnce(mockBase64);
+        vi.spyOn(imageToBase64Module, 'imageUrlToBase64').mockResolvedValueOnce({
+          base64: mockBase64,
+          mimeType: 'image/png',
+        });
 
         const result = await instance['convertContentToGooglePart']({
           type: 'image_url',

package/src/libs/agent-runtime/google/index.ts

@@ -133,12 +133,12 @@ export class LobeGoogleAI implements LobeRuntimeAI {
         }
 
         if (type === 'url') {
-          const base64Image = await imageUrlToBase64(content.image_url.url);
+          const { base64, mimeType } = await imageUrlToBase64(content.image_url.url);
 
           return {
             inlineData: {
-              data: base64Image,
-              mimeType: mimeType || 'image/png',
+              data: base64,
+              mimeType,
             },
           };
         }

package/src/libs/agent-runtime/utils/anthropicHelpers.test.ts

@@ -53,7 +53,10 @@ describe('anthropicHelpers', () => {
         base64: null,
         type: 'url',
       });
-      vi.mocked(imageUrlToBase64).mockResolvedValue('convertedBase64String');
+      vi.mocked(imageUrlToBase64).mockResolvedValue({
+        base64: 'convertedBase64String',
+        mimeType: 'image/jpg',
+      });
 
       const content = {
         type: 'image_url',
@@ -67,7 +70,7 @@ describe('anthropicHelpers', () => {
       expect(result).toEqual({
         source: {
           data: 'convertedBase64String',
-          media_type: 'image/png',
+          media_type: 'image/jpg',
           type: 'base64',
         },
         type: 'image',
@@ -80,7 +83,10 @@ describe('anthropicHelpers', () => {
         base64: null,
         type: 'url',
       });
-      vi.mocked(imageUrlToBase64).mockResolvedValue('convertedBase64String');
+      vi.mocked(imageUrlToBase64).mockResolvedValue({
+        base64: 'convertedBase64String',
+        mimeType: 'image/png',
+      });
 
       const content = {
         type: 'image_url',

package/src/libs/agent-runtime/utils/anthropicHelpers.ts

@@ -28,11 +28,11 @@ export const buildAnthropicBlock = async (
         };
 
       if (type === 'url') {
-        const base64 = await imageUrlToBase64(content.image_url.url);
+        const { base64, mimeType } = await imageUrlToBase64(content.image_url.url);
         return {
           source: {
             data: base64 as string,
-            media_type: (mimeType as Anthropic.ImageBlockParam.Source['media_type']) || 'image/png',
+            media_type: mimeType as Anthropic.ImageBlockParam.Source['media_type'],
             type: 'base64',
           },
           type: 'image',

package/src/libs/agent-runtime/utils/openaiCompatibleFactory/index.test.ts

@@ -8,6 +8,7 @@ import {
   LobeOpenAICompatibleRuntime,
   ModelProvider,
 } from '@/libs/agent-runtime';
+import { sleep } from '@/utils/sleep';
 
 import * as debugStreamModule from '../debugStream';
 import { LobeOpenAICompatibleFactory } from './index';
@@ -512,9 +513,18 @@ describe('LobeOpenAICompatibleFactory', () => {
     describe('cancel request', () => {
       it('should cancel ongoing request correctly', async () => {
         const controller = new AbortController();
-        const mockCreateMethod = vi.spyOn(instance['client'].chat.completions, 'create');
+        const mockCreateMethod = vi
+          .spyOn(instance['client'].chat.completions, 'create')
+          .mockImplementation(
+            () =>
+              new Promise((_, reject) => {
+                setTimeout(() => {
+                  reject(new DOMException('The user aborted a request.', 'AbortError'));
+                }, 100);
+              }) as any,
+          );
 
-        instance.chat(
+        const chatPromise = instance.chat(
           {
             messages: [{ content: 'Hello', role: 'user' }],
             model: 'mistralai/mistral-7b-instruct:free',
@@ -523,8 +533,22 @@ describe('LobeOpenAICompatibleFactory', () => {
           { signal: controller.signal },
         );
 
+        // 给一些时间让请求开始
+        await sleep(50);
+
         controller.abort();
 
+        // 等待并断言 Promise 被拒绝
+        // 使用 try-catch 来捕获和验证错误
+        try {
+          await chatPromise;
+          // 如果 Promise 没有被拒绝，测试应该失败
+          expect.fail('Expected promise to be rejected');
+        } catch (error) {
+          expect((error as any).errorType).toBe('AgentRuntimeError');
+          expect((error as any).error.name).toBe('AbortError');
+          expect((error as any).error.message).toBe('The user aborted a request.');
+        }
         expect(mockCreateMethod).toHaveBeenCalledWith(
           expect.anything(),
           expect.objectContaining({

package/src/libs/agent-runtime/utils/openaiCompatibleFactory/index.ts

@@ -20,6 +20,7 @@ import { desensitizeUrl } from '../desensitizeUrl';
 import { handleOpenAIError } from '../handleOpenAIError';
 import { StreamingResponse } from '../response';
 import { OpenAIStream } from '../streams';
+import { convertOpenAIMessages } from '../openaiHelpers';
 
 // the model contains the following keywords is not a chat model, so we should filter them out
 const CHAT_MODELS_BLOCK_LIST = [
@@ -158,9 +159,12 @@ export const LobeOpenAICompatibleFactory = <T extends Record<string, any> = any>
               stream: payload.stream ?? true,
             } as OpenAI.ChatCompletionCreateParamsStreaming);
 
+        const messages = await convertOpenAIMessages(postPayload.messages);
+
         const response = await this.client.chat.completions.create(
           {
             ...postPayload,
+            messages,
             ...(chatCompletion?.noUserId ? {} : { user: options?.user }),
           },
           {

package/src/libs/agent-runtime/utils/openaiHelpers.test.ts

@@ -0,0 +1,146 @@
+import OpenAI from 'openai';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { imageUrlToBase64 } from '@/utils/imageToBase64';
+
+import { convertMessageContent, convertOpenAIMessages } from './openaiHelpers';
+import { parseDataUri } from './uriParser';
+
+// 模拟依赖
+vi.mock('@/utils/imageToBase64');
+vi.mock('./uriParser');
+
+describe('convertMessageContent', () => {
+  beforeEach(() => {
+    vi.resetAllMocks();
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it('should return the same content if not image_url type', async () => {
+    const content = { type: 'text', text: 'Hello' } as OpenAI.ChatCompletionContentPart;
+    const result = await convertMessageContent(content);
+    expect(result).toEqual(content);
+  });
+
+  it('should convert image URL to base64 when necessary', async () => {
+    // 设置环境变量
+    process.env.LLM_VISION_IMAGE_USE_BASE64 = '1';
+
+    const content = {
+      type: 'image_url',
+      image_url: { url: 'https://example.com/image.jpg' },
+    } as OpenAI.ChatCompletionContentPart;
+
+    vi.mocked(parseDataUri).mockReturnValue({ type: 'url', base64: null, mimeType: null });
+    vi.mocked(imageUrlToBase64).mockResolvedValue({
+      base64: 'base64String',
+      mimeType: 'image/jpeg',
+    });
+
+    const result = await convertMessageContent(content);
+
+    expect(result).toEqual({
+      type: 'image_url',
+      image_url: { url: 'data:image/jpeg;base64,base64String' },
+    });
+
+    expect(parseDataUri).toHaveBeenCalledWith('https://example.com/image.jpg');
+    expect(imageUrlToBase64).toHaveBeenCalledWith('https://example.com/image.jpg');
+  });
+
+  it('should not convert image URL when not necessary', async () => {
+    process.env.LLM_VISION_IMAGE_USE_BASE64 = undefined;
+
+    const content = {
+      type: 'image_url',
+      image_url: { url: 'https://example.com/image.jpg' },
+    } as OpenAI.ChatCompletionContentPart;
+
+    vi.mocked(parseDataUri).mockReturnValue({ type: 'url', base64: null, mimeType: null });
+
+    const result = await convertMessageContent(content);
+
+    expect(result).toEqual(content);
+    expect(imageUrlToBase64).not.toHaveBeenCalled();
+  });
+});
+
+describe('convertOpenAIMessages', () => {
+  it('should convert string content messages', async () => {
+    const messages = [
+      { role: 'user', content: 'Hello' },
+      { role: 'assistant', content: 'Hi there' },
+    ] as OpenAI.ChatCompletionMessageParam[];
+
+    const result = await convertOpenAIMessages(messages);
+
+    expect(result).toEqual(messages);
+  });
+
+  it('should convert array content messages', async () => {
+    const messages = [
+      {
+        role: 'user',
+        content: [
+          { type: 'text', text: 'Hello' },
+          { type: 'image_url', image_url: { url: 'https://example.com/image.jpg' } },
+        ],
+      },
+    ] as OpenAI.ChatCompletionMessageParam[];
+
+    vi.spyOn(Promise, 'all');
+    vi.mocked(parseDataUri).mockReturnValue({ type: 'url', base64: null, mimeType: null });
+    vi.mocked(imageUrlToBase64).mockResolvedValue({
+      base64: 'base64String',
+      mimeType: 'image/jpeg',
+    });
+
+    process.env.LLM_VISION_IMAGE_USE_BASE64 = '1';
+
+    const result = await convertOpenAIMessages(messages);
+
+    expect(result).toEqual([
+      {
+        role: 'user',
+        content: [
+          { type: 'text', text: 'Hello' },
+          {
+            type: 'image_url',
+            image_url: { url: 'data:image/jpeg;base64,base64String' },
+          },
+        ],
+      },
+    ]);
+
+    expect(Promise.all).toHaveBeenCalledTimes(2); // 一次用于消息数组，一次用于内容数组
+
+    process.env.LLM_VISION_IMAGE_USE_BASE64 = undefined;
+  });
+  it('should convert array content messages', async () => {
+    const messages = [
+      {
+        role: 'user',
+        content: [
+          { type: 'text', text: 'Hello' },
+          { type: 'image_url', image_url: { url: 'https://example.com/image.jpg' } },
+        ],
+      },
+    ] as OpenAI.ChatCompletionMessageParam[];
+
+    vi.spyOn(Promise, 'all');
+    vi.mocked(parseDataUri).mockReturnValue({ type: 'url', base64: null, mimeType: null });
+    vi.mocked(imageUrlToBase64).mockResolvedValue({
+      base64: 'base64String',
+      mimeType: 'image/jpeg',
+    });
+
+    const result = await convertOpenAIMessages(messages);
+
+    expect(result).toEqual(messages);
+
+    expect(Promise.all).toHaveBeenCalledTimes(2); // 一次用于消息数组，一次用于内容数组
+  });
+});

package/src/libs/agent-runtime/utils/openaiHelpers.ts

@@ -0,0 +1,40 @@
+import OpenAI from 'openai';
+
+import { imageUrlToBase64 } from '@/utils/imageToBase64';
+
+import { parseDataUri } from './uriParser';
+
+export const convertMessageContent = async (
+  content: OpenAI.ChatCompletionContentPart,
+): Promise<OpenAI.ChatCompletionContentPart> => {
+  if (content.type === 'image_url') {
+    const { type } = parseDataUri(content.image_url.url);
+
+    if (type === 'url' && process.env.LLM_VISION_IMAGE_USE_BASE64 === '1') {
+      const { base64, mimeType } = await imageUrlToBase64(content.image_url.url);
+
+      return {
+        ...content,
+        image_url: { ...content.image_url, url: `data:${mimeType};base64,${base64}` },
+      };
+    }
+  }
+
+  return content;
+};
+
+export const convertOpenAIMessages = async (messages: OpenAI.ChatCompletionMessageParam[]) => {
+  return (await Promise.all(
+    messages.map(async (message) => ({
+      ...message,
+      content:
+        typeof message.content === 'string'
+          ? message.content
+          : await Promise.all(
+              (message.content || []).map((c) =>
+                convertMessageContent(c as OpenAI.ChatCompletionContentPart),
+              ),
+            ),
+    })),
+  )) as OpenAI.ChatCompletionMessageParam[];
+};

package/src/libs/agent-runtime/zhipu/index.ts

@@ -2,19 +2,14 @@ import OpenAI, { ClientOptions } from 'openai';
 
 import { LobeRuntimeAI } from '../BaseAI';
 import { AgentRuntimeErrorType } from '../error';
-import {
-  ChatCompetitionOptions,
-  ChatStreamPayload,
-  ModelProvider,
-  OpenAIChatMessage,
-} from '../types';
+import { ChatCompetitionOptions, ChatStreamPayload, ModelProvider } from '../types';
 import { AgentRuntimeError } from '../utils/createError';
 import { debugStream } from '../utils/debugStream';
 import { desensitizeUrl } from '../utils/desensitizeUrl';
 import { handleOpenAIError } from '../utils/handleOpenAIError';
+import { convertOpenAIMessages } from '../utils/openaiHelpers';
 import { StreamingResponse } from '../utils/response';
 import { OpenAIStream } from '../utils/streams';
-import { parseDataUri } from '../utils/uriParser';
 import { generateApiToken } from './authToken';
 
 const DEFAULT_BASE_URL = 'https://open.bigmodel.cn/api/paas/v4';
@@ -52,7 +47,7 @@ export class LobeZhipuAI implements LobeRuntimeAI {
 
   async chat(payload: ChatStreamPayload, options?: ChatCompetitionOptions) {
     try {
-      const params = this.buildCompletionsParams(payload);
+      const params = await this.buildCompletionsParams(payload);
 
       const response = await this.client.chat.completions.create(
         params as unknown as OpenAI.ChatCompletionCreateParamsStreaming,
@@ -85,11 +80,11 @@ export class LobeZhipuAI implements LobeRuntimeAI {
     }
   }
 
-  private buildCompletionsParams(payload: ChatStreamPayload) {
+  private async buildCompletionsParams(payload: ChatStreamPayload) {
     const { messages, temperature, top_p, ...params } = payload;
 
     return {
-      messages: messages.map((m) => this.transformMessage(m)) as any,
+      messages: await convertOpenAIMessages(messages as any),
       ...params,
       do_sample: temperature === 0,
       stream: true,
@@ -99,32 +94,6 @@ export class LobeZhipuAI implements LobeRuntimeAI {
       top_p: top_p === 1 ? 0.99 : top_p,
     };
   }
-
-  // TODO: 临时处理，后续需要移除
-  private transformMessage = (message: OpenAIChatMessage): OpenAIChatMessage => {
-    return {
-      ...message,
-      content:
-        typeof message.content === 'string'
-          ? message.content
-          : message.content.map((c) => {
-              switch (c.type) {
-                default:
-                case 'text': {
-                  return c;
-                }
-
-                case 'image_url': {
-                  const { base64 } = parseDataUri(c.image_url.url);
-                  return {
-                    image_url: { ...c.image_url, url: base64 || c.image_url.url },
-                    type: 'image_url',
-                  };
-                }
-              }
-            }),
-    };
-  };
 }
 
 export default LobeZhipuAI;

package/src/utils/imageToBase64.test.ts

@@ -72,13 +72,14 @@ describe('imageUrlToBase64', () => {
   it('should convert image URL to base64 string', async () => {
     mockFetch.mockResolvedValue({
       arrayBuffer: () => Promise.resolve(mockArrayBuffer),
+      blob: () => Promise.resolve(new Blob([mockArrayBuffer], { type: 'image/jpg' })),
     });
 
     const result = await imageUrlToBase64('https://example.com/image.jpg');
 
     expect(mockFetch).toHaveBeenCalledWith('https://example.com/image.jpg');
     expect(global.btoa).toHaveBeenCalled();
-    expect(result).toBe('mockBase64String');
+    expect(result).toEqual({ base64: 'mockBase64String', mimeType: 'image/jpg' });
   });
 
   it('should throw an error when fetch fails', async () => {

package/src/utils/imageToBase64.ts

@@ -36,16 +36,25 @@ export const imageToBase64 = ({
   return canvas.toDataURL(type);
 };
 
-export const imageUrlToBase64 = async (imageUrl: string): Promise<string> => {
+export const imageUrlToBase64 = async (
+  imageUrl: string,
+): Promise<{ base64: string; mimeType: string }> => {
   try {
     const res = await fetch(imageUrl);
-    const arrayBuffer = await res.arrayBuffer();
+    const blob = await res.blob();
+    const arrayBuffer = await blob.arrayBuffer();
 
-    return typeof btoa === 'function'
-      ? btoa(
-          new Uint8Array(arrayBuffer).reduce((data, byte) => data + String.fromCharCode(byte), ''),
-        )
-      : Buffer.from(arrayBuffer).toString('base64');
+    const base64 =
+      typeof btoa === 'function'
+        ? btoa(
+            new Uint8Array(arrayBuffer).reduce(
+              (data, byte) => data + String.fromCharCode(byte),
+              '',
+            ),
+          )
+        : Buffer.from(arrayBuffer).toString('base64');
+
+    return { base64, mimeType: blob.type };
   } catch (error) {
     console.error('Error converting image to base64:', error);
     throw error;