@lobehub/chat 1.99.2 → 1.99.4

package/src/libs/model-runtime/openai/index.ts

@@ -1,9 +1,4 @@
-import debug from 'debug';
-import { toFile } from 'openai';
-import { FileLike } from 'openai/uploads';
-
 import { responsesAPIModels } from '@/const/models';
-import { RuntimeImageGenParamsValue } from '@/libs/standard-parameters/meta-schema';
 
 import { ChatStreamPayload, ModelProvider } from '../types';
 import { processMultiProviderModelList } from '../utils/modelParse';
@@ -15,45 +10,8 @@ export interface OpenAIModelCard {
 }
 
 const prunePrefixes = ['o1', 'o3', 'o4', 'codex', 'computer-use'];
-
 const oaiSearchContextSize = process.env.OPENAI_SEARCH_CONTEXT_SIZE; // low, medium, high
 
-const log = debug('lobe-image:openai');
-
-/**
- * 将图片 URL 转换为 File 对象
- * @param imageUrl - 图片 URL（可以是 HTTP URL 或 base64 data URL）
- * @returns FileLike 对象
- */
-export const convertImageUrlToFile = async (imageUrl: string): Promise<FileLike> => {
-  log('Converting image URL to File: %s', imageUrl.startsWith('data:') ? 'base64 data' : imageUrl);
-
-  let buffer: Buffer;
-  let mimeType: string;
-
-  if (imageUrl.startsWith('data:')) {
-    // 处理 base64 data URL
-    log('Processing base64 image data');
-    const [mimeTypePart, base64Data] = imageUrl.split(',');
-    mimeType = mimeTypePart.split(':')[1].split(';')[0];
-    buffer = Buffer.from(base64Data, 'base64');
-  } else {
-    // 处理 HTTP URL
-    log('Fetching image from URL: %s', imageUrl);
-    const response = await fetch(imageUrl);
-    if (!response.ok) {
-      throw new Error(`Failed to fetch image from ${imageUrl}: ${response.statusText}`);
-    }
-    buffer = Buffer.from(await response.arrayBuffer());
-    mimeType = response.headers.get('content-type') || 'image/png';
-  }
-
-  log('Successfully converted image to buffer, size: %s, mimeType: %s', buffer.length, mimeType);
-
-  // 使用 OpenAI 的 toFile 方法创建 File 对象
-  return toFile(buffer, `image.${mimeType.split('/')[1]}`, { type: mimeType });
-};
-
 export const LobeOpenAI = createOpenAICompatibleRuntime({
   baseURL: 'https://api.openai.com/v1',
   chatCompletion: {
@@ -88,91 +46,6 @@ export const LobeOpenAI = createOpenAICompatibleRuntime({
       return { ...rest, model, stream: payload.stream ?? true };
     },
   },
-  createImage: async (payload) => {
-    const { model, params, client } = payload;
-    log('Creating image with model: %s and params: %O', model, params);
-
-    const defaultInput = {
-      n: 1,
-    };
-
-    // 映射参数名称，将 imageUrls 映射为 image
-    const paramsMap = new Map<RuntimeImageGenParamsValue, string>([['imageUrls', 'image']]);
-    const userInput: Record<string, any> = Object.fromEntries(
-      Object.entries(params).map(([key, value]) => [
-        paramsMap.get(key as RuntimeImageGenParamsValue) ?? key,
-        value,
-      ]),
-    );
-
-    const isImageEdit = Array.isArray(userInput.image) && userInput.image.length > 0;
-    // 如果有 imageUrls 参数，将其转换为 File 对象
-    if (isImageEdit) {
-      log('Converting imageUrls to File objects: %O', userInput.image);
-      try {
-        // 转换所有图片 URL 为 File 对象
-        const imageFiles = await Promise.all(
-          userInput.image.map((url: string) => convertImageUrlToFile(url)),
-        );
-
-        log('Successfully converted %d images to File objects', imageFiles.length);
-
-        // 根据官方文档，如果有多个图片，传递数组；如果只有一个，传递单个 File
-        userInput.image = imageFiles.length === 1 ? imageFiles[0] : imageFiles;
-      } catch (error) {
-        log('Error converting imageUrls to File objects: %O', error);
-        throw new Error(`Failed to convert image URLs to File objects: ${error}`);
-      }
-    } else {
-      delete userInput.image;
-    }
-
-    if (userInput.size === 'auto') {
-      delete userInput.size;
-    }
-
-    const options = {
-      model,
-      ...defaultInput,
-      ...(userInput as any),
-    };
-
-    log('options: %O', options);
-
-    // 判断是否为图片编辑操作
-    const img = isImageEdit
-      ? await client.images.edit(options)
-      : await client.images.generate(options);
-
-    // 检查响应数据的完整性
-    if (!img || !img.data || !Array.isArray(img.data) || img.data.length === 0) {
-      log('Invalid image response: missing data array');
-      throw new Error('Invalid image response: missing or empty data array');
-    }
-
-    const imageData = img.data[0];
-    if (!imageData) {
-      log('Invalid image response: first data item is null/undefined');
-      throw new Error('Invalid image response: first data item is null or undefined');
-    }
-
-    if (!imageData.b64_json) {
-      log('Invalid image response: missing b64_json field');
-      throw new Error('Invalid image response: missing b64_json field');
-    }
-
-    // 确定图片的 MIME 类型，默认为 PNG
-    const mimeType = 'image/png'; // OpenAI 图片生成默认返回 PNG 格式
-
-    // 将 base64 字符串转换为完整的 data URL
-    const dataUrl = `data:${mimeType};base64,${imageData.b64_json}`;
-
-    log('Successfully converted base64 to data URL, length: %d', dataUrl.length);
-
-    return {
-      imageUrl: dataUrl,
-    };
-  },
   debug: {
     chatCompletion: () => process.env.DEBUG_OPENAI_CHAT_COMPLETION === '1',
     responses: () => process.env.DEBUG_OPENAI_RESPONSES === '1',

package/src/libs/model-runtime/openrouter/__snapshots__/index.test.ts.snap

@@ -21,6 +21,7 @@ _These are free, rate-limited endpoints for [Reflection 70B](/models/mattshumer/
     },
     "reasoning": true,
     "releasedAt": "2024-09-06",
+    "type": "chat",
     "vision": false,
   },
 ]
@@ -47,6 +48,7 @@ _These are free, rate-limited endpoints for [Reflection 70B](/models/mattshumer/
     },
     "reasoning": false,
     "releasedAt": "2024-09-06",
+    "type": "chat",
     "vision": false,
   },
 ]
@@ -73,6 +75,7 @@ _These are free, rate-limited endpoints for [Reflection 70B](/models/mattshumer/
     },
     "reasoning": false,
     "releasedAt": "2024-09-06",
+    "type": "chat",
     "vision": false,
   },
 ]

package/src/libs/model-runtime/ppio/__snapshots__/index.test.ts.snap

@@ -10,6 +10,7 @@ exports[`PPIO > models > should get models 1`] = `
     "functionCall": false,
     "id": "deepseek/deepseek-r1/community",
     "reasoning": true,
+    "type": "chat",
     "vision": false,
   },
   {
@@ -20,6 +21,7 @@ exports[`PPIO > models > should get models 1`] = `
     "functionCall": false,
     "id": "deepseek/deepseek-v3/community",
     "reasoning": false,
+    "type": "chat",
     "vision": false,
   },
 ]

package/src/libs/model-runtime/utils/modelParse.ts

@@ -136,6 +136,7 @@ const processModelCard = (
       reasoningKeywords.some((keyword) => model.id.toLowerCase().includes(keyword)) ||
       knownModel?.abilities?.reasoning ||
       false,
+    type: model.type || knownModel?.type || 'chat',
     vision:
       (visionKeywords.some((keyword) => model.id.toLowerCase().includes(keyword)) &&
         !isExcludedModel) ||

package/src/libs/model-runtime/utils/openaiCompatibleFactory/index.test.ts

@@ -14,6 +14,7 @@ import officalOpenAIModels from '@/libs/model-runtime/openai/fixtures/openai-mod
 import { sleep } from '@/utils/sleep';
 
 import * as debugStreamModule from '../debugStream';
+import * as openaiHelpers from '../openaiHelpers';
 import { createOpenAICompatibleRuntime } from './index';
 
 const provider = 'groq';
@@ -978,6 +979,329 @@ describe('LobeOpenAICompatibleFactory', () => {
     });
   });
 
+  describe('createImage', () => {
+    beforeEach(() => {
+      // Mock convertImageUrlToFile since it's already tested in openaiHelpers.test.ts
+      vi.spyOn(openaiHelpers, 'convertImageUrlToFile').mockResolvedValue(
+        new File(['mock-file-content'], 'test-image.jpg', { type: 'image/jpeg' }),
+      );
+    });
+
+    describe('basic image generation', () => {
+      it('should generate image successfully without imageUrls', async () => {
+        const mockResponse = {
+          data: [
+            {
+              b64_json:
+                'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==',
+            },
+          ],
+        };
+
+        vi.spyOn(instance['client'].images, 'generate').mockResolvedValue(mockResponse as any);
+
+        const payload = {
+          model: 'dall-e-3',
+          params: {
+            prompt: 'A beautiful sunset',
+            size: '1024x1024',
+            quality: 'standard',
+          },
+        };
+
+        const result = await (instance as any).createImage(payload);
+
+        expect(instance['client'].images.generate).toHaveBeenCalledWith({
+          model: 'dall-e-3',
+          n: 1,
+          prompt: 'A beautiful sunset',
+          size: '1024x1024',
+          quality: 'standard',
+          response_format: 'b64_json',
+        });
+
+        expect(result).toEqual({
+          imageUrl:
+            'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==',
+        });
+      });
+
+      it('should handle size auto parameter correctly', async () => {
+        const mockResponse = {
+          data: [{ b64_json: 'mock-base64-data' }],
+        };
+
+        vi.spyOn(instance['client'].images, 'generate').mockResolvedValue(mockResponse as any);
+
+        const payload = {
+          model: 'dall-e-3',
+          params: {
+            prompt: 'A beautiful sunset',
+            size: 'auto',
+          },
+        };
+
+        await (instance as any).createImage(payload);
+
+        // size: 'auto' should be removed from the options
+        expect(instance['client'].images.generate).toHaveBeenCalledWith({
+          model: 'dall-e-3',
+          n: 1,
+          prompt: 'A beautiful sunset',
+          response_format: 'b64_json',
+        });
+      });
+
+      it('should not add response_format parameter for gpt-image-1 model', async () => {
+        const mockResponse = {
+          data: [{ b64_json: 'gpt-image-1-base64-data' }],
+        };
+
+        vi.spyOn(instance['client'].images, 'generate').mockResolvedValue(mockResponse as any);
+
+        const payload = {
+          model: 'gpt-image-1',
+          params: {
+            prompt: 'A modern digital artwork',
+            size: '1024x1024',
+          },
+        };
+
+        const result = await (instance as any).createImage(payload);
+
+        // gpt-image-1 model should not include response_format parameter
+        expect(instance['client'].images.generate).toHaveBeenCalledWith({
+          model: 'gpt-image-1',
+          n: 1,
+          prompt: 'A modern digital artwork',
+          size: '1024x1024',
+        });
+
+        expect(result).toEqual({
+          imageUrl: 'data:image/png;base64,gpt-image-1-base64-data',
+        });
+      });
+    });
+
+    describe('image editing', () => {
+      it('should edit image with single imageUrl', async () => {
+        const mockResponse = {
+          data: [{ b64_json: 'edited-image-base64' }],
+        };
+
+        vi.spyOn(instance['client'].images, 'edit').mockResolvedValue(mockResponse as any);
+
+        const payload = {
+          model: 'dall-e-2',
+          params: {
+            prompt: 'Add a rainbow to this image',
+            imageUrls: ['https://example.com/image1.jpg'],
+            mask: 'https://example.com/mask.jpg',
+          },
+        };
+
+        const result = await (instance as any).createImage(payload);
+
+        expect(openaiHelpers.convertImageUrlToFile).toHaveBeenCalledWith(
+          'https://example.com/image1.jpg',
+        );
+        expect(instance['client'].images.edit).toHaveBeenCalledWith({
+          model: 'dall-e-2',
+          n: 1,
+          prompt: 'Add a rainbow to this image',
+          image: expect.any(File),
+          mask: 'https://example.com/mask.jpg',
+          response_format: 'b64_json',
+        });
+
+        expect(result).toEqual({
+          imageUrl: 'data:image/png;base64,edited-image-base64',
+        });
+      });
+
+      it('should edit image with multiple imageUrls', async () => {
+        const mockResponse = {
+          data: [{ b64_json: 'edited-multiple-images-base64' }],
+        };
+
+        const mockFile1 = new File(['content1'], 'image1.jpg', { type: 'image/jpeg' });
+        const mockFile2 = new File(['content2'], 'image2.jpg', { type: 'image/jpeg' });
+
+        vi.mocked(openaiHelpers.convertImageUrlToFile)
+          .mockResolvedValueOnce(mockFile1)
+          .mockResolvedValueOnce(mockFile2);
+
+        vi.spyOn(instance['client'].images, 'edit').mockResolvedValue(mockResponse as any);
+
+        const payload = {
+          model: 'dall-e-2',
+          params: {
+            prompt: 'Merge these images',
+            imageUrls: ['https://example.com/image1.jpg', 'https://example.com/image2.jpg'],
+          },
+        };
+
+        const result = await (instance as any).createImage(payload);
+
+        expect(openaiHelpers.convertImageUrlToFile).toHaveBeenCalledTimes(2);
+        expect(openaiHelpers.convertImageUrlToFile).toHaveBeenCalledWith(
+          'https://example.com/image1.jpg',
+        );
+        expect(openaiHelpers.convertImageUrlToFile).toHaveBeenCalledWith(
+          'https://example.com/image2.jpg',
+        );
+
+        expect(instance['client'].images.edit).toHaveBeenCalledWith({
+          model: 'dall-e-2',
+          n: 1,
+          prompt: 'Merge these images',
+          image: [mockFile1, mockFile2],
+          response_format: 'b64_json',
+        });
+
+        expect(result).toEqual({
+          imageUrl: 'data:image/png;base64,edited-multiple-images-base64',
+        });
+      });
+
+      it('should handle convertImageUrlToFile error', async () => {
+        vi.mocked(openaiHelpers.convertImageUrlToFile).mockRejectedValue(
+          new Error('Failed to download image'),
+        );
+
+        const payload = {
+          model: 'dall-e-2',
+          params: {
+            prompt: 'Edit this image',
+            imageUrls: ['https://invalid-url.com/image.jpg'],
+          },
+        };
+
+        await expect((instance as any).createImage(payload)).rejects.toThrow(
+          'Failed to convert image URLs to File objects: Error: Failed to download image',
+        );
+      });
+    });
+
+    describe('error handling', () => {
+      it('should throw error when API response is invalid - no data', async () => {
+        vi.spyOn(instance['client'].images, 'generate').mockResolvedValue({} as any);
+
+        const payload = {
+          model: 'dall-e-3',
+          params: { prompt: 'Test prompt' },
+        };
+
+        await expect((instance as any).createImage(payload)).rejects.toThrow(
+          'Invalid image response: missing or empty data array',
+        );
+      });
+
+      it('should throw error when API response is invalid - empty data array', async () => {
+        vi.spyOn(instance['client'].images, 'generate').mockResolvedValue({
+          data: [],
+        } as any);
+
+        const payload = {
+          model: 'dall-e-3',
+          params: { prompt: 'Test prompt' },
+        };
+
+        await expect((instance as any).createImage(payload)).rejects.toThrow(
+          'Invalid image response: missing or empty data array',
+        );
+      });
+
+      it('should throw error when first data item is null', async () => {
+        vi.spyOn(instance['client'].images, 'generate').mockResolvedValue({
+          data: [null],
+        } as any);
+
+        const payload = {
+          model: 'dall-e-3',
+          params: { prompt: 'Test prompt' },
+        };
+
+        await expect((instance as any).createImage(payload)).rejects.toThrow(
+          'Invalid image response: first data item is null or undefined',
+        );
+      });
+
+      it('should throw error when b64_json is missing', async () => {
+        vi.spyOn(instance['client'].images, 'generate').mockResolvedValue({
+          data: [{ url: 'https://example.com/image.jpg' }],
+        } as any);
+
+        const payload = {
+          model: 'dall-e-3',
+          params: { prompt: 'Test prompt' },
+        };
+
+        await expect((instance as any).createImage(payload)).rejects.toThrow(
+          'Invalid image response: missing b64_json field',
+        );
+      });
+    });
+
+    describe('parameter mapping', () => {
+      it('should map imageUrls parameter to image', async () => {
+        const mockResponse = {
+          data: [{ b64_json: 'test-base64' }],
+        };
+
+        vi.spyOn(instance['client'].images, 'edit').mockResolvedValue(mockResponse as any);
+
+        const payload = {
+          model: 'dall-e-2',
+          params: {
+            prompt: 'Test prompt',
+            imageUrls: ['https://example.com/image.jpg'],
+            customParam: 'should remain unchanged',
+          },
+        };
+
+        await (instance as any).createImage(payload);
+
+        expect(instance['client'].images.edit).toHaveBeenCalledWith({
+          model: 'dall-e-2',
+          n: 1,
+          prompt: 'Test prompt',
+          image: expect.any(File),
+          customParam: 'should remain unchanged',
+          response_format: 'b64_json',
+        });
+      });
+
+      it('should handle parameters without imageUrls', async () => {
+        const mockResponse = {
+          data: [{ b64_json: 'test-base64' }],
+        };
+
+        vi.spyOn(instance['client'].images, 'generate').mockResolvedValue(mockResponse as any);
+
+        const payload = {
+          model: 'dall-e-3',
+          params: {
+            prompt: 'Test prompt',
+            quality: 'hd',
+            style: 'vivid',
+          },
+        };
+
+        await (instance as any).createImage(payload);
+
+        expect(instance['client'].images.generate).toHaveBeenCalledWith({
+          model: 'dall-e-3',
+          n: 1,
+          prompt: 'Test prompt',
+          quality: 'hd',
+          style: 'vivid',
+          response_format: 'b64_json',
+        });
+      });
+    });
+  });
+
   describe('models', () => {
     it('should get models with third party model list', async () => {
       vi.spyOn(instance['client'].models, 'list').mockResolvedValue({
@@ -993,54 +1317,82 @@ describe('LobeOpenAICompatibleFactory', () => {
 
       expect(list).toEqual([
         {
+          abilities: {
+            functionCall: true,
+            vision: true,
+          },
+          config: {
+            deploymentName: 'gpt-4o',
+          },
           contextWindowTokens: 128000,
-          releasedAt: '2023-10-25',
           description:
             'ChatGPT-4o 是一款动态模型，实时更新以保持当前最新版本。它结合了强大的语言理解与生成能力，适合于大规模应用场景，包括客户服务、教育和技术支持。',
           displayName: 'GPT-4o',
           enabled: true,
-          functionCall: true,
           id: 'gpt-4o',
+          maxOutput: 4096,
           pricing: {
+            cachedInput: 1.25,
             input: 2.5,
             output: 10,
           },
-          vision: true,
+          providerId: 'azure',
+          releasedAt: '2024-05-13',
+          source: 'builtin',
+          type: 'chat',
         },
         {
+          abilities: {
+            functionCall: true,
+            vision: true,
+          },
           contextWindowTokens: 200000,
           description:
             'Claude 3 Haiku 是 Anthropic 的最快且最紧凑的模型，旨在实现近乎即时的响应。它具有快速且准确的定向性能。',
           displayName: 'Claude 3 Haiku',
-          functionCall: true,
+          enabled: false,
           id: 'claude-3-haiku-20240307',
           maxOutput: 4096,
           pricing: {
             input: 0.25,
             output: 1.25,
           },
+          providerId: 'anthropic',
           releasedAt: '2024-03-07',
-          vision: true,
+          settings: {
+            extendParams: ['disableContextCaching'],
+          },
+          source: 'builtin',
+          type: 'chat',
         },
         {
+          abilities: {
+            functionCall: true,
+            vision: true,
+          },
+          config: {
+            deploymentName: 'gpt-4o-mini',
+          },
           contextWindowTokens: 128000,
-          description:
-            'GPT-4o mini是OpenAI在GPT-4 Omni之后推出的最新模型，支持图文输入并输出文本。作为他们最先进的小型模型，它比其他近期的前沿模型便宜很多，并且比GPT-3.5 Turbo便宜超过60%。它保持了最先进的智能，同时具有显著的性价比。GPT-4o mini在MMLU测试中获得了 82% 的得分，目前在聊天偏好上排名高于 GPT-4。',
-          displayName: 'GPT-4o mini',
-          enabled: true,
-          functionCall: true,
+          description: 'GPT-4o Mini，小型高效模型，具备与GPT-4o相似的卓越性能。',
+          displayName: 'GPT 4o Mini',
+          enabled: false,
           id: 'gpt-4o-mini',
-          maxOutput: 16385,
+          maxOutput: 4096,
           pricing: {
+            cachedInput: 0.075,
             input: 0.15,
             output: 0.6,
           },
+          providerId: 'azure',
           releasedAt: '2023-10-26',
-          vision: true,
+          source: 'builtin',
+          type: 'chat',
         },
         {
           id: 'gemini',
           releasedAt: '2025-01-10',
+          type: undefined,
         },
       ]);
     });