@lobehub/chat 1.99.2 → 1.99.4

package/src/libs/model-runtime/utils/openaiCompatibleFactory/index.ts

@@ -1,9 +1,11 @@
 import dayjs from 'dayjs';
 import utc from 'dayjs/plugin/utc';
+import createDebug from 'debug';
 import OpenAI, { ClientOptions } from 'openai';
 import { Stream } from 'openai/streaming';
 
-import { LOBE_DEFAULT_MODEL_LIST } from '@/config/modelProviders';
+import { LOBE_DEFAULT_MODEL_LIST } from '@/config/aiModels';
+import { RuntimeImageGenParamsValue } from '@/libs/standard-parameters/meta-schema';
 import type { ChatModelCard } from '@/types/llm';
 
 import { LobeRuntimeAI } from '../../BaseAI';
@@ -27,7 +29,11 @@ import { AgentRuntimeError } from '../createError';
 import { debugResponse, debugStream } from '../debugStream';
 import { desensitizeUrl } from '../desensitizeUrl';
 import { handleOpenAIError } from '../handleOpenAIError';
-import { convertOpenAIMessages, convertOpenAIResponseInputs } from '../openaiHelpers';
+import {
+  convertImageUrlToFile,
+  convertOpenAIMessages,
+  convertOpenAIResponseInputs,
+} from '../openaiHelpers';
 import { StreamingResponse } from '../response';
 import { OpenAIResponsesStream, OpenAIStream, OpenAIStreamOptions } from '../streams';
 
@@ -168,7 +174,6 @@ export const createOpenAICompatibleRuntime = <T extends Record<string, any> = an
   debug,
   constructorOptions,
   chatCompletion,
-  createImage,
   models,
   customClient,
   responses,
@@ -311,58 +316,155 @@ export const createOpenAICompatibleRuntime = <T extends Record<string, any> = an
     }
 
     async createImage(payload: CreateImagePayload) {
-      return createImage!({
-        ...payload,
-        client: this.client,
-      });
-    }
+      const { model, params } = payload;
+      const log = createDebug(`lobe-image:model-runtime`);
 
-    async models() {
-      if (typeof models === 'function') return models({ client: this.client });
+      log('Creating image with model: %s and params: %O', model, params);
 
-      const list = await this.client.models.list();
+      const defaultInput = {
+        n: 1,
+        ...(model.includes('dall-e') ? { response_format: 'b64_json' } : {}),
+      };
 
-      return list.data
-        .filter((model) => {
-          return CHAT_MODELS_BLOCK_LIST.every(
-            (keyword) => !model.id.toLowerCase().includes(keyword),
+      // 映射参数名称，将 imageUrls 映射为 image
+      const paramsMap = new Map<RuntimeImageGenParamsValue, string>([
+        ['imageUrls', 'image'],
+        ['imageUrl', 'image'],
+      ]);
+      const userInput: Record<string, any> = Object.fromEntries(
+        Object.entries(params).map(([key, value]) => [
+          paramsMap.get(key as RuntimeImageGenParamsValue) ?? key,
+          value,
+        ]),
+      );
+
+      const isImageEdit = Array.isArray(userInput.image) && userInput.image.length > 0;
+      // 如果有 imageUrls 参数，将其转换为 File 对象
+      if (isImageEdit) {
+        log('Converting imageUrls to File objects: %O', userInput.image);
+        try {
+          // 转换所有图片 URL 为 File 对象
+          const imageFiles = await Promise.all(
+            userInput.image.map((url: string) => convertImageUrlToFile(url)),
           );
-        })
-        .map((item) => {
-          if (models?.transformModel) {
-            return models.transformModel(item);
-          }
 
-          const toReleasedAt = () => {
-            if (!item.created) return;
-            dayjs.extend(utc);
-
-            // guarantee item.created in Date String format
-            if (
-              typeof (item.created as any) === 'string' ||
-              // or in milliseconds
-              item.created.toFixed(0).length === 13
-            ) {
-              return dayjs.utc(item.created).format('YYYY-MM-DD');
+          log('Successfully converted %d images to File objects', imageFiles.length);
+
+          // 根据官方文档，如果有多个图片，传递数组；如果只有一个，传递单个 File
+          userInput.image = imageFiles.length === 1 ? imageFiles[0] : imageFiles;
+        } catch (error) {
+          log('Error converting imageUrls to File objects: %O', error);
+          throw new Error(`Failed to convert image URLs to File objects: ${error}`);
+        }
+      } else {
+        delete userInput.image;
+      }
+
+      if (userInput.size === 'auto') {
+        delete userInput.size;
+      }
+
+      const options = {
+        model,
+        ...defaultInput,
+        ...userInput,
+      };
+
+      log('options: %O', options);
+
+      // 判断是否为图片编辑操作
+      const img = isImageEdit
+        ? await this.client.images.edit(options as any)
+        : await this.client.images.generate(options as any);
+
+      // 检查响应数据的完整性
+      if (!img || !img.data || !Array.isArray(img.data) || img.data.length === 0) {
+        log('Invalid image response: missing data array');
+        throw new Error('Invalid image response: missing or empty data array');
+      }
+
+      const imageData = img.data[0];
+      if (!imageData) {
+        log('Invalid image response: first data item is null/undefined');
+        throw new Error('Invalid image response: first data item is null or undefined');
+      }
+
+      if (!imageData.b64_json) {
+        log('Invalid image response: missing b64_json field');
+        throw new Error('Invalid image response: missing b64_json field');
+      }
+
+      // 确定图片的 MIME 类型，默认为 PNG
+      const mimeType = 'image/png'; // OpenAI 图片生成默认返回 PNG 格式
+
+      // 将 base64 字符串转换为完整的 data URL
+      const dataUrl = `data:${mimeType};base64,${imageData.b64_json}`;
+
+      log('Successfully converted base64 to data URL, length: %d', dataUrl.length);
+
+      return {
+        imageUrl: dataUrl,
+      };
+    }
+
+    async models() {
+      let resultModels: ChatModelCard[] = [];
+      if (typeof models === 'function') {
+        resultModels = await models({ client: this.client });
+      } else {
+        const list = await this.client.models.list();
+        resultModels = list.data
+          .filter((model) => {
+            return CHAT_MODELS_BLOCK_LIST.every(
+              (keyword) => !model.id.toLowerCase().includes(keyword),
+            );
+          })
+          .map((item) => {
+            if (models?.transformModel) {
+              return models.transformModel(item);
             }
 
-            // by default, the created time is in seconds
-            return dayjs.utc(item.created * 1000).format('YYYY-MM-DD');
-          };
+            const toReleasedAt = () => {
+              if (!item.created) return;
+              dayjs.extend(utc);
 
-          // TODO: should refactor after remove v1 user/modelList code
-          const knownModel = LOBE_DEFAULT_MODEL_LIST.find((model) => model.id === item.id);
+              // guarantee item.created in Date String format
+              if (
+                typeof (item.created as any) === 'string' ||
+                // or in milliseconds
+                item.created.toFixed(0).length === 13
+              ) {
+                return dayjs.utc(item.created).format('YYYY-MM-DD');
+              }
 
-          if (knownModel) {
-            const releasedAt = knownModel.releasedAt ?? toReleasedAt();
+              // by default, the created time is in seconds
+              return dayjs.utc(item.created * 1000).format('YYYY-MM-DD');
+            };
 
-            return { ...knownModel, releasedAt };
-          }
+            // TODO: should refactor after remove v1 user/modelList code
+            const knownModel = LOBE_DEFAULT_MODEL_LIST.find((model) => model.id === item.id);
 
-          return { id: item.id, releasedAt: toReleasedAt() };
-        })
+            if (knownModel) {
+              const releasedAt = knownModel.releasedAt ?? toReleasedAt();
+
+              return { ...knownModel, releasedAt };
+            }
 
-        .filter(Boolean) as ChatModelCard[];
+            return {
+              id: item.id,
+              releasedAt: toReleasedAt(),
+            };
+          })
+
+          .filter(Boolean) as ChatModelCard[];
+      }
+
+      return resultModels.map((model) => {
+        return {
+          ...model,
+          type: model.type || LOBE_DEFAULT_MODEL_LIST.find((m) => m.id === model.id)?.type,
+        };
+      }) as ChatModelCard[];
     }
 
     async embeddings(

package/src/libs/model-runtime/utils/openaiHelpers.test.ts

@@ -4,6 +4,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 import { imageUrlToBase64 } from '@/utils/imageToBase64';
 
 import {
+  convertImageUrlToFile,
   convertMessageContent,
   convertOpenAIMessages,
   convertOpenAIResponseInputs,
@@ -288,3 +289,153 @@ describe('convertOpenAIResponseInputs', () => {
     ]);
   });
 });
+
+describe('convertImageUrlToFile', () => {
+  beforeEach(() => {
+    vi.resetAllMocks();
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  describe('Data URL handling', () => {
+    it('should convert PNG data URL to File object correctly', async () => {
+      const base64Data =
+        'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==';
+      const dataUrl = `data:image/png;base64,${base64Data}`;
+
+      const result = await convertImageUrlToFile(dataUrl);
+
+      expect(result).toBeDefined();
+      expect(result).toHaveProperty('name', 'image.png');
+      expect(result).toHaveProperty('type', 'image/png');
+      expect(result).toHaveProperty('size');
+      expect(result.size).toBeGreaterThan(0);
+    });
+
+    it('should convert JPEG data URL to File object correctly', async () => {
+      const base64Data =
+        '/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/2wBDAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAARCAABAAEDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAv/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFQEBAQAAAAAAAAAAAAAAAAAAAAX/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwA9BQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==';
+      const dataUrl = `data:image/jpeg;base64,${base64Data}`;
+
+      const result = await convertImageUrlToFile(dataUrl);
+
+      expect(result).toBeDefined();
+      expect(result).toHaveProperty('name', 'image.jpeg');
+      expect(result).toHaveProperty('type', 'image/jpeg');
+      expect(result).toHaveProperty('size');
+      expect(result.size).toBeGreaterThan(0);
+    });
+
+    it('should convert WebP data URL to File object correctly', async () => {
+      const base64Data = 'UklGRiQAAABXRUJQVlA4IBgAAAAwAQCdASoBAAEAAAAAJaQAA6g=';
+      const dataUrl = `data:image/webp;base64,${base64Data}`;
+
+      const result = await convertImageUrlToFile(dataUrl);
+
+      expect(result).toBeDefined();
+      expect(result).toHaveProperty('name', 'image.webp');
+      expect(result).toHaveProperty('type', 'image/webp');
+      expect(result).toHaveProperty('size');
+      expect(result.size).toBeGreaterThan(0);
+    });
+  });
+
+  describe('HTTP URL handling', () => {
+    const mockFetch = vi.fn();
+
+    beforeEach(() => {
+      // Mock global fetch using vi.stubGlobal for better isolation
+      vi.stubGlobal('fetch', mockFetch);
+    });
+
+    afterEach(() => {
+      vi.unstubAllGlobals();
+      vi.clearAllMocks();
+    });
+
+    it('should convert HTTP URL to File object correctly', async () => {
+      const mockArrayBuffer = new ArrayBuffer(8);
+      const mockHeaders = new Headers();
+      mockHeaders.set('content-type', 'image/jpeg');
+
+      mockFetch.mockResolvedValue({
+        ok: true,
+        arrayBuffer: () => Promise.resolve(mockArrayBuffer),
+        headers: mockHeaders,
+      } satisfies Partial<Response>);
+
+      const result = await convertImageUrlToFile('https://example.com/image.jpg');
+
+      expect(mockFetch).toHaveBeenCalledWith('https://example.com/image.jpg');
+      expect(result).toBeDefined();
+      expect(result).toHaveProperty('name', 'image.jpeg');
+      expect(result).toHaveProperty('type', 'image/jpeg');
+      expect(result).toHaveProperty('size');
+      expect(result.size).toEqual(8);
+    });
+
+    it('should handle different content types from HTTP response headers', async () => {
+      const testCases = [
+        { contentType: 'image/jpeg', expectedExtension: 'jpeg' },
+        { contentType: 'image/png', expectedExtension: 'png' },
+        { contentType: 'image/webp', expectedExtension: 'webp' },
+        { contentType: null, expectedExtension: 'png' }, // default fallback
+      ];
+
+      for (const testCase of testCases) {
+        const mockArrayBuffer = new ArrayBuffer(8);
+        const mockHeaders = new Headers();
+        if (testCase.contentType) {
+          mockHeaders.set('content-type', testCase.contentType);
+        }
+
+        mockFetch.mockResolvedValue({
+          ok: true,
+          arrayBuffer: () => Promise.resolve(mockArrayBuffer),
+          headers: mockHeaders,
+        } satisfies Partial<Response>);
+
+        const result = await convertImageUrlToFile('https://example.com/image.jpg');
+
+        expect(result).toHaveProperty('name', `image.${testCase.expectedExtension}`);
+        expect(result).toHaveProperty('type', testCase.contentType || 'image/png');
+
+        vi.clearAllMocks();
+      }
+    });
+
+    it('should throw error when HTTP request fails', async () => {
+      mockFetch.mockResolvedValue({
+        ok: false,
+        statusText: 'Not Found',
+      } satisfies Partial<Response>);
+
+      await expect(convertImageUrlToFile('https://example.com/nonexistent.jpg')).rejects.toThrow(
+        'Failed to fetch image from https://example.com/nonexistent.jpg: Not Found',
+      );
+
+      expect(mockFetch).toHaveBeenCalledWith('https://example.com/nonexistent.jpg');
+    });
+
+    it('should throw error when network request fails', async () => {
+      mockFetch.mockRejectedValue(new Error('Network error'));
+
+      await expect(convertImageUrlToFile('https://example.com/image.jpg')).rejects.toThrow(
+        'Network error',
+      );
+
+      expect(mockFetch).toHaveBeenCalledWith('https://example.com/image.jpg');
+    });
+  });
+
+  describe('Edge cases', () => {
+    it('should handle malformed data URL gracefully', async () => {
+      const malformedDataUrl = 'data:invalid-format';
+
+      // 这个测试可能会抛出错误，我们需要适当处理
+      await expect(convertImageUrlToFile(malformedDataUrl)).rejects.toThrow();
+    });
+  });
+});

package/src/libs/model-runtime/utils/openaiHelpers.ts

@@ -1,4 +1,4 @@
-import OpenAI from 'openai';
+import OpenAI, { toFile } from 'openai';
 
 import { disableStreamModels, systemToUserModels } from '@/const/models';
 import { ChatStreamPayload, OpenAIChatMessage } from '@/libs/model-runtime';
@@ -119,3 +119,28 @@ export const pruneReasoningPayload = (payload: ChatStreamPayload) => {
     top_p: 1,
   };
 };
+
+/**
+ * Convert image URL (data URL or HTTP URL) to File object for OpenAI API
+ */
+export const convertImageUrlToFile = async (imageUrl: string) => {
+  let buffer: Buffer;
+  let mimeType: string;
+
+  if (imageUrl.startsWith('data:')) {
+    // a base64 image
+    const [mimeTypePart, base64Data] = imageUrl.split(',');
+    mimeType = mimeTypePart.split(':')[1].split(';')[0];
+    buffer = Buffer.from(base64Data, 'base64');
+  } else {
+    // a http url
+    const response = await fetch(imageUrl);
+    if (!response.ok) {
+      throw new Error(`Failed to fetch image from ${imageUrl}: ${response.statusText}`);
+    }
+    buffer = Buffer.from(await response.arrayBuffer());
+    mimeType = response.headers.get('content-type') || 'image/png';
+  }
+
+  return toFile(buffer, `image.${mimeType.split('/')[1]}`, { type: mimeType });
+};

package/src/libs/model-runtime/xai/index.ts

@@ -7,10 +7,7 @@ export interface XAIModelCard {
   id: string;
 }
 
-export const GrokReasoningModels = new Set([
-  'grok-3-mini',
-  'grok-4',
-]);
+export const GrokReasoningModels = new Set(['grok-3-mini', 'grok-4']);
 
 export const isGrokReasoningModel = (model: string) =>
   Array.from(GrokReasoningModels).some((id) => model.includes(id));

package/src/store/aiInfra/slices/aiModel/action.ts

@@ -84,7 +84,7 @@ export const createAiModelSlice: StateCreator<
           },
           enabled: model.enabled || false,
           source: 'remote',
-          type: 'chat',
+          type: model.type || 'chat',
         })),
       );
 

package/src/store/aiInfra/slices/aiProvider/action.ts

@@ -203,6 +203,8 @@ export const createAiProviderSlice: StateCreator<
         onSuccess: async (data) => {
           if (!data) return;
 
+          const { LOBE_DEFAULT_MODEL_LIST } = await import('@/config/aiModels');
+
           const getModelListByType = (providerId: string, type: string) => {
             const models = data.enabledAiModels
               .filter((model) => model.providerId === providerId && model.type === type)
@@ -212,7 +214,9 @@ export const createAiProviderSlice: StateCreator<
                 displayName: model.displayName ?? '',
                 id: model.id,
                 ...(model.type === 'image' && {
-                  parameters: (model as AIImageModelCard).parameters,
+                  parameters:
+                    (model as AIImageModelCard).parameters ||
+                    LOBE_DEFAULT_MODEL_LIST.find((m) => m.id === model.id)?.parameters,
                 }),
               }));
 
@@ -231,7 +235,6 @@ export const createAiProviderSlice: StateCreator<
             children: getModelListByType(provider.id, 'image'),
             name: provider.name || provider.id,
           }));
-          const { LOBE_DEFAULT_MODEL_LIST } = await import('@/config/aiModels');
 
           set(
             {

package/src/types/aiModel.ts

@@ -268,6 +268,7 @@ export interface AiFullModelCard extends AIBaseModelCard {
   displayName?: string;
   id: string;
   maxDimension?: number;
+  parameters?: ModelParamsSchema;
   pricing?: ChatModelPricing;
   type: AiModelType;
 }

package/src/types/llm.ts

@@ -1,6 +1,6 @@
 import { ReactNode } from 'react';
 
-import { ChatModelPricing } from '@/types/aiModel';
+import { AiModelType, ChatModelPricing } from '@/types/aiModel';
 import { AiProviderSettings } from '@/types/aiProvider';
 
 export type ModelPriceCurrency = 'CNY' | 'USD';
@@ -53,6 +53,8 @@ export interface ChatModelCard {
    */
   releasedAt?: string;
 
+  type?: AiModelType;
+
   /**
    *  whether model supports vision
    */