@lobehub/chat 1.116.4 → 1.117.0

package/packages/model-runtime/src/utils/streams/openai/openai.test.ts

@@ -2271,4 +2271,45 @@ describe('OpenAIStream', () => {
       );
     });
   });
+
+  it('should handle base64_image in delta.images (image_url shape)', async () => {
+    const base64 =
+      'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==';
+
+    const mockOpenAIStream = new ReadableStream({
+      start(controller) {
+        controller.enqueue({
+          choices: [
+            {
+              delta: {
+                images: [
+                  {
+                    type: 'image_url',
+                    image_url: { url: base64 },
+                    index: 0,
+                  },
+                ],
+              },
+              index: 0,
+            },
+          ],
+          id: '6',
+        });
+
+        controller.close();
+      },
+    });
+
+    const protocolStream = OpenAIStream(mockOpenAIStream);
+
+    const decoder = new TextDecoder();
+    const chunks = [];
+
+    // @ts-ignore
+    for await (const chunk of protocolStream) {
+      chunks.push(decoder.decode(chunk, { stream: true }));
+    }
+
+    expect(chunks).toEqual(['id: 6\n', 'event: base64_image\n', `data: "${base64}"\n\n`]);
+  });
 });

package/packages/model-runtime/src/utils/streams/openai/openai.ts

@@ -96,6 +96,36 @@ const transformOpenAIStream = (
       }
     }
 
+    // Handle image preview chunks (e.g. Gemini 2.5 flash image preview)
+    // Example shape:
+    // choices[0].delta.images = [{ type: 'image_url', image_url: { url: 'data:image/png;base64,...' }, index: 0 }]
+    if (
+      (item as any).delta &&
+      Array.isArray((item as any).delta.images) &&
+      (item as any).delta.images.length > 0
+    ) {
+      const images = (item as any).delta.images as any[];
+
+      return images
+        .map((img) => {
+          // support multiple possible shapes for the url
+          const url =
+            img?.image_url?.url ||
+            img?.image_url?.image_url?.url ||
+            img?.url ||
+            (typeof img === 'string' ? img : undefined);
+
+          if (!url) return null;
+
+          return {
+            data: url,
+            id: chunk.id,
+            type: 'base64_image',
+          } as StreamProtocolChunk;
+        })
+        .filter(Boolean) as StreamProtocolChunk[];
+    }
+
     // 给定结束原因
     if (item.finish_reason) {
       // one-api 的流式接口，会出现既有 finish_reason ，也有 content 的情况
@@ -192,11 +222,11 @@ const transformOpenAIStream = (
         if ('content' in item.delta && Array.isArray(item.delta.content)) {
           return item.delta.content
             .filter((block: any) => block.type === 'thinking' && Array.isArray(block.thinking))
-            .map((block: any) => 
+            .map((block: any) =>
               block.thinking
                 .filter((thinkItem: any) => thinkItem.type === 'text' && thinkItem.text)
                 .map((thinkItem: any) => thinkItem.text)
-                .join('')
+                .join(''),
             )
             .join('');
         }
@@ -233,6 +263,12 @@ const transformOpenAIStream = (
           streamContext.thinkingInContent = false;
         }
 
+        // 如果 content 是空字符串但 chunk 带有 usage，则优先返回 usage（例如 Gemini image-preview 最终会在单独的 chunk 中返回 usage）
+        if (content === '' && chunk.usage) {
+          const usage = chunk.usage;
+          return { data: convertUsage(usage, provider), id: chunk.id, type: 'usage' };
+        }
+
         // 判断是否有 citations 内容，更新 returnedCitation 状态
         if (!streamContext?.returnedCitation) {
           const citations =

package/packages/model-runtime/src/utils/streams/protocol.test.ts

@@ -200,4 +200,36 @@ describe('createTokenSpeedCalculator', async () => {
     const results = await processChunk(transformer, chunks);
     expect(results).toHaveLength(chunks.length);
   });
+
+  it('should calculate token speed considering outputImageTokens when totalOutputTokens is missing', async () => {
+    const chunks = [
+      { data: '', id: 'chatcmpl-image-1', type: 'text' },
+      { data: 'hi', id: 'chatcmpl-image-1', type: 'text' },
+      { data: 'stop', id: 'chatcmpl-image-1', type: 'stop' },
+      {
+        data: {
+          inputTextTokens: 9,
+          outputTextTokens: 1,
+          outputImageTokens: 4,
+          totalInputTokens: 9,
+          // totalOutputTokens intentionally omitted to force summation path
+          totalTokens: 13,
+        },
+        id: 'chatcmpl-image-1',
+        type: 'usage',
+      },
+    ];
+
+    const transformer = createTokenSpeedCalculator((v) => v, { inputStartAt });
+    const results = await processChunk(transformer, chunks);
+
+    // should push an extra speed chunk
+    expect(results).toHaveLength(chunks.length + 1);
+    const speedChunk = results.slice(-1)[0];
+    expect(speedChunk.id).toBe('output_speed');
+    expect(speedChunk.type).toBe('speed');
+    // tps and ttft should be numeric (avoid flakiness if interval is 0ms)
+    expect(speedChunk.data.tps).not.toBeNaN();
+    expect(speedChunk.data.ttft).not.toBeNaN();
+  });
 });

package/packages/model-runtime/src/utils/streams/protocol.ts

@@ -364,10 +364,14 @@ export const createTokenSpeedCalculator = (
     }
     // if the chunk is the stop chunk, set as output finish
     if (inputStartAt && outputStartAt && chunk.type === 'usage') {
-      const totalOutputTokens = chunk.data?.totalOutputTokens || chunk.data?.outputTextTokens;
-      const reasoningTokens = chunk.data?.outputReasoningTokens || 0;
+      const totalOutputTokens =
+        chunk.data?.totalOutputTokens ??
+        (chunk.data?.outputTextTokens ?? 0) + (chunk.data?.outputImageTokens ?? 0);
+      const reasoningTokens = chunk.data?.outputReasoningTokens ?? 0;
       const outputTokens =
-        (outputThinking ?? false) ? totalOutputTokens : totalOutputTokens - reasoningTokens;
+        (outputThinking ?? false)
+          ? totalOutputTokens
+          : Math.max(0, totalOutputTokens - reasoningTokens);
       result.push({
         data: {
           tps: (outputTokens / (Date.now() - outputStartAt)) * 1000,

package/packages/model-runtime/src/utils/usageConverter.test.ts

@@ -290,4 +290,62 @@ describe('convertUsage', () => {
       totalTokens: 6550,
     });
   });
+
+  it('should handle output image tokens correctly', () => {
+    // Arrange
+    const usageWithImage = {
+      prompt_tokens: 100,
+      completion_tokens: 200,
+      completion_tokens_details: {
+        image_tokens: 60,
+        reasoning_tokens: 30,
+      },
+      total_tokens: 300,
+    } as OpenAI.Completions.CompletionUsage;
+
+    // Act
+    const result = convertUsage(usageWithImage);
+
+    // Assert
+    expect(result).toEqual({
+      inputTextTokens: 100,
+      totalInputTokens: 100,
+      totalOutputTokens: 200,
+      outputImageTokens: 60,
+      outputReasoningTokens: 30,
+      outputTextTokens: 110, // 200 - 60 - 30
+      totalTokens: 300,
+    });
+  });
+
+  it('should handle response output image tokens correctly for ResponseUsage', () => {
+    // Arrange
+    const responseUsage = {
+      input_tokens: 100,
+      input_tokens_details: {
+        cached_tokens: 0,
+      },
+      output_tokens: 200,
+      output_tokens_details: {
+        image_tokens: 60,
+        reasoning_tokens: 30,
+      },
+      total_tokens: 300,
+    } as OpenAI.Responses.ResponseUsage;
+
+    // Act
+    const result = convertResponseUsage(responseUsage);
+
+    // Assert
+    expect(result).toEqual({
+      inputTextTokens: 100,
+      inputCacheMissTokens: 100, // 100 - 0
+      totalInputTokens: 100,
+      totalOutputTokens: 200,
+      outputImageTokens: 60,
+      outputReasoningTokens: 30,
+      outputTextTokens: 170, // 200 - 30
+      totalTokens: 300,
+    });
+  });
 });

package/packages/model-runtime/src/utils/usageConverter.ts

@@ -20,12 +20,13 @@ export const convertUsage = (
   const totalOutputTokens = usage.completion_tokens;
   const outputReasoning = usage.completion_tokens_details?.reasoning_tokens || 0;
   const outputAudioTokens = usage.completion_tokens_details?.audio_tokens || 0;
+  const outputImageTokens = (usage.completion_tokens_details as any)?.image_tokens || 0;
 
   // XAI 的 completion_tokens 不包含 reasoning_tokens，需要特殊处理
   const outputTextTokens =
     provider === 'xai'
       ? totalOutputTokens - outputAudioTokens
-      : totalOutputTokens - outputReasoning - outputAudioTokens;
+      : totalOutputTokens - outputReasoning - outputAudioTokens - outputImageTokens;
 
   const totalTokens = inputCitationTokens + usage.total_tokens;
 
@@ -37,6 +38,7 @@ export const convertUsage = (
     inputCitationTokens: inputCitationTokens,
     inputTextTokens: inputTextTokens,
     outputAudioTokens: outputAudioTokens,
+    outputImageTokens: outputImageTokens,
     outputReasoningTokens: outputReasoning,
     outputTextTokens: outputTextTokens,
     rejectedPredictionTokens: usage.completion_tokens_details?.rejected_prediction_tokens,
@@ -75,6 +77,7 @@ export const convertResponseUsage = (usage: OpenAI.Responses.ResponseUsage): Mod
 
   // For ResponseUsage, outputTextTokens is totalOutputTokens minus reasoning, as no audio output tokens are specified.
   const outputTextTokens = totalOutputTokens - outputReasoningTokens;
+  const outputImageTokens = (usage.output_tokens_details as any)?.image_tokens || 0;
 
   // 3. Construct the comprehensive data object (matching ModelTokensUsage structure)
   const data = {
@@ -87,6 +90,7 @@ export const convertResponseUsage = (usage: OpenAI.Responses.ResponseUsage): Mod
     inputCitationTokens: undefined, // Not in ResponseUsage
     inputTextTokens: inputTextTokens,
     outputAudioTokens: undefined, // Not in ResponseUsage
+    outputImageTokens: outputImageTokens,
     outputReasoningTokens: outputReasoningTokens,
     outputTextTokens: outputTextTokens,
     rejectedPredictionTokens: undefined, // Not in ResponseUsage

package/packages/utils/vitest.config.mts

@@ -7,6 +7,7 @@ export default defineConfig({
       /* eslint-disable sort-keys-fix/sort-keys-fix */
       '@/types': resolve(__dirname, '../types/src'),
       '@/const': resolve(__dirname, '../const/src'),
+      '@/libs/model-runtime': resolve(__dirname, '../model-runtime/src'),
       '@': resolve(__dirname, '../../src'),
       /* eslint-enable */
     },

package/src/config/aiModels/google.ts

@@ -1,3 +1,4 @@
+import { CHAT_MODEL_IMAGE_GENERATION_PARAMS } from '@/const/image';
 import { ModelParamsSchema } from '@/libs/standard-parameters';
 import { AIChatModelCard, AIImageModelCard } from '@/types/aiModel';
 
@@ -194,21 +195,21 @@ const googleChatModels: AIChatModelCard[] = [
       imageOutput: true,
       vision: true,
     },
-    contextWindowTokens: 32_768 + 32_768,
+    contextWindowTokens: 32_768 + 8192,
     description:
       'Gemini 2.5 Flash Image Preview 是 Google 最新、最快、最高效的原生多模态模型，它允许您通过对话生成和编辑图像。',
     displayName: 'Gemini 2.5 Flash Image Preview',
     enabled: true,
     id: 'gemini-2.5-flash-image-preview',
-    maxOutput: 32_768,
+    maxOutput: 8192,
     pricing: {
       units: [
         { name: 'textInput', rate: 0.3, strategy: 'fixed', unit: 'millionTokens' },
         { name: 'textOutput', rate: 2.5, strategy: 'fixed', unit: 'millionTokens' },
-        { name: 'imageOutput', rate: 3, strategy: 'fixed', unit: 'millionTokens' },
+        { name: 'imageOutput', rate: 30, strategy: 'fixed', unit: 'millionTokens' },
       ],
     },
-    releasedAt: '2025-08-27',
+    releasedAt: '2025-08-26',
     type: 'chat',
   },
   {
@@ -605,71 +606,90 @@ const imagenBaseParameters: ModelParamsSchema = {
   prompt: { default: '' },
 };
 
+/* eslint-disable sort-keys-fix/sort-keys-fix */
 const googleImageModels: AIImageModelCard[] = [
   {
-    description: 'Imagen 4th generation text-to-image model series',
-    displayName: 'Imagen 4',
+    displayName: 'Gemini 2.5 Flash Image Preview',
+    id: 'gemini-2.5-flash-image-preview:image',
     enabled: true,
+    type: 'image',
+    description:
+      'Gemini 2.5 Flash Image Preview 是 Google 最新、最快、最高效的原生多模态模型，它允许您通过对话生成和编辑图像。',
+    releasedAt: '2025-08-26',
+    parameters: CHAT_MODEL_IMAGE_GENERATION_PARAMS,
+    pricing: {
+      units: [
+        { name: 'textInput', rate: 0.3, strategy: 'fixed', unit: 'millionTokens' },
+        { name: 'textOutput', rate: 2.5, strategy: 'fixed', unit: 'millionTokens' },
+        { name: 'imageOutput', rate: 3, strategy: 'fixed', unit: 'millionTokens' },
+      ],
+    },
+  },
+  {
+    displayName: 'Imagen 4',
     id: 'imagen-4.0-generate-001',
+    enabled: true,
+    type: 'image',
+    description: 'Imagen 4th generation text-to-image model series',
     organization: 'Deepmind',
+    releasedAt: '2025-08-15',
     parameters: imagenBaseParameters,
     pricing: {
       units: [{ name: 'imageGeneration', rate: 0.04, strategy: 'fixed', unit: 'image' }],
     },
-    releasedAt: '2024-08-14',
-    type: 'image',
   },
   {
-    description: 'Imagen 4th generation text-to-image model series Ultra version',
     displayName: 'Imagen 4 Ultra',
-    enabled: true,
     id: 'imagen-4.0-ultra-generate-001',
+    enabled: true,
+    type: 'image',
+    description: 'Imagen 4th generation text-to-image model series Ultra version',
     organization: 'Deepmind',
+    releasedAt: '2025-08-15',
     parameters: imagenBaseParameters,
     pricing: {
       units: [{ name: 'imageGeneration', rate: 0.06, strategy: 'fixed', unit: 'image' }],
     },
-    releasedAt: '2024-08-14',
-    type: 'image',
   },
   {
-    description: 'Imagen 4th generation text-to-image model series Fast version',
     displayName: 'Imagen 4 Fast',
-    enabled: true,
     id: 'imagen-4.0-fast-generate-001',
+    enabled: true,
+    type: 'image',
+    description: 'Imagen 4th generation text-to-image model series Fast version',
     organization: 'Deepmind',
+    releasedAt: '2025-08-15',
     parameters: imagenBaseParameters,
     pricing: {
       units: [{ name: 'imageGeneration', rate: 0.02, strategy: 'fixed', unit: 'image' }],
     },
-    releasedAt: '2024-08-14',
-    type: 'image',
   },
   {
-    description: 'Imagen 4th generation text-to-image model series',
     displayName: 'Imagen 4 Preview 06-06',
     id: 'imagen-4.0-generate-preview-06-06',
+    type: 'image',
+    description: 'Imagen 4th generation text-to-image model series',
     organization: 'Deepmind',
+    releasedAt: '2024-06-06',
     parameters: imagenBaseParameters,
     pricing: {
       units: [{ name: 'imageGeneration', rate: 0.04, strategy: 'fixed', unit: 'image' }],
     },
-    releasedAt: '2024-06-06',
-    type: 'image',
   },
   {
-    description: 'Imagen 4th generation text-to-image model series Ultra version',
     displayName: 'Imagen 4 Ultra Preview 06-06',
     id: 'imagen-4.0-ultra-generate-preview-06-06',
+    type: 'image',
+    description: 'Imagen 4th generation text-to-image model series Ultra version',
     organization: 'Deepmind',
+    releasedAt: '2025-06-11',
     parameters: imagenBaseParameters,
     pricing: {
       units: [{ name: 'imageGeneration', rate: 0.06, strategy: 'fixed', unit: 'image' }],
     },
-    releasedAt: '2024-06-06',
-    type: 'image',
   },
 ];
+/* eslint-enable sort-keys-fix/sort-keys-fix */
 
 export const allModels = [...googleChatModels, ...googleImageModels];
 

package/src/config/aiModels/openrouter.ts

@@ -11,6 +11,39 @@ const openrouterChatModels: AIChatModelCard[] = [
     id: 'openrouter/auto',
     type: 'chat',
   },
+  {
+    abilities: {
+      imageOutput: true,
+      vision: true,
+    },
+    contextWindowTokens: 32_768 + 8192,
+    description: 'Gemini 2.5 Flash 实验模型，支持图像生成',
+    displayName: 'Gemini 2.5 Flash Image Preview',
+    id: 'google/gemini-2.5-flash-image-preview',
+    maxOutput: 8192,
+    pricing: {
+      units: [
+        { name: 'imageOutput', rate: 30, strategy: 'fixed', unit: 'millionTokens' },
+        { name: 'textInput', rate: 0.3, strategy: 'fixed', unit: 'millionTokens' },
+        { name: 'textOutput', rate: 2.5, strategy: 'fixed', unit: 'millionTokens' },
+      ],
+    },
+    releasedAt: '2025-08-26',
+    type: 'chat',
+  },
+  {
+    abilities: {
+      imageOutput: true,
+      vision: true,
+    },
+    contextWindowTokens: 32_768 + 8192,
+    description: 'Gemini 2.5 Flash 实验模型，支持图像生成',
+    displayName: 'Gemini 2.5 Flash Image Preview (free)',
+    id: 'google/gemini-2.5-flash-image-preview:free',
+    maxOutput: 8192,
+    releasedAt: '2025-08-26',
+    type: 'chat',
+  },
   {
     abilities: {
       reasoning: true,

package/src/config/aiModels/vertexai.ts

@@ -126,21 +126,21 @@ const vertexaiChatModels: AIChatModelCard[] = [
       imageOutput: true,
       vision: true,
     },
-    contextWindowTokens: 32_768 + 32_768,
+    contextWindowTokens: 32_768 + 8192,
     description:
       'Gemini 2.5 Flash Image Preview 是 Google 最新、最快、最高效的原生多模态模型，它允许您通过对话生成和编辑图像。',
     displayName: 'Gemini 2.5 Flash Image Preview',
     enabled: true,
     id: 'gemini-2.5-flash-image-preview',
-    maxOutput: 32_768,
+    maxOutput: 8192,
     pricing: {
       units: [
         { name: 'textInput', rate: 0.3, strategy: 'fixed', unit: 'millionTokens' },
         { name: 'textOutput', rate: 2.5, strategy: 'fixed', unit: 'millionTokens' },
-        { name: 'imageOutput', rate: 3, strategy: 'fixed', unit: 'millionTokens' },
+        { name: 'imageOutput', rate: 30, strategy: 'fixed', unit: 'millionTokens' },
       ],
     },
-    releasedAt: '2025-08-27',
+    releasedAt: '2025-08-26',
     type: 'chat',
   },
   {

package/src/features/Conversation/Extras/Usage/UsageDetail/index.tsx

@@ -61,6 +61,12 @@ const TokenDetail = memo<TokenDetailProps>(({ meta, model, provider }) => {
         ? detailTokens.outputReasoning.credit
         : detailTokens.outputReasoning.token,
     },
+    !!detailTokens.outputImage && {
+      color: theme.purple,
+      id: 'outputImage',
+      title: t('messages.tokenDetails.outputImage'),
+      value: isShowCredit ? detailTokens.outputImage.credit : detailTokens.outputImage.token,
+    },
     !!detailTokens.outputAudio && {
       color: theme.cyan9,
       id: 'outputAudio',

package/src/features/Conversation/Extras/Usage/UsageDetail/tokens.test.ts

@@ -143,6 +143,44 @@ describe('getDetailsToken', () => {
     });
   });
 
+  it('should handle outputImageTokens correctly', () => {
+    const usage = {
+      inputTextTokens: 100,
+      outputImageTokens: 60,
+      outputReasoningTokens: 30,
+      totalOutputTokens: 200,
+      totalTokens: 300,
+    } as ModelTokensUsage;
+
+    const result = getDetailsToken(usage, mockModelCard);
+
+    expect(result.outputImage).toEqual({
+      credit: 1, // 60 * 0.02 = 1.2 -> 1
+      id: 'outputImage',
+      token: 60,
+    });
+
+    expect(result.outputReasoning).toEqual({
+      credit: 1, // 30 * 0.02 = 0.6 -> 1
+      token: 30,
+    });
+
+    expect(result.outputText).toEqual({
+      credit: 2, // (200 - 30 - 60) * 0.02 = 2.2 -> 2
+      token: 110,
+    });
+
+    expect(result.totalOutput).toEqual({
+      credit: 4, // 200 * 0.02 = 4
+      token: 200,
+    });
+
+    expect(result.totalTokens).toEqual({
+      credit: 4, // total credit equals totalOutputCredit here
+      token: 300,
+    });
+  });
+
   it('should handle inputCitationTokens correctly', () => {
     const usage: ModelTokensUsage = {
       inputCitationTokens: 75,

package/src/features/Conversation/Extras/Usage/UsageDetail/tokens.ts

@@ -21,9 +21,14 @@ export const getDetailsToken = (
 
   const outputReasoningTokens = usage.outputReasoningTokens || (usage as any).reasoningTokens || 0;
 
+  const outputImageTokens = usage.outputImageTokens || (usage as any).imageTokens || 0;
+
   const outputTextTokens = usage.outputTextTokens
     ? usage.outputTextTokens
-    : totalOutputTokens - outputReasoningTokens - (usage.outputAudioTokens || 0);
+    : totalOutputTokens -
+      outputReasoningTokens -
+      (usage.outputAudioTokens || 0) -
+      outputImageTokens;
 
   const inputWriteCacheTokens = usage.inputWriteCacheTokens || 0;
   const inputCacheTokens = usage.inputCachedTokens || (usage as any).cachedTokens || 0;
@@ -93,6 +98,13 @@ export const getDetailsToken = (
           token: usage.outputAudioTokens,
         }
       : undefined,
+    outputImage: !!outputImageTokens
+      ? {
+          credit: calcCredit(outputImageTokens, formatPrice.output),
+          id: 'outputImage',
+          token: outputImageTokens,
+        }
+      : undefined,
     outputReasoning: !!outputReasoningTokens
       ? {
           credit: calcCredit(outputReasoningTokens, formatPrice.output),

package/src/locales/default/chat.ts

@@ -128,6 +128,7 @@ export default {
       inputWriteCached: '输入缓存写入',
       output: '输出',
       outputAudio: '音频输出',
+      outputImage: '图像输出',
       outputText: '文本输出',
       outputTitle: '输出明细',
       reasoning: '深度思考',