@lobehub/chat 1.116.3 → 1.117.0

package/packages/model-runtime/src/utils/streams/openai/openai.test.ts

@@ -2271,4 +2271,45 @@ describe('OpenAIStream', () => {
       );
     });
   });
+
+  it('should handle base64_image in delta.images (image_url shape)', async () => {
+    const base64 =
+      'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==';
+
+    const mockOpenAIStream = new ReadableStream({
+      start(controller) {
+        controller.enqueue({
+          choices: [
+            {
+              delta: {
+                images: [
+                  {
+                    type: 'image_url',
+                    image_url: { url: base64 },
+                    index: 0,
+                  },
+                ],
+              },
+              index: 0,
+            },
+          ],
+          id: '6',
+        });
+
+        controller.close();
+      },
+    });
+
+    const protocolStream = OpenAIStream(mockOpenAIStream);
+
+    const decoder = new TextDecoder();
+    const chunks = [];
+
+    // @ts-ignore
+    for await (const chunk of protocolStream) {
+      chunks.push(decoder.decode(chunk, { stream: true }));
+    }
+
+    expect(chunks).toEqual(['id: 6\n', 'event: base64_image\n', `data: "${base64}"\n\n`]);
+  });
 });

package/packages/model-runtime/src/utils/streams/openai/openai.ts

@@ -96,6 +96,36 @@ const transformOpenAIStream = (
       }
     }
 
+    // Handle image preview chunks (e.g. Gemini 2.5 flash image preview)
+    // Example shape:
+    // choices[0].delta.images = [{ type: 'image_url', image_url: { url: 'data:image/png;base64,...' }, index: 0 }]
+    if (
+      (item as any).delta &&
+      Array.isArray((item as any).delta.images) &&
+      (item as any).delta.images.length > 0
+    ) {
+      const images = (item as any).delta.images as any[];
+
+      return images
+        .map((img) => {
+          // support multiple possible shapes for the url
+          const url =
+            img?.image_url?.url ||
+            img?.image_url?.image_url?.url ||
+            img?.url ||
+            (typeof img === 'string' ? img : undefined);
+
+          if (!url) return null;
+
+          return {
+            data: url,
+            id: chunk.id,
+            type: 'base64_image',
+          } as StreamProtocolChunk;
+        })
+        .filter(Boolean) as StreamProtocolChunk[];
+    }
+
     // 给定结束原因
     if (item.finish_reason) {
       // one-api 的流式接口，会出现既有 finish_reason ，也有 content 的情况
@@ -192,11 +222,11 @@ const transformOpenAIStream = (
         if ('content' in item.delta && Array.isArray(item.delta.content)) {
           return item.delta.content
             .filter((block: any) => block.type === 'thinking' && Array.isArray(block.thinking))
-            .map((block: any) => 
+            .map((block: any) =>
               block.thinking
                 .filter((thinkItem: any) => thinkItem.type === 'text' && thinkItem.text)
                 .map((thinkItem: any) => thinkItem.text)
-                .join('')
+                .join(''),
             )
             .join('');
         }
@@ -233,6 +263,12 @@ const transformOpenAIStream = (
           streamContext.thinkingInContent = false;
         }
 
+        // 如果 content 是空字符串但 chunk 带有 usage，则优先返回 usage（例如 Gemini image-preview 最终会在单独的 chunk 中返回 usage）
+        if (content === '' && chunk.usage) {
+          const usage = chunk.usage;
+          return { data: convertUsage(usage, provider), id: chunk.id, type: 'usage' };
+        }
+
         // 判断是否有 citations 内容，更新 returnedCitation 状态
         if (!streamContext?.returnedCitation) {
           const citations =

package/packages/model-runtime/src/utils/streams/protocol.test.ts

@@ -200,4 +200,36 @@ describe('createTokenSpeedCalculator', async () => {
     const results = await processChunk(transformer, chunks);
     expect(results).toHaveLength(chunks.length);
   });
+
+  it('should calculate token speed considering outputImageTokens when totalOutputTokens is missing', async () => {
+    const chunks = [
+      { data: '', id: 'chatcmpl-image-1', type: 'text' },
+      { data: 'hi', id: 'chatcmpl-image-1', type: 'text' },
+      { data: 'stop', id: 'chatcmpl-image-1', type: 'stop' },
+      {
+        data: {
+          inputTextTokens: 9,
+          outputTextTokens: 1,
+          outputImageTokens: 4,
+          totalInputTokens: 9,
+          // totalOutputTokens intentionally omitted to force summation path
+          totalTokens: 13,
+        },
+        id: 'chatcmpl-image-1',
+        type: 'usage',
+      },
+    ];
+
+    const transformer = createTokenSpeedCalculator((v) => v, { inputStartAt });
+    const results = await processChunk(transformer, chunks);
+
+    // should push an extra speed chunk
+    expect(results).toHaveLength(chunks.length + 1);
+    const speedChunk = results.slice(-1)[0];
+    expect(speedChunk.id).toBe('output_speed');
+    expect(speedChunk.type).toBe('speed');
+    // tps and ttft should be numeric (avoid flakiness if interval is 0ms)
+    expect(speedChunk.data.tps).not.toBeNaN();
+    expect(speedChunk.data.ttft).not.toBeNaN();
+  });
 });

package/packages/model-runtime/src/utils/streams/protocol.ts

@@ -364,10 +364,14 @@ export const createTokenSpeedCalculator = (
     }
     // if the chunk is the stop chunk, set as output finish
     if (inputStartAt && outputStartAt && chunk.type === 'usage') {
-      const totalOutputTokens = chunk.data?.totalOutputTokens || chunk.data?.outputTextTokens;
-      const reasoningTokens = chunk.data?.outputReasoningTokens || 0;
+      const totalOutputTokens =
+        chunk.data?.totalOutputTokens ??
+        (chunk.data?.outputTextTokens ?? 0) + (chunk.data?.outputImageTokens ?? 0);
+      const reasoningTokens = chunk.data?.outputReasoningTokens ?? 0;
       const outputTokens =
-        (outputThinking ?? false) ? totalOutputTokens : totalOutputTokens - reasoningTokens;
+        (outputThinking ?? false)
+          ? totalOutputTokens
+          : Math.max(0, totalOutputTokens - reasoningTokens);
       result.push({
         data: {
           tps: (outputTokens / (Date.now() - outputStartAt)) * 1000,

package/packages/model-runtime/src/utils/usageConverter.test.ts

@@ -290,4 +290,62 @@ describe('convertUsage', () => {
       totalTokens: 6550,
     });
   });
+
+  it('should handle output image tokens correctly', () => {
+    // Arrange
+    const usageWithImage = {
+      prompt_tokens: 100,
+      completion_tokens: 200,
+      completion_tokens_details: {
+        image_tokens: 60,
+        reasoning_tokens: 30,
+      },
+      total_tokens: 300,
+    } as OpenAI.Completions.CompletionUsage;
+
+    // Act
+    const result = convertUsage(usageWithImage);
+
+    // Assert
+    expect(result).toEqual({
+      inputTextTokens: 100,
+      totalInputTokens: 100,
+      totalOutputTokens: 200,
+      outputImageTokens: 60,
+      outputReasoningTokens: 30,
+      outputTextTokens: 110, // 200 - 60 - 30
+      totalTokens: 300,
+    });
+  });
+
+  it('should handle response output image tokens correctly for ResponseUsage', () => {
+    // Arrange
+    const responseUsage = {
+      input_tokens: 100,
+      input_tokens_details: {
+        cached_tokens: 0,
+      },
+      output_tokens: 200,
+      output_tokens_details: {
+        image_tokens: 60,
+        reasoning_tokens: 30,
+      },
+      total_tokens: 300,
+    } as OpenAI.Responses.ResponseUsage;
+
+    // Act
+    const result = convertResponseUsage(responseUsage);
+
+    // Assert
+    expect(result).toEqual({
+      inputTextTokens: 100,
+      inputCacheMissTokens: 100, // 100 - 0
+      totalInputTokens: 100,
+      totalOutputTokens: 200,
+      outputImageTokens: 60,
+      outputReasoningTokens: 30,
+      outputTextTokens: 170, // 200 - 30
+      totalTokens: 300,
+    });
+  });
 });

package/packages/model-runtime/src/utils/usageConverter.ts

@@ -20,12 +20,13 @@ export const convertUsage = (
   const totalOutputTokens = usage.completion_tokens;
   const outputReasoning = usage.completion_tokens_details?.reasoning_tokens || 0;
   const outputAudioTokens = usage.completion_tokens_details?.audio_tokens || 0;
+  const outputImageTokens = (usage.completion_tokens_details as any)?.image_tokens || 0;
 
   // XAI 的 completion_tokens 不包含 reasoning_tokens，需要特殊处理
   const outputTextTokens =
     provider === 'xai'
       ? totalOutputTokens - outputAudioTokens
-      : totalOutputTokens - outputReasoning - outputAudioTokens;
+      : totalOutputTokens - outputReasoning - outputAudioTokens - outputImageTokens;
 
   const totalTokens = inputCitationTokens + usage.total_tokens;
 
@@ -37,6 +38,7 @@ export const convertUsage = (
     inputCitationTokens: inputCitationTokens,
     inputTextTokens: inputTextTokens,
     outputAudioTokens: outputAudioTokens,
+    outputImageTokens: outputImageTokens,
     outputReasoningTokens: outputReasoning,
     outputTextTokens: outputTextTokens,
     rejectedPredictionTokens: usage.completion_tokens_details?.rejected_prediction_tokens,
@@ -75,6 +77,7 @@ export const convertResponseUsage = (usage: OpenAI.Responses.ResponseUsage): Mod
 
   // For ResponseUsage, outputTextTokens is totalOutputTokens minus reasoning, as no audio output tokens are specified.
   const outputTextTokens = totalOutputTokens - outputReasoningTokens;
+  const outputImageTokens = (usage.output_tokens_details as any)?.image_tokens || 0;
 
   // 3. Construct the comprehensive data object (matching ModelTokensUsage structure)
   const data = {
@@ -87,6 +90,7 @@ export const convertResponseUsage = (usage: OpenAI.Responses.ResponseUsage): Mod
     inputCitationTokens: undefined, // Not in ResponseUsage
     inputTextTokens: inputTextTokens,
     outputAudioTokens: undefined, // Not in ResponseUsage
+    outputImageTokens: outputImageTokens,
     outputReasoningTokens: outputReasoningTokens,
     outputTextTokens: outputTextTokens,
     rejectedPredictionTokens: undefined, // Not in ResponseUsage

package/packages/model-runtime/vitest.config.mts

@@ -13,6 +13,9 @@ export default defineConfig({
       '@': resolve(__dirname, '../../src'),
       /* eslint-enable */
     },
+    coverage: {
+      reporter: ['text', 'json', 'lcov', 'text-summary'],
+    },
     environment: 'happy-dom',
   },
 });

package/packages/prompts/package.json

@@ -8,7 +8,6 @@
     "test:coverage": "vitest --coverage"
   },
   "dependencies": {
-    "@lobechat/const": "workspace:*",
     "@lobechat/types": "workspace:*"
   }
 }

package/packages/prompts/src/chains/__tests__/abstractChunk.test.ts

@@ -0,0 +1,52 @@
+import { describe, expect, it, vi } from 'vitest';
+
+import { chainAbstractChunkText } from '../abstractChunk';
+
+describe('chainAbstractChunkText', () => {
+  it('should generate correct chat payload for chunk text', () => {
+    const testText = 'This is a sample chunk of text that needs to be summarized.';
+
+    const result = chainAbstractChunkText(testText);
+
+    expect(result).toEqual({
+      messages: [
+        {
+          content:
+            '你是一名擅长从 chunk 中提取摘要的助理，你需要将用户的会话总结为 1~2 句话的摘要，输出成 chunk 所使用的语种',
+          role: 'system',
+        },
+        {
+          content: `chunk: ${testText}`,
+          role: 'user',
+        },
+      ],
+    });
+  });
+
+  it('should handle empty text', () => {
+    const result = chainAbstractChunkText('');
+
+    expect(result.messages).toHaveLength(2);
+    expect(result.messages![1].content).toBe('chunk: ');
+  });
+
+  it('should handle text with special characters', () => {
+    const testText = 'Text with special chars: @#$%^&*()';
+
+    const result = chainAbstractChunkText(testText);
+
+    expect(result.messages![1].content).toBe(`chunk: ${testText}`);
+  });
+
+  it('should always use system role for first message', () => {
+    const result = chainAbstractChunkText('test');
+
+    expect(result.messages![0].role).toBe('system');
+  });
+
+  it('should always use user role for second message', () => {
+    const result = chainAbstractChunkText('test');
+
+    expect(result.messages![1].role).toBe('user');
+  });
+});

package/packages/prompts/src/chains/__tests__/answerWithContext.test.ts

@@ -0,0 +1,100 @@
+import { describe, expect, it } from 'vitest';
+
+import { chainAnswerWithContext } from '../answerWithContext';
+
+describe('chainAnswerWithContext', () => {
+  it('should generate correct chat payload with context and knowledge', () => {
+    const testParams = {
+      context: ['Context passage 1', 'Context passage 2'],
+      knowledge: ['AI', 'Machine Learning'],
+      question: 'What is artificial intelligence?',
+    };
+
+    const result = chainAnswerWithContext(testParams);
+
+    expect(result.messages).toHaveLength(1);
+    expect(result.messages![0].role).toBe('user');
+    expect(result.messages![0].content).toContain('AI/Machine Learning');
+    expect(result.messages![0].content).toContain('Context passage 1');
+    expect(result.messages![0].content).toContain('Context passage 2');
+    expect(result.messages![0].content).toContain('What is artificial intelligence?');
+  });
+
+  it('should handle single knowledge area', () => {
+    const testParams = {
+      context: ['Single context'],
+      knowledge: ['Technology'],
+      question: 'How does it work?',
+    };
+
+    const result = chainAnswerWithContext(testParams);
+
+    expect(result.messages![0].content).toContain('Technology');
+  });
+
+  it('should handle multiple knowledge areas', () => {
+    const testParams = {
+      context: ['Context'],
+      knowledge: ['AI', 'ML', 'NLP', 'Computer Vision'],
+      question: 'Tell me about these fields',
+    };
+
+    const result = chainAnswerWithContext(testParams);
+
+    expect(result.messages![0].content).toContain('AI/ML/NLP/Computer Vision');
+  });
+
+  it('should handle empty context array', () => {
+    const testParams = {
+      context: [],
+      knowledge: ['AI'],
+      question: 'What is AI?',
+    };
+
+    const result = chainAnswerWithContext(testParams);
+
+    expect(result.messages![0].content).toContain('<Context>');
+    expect(result.messages![0].content).toContain('</Context>');
+  });
+
+  it('should include proper context formatting', () => {
+    const testParams = {
+      context: ['First passage', 'Second passage'],
+      knowledge: ['Test'],
+      question: 'Test question',
+    };
+
+    const result = chainAnswerWithContext(testParams);
+
+    expect(result.messages![0].content).toContain(
+      '<Context>\nFirst passage\nSecond passage\n</Context>',
+    );
+  });
+
+  it('should include proper instructions about using passages', () => {
+    const testParams = {
+      context: ['Context'],
+      knowledge: ['Knowledge'],
+      question: 'Question',
+    };
+
+    const result = chainAnswerWithContext(testParams);
+    const content = result.messages![0].content;
+
+    expect(content).toContain('passages might not be relevant');
+    expect(content).toContain('please only use the passages that are relevant');
+    expect(content).toContain('answer using your knowledge');
+  });
+
+  it('should include markdown formatting instruction', () => {
+    const testParams = {
+      context: ['Context'],
+      knowledge: ['Knowledge'],
+      question: 'Question',
+    };
+
+    const result = chainAnswerWithContext(testParams);
+
+    expect(result.messages![0].content).toContain('follow markdown syntax');
+  });
+});

package/packages/prompts/src/chains/__tests__/rewriteQuery.test.ts

@@ -0,0 +1,88 @@
+import { describe, expect, it, vi } from 'vitest';
+
+// Mock DEFAULT_REWRITE_QUERY
+
+import { DEFAULT_REWRITE_QUERY, chainRewriteQuery } from '../rewriteQuery';
+
+describe('chainRewriteQuery', () => {
+  it('should generate correct chat payload with default instruction', () => {
+    const query = 'What about the weather?';
+    const context = ['Previous message 1', 'Previous message 2'];
+
+    const result = chainRewriteQuery(query, context);
+
+    expect(result.messages).toHaveLength(2);
+    expect(result.messages![0].role).toBe('system');
+    expect(result.messages![1].role).toBe('user');
+    expect(result.messages![0].content).toContain(DEFAULT_REWRITE_QUERY);
+    expect(result.messages![1].content).toContain(query);
+  });
+
+  it('should include chat history in system message', () => {
+    const query = 'Follow up question';
+    const context = ['User: Hello', 'Assistant: Hi there'];
+
+    const result = chainRewriteQuery(query, context);
+
+    expect(result.messages![0].content).toContain('<chatHistory>');
+    expect(result.messages![0].content).toContain('User: Hello');
+    expect(result.messages![0].content).toContain('Assistant: Hi there');
+    expect(result.messages![0].content).toContain('</chatHistory>');
+  });
+
+  it('should use custom instruction when provided', () => {
+    const query = 'Test query';
+    const context = ['Context'];
+    const customInstruction = 'Custom rewrite instruction';
+
+    const result = chainRewriteQuery(query, context, customInstruction);
+
+    expect(result.messages![0].content).toContain(customInstruction);
+    expect(result.messages![0].content).not.toContain(DEFAULT_REWRITE_QUERY);
+  });
+
+  it('should format user message correctly', () => {
+    const query = 'What is the status?';
+    const context = ['Previous context'];
+
+    const result = chainRewriteQuery(query, context);
+
+    expect(result.messages![1].content).toBe(`Follow Up Input: ${query}, it's standalone query:`);
+  });
+
+  it('should handle empty context array', () => {
+    const query = 'Empty context query';
+    const context: string[] = [];
+
+    const result = chainRewriteQuery(query, context);
+
+    expect(result.messages![0].content).toContain('<chatHistory>\n\n</chatHistory>');
+  });
+
+  it('should handle single context item', () => {
+    const query = 'Single context query';
+    const context = ['Only one message'];
+
+    const result = chainRewriteQuery(query, context);
+
+    expect(result.messages![0].content).toContain('Only one message');
+  });
+
+  it('should join multiple context items with newlines', () => {
+    const query = 'Multi context query';
+    const context = ['Message 1', 'Message 2', 'Message 3'];
+
+    const result = chainRewriteQuery(query, context);
+
+    expect(result.messages![0].content).toContain('Message 1\nMessage 2\nMessage 3');
+  });
+
+  it('should handle special characters in query', () => {
+    const query = 'Query with special chars: @#$%^&*()';
+    const context = ['Context'];
+
+    const result = chainRewriteQuery(query, context);
+
+    expect(result.messages![1].content).toContain(query);
+  });
+});

package/packages/prompts/src/chains/__tests__/summaryGenerationTitle.test.ts

@@ -0,0 +1,107 @@
+import { describe, expect, it } from 'vitest';
+
+import { chainSummaryGenerationTitle } from '../summaryGenerationTitle';
+
+describe('chainSummaryGenerationTitle', () => {
+  it('should generate correct chat payload for image modal', () => {
+    const prompts = ['A beautiful sunset', 'Mountain landscape'];
+    const modal = 'image' as const;
+    const locale = 'zh-CN';
+
+    const result = chainSummaryGenerationTitle(prompts, modal, locale);
+
+    expect(result.messages).toHaveLength(2);
+    expect(result.messages![0].role).toBe('system');
+    expect(result.messages![1].role).toBe('user');
+    expect(result.messages![0].content).toContain('AI image prompt');
+    expect(result.messages![0].content).toContain(locale);
+  });
+
+  it('should generate correct chat payload for video modal', () => {
+    const prompts = ['Dancing in the rain'];
+    const modal = 'video' as const;
+    const locale = 'en-US';
+
+    const result = chainSummaryGenerationTitle(prompts, modal, locale);
+
+    expect(result.messages![0].content).toContain('AI video prompt');
+    expect(result.messages![0].content).toContain(locale);
+  });
+
+  it('should format single prompt correctly', () => {
+    const prompts = ['Single prompt'];
+    const modal = 'image' as const;
+    const locale = 'zh-CN';
+
+    const result = chainSummaryGenerationTitle(prompts, modal, locale);
+
+    expect(result.messages![1].content).toContain('1. Single prompt');
+  });
+
+  it('should format multiple prompts with numbering', () => {
+    const prompts = ['First prompt', 'Second prompt', 'Third prompt'];
+    const modal = 'image' as const;
+    const locale = 'zh-CN';
+
+    const result = chainSummaryGenerationTitle(prompts, modal, locale);
+
+    const userMessage = result.messages![1].content;
+    expect(userMessage).toContain('1. First prompt');
+    expect(userMessage).toContain('2. Second prompt');
+    expect(userMessage).toContain('3. Third prompt');
+  });
+
+  it('should include system instructions about title requirements', () => {
+    const prompts = ['Test prompt'];
+    const modal = 'image' as const;
+    const locale = 'zh-CN';
+
+    const result = chainSummaryGenerationTitle(prompts, modal, locale);
+
+    const systemMessage = result.messages![0].content;
+    expect(systemMessage).toContain('资深的 AI 艺术创作者');
+    expect(systemMessage).toContain('10个字以内');
+    expect(systemMessage).toContain('不需要包含标点符号');
+  });
+
+  it('should handle empty prompts array', () => {
+    const prompts: string[] = [];
+    const modal = 'image' as const;
+    const locale = 'zh-CN';
+
+    const result = chainSummaryGenerationTitle(prompts, modal, locale);
+
+    expect(result.messages![1].content).toContain('提示词：\n');
+  });
+
+  it('should handle different locales', () => {
+    const prompts = ['Test'];
+    const modal = 'image' as const;
+    const customLocale = 'ja-JP';
+
+    const result = chainSummaryGenerationTitle(prompts, modal, customLocale);
+
+    expect(result.messages![0].content).toContain(customLocale);
+  });
+
+  it('should differentiate between image and video modals in instructions', () => {
+    const prompts = ['Test prompt'];
+    const locale = 'zh-CN';
+
+    const imageResult = chainSummaryGenerationTitle(prompts, 'image', locale);
+    const videoResult = chainSummaryGenerationTitle(prompts, 'video', locale);
+
+    expect(imageResult.messages![0].content).toContain('AI image prompt');
+    expect(videoResult.messages![0].content).toContain('AI video prompt');
+  });
+
+  it('should format prompts with newlines between them', () => {
+    const prompts = ['Prompt one', 'Prompt two'];
+    const modal = 'image' as const;
+    const locale = 'zh-CN';
+
+    const result = chainSummaryGenerationTitle(prompts, modal, locale);
+
+    expect(result.messages![1].content).toContain('1. Prompt one\n2. Prompt two');
+  });
+});

package/packages/prompts/src/chains/abstractChunk.ts

@@ -1,4 +1,3 @@
-import { DEFAULT_MODEL } from '@lobechat/const';
 import { ChatStreamPayload } from '@lobechat/types';
 
 export const chainAbstractChunkText = (text: string): Partial<ChatStreamPayload> => {
@@ -14,6 +13,5 @@ export const chainAbstractChunkText = (text: string): Partial<ChatStreamPayload>
         role: 'user',
       },
     ],
-    model: DEFAULT_MODEL,
   };
 };

package/packages/prompts/src/chains/rewriteQuery.ts

@@ -1,6 +1,8 @@
-import { DEFAULT_REWRITE_QUERY } from '@lobechat/const';
 import { ChatStreamPayload } from '@lobechat/types';
 
+export const DEFAULT_REWRITE_QUERY =
+  'Given the following conversation and a follow-up question, rephrase the follow up question to be a standalone question, in its original language. Keep as much details as possible from previous messages. Keep entity names and all.';
+
 export const chainRewriteQuery = (
   query: string,
   context: string[],