@lobehub/chat 1.4.3 → 1.5.0

package/CHANGELOG.md

@@ -2,6 +2,31 @@
 
 # Changelog
 
+## [Version 1.5.0](https://github.com/lobehub/lobe-chat/compare/v1.4.3...v1.5.0)
+
+<sup>Released on **2024-07-17**</sup>
+
+#### ✨ Features
+
+- **misc**: Spport qwen-vl and tool call for qwen.
+
+<br/>
+
+<details>
+<summary><kbd>Improvements and Fixes</kbd></summary>
+
+#### What's improved
+
+- **misc**: Spport qwen-vl and tool call for qwen, closes [#3114](https://github.com/lobehub/lobe-chat/issues/3114) ([5216a85](https://github.com/lobehub/lobe-chat/commit/5216a85))
+
+</details>
+
+<div align="right">
+
+[![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top)
+
+</div>
+
 ### [Version 1.4.3](https://github.com/lobehub/lobe-chat/compare/v1.4.2...v1.4.3)
 
 <sup>Released on **2024-07-15**</sup>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lobehub/chat",
-  "version": "1.4.3",
+  "version": "1.5.0",
   "description": "Lobe Chat - an open-source, high-performance chatbot framework that supports speech synthesis, multimodal, and extensible Function Call plugin system. Supports one-click free deployment of your private ChatGPT/LLM web application.",
   "keywords": [
     "framework",
@@ -267,6 +267,7 @@
     "vitest": "~1.2.2",
     "vitest-canvas-mock": "^0.3.3"
   },
+  "packageManager": "pnpm@9.5.0",
   "publishConfig": {
     "access": "public",
     "registry": "https://registry.npmjs.org"

package/src/config/modelProviders/qwen.ts

@@ -7,6 +7,7 @@ const Qwen: ModelProviderCard = {
       description: '通义千问超大规模语言模型，支持中文、英文等不同语言输入',
       displayName: 'Qwen Turbo',
       enabled: true,
+      functionCall: true,
       id: 'qwen-turbo',
       tokens: 8000,
     },
@@ -14,6 +15,7 @@ const Qwen: ModelProviderCard = {
       description: '通义千问超大规模语言模型增强版，支持中文、英文等不同语言输入',
       displayName: 'Qwen Plus',
       enabled: true,
+      functionCall: true,
       id: 'qwen-plus',
       tokens: 32_000,
     },
@@ -22,6 +24,7 @@ const Qwen: ModelProviderCard = {
         '通义千问千亿级别超大规模语言模型，支持中文、英文等不同语言输入，当前通义千问2.5产品版本背后的API模型',
       displayName: 'Qwen Max',
       enabled: true,
+      functionCall: true,
       id: 'qwen-max',
       tokens: 8000,
     },
@@ -29,6 +32,7 @@ const Qwen: ModelProviderCard = {
       description:
         '通义千问千亿级别超大规模语言模型，支持中文、英文等不同语言输入，扩展了上下文窗口',
       displayName: 'Qwen Max LongContext',
+      functionCall: true,
       id: 'qwen-max-longcontext',
       tokens: 30_000,
     },
@@ -50,6 +54,24 @@ const Qwen: ModelProviderCard = {
       id: 'qwen2-72b-instruct',
       tokens: 131_072,
     },
+    {
+      description:
+        '通义千问大规模视觉语言模型增强版。大幅提升细节识别能力和文字识别能力，支持超百万像素分辨率和任意长宽比规格的图像。',
+      displayName: 'Qwen VL Plus',
+      enabled: true,
+      id: 'qwen-vl-plus',
+      tokens: 6144,
+      vision: true,
+    },
+    {
+      description:
+        '通义千问超大规模视觉语言模型。相比增强版，再次提升视觉推理能力和指令遵循能力，提供更高的视觉感知和认知水平。',
+      displayName: 'Qwen VL Max',
+      enabled: true,
+      id: 'qwen-vl-max',
+      tokens: 6144,
+      vision: true,
+    },
   ],
   checkModel: 'qwen-turbo',
   disableBrowserRequest: true,

package/src/libs/agent-runtime/qwen/index.test.ts

@@ -2,6 +2,7 @@
 import OpenAI from 'openai';
 import { Mock, afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 
+import Qwen from '@/config/modelProviders/qwen';
 import { LobeOpenAICompatibleRuntime } from '@/libs/agent-runtime';
 import { ModelProvider } from '@/libs/agent-runtime';
 import { AgentRuntimeErrorType } from '@/libs/agent-runtime';
@@ -17,7 +18,7 @@ const invalidErrorType = AgentRuntimeErrorType.InvalidProviderAPIKey;
 // Mock the console.error to avoid polluting test output
 vi.spyOn(console, 'error').mockImplementation(() => {});
 
-let instance: LobeOpenAICompatibleRuntime;
+let instance: LobeQwenAI;
 
 beforeEach(() => {
   instance = new LobeQwenAI({ apiKey: 'test' });
@@ -41,7 +42,116 @@ describe('LobeQwenAI', () => {
     });
   });
 
+  describe('models', () => {
+    it('should correctly list available models', async () => {
+      const instance = new LobeQwenAI({ apiKey: 'test_api_key' });
+      vi.spyOn(instance, 'models').mockResolvedValue(Qwen.chatModels);
+
+      const models = await instance.models();
+      expect(models).toEqual(Qwen.chatModels);
+    });
+  });
+
   describe('chat', () => {
+    describe('Params', () => {
+      it('should call llms with proper options', async () => {
+        const mockStream = new ReadableStream();
+        const mockResponse = Promise.resolve(mockStream);
+
+        (instance['client'].chat.completions.create as Mock).mockResolvedValue(mockResponse);
+
+        const result = await instance.chat({
+          messages: [{ content: 'Hello', role: 'user' }],
+          model: 'qwen-turbo',
+          temperature: 0.6,
+          top_p: 0.7,
+        });
+
+        // Assert
+        expect(instance['client'].chat.completions.create).toHaveBeenCalledWith(
+          {
+            messages: [{ content: 'Hello', role: 'user' }],
+            model: 'qwen-turbo',
+            temperature: 0.6,
+            stream: true,
+            top_p: 0.7,
+            result_format: 'message',
+          },
+          { headers: { Accept: '*/*' } },
+        );
+        expect(result).toBeInstanceOf(Response);
+      });
+
+      it('should call vlms with proper options', async () => {
+        const mockStream = new ReadableStream();
+        const mockResponse = Promise.resolve(mockStream);
+
+        (instance['client'].chat.completions.create as Mock).mockResolvedValue(mockResponse);
+
+        const result = await instance.chat({
+          messages: [{ content: 'Hello', role: 'user' }],
+          model: 'qwen-vl-plus',
+          temperature: 0.6,
+          top_p: 0.7,
+        });
+
+        // Assert
+        expect(instance['client'].chat.completions.create).toHaveBeenCalledWith(
+          {
+            messages: [{ content: 'Hello', role: 'user' }],
+            model: 'qwen-vl-plus',
+            stream: true,
+          },
+          { headers: { Accept: '*/*' } },
+        );
+        expect(result).toBeInstanceOf(Response);
+      });
+
+      it('should transform non-streaming response to stream correctly', async () => {
+        const mockResponse: OpenAI.ChatCompletion = {
+          id: 'chatcmpl-fc539f49-51a8-94be-8061',
+          object: 'chat.completion',
+          created: 1719901794,
+          model: 'qwen-turbo',
+          choices: [
+            {
+              index: 0,
+              message: { role: 'assistant', content: 'Hello' },
+              finish_reason: 'stop',
+              logprobs: null,
+            },
+          ],
+        };
+        vi.spyOn(instance['client'].chat.completions, 'create').mockResolvedValue(
+          mockResponse as any,
+        );
+
+        const result = await instance.chat({
+          messages: [{ content: 'Hello', role: 'user' }],
+          model: 'qwen-turbo',
+          temperature: 0.6,
+          stream: false,
+        });
+
+        const decoder = new TextDecoder();
+
+        const reader = result.body!.getReader();
+        expect(decoder.decode((await reader.read()).value)).toContain(
+          'id: chatcmpl-fc539f49-51a8-94be-8061\n',
+        );
+        expect(decoder.decode((await reader.read()).value)).toContain('event: text\n');
+        expect(decoder.decode((await reader.read()).value)).toContain('data: "Hello"\n\n');
+
+        expect(decoder.decode((await reader.read()).value)).toContain(
+          'id: chatcmpl-fc539f49-51a8-94be-8061\n',
+        );
+        expect(decoder.decode((await reader.read()).value)).toContain('event: stop\n');
+        expect(decoder.decode((await reader.read()).value)).toContain('');
+
+        expect((await reader.read()).done).toBe(true);
+      });
+    });
+
     describe('Error', () => {
       it('should return QwenBizError with an openai error response when OpenAI.APIError is thrown', async () => {
         // Arrange
@@ -129,8 +239,7 @@ describe('LobeQwenAI', () => {
 
         instance = new LobeQwenAI({
           apiKey: 'test',
-
-          baseURL: 'https://api.abc.com/v1',
+          baseURL: defaultBaseURL,
         });
 
         vi.spyOn(instance['client'].chat.completions, 'create').mockRejectedValue(apiError);
@@ -144,7 +253,8 @@ describe('LobeQwenAI', () => {
           });
         } catch (e) {
           expect(e).toEqual({
-            endpoint: 'https://api.***.com/v1',
+            /* Desensitizing is unnecessary for a public-accessible gateway endpoint. */
+            endpoint: defaultBaseURL,
             error: {
               cause: { message: 'api is undefined' },
               stack: 'abc',

package/src/libs/agent-runtime/qwen/index.ts

@@ -1,28 +1,128 @@
-import OpenAI from 'openai';
-
-import { ModelProvider } from '../types';
-import { LobeOpenAICompatibleFactory } from '../utils/openaiCompatibleFactory';
-
-export const LobeQwenAI = LobeOpenAICompatibleFactory({
-  baseURL: 'https://dashscope.aliyuncs.com/compatible-mode/v1',
-  chatCompletion: {
-    handlePayload: (payload) => {
-      const top_p = payload.top_p;
-      return {
-        ...payload,
-        stream: payload.stream ?? true,
-        top_p: top_p && top_p >= 1 ? 0.9999 : top_p,
-      } as OpenAI.ChatCompletionCreateParamsStreaming;
-    },
-  },
-  constructorOptions: {
-    defaultHeaders: {
-      'Content-Type': 'application/json',
-    },
-  },
-  debug: {
-    chatCompletion: () => process.env.DEBUG_QWEN_CHAT_COMPLETION === '1',
-  },
-
-  provider: ModelProvider.Qwen,
-});
+import { omit } from 'lodash-es';
+import OpenAI, { ClientOptions } from 'openai';
+
+import Qwen from '@/config/modelProviders/qwen';
+
+import { LobeOpenAICompatibleRuntime, LobeRuntimeAI } from '../BaseAI';
+import { AgentRuntimeErrorType } from '../error';
+import { ChatCompetitionOptions, ChatStreamPayload, ModelProvider } from '../types';
+import { AgentRuntimeError } from '../utils/createError';
+import { debugStream } from '../utils/debugStream';
+import { handleOpenAIError } from '../utils/handleOpenAIError';
+import { transformResponseToStream } from '../utils/openaiCompatibleFactory';
+import { StreamingResponse } from '../utils/response';
+import { QwenAIStream } from '../utils/streams';
+
+const DEFAULT_BASE_URL = 'https://dashscope.aliyuncs.com/compatible-mode/v1';
+
+/**
+ * Use DashScope OpenAI compatible mode for now.
+ * DashScope OpenAI [compatible mode](https://help.aliyun.com/zh/dashscope/developer-reference/tongyi-qianwen-vl-plus-api) currently supports base64 image input for vision models e.g. qwen-vl-plus.
+ * You can use images input either:
+ * 1. Use qwen-vl-* out of box with base64 image_url input;
+ * or
+ * 2. Set S3-* enviroment variables properly to store all uploaded files.
+ */
+export class LobeQwenAI extends LobeOpenAICompatibleRuntime implements LobeRuntimeAI {
+  client: OpenAI;
+  baseURL: string;
+
+  constructor({
+    apiKey,
+    baseURL = DEFAULT_BASE_URL,
+    ...res
+  }: ClientOptions & Record<string, any> = {}) {
+    super();
+    if (!apiKey) throw AgentRuntimeError.createError(AgentRuntimeErrorType.InvalidProviderAPIKey);
+    this.client = new OpenAI({ apiKey, baseURL, ...res });
+    this.baseURL = this.client.baseURL;
+  }
+
+  async models() {
+    return Qwen.chatModels;
+  }
+
+  async chat(payload: ChatStreamPayload, options?: ChatCompetitionOptions) {
+    try {
+      const params = this.buildCompletionParamsByModel(payload);
+
+      const response = await this.client.chat.completions.create(
+        params as OpenAI.ChatCompletionCreateParamsStreaming & { result_format: string },
+        {
+          headers: { Accept: '*/*' },
+          signal: options?.signal,
+        },
+      );
+
+      if (params.stream) {
+        const [prod, debug] = response.tee();
+
+        if (process.env.DEBUG_QWEN_CHAT_COMPLETION === '1') {
+          debugStream(debug.toReadableStream()).catch(console.error);
+        }
+
+        return StreamingResponse(QwenAIStream(prod, options?.callback), {
+          headers: options?.headers,
+        });
+      }
+
+      const stream = transformResponseToStream(response as unknown as OpenAI.ChatCompletion);
+
+      return StreamingResponse(QwenAIStream(stream, options?.callback), {
+        headers: options?.headers,
+      });
+    } catch (error) {
+      if ('status' in (error as any)) {
+        switch ((error as Response).status) {
+          case 401: {
+            throw AgentRuntimeError.chat({
+              endpoint: this.baseURL,
+              error: error as any,
+              errorType: AgentRuntimeErrorType.InvalidProviderAPIKey,
+              provider: ModelProvider.Qwen,
+            });
+          }
+
+          default: {
+            break;
+          }
+        }
+      }
+      const { errorResult, RuntimeError } = handleOpenAIError(error);
+      const errorType = RuntimeError || AgentRuntimeErrorType.ProviderBizError;
+
+      throw AgentRuntimeError.chat({
+        endpoint: this.baseURL,
+        error: errorResult,
+        errorType,
+        provider: ModelProvider.Qwen,
+      });
+    }
+  }
+
+  private buildCompletionParamsByModel(payload: ChatStreamPayload) {
+    const { model, top_p, stream, messages, tools } = payload;
+    const isVisionModel = model.startsWith('qwen-vl');
+
+    const params = {
+      ...payload,
+      messages,
+      result_format: 'message',
+      stream: !!tools?.length ? false : stream ?? true,
+      top_p: top_p && top_p >= 1 ? 0.999 : top_p,
+    };
+
+    /* Qwen-vl models temporarily do not support parameters below. */
+    /* Notice: `top_p` imposes significant impact on the result，the default 1 or 0.999 is not a proper choice. */
+    return isVisionModel
+      ? omit(
+          params,
+          'presence_penalty',
+          'frequency_penalty',
+          'temperature',
+          'result_format',
+          'top_p',
+        )
+      : params;
+  }
+}

package/src/libs/agent-runtime/utils/openaiCompatibleFactory/index.ts

@@ -54,6 +54,59 @@ interface OpenAICompatibleFactoryOptions<T extends Record<string, any> = any> {
   provider: string;
 }
 
+/**
+ * make the OpenAI response data as a stream
+ */
+export function transformResponseToStream(data: OpenAI.ChatCompletion) {
+  return new ReadableStream({
+    start(controller) {
+      const chunk: OpenAI.ChatCompletionChunk = {
+        choices: data.choices.map((choice: OpenAI.ChatCompletion.Choice) => ({
+          delta: {
+            content: choice.message.content,
+            role: choice.message.role,
+            tool_calls: choice.message.tool_calls?.map(
+              (tool, index): OpenAI.ChatCompletionChunk.Choice.Delta.ToolCall => ({
+                function: tool.function,
+                id: tool.id,
+                index,
+                type: tool.type,
+              }),
+            ),
+          },
+          finish_reason: null,
+          index: choice.index,
+          logprobs: choice.logprobs,
+        })),
+        created: data.created,
+        id: data.id,
+        model: data.model,
+        object: 'chat.completion.chunk',
+      };
+
+      controller.enqueue(chunk);
+
+      controller.enqueue({
+        choices: data.choices.map((choice: OpenAI.ChatCompletion.Choice) => ({
+          delta: {
+            content: choice.message.content,
+            role: choice.message.role,
+          },
+          finish_reason: choice.finish_reason,
+          index: choice.index,
+          logprobs: choice.logprobs,
+        })),
+        created: data.created,
+        id: data.id,
+        model: data.model,
+        object: 'chat.completion.chunk',
+        system_fingerprint: data.system_fingerprint,
+      } as OpenAI.ChatCompletionChunk);
+      controller.close();
+    },
+  });
+}
+
 export const LobeOpenAICompatibleFactory = <T extends Record<string, any> = any>({
   provider,
   baseURL: DEFAULT_BASE_URL,
@@ -117,7 +170,7 @@ export const LobeOpenAICompatibleFactory = <T extends Record<string, any> = any>
           debugResponse(response);
         }
 
-        const stream = this.transformResponseToStream(response as unknown as OpenAI.ChatCompletion);
+        const stream = transformResponseToStream(response as unknown as OpenAI.ChatCompletion);
 
         return StreamingResponse(OpenAIStream(stream, options?.callback), {
           headers: options?.headers,
@@ -162,60 +215,6 @@ export const LobeOpenAICompatibleFactory = <T extends Record<string, any> = any>
       }
     }
 
-    /**
-     * make the OpenAI response data as a stream
-     * @private
-     */
-    private transformResponseToStream(data: OpenAI.ChatCompletion) {
-      return new ReadableStream({
-        start(controller) {
-          const chunk: OpenAI.ChatCompletionChunk = {
-            choices: data.choices.map((choice: OpenAI.ChatCompletion.Choice) => ({
-              delta: {
-                content: choice.message.content,
-                role: choice.message.role,
-                tool_calls: choice.message.tool_calls?.map(
-                  (tool, index): OpenAI.ChatCompletionChunk.Choice.Delta.ToolCall => ({
-                    function: tool.function,
-                    id: tool.id,
-                    index,
-                    type: tool.type,
-                  }),
-                ),
-              },
-              finish_reason: null,
-              index: choice.index,
-              logprobs: choice.logprobs,
-            })),
-            created: data.created,
-            id: data.id,
-            model: data.model,
-            object: 'chat.completion.chunk',
-          };
-
-          controller.enqueue(chunk);
-
-          controller.enqueue({
-            choices: data.choices.map((choice: OpenAI.ChatCompletion.Choice) => ({
-              delta: {
-                content: choice.message.content,
-                role: choice.message.role,
-              },
-              finish_reason: choice.finish_reason,
-              index: choice.index,
-              logprobs: choice.logprobs,
-            })),
-            created: data.created,
-            id: data.id,
-            model: data.model,
-            object: 'chat.completion.chunk',
-            system_fingerprint: data.system_fingerprint,
-          } as OpenAI.ChatCompletionChunk);
-          controller.close();
-        },
-      });
-    }
-
     private handleError(error: any): ChatCompletionErrorPayload {
       let desensitizedEndpoint = this.baseURL;
 

package/src/libs/agent-runtime/utils/streams/index.ts

@@ -6,3 +6,4 @@ export * from './minimax';
 export * from './ollama';
 export * from './openai';
 export * from './protocol';
+export * from './qwen';

package/src/libs/agent-runtime/utils/streams/qwen.test.ts

@@ -0,0 +1,350 @@
+import { beforeAll, describe, expect, it, vi } from 'vitest';
+
+import { QwenAIStream } from './qwen';
+
+describe('QwenAIStream', () => {
+  beforeAll(() => {});
+
+  it('should transform OpenAI stream to protocol stream', async () => {
+    const mockOpenAIStream = new ReadableStream({
+      start(controller) {
+        controller.enqueue({
+          choices: [
+            {
+              delta: { content: 'Hello' },
+              index: 0,
+            },
+          ],
+          id: '1',
+        });
+        controller.enqueue({
+          choices: [
+            {
+              delta: { content: ' world!' },
+              index: 1,
+            },
+          ],
+          id: '1',
+        });
+        controller.enqueue({
+          choices: [
+            {
+              delta: null,
+              finish_reason: 'stop',
+              index: 2,
+            },
+          ],
+          id: '1',
+        });
+
+        controller.close();
+      },
+    });
+
+    const onStartMock = vi.fn();
+    const onTextMock = vi.fn();
+    const onTokenMock = vi.fn();
+    const onCompletionMock = vi.fn();
+
+    const protocolStream = QwenAIStream(mockOpenAIStream, {
+      onStart: onStartMock,
+      onText: onTextMock,
+      onToken: onTokenMock,
+      onCompletion: onCompletionMock,
+    });
+
+    const decoder = new TextDecoder();
+    const chunks = [];
+
+    // @ts-ignore
+    for await (const chunk of protocolStream) {
+      chunks.push(decoder.decode(chunk, { stream: true }));
+    }
+
+    expect(chunks).toEqual([
+      'id: 1\n',
+      'event: text\n',
+      `data: "Hello"\n\n`,
+      'id: 1\n',
+      'event: text\n',
+      `data: " world!"\n\n`,
+      'id: 1\n',
+      'event: stop\n',
+      `data: "stop"\n\n`,
+    ]);
+
+    expect(onStartMock).toHaveBeenCalledTimes(1);
+    expect(onTextMock).toHaveBeenNthCalledWith(1, '"Hello"');
+    expect(onTextMock).toHaveBeenNthCalledWith(2, '" world!"');
+    expect(onTokenMock).toHaveBeenCalledTimes(2);
+    expect(onCompletionMock).toHaveBeenCalledTimes(1);
+  });
+
+  it('should handle tool calls', async () => {
+    const mockOpenAIStream = new ReadableStream({
+      start(controller) {
+        controller.enqueue({
+          choices: [
+            {
+              delta: {
+                tool_calls: [
+                  {
+                    function: { name: 'tool1', arguments: '{}' },
+                    id: 'call_1',
+                    index: 0,
+                    type: 'function',
+                  },
+                  {
+                    function: { name: 'tool2', arguments: '{}' },
+                    id: 'call_2',
+                    index: 1,
+                  },
+                ],
+              },
+              index: 0,
+            },
+          ],
+          id: '2',
+        });
+
+        controller.close();
+      },
+    });
+
+    const onToolCallMock = vi.fn();
+
+    const protocolStream = QwenAIStream(mockOpenAIStream, {
+      onToolCall: onToolCallMock,
+    });
+
+    const decoder = new TextDecoder();
+    const chunks = [];
+
+    // @ts-ignore
+    for await (const chunk of protocolStream) {
+      chunks.push(decoder.decode(chunk, { stream: true }));
+    }
+
+    expect(chunks).toEqual([
+      'id: 2\n',
+      'event: tool_calls\n',
+      `data: [{"function":{"name":"tool1","arguments":"{}"},"id":"call_1","index":0,"type":"function"},{"function":{"name":"tool2","arguments":"{}"},"id":"call_2","index":1,"type":"function"}]\n\n`,
+    ]);
+
+    expect(onToolCallMock).toHaveBeenCalledTimes(1);
+  });
+
+  it('should handle empty stream', async () => {
+    const mockStream = new ReadableStream({
+      start(controller) {
+        controller.close();
+      },
+    });
+
+    const protocolStream = QwenAIStream(mockStream);
+
+    const decoder = new TextDecoder();
+    const chunks = [];
+
+    // @ts-ignore
+    for await (const chunk of protocolStream) {
+      chunks.push(decoder.decode(chunk, { stream: true }));
+    }
+
+    expect(chunks).toEqual([]);
+  });
+
+  it('should handle chunk with no choices', async () => {
+    const mockStream = new ReadableStream({
+      start(controller) {
+        controller.enqueue({
+          choices: [],
+          id: '1',
+        });
+
+        controller.close();
+      },
+    });
+
+    const protocolStream = QwenAIStream(mockStream);
+
+    const decoder = new TextDecoder();
+    const chunks = [];
+
+    // @ts-ignore
+    for await (const chunk of protocolStream) {
+      chunks.push(decoder.decode(chunk, { stream: true }));
+    }
+
+    expect(chunks).toEqual(['id: 1\n', 'event: data\n', 'data: {"choices":[],"id":"1"}\n\n']);
+  });
+
+  it('should handle vision model stream', async () => {
+    const mockStream = new ReadableStream({
+      start(controller) {
+        controller.enqueue({
+          choices: [
+            {
+              delta: {
+                content: [
+                  {
+                    text: '图中是一只小狗',
+                  },
+                ],
+              },
+            },
+          ],
+          id: '3',
+        });
+
+        /**
+         * Just for test against the description of 'output.choices[x].message.content' in [documents](https://help.aliyun.com/zh/dashscope/developer-reference/tongyi-qianwen-vl-plus-api)
+         * You're not likely to get image outputs from current versions of vl models.
+         */
+        controller.enqueue({
+          choices: [
+            {
+              delta: {
+                content: [
+                  {
+                    image: 'https://hello.mock/test.png',
+                  },
+                ],
+              },
+            },
+          ],
+          id: '3',
+        });
+        controller.close();
+      },
+    });
+
+    const protocolStream = QwenAIStream(mockStream);
+
+    const decoder = new TextDecoder();
+    const chunks = [];
+
+    // @ts-ignore
+    for await (const chunk of protocolStream) {
+      chunks.push(decoder.decode(chunk, { stream: true }));
+    }
+
+    expect(chunks).toEqual([
+      'id: 3\n',
+      'event: text\n',
+      'data: "图中是一只小狗"\n\n',
+      'id: 3\n',
+      'event: text\n',
+      'data: "![image](https://hello.mock/test.png)"\n\n',
+    ]);
+  });
+
+  it('should delta content null', async () => {
+    const mockOpenAIStream = new ReadableStream({
+      start(controller) {
+        controller.enqueue({
+          choices: [
+            {
+              delta: { content: null },
+              index: 0,
+            },
+          ],
+          id: '3',
+        });
+
+        controller.close();
+      },
+    });
+
+    const protocolStream = QwenAIStream(mockOpenAIStream);
+
+    const decoder = new TextDecoder();
+    const chunks = [];
+
+    // @ts-ignore
+    for await (const chunk of protocolStream) {
+      chunks.push(decoder.decode(chunk, { stream: true }));
+    }
+
+    expect(chunks).toEqual(['id: 3\n', 'event: data\n', `data: {"content":null}\n\n`]);
+  });
+
+  it('should handle other delta data', async () => {
+    const mockOpenAIStream = new ReadableStream({
+      start(controller) {
+        controller.enqueue({
+          choices: [
+            {
+              delta: { custom_field: 'custom_value' },
+              index: 0,
+            },
+          ],
+          id: '4',
+        });
+
+        controller.close();
+      },
+    });
+
+    const protocolStream = QwenAIStream(mockOpenAIStream);
+
+    const decoder = new TextDecoder();
+    const chunks = [];
+
+    // @ts-ignore
+    for await (const chunk of protocolStream) {
+      chunks.push(decoder.decode(chunk, { stream: true }));
+    }
+
+    expect(chunks).toEqual([
+      'id: 4\n',
+      'event: data\n',
+      `data: {"delta":{"custom_field":"custom_value"},"id":"4","index":0}\n\n`,
+    ]);
+  });
+
+  it('should handle tool calls without index and type', async () => {
+    const mockOpenAIStream = new ReadableStream({
+      start(controller) {
+        controller.enqueue({
+          choices: [
+            {
+              delta: {
+                tool_calls: [
+                  {
+                    function: { name: 'tool1', arguments: '{}' },
+                    id: 'call_1',
+                  },
+                  {
+                    function: { name: 'tool2', arguments: '{}' },
+                    id: 'call_2',
+                  },
+                ],
+              },
+              index: 0,
+            },
+          ],
+          id: '5',
+        });
+
+        controller.close();
+      },
+    });
+
+    const protocolStream = QwenAIStream(mockOpenAIStream);
+
+    const decoder = new TextDecoder();
+    const chunks = [];
+
+    // @ts-ignore
+    for await (const chunk of protocolStream) {
+      chunks.push(decoder.decode(chunk, { stream: true }));
+    }
+
+    expect(chunks).toEqual([
+      'id: 5\n',
+      'event: tool_calls\n',
+      `data: [{"function":{"name":"tool1","arguments":"{}"},"id":"call_1","index":0,"type":"function"},{"function":{"name":"tool2","arguments":"{}"},"id":"call_2","index":1,"type":"function"}]\n\n`,
+    ]);
+  });
+});

package/src/libs/agent-runtime/utils/streams/qwen.ts

@@ -0,0 +1,94 @@
+import { readableFromAsyncIterable } from 'ai';
+import { ChatCompletionContentPartText } from 'ai/prompts';
+import OpenAI from 'openai';
+import { ChatCompletionContentPart } from 'openai/resources/index.mjs';
+import type { Stream } from 'openai/streaming';
+
+import { ChatStreamCallbacks } from '../../types';
+import {
+  StreamProtocolChunk,
+  StreamProtocolToolCallChunk,
+  StreamToolCallChunkData,
+  chatStreamable,
+  createCallbacksTransformer,
+  createSSEProtocolTransformer,
+  generateToolCallId,
+} from './protocol';
+
+export const transformQwenStream = (chunk: OpenAI.ChatCompletionChunk): StreamProtocolChunk => {
+  const item = chunk.choices[0];
+
+  if (!item) {
+    return { data: chunk, id: chunk.id, type: 'data' };
+  }
+
+  if (Array.isArray(item.delta?.content)) {
+    const part = item.delta.content[0];
+    const process = (part: ChatCompletionContentPart): ChatCompletionContentPartText => {
+      let [key, value] = Object.entries(part)[0];
+      if (key === 'image') {
+        return {
+          text: `![image](${value})`,
+          type: 'text',
+        };
+      }
+      return {
+        text: value,
+        type: 'text',
+      };
+    };
+
+    const data = process(part);
+
+    return {
+      data: data.text,
+      id: chunk.id,
+      type: 'text',
+    };
+  }
+
+  if (item.delta?.tool_calls) {
+    return {
+      data: item.delta.tool_calls.map(
+        (value, index): StreamToolCallChunkData => ({
+          function: value.function,
+          id: value.id || generateToolCallId(index, value.function?.name),
+          index: typeof value.index !== 'undefined' ? value.index : index,
+          type: value.type || 'function',
+        }),
+      ),
+      id: chunk.id,
+      type: 'tool_calls',
+    } as StreamProtocolToolCallChunk;
+  }
+
+  if (item.finish_reason) {
+    return { data: item.finish_reason, id: chunk.id, type: 'stop' };
+  }
+
+  if (typeof item.delta?.content === 'string') {
+    return { data: item.delta.content, id: chunk.id, type: 'text' };
+  }
+
+  if (item.delta?.content === null) {
+    return { data: item.delta, id: chunk.id, type: 'data' };
+  }
+
+  return {
+    data: { delta: item.delta, id: chunk.id, index: item.index },
+    id: chunk.id,
+    type: 'data',
+  };
+};
+
+export const QwenAIStream = (
+  stream: Stream<OpenAI.ChatCompletionChunk> | ReadableStream,
+  callbacks?: ChatStreamCallbacks,
+) => {
+  const readableStream =
+    stream instanceof ReadableStream ? stream : readableFromAsyncIterable(chatStreamable(stream));
+
+  return readableStream
+    .pipeThrough(createSSEProtocolTransformer(transformQwenStream))
+    .pipeThrough(createCallbacksTransformer(callbacks));
+};

package/vitest.config.ts

@@ -24,9 +24,6 @@ export default defineConfig({
       reporter: ['text', 'json', 'lcov', 'text-summary'],
       reportsDirectory: './coverage/app',
     },
-    deps: {
-      inline: ['vitest-canvas-mock'],
-    },
     environment: 'happy-dom',
     exclude: [
       '**/node_modules/**',
@@ -36,6 +33,11 @@ export default defineConfig({
       'src/server/modules/**/**',
     ],
     globals: true,
+    server: {
+      deps: {
+        inline: ['vitest-canvas-mock'],
+      },
+    },
     setupFiles: './tests/setup.ts',
   },
 });