@lobehub/chat 1.54.0 → 1.55.1

package/src/libs/agent-runtime/AgentRuntime.ts

@@ -49,6 +49,7 @@ import {
   TextToSpeechPayload,
 } from './types';
 import { LobeUpstageAI } from './upstage';
+import { LobeVLLMAI } from './vllm';
 import { LobeWenxinAI } from './wenxin';
 import { LobeXAI } from './xai';
 import { LobeZeroOneAI } from './zeroone';
@@ -172,6 +173,7 @@ class AgentRuntime {
       tencentcloud: Partial<ClientOptions>;
       togetherai: Partial<ClientOptions>;
       upstage: Partial<ClientOptions>;
+      vllm: Partial<ClientOptions>;
       wenxin: Partial<ClientOptions>;
       xai: Partial<ClientOptions>;
       zeroone: Partial<ClientOptions>;
@@ -227,6 +229,11 @@ class AgentRuntime {
         break;
       }
 
+      case ModelProvider.VLLM: {
+        runtimeModel = new LobeVLLMAI(params.vllm);
+        break;
+      }
+
       case ModelProvider.Perplexity: {
         runtimeModel = new LobePerplexityAI(params.perplexity);
         break;

package/src/libs/agent-runtime/azureOpenai/index.test.ts

@@ -1,9 +1,9 @@
 // @vitest-environment node
-import { AzureKeyCredential, OpenAIClient } from '@azure/openai';
-import OpenAI from 'openai';
+import { AzureOpenAI } from 'openai';
 import { Mock, afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 
 import * as debugStreamModule from '../utils/debugStream';
+import * as openaiCompatibleFactoryModule from '../utils/openaiCompatibleFactory';
 import { LobeAzureOpenAI } from './index';
 
 const bizErrorType = 'ProviderBizError';
@@ -23,7 +23,7 @@ describe('LobeAzureOpenAI', () => {
     );
 
     // 使用 vi.spyOn 来模拟 streamChatCompletions 方法
-    vi.spyOn(instance['client'], 'streamChatCompletions').mockResolvedValue(
+    vi.spyOn(instance['client'].chat.completions, 'create').mockResolvedValue(
       new ReadableStream() as any,
     );
   });
@@ -48,7 +48,7 @@ describe('LobeAzureOpenAI', () => {
 
       const instance = new LobeAzureOpenAI(endpoint, apikey, apiVersion);
 
-      expect(instance.client).toBeInstanceOf(OpenAIClient);
+      expect(instance.client).toBeInstanceOf(AzureOpenAI);
       expect(instance.baseURL).toBe(endpoint);
     });
   });
@@ -59,7 +59,7 @@ describe('LobeAzureOpenAI', () => {
       const mockStream = new ReadableStream();
       const mockResponse = Promise.resolve(mockStream);
 
-      (instance['client'].streamChatCompletions as Mock).mockResolvedValue(mockResponse);
+      (instance['client'].chat.completions.create as Mock).mockResolvedValue(mockResponse);
 
       // Act
       const result = await instance.chat({
@@ -164,7 +164,9 @@ describe('LobeAzureOpenAI', () => {
             controller.close();
           },
         });
-        vi.spyOn(instance['client'], 'streamChatCompletions').mockResolvedValue(mockStream as any);
+        vi.spyOn(instance['client'].chat.completions, 'create').mockResolvedValue(
+          mockStream as any,
+        );
 
         const result = await instance.chat({
           stream: true,
@@ -204,6 +206,42 @@ describe('LobeAzureOpenAI', () => {
           ].map((item) => `${item}\n`),
         );
       });
+
+      it('should handle non-streaming response', async () => {
+        vi.spyOn(openaiCompatibleFactoryModule, 'transformResponseToStream').mockImplementation(
+          () => {
+            return new ReadableStream();
+          },
+        );
+        // Act
+        await instance.chat({
+          stream: false,
+          temperature: 0.6,
+          model: 'gpt-35-turbo-16k',
+          messages: [{ role: 'user', content: '你好' }],
+        });
+
+        // Assert
+        expect(openaiCompatibleFactoryModule.transformResponseToStream).toHaveBeenCalled();
+      });
+    });
+
+    it('should handle o1 series models without streaming', async () => {
+      vi.spyOn(openaiCompatibleFactoryModule, 'transformResponseToStream').mockImplementation(
+        () => {
+          return new ReadableStream();
+        },
+      );
+
+      // Act
+      await instance.chat({
+        temperature: 0.6,
+        model: 'o1-preview',
+        messages: [{ role: 'user', content: '你好' }],
+      });
+
+      // Assert
+      expect(openaiCompatibleFactoryModule.transformResponseToStream).toHaveBeenCalled();
     });
 
     describe('Error', () => {
@@ -214,7 +252,7 @@ describe('LobeAzureOpenAI', () => {
           message: 'Deployment not found',
         };
 
-        (instance['client'].streamChatCompletions as Mock).mockRejectedValue(error);
+        (instance['client'].chat.completions.create as Mock).mockRejectedValue(error);
 
         // Act
         try {
@@ -242,7 +280,7 @@ describe('LobeAzureOpenAI', () => {
         // Arrange
         const genericError = new Error('Generic Error');
 
-        (instance['client'].streamChatCompletions as Mock).mockRejectedValue(genericError);
+        (instance['client'].chat.completions.create as Mock).mockRejectedValue(genericError);
 
         // Act
         try {
@@ -279,7 +317,7 @@ describe('LobeAzureOpenAI', () => {
         }) as any;
         mockDebugStream.toReadableStream = () => mockDebugStream;
 
-        (instance['client'].streamChatCompletions as Mock).mockResolvedValue({
+        (instance['client'].chat.completions.create as Mock).mockResolvedValue({
           tee: () => [mockProdStream, { toReadableStream: () => mockDebugStream }],
         });
 

package/src/libs/agent-runtime/azureOpenai/index.ts

@@ -1,26 +1,28 @@
-import {
-  AzureKeyCredential,
-  ChatRequestMessage,
-  GetChatCompletionsOptions,
-  OpenAIClient,
-} from '@azure/openai';
+import OpenAI, { AzureOpenAI } from 'openai';
+import type { Stream } from 'openai/streaming';
 
 import { LobeRuntimeAI } from '../BaseAI';
 import { AgentRuntimeErrorType } from '../error';
 import { ChatCompetitionOptions, ChatStreamPayload, ModelProvider } from '../types';
 import { AgentRuntimeError } from '../utils/createError';
 import { debugStream } from '../utils/debugStream';
+import { transformResponseToStream } from '../utils/openaiCompatibleFactory';
 import { StreamingResponse } from '../utils/response';
-import { AzureOpenAIStream } from '../utils/streams';
+import { OpenAIStream } from '../utils/streams';
 
 export class LobeAzureOpenAI implements LobeRuntimeAI {
-  client: OpenAIClient;
+  client: AzureOpenAI;
 
   constructor(endpoint?: string, apikey?: string, apiVersion?: string) {
     if (!apikey || !endpoint)
       throw AgentRuntimeError.createError(AgentRuntimeErrorType.InvalidProviderAPIKey);
 
-    this.client = new OpenAIClient(endpoint, new AzureKeyCredential(apikey), { apiVersion });
+    this.client = new AzureOpenAI({
+      apiKey: apikey,
+      apiVersion,
+      dangerouslyAllowBrowser: true,
+      endpoint,
+    });
 
     this.baseURL = endpoint;
   }
@@ -28,28 +30,33 @@ export class LobeAzureOpenAI implements LobeRuntimeAI {
   baseURL: string;
 
   async chat(payload: ChatStreamPayload, options?: ChatCompetitionOptions) {
-    // ============  1. preprocess messages   ============ //
-    const camelCasePayload = this.camelCaseKeys(payload);
-    const { messages, model, maxTokens = 2048, ...params } = camelCasePayload;
-
-    // ============  2. send api   ============ //
-
+    const { messages, model, ...params } = payload;
+    // o1 series models on Azure OpenAI does not support streaming currently
+    const enableStreaming = model.startsWith('o1') ? false : (params.stream ?? true);
     try {
-      const response = await this.client.streamChatCompletions(
+      const response = await this.client.chat.completions.create({
+        messages: messages as OpenAI.ChatCompletionMessageParam[],
         model,
-        messages as ChatRequestMessage[],
-        { ...params, abortSignal: options?.signal, maxTokens } as GetChatCompletionsOptions,
-      );
-
-      const [debug, prod] = response.tee();
-
-      if (process.env.DEBUG_AZURE_CHAT_COMPLETION === '1') {
-        debugStream(debug).catch(console.error);
-      }
-
-      return StreamingResponse(AzureOpenAIStream(prod, options?.callback), {
-        headers: options?.headers,
+        ...params,
+        max_completion_tokens: 2048,
+        stream: enableStreaming,
+        tool_choice: params.tools ? 'auto' : undefined,
       });
+      if (enableStreaming) {
+        const stream = response as Stream<OpenAI.ChatCompletionChunk>;
+        const [prod, debug] = stream.tee();
+        if (process.env.DEBUG_AZURE_CHAT_COMPLETION === '1') {
+          debugStream(debug.toReadableStream()).catch(console.error);
+        }
+        return StreamingResponse(OpenAIStream(prod, { callbacks: options?.callback }), {
+          headers: options?.headers,
+        });
+      } else {
+        const stream = transformResponseToStream(response as OpenAI.ChatCompletion);
+        return StreamingResponse(OpenAIStream(stream, { callbacks: options?.callback }), {
+          headers: options?.headers,
+        });
+      }
     } catch (e) {
       let error = e as { [key: string]: any; code: string; message: string };
 

package/src/libs/agent-runtime/types/type.ts

@@ -59,6 +59,7 @@ export enum ModelProvider {
   TencentCloud = 'tencentcloud',
   TogetherAI = 'togetherai',
   Upstage = 'upstage',
+  VLLM = 'vllm',
   Wenxin = 'wenxin',
   XAI = 'xai',
   ZeroOne = 'zeroone',

package/src/libs/agent-runtime/utils/streams/index.ts

@@ -1,5 +1,4 @@
 export * from './anthropic';
-export * from './azureOpenai';
 export * from './bedrock';
 export * from './google-ai';
 export * from './ollama';

package/src/libs/agent-runtime/vllm/index.ts

@@ -0,0 +1,44 @@
+import { ModelProvider } from '../types';
+import { LobeOpenAICompatibleFactory } from '../utils/openaiCompatibleFactory';
+
+import type { ChatModelCard } from '@/types/llm';
+
+export interface VLLMModelCard {
+  id: string;
+}
+
+export const LobeVLLMAI = LobeOpenAICompatibleFactory({
+  baseURL: 'http://localhost:8000/v1',
+  debug: {
+    chatCompletion: () => process.env.DEBUG_VLLM_CHAT_COMPLETION === '1',
+  },
+  models: async ({ client }) => {
+    const { LOBE_DEFAULT_MODEL_LIST } = await import('@/config/aiModels');
+
+    const modelsPage = await client.models.list() as any;
+    const modelList: VLLMModelCard[] = modelsPage.data;
+
+    return modelList
+      .map((model) => {
+        const knownModel = LOBE_DEFAULT_MODEL_LIST.find((m) => model.id.toLowerCase() === m.id.toLowerCase());
+
+        return {
+          contextWindowTokens: knownModel?.contextWindowTokens ?? undefined,
+          displayName: knownModel?.displayName ?? undefined,
+          enabled: knownModel?.enabled || false,
+          functionCall:
+            knownModel?.abilities?.functionCall
+            || false,
+          id: model.id,
+          reasoning:
+            knownModel?.abilities?.reasoning
+            || false,
+          vision:
+            knownModel?.abilities?.vision
+            || false,
+        };
+      })
+      .filter(Boolean) as ChatModelCard[];
+  },
+  provider: ModelProvider.VLLM,
+});

package/src/server/modules/AgentRuntime/index.test.ts

@@ -223,7 +223,9 @@ describe('initAgentRuntimeWithUserPayload method', () => {
     });
 
     it('Azure AI Provider: without apikey', async () => {
-      const jwtPayload: JWTPayload = {};
+      const jwtPayload: JWTPayload = {
+        azureApiVersion: 'test-azure-api-version',
+      };
       const runtime = await initAgentRuntimeWithUserPayload(ModelProvider.Azure, jwtPayload);
 
       expect(runtime['_runtime']).toBeInstanceOf(LobeAzureOpenAI);

package/src/server/routers/lambda/aiModel.test.ts

@@ -0,0 +1,240 @@
+import { describe, expect, it, vi } from 'vitest';
+
+import { AiInfraRepos } from '@/database/repositories/aiInfra';
+import { AiModelModel } from '@/database/server/models/aiModel';
+import { UserModel } from '@/database/server/models/user';
+
+import { aiModelRouter } from './aiModel';
+
+vi.mock('@/database/server/models/aiModel');
+vi.mock('@/database/server/models/user');
+vi.mock('@/database/repositories/aiInfra');
+vi.mock('@/server/globalConfig', () => ({
+  getServerGlobalConfig: vi.fn().mockReturnValue({
+    aiProvider: {},
+  }),
+}));
+vi.mock('@/server/modules/KeyVaultsEncrypt', () => ({
+  KeyVaultsGateKeeper: {
+    initWithEnvKey: vi.fn().mockResolvedValue({
+      encrypt: vi.fn(),
+      decrypt: vi.fn(),
+    }),
+  },
+}));
+
+describe('aiModelRouter', () => {
+  const mockCtx = {
+    userId: 'test-user',
+  };
+
+  it('should create ai model', async () => {
+    const mockCreate = vi.fn().mockResolvedValue({ id: 'model-1' });
+    vi.mocked(AiModelModel).mockImplementation(
+      () =>
+        ({
+          create: mockCreate,
+        }) as any,
+    );
+
+    const caller = aiModelRouter.createCaller(mockCtx);
+
+    const result = await caller.createAiModel({
+      id: 'test-model',
+      providerId: 'test-provider',
+    });
+
+    expect(result).toBe('model-1');
+    expect(mockCreate).toHaveBeenCalledWith({
+      id: 'test-model',
+      providerId: 'test-provider',
+    });
+  });
+
+  it('should get ai model by id', async () => {
+    const mockModel = {
+      id: 'model-1',
+      name: 'Test Model',
+    };
+    const mockFindById = vi.fn().mockResolvedValue(mockModel);
+    vi.mocked(AiModelModel).mockImplementation(
+      () =>
+        ({
+          findById: mockFindById,
+        }) as any,
+    );
+
+    const caller = aiModelRouter.createCaller(mockCtx);
+
+    const result = await caller.getAiModelById({ id: 'model-1' });
+
+    expect(result).toEqual(mockModel);
+    expect(mockFindById).toHaveBeenCalledWith('model-1');
+  });
+
+  it('should get ai provider model list', async () => {
+    const mockModelList = [
+      { id: 'model-1', name: 'Model 1' },
+      { id: 'model-2', name: 'Model 2' },
+    ];
+    const mockGetList = vi.fn().mockResolvedValue(mockModelList);
+    vi.mocked(AiInfraRepos).mockImplementation(
+      () =>
+        ({
+          getAiProviderModelList: mockGetList,
+        }) as any,
+    );
+
+    const caller = aiModelRouter.createCaller(mockCtx);
+
+    const result = await caller.getAiProviderModelList({ id: 'provider-1' });
+
+    expect(result).toEqual(mockModelList);
+    expect(mockGetList).toHaveBeenCalledWith('provider-1');
+  });
+
+  it('should remove ai model', async () => {
+    const mockDelete = vi.fn().mockResolvedValue(true);
+    vi.mocked(AiModelModel).mockImplementation(
+      () =>
+        ({
+          delete: mockDelete,
+        }) as any,
+    );
+
+    const caller = aiModelRouter.createCaller(mockCtx);
+
+    await caller.removeAiModel({
+      id: 'model-1',
+      providerId: 'provider-1',
+    });
+
+    expect(mockDelete).toHaveBeenCalledWith('model-1', 'provider-1');
+  });
+
+  it('should update ai model', async () => {
+    const mockUpdate = vi.fn().mockResolvedValue(true);
+    vi.mocked(AiModelModel).mockImplementation(
+      () =>
+        ({
+          update: mockUpdate,
+        }) as any,
+    );
+
+    const caller = aiModelRouter.createCaller(mockCtx);
+
+    await caller.updateAiModel({
+      id: 'model-1',
+      providerId: 'provider-1',
+      value: {
+        displayName: 'Updated Model',
+      },
+    });
+
+    expect(mockUpdate).toHaveBeenCalledWith('model-1', 'provider-1', {
+      displayName: 'Updated Model',
+    });
+  });
+
+  it('should toggle model enabled status', async () => {
+    const mockToggle = vi.fn().mockResolvedValue(true);
+    vi.mocked(AiModelModel).mockImplementation(
+      () =>
+        ({
+          toggleModelEnabled: mockToggle,
+        }) as any,
+    );
+
+    const caller = aiModelRouter.createCaller(mockCtx);
+
+    await caller.toggleModelEnabled({
+      id: 'model-1',
+      providerId: 'provider-1',
+      enabled: true,
+    });
+
+    expect(mockToggle).toHaveBeenCalledWith({
+      id: 'model-1',
+      providerId: 'provider-1',
+      enabled: true,
+    });
+  });
+
+  it('should batch toggle ai models', async () => {
+    const mockBatchToggle = vi.fn().mockResolvedValue(true);
+    vi.mocked(AiModelModel).mockImplementation(
+      () =>
+        ({
+          batchToggleAiModels: mockBatchToggle,
+        }) as any,
+    );
+
+    const caller = aiModelRouter.createCaller(mockCtx);
+
+    await caller.batchToggleAiModels({
+      id: 'provider-1',
+      models: ['model-1', 'model-2'],
+      enabled: true,
+    });
+
+    expect(mockBatchToggle).toHaveBeenCalledWith('provider-1', ['model-1', 'model-2'], true);
+  });
+
+  it('should batch update ai models', async () => {
+    const mockBatchUpdate = vi.fn().mockResolvedValue([]);
+    vi.mocked(AiModelModel).mockImplementation(
+      () =>
+        ({
+          batchUpdateAiModels: mockBatchUpdate,
+        }) as any,
+    );
+
+    const caller = aiModelRouter.createCaller(mockCtx);
+
+    await caller.batchUpdateAiModels({
+      id: 'provider-1',
+      models: [{ id: 'model-1' }, { id: 'model-2' }],
+    });
+
+    expect(mockBatchUpdate).toHaveBeenCalledWith('provider-1', [
+      { id: 'model-1' },
+      { id: 'model-2' },
+    ]);
+  });
+
+  it('should clear models by provider', async () => {
+    const mockClear = vi.fn().mockResolvedValue(true);
+    vi.mocked(AiModelModel).mockImplementation(
+      () =>
+        ({
+          clearModelsByProvider: mockClear,
+        }) as any,
+    );
+
+    const caller = aiModelRouter.createCaller(mockCtx);
+
+    await caller.clearModelsByProvider({
+      providerId: 'provider-1',
+    });
+
+    expect(mockClear).toHaveBeenCalledWith('provider-1');
+  });
+
+  it('should clear remote models', async () => {
+    const mockClearRemote = vi.fn().mockResolvedValue(true);
+    vi.mocked(AiModelModel).mockImplementation(
+      () =>
+        ({
+          clearRemoteModels: mockClearRemote,
+        }) as any,
+    );
+
+    const caller = aiModelRouter.createCaller(mockCtx);
+
+    await caller.clearRemoteModels({
+      providerId: 'provider-1',
+    });
+
+    expect(mockClearRemote).toHaveBeenCalledWith('provider-1');
+  });
+});