@lobehub/chat 1.21.16 → 1.22.1

package/locales/zh-TW/models.json

@@ -458,6 +458,9 @@
   "google/gemma-2-27b-it": {
     "description": "Gemma 2 延續了輕量化與高效的設計理念。"
   },
+  "google/gemma-2-2b-it": {
+    "description": "Google的輕量級指令調優模型"
+  },
   "google/gemma-2-9b-it": {
     "description": "Gemma 2 是 Google 輕量化的開源文本模型系列。"
   },
@@ -589,6 +592,12 @@
   "llama-3.1-sonar-small-128k-online": {
     "description": "Llama 3.1 Sonar Small Online 模型，具備 8B 參數，支持約 127,000 個標記的上下文長度，專為在線聊天設計，能高效處理各種文本交互。"
   },
+  "llama-3.2-11b-vision-preview": {
+    "description": "Llama 3.2 旨在處理結合視覺和文本數據的任務。它在圖像描述和視覺問答等任務中表現出色，跨越了語言生成和視覺推理之間的鴻溝。"
+  },
+  "llama-3.2-90b-vision-preview": {
+    "description": "Llama 3.2 旨在處理結合視覺和文本數據的任務。它在圖像描述和視覺問答等任務中表現出色，跨越了語言生成和視覺推理之間的鴻溝。"
+  },
   "llama3-70b-8192": {
     "description": "Meta Llama 3 70B 提供無與倫比的複雜性處理能力，為高要求項目量身定制。"
   },
@@ -643,6 +652,9 @@
   "meta-llama/Llama-2-13b-chat-hf": {
     "description": "LLaMA-2 Chat (13B) 提供優秀的語言處理能力和出色的互動體驗。"
   },
+  "meta-llama/Llama-2-7b-chat-hf": {
+    "description": "最佳對話模型之一"
+  },
   "meta-llama/Llama-3-70b-chat-hf": {
     "description": "LLaMA-3 Chat (70B) 是功能強大的聊天模型，支持複雜的對話需求。"
   },
@@ -811,7 +823,7 @@
   "open-mixtral-8x7b": {
     "description": "Mixtral 8x7B 是一個稀疏專家模型，利用多個參數提高推理速度，適合處理多語言和代碼生成任務。"
   },
-  "openai/gpt-4o-2024-08-06": {
+  "openai/gpt-4o": {
     "description": "ChatGPT-4o 是一款動態模型，實時更新以保持當前最新版本。它結合了強大的語言理解與生成能力，適合於大規模應用場景，包括客戶服務、教育和技術支持。"
   },
   "openai/gpt-4o-mini": {
@@ -862,11 +874,11 @@
   "qwen-vl-chat-v1": {
     "description": "通義千問VL支持靈活的交互方式，包括多圖、多輪問答、創作等能力的模型。"
   },
-  "qwen-vl-max": {
+  "qwen-vl-max-latest": {
     "description": "通義千問超大規模視覺語言模型。相比增強版，再次提升視覺推理能力和指令遵循能力，提供更高的視覺感知和認知水平。"
   },
-  "qwen-vl-plus": {
-    "description": "通義千問大規模視覺語言模型增強版。大幅提升細節識別能力和文字識別能力，支持超百萬像素分辨率和任意長寬比規格的圖像。"
+  "qwen-vl-plus-latest": {
+    "description": "通義千問大規模視覺語言模型增強版。大幅提升細節識別能力和文字識別能力，支持超百萬像素解析度和任意長寬比規格的圖像。"
   },
   "qwen-vl-v1": {
     "description": "以Qwen-7B語言模型初始化，添加圖像模型，圖像輸入分辨率為448的預訓練模型。"

package/locales/zh-TW/providers.json

@@ -30,6 +30,9 @@
   "groq": {
     "description": "Groq 的 LPU 推理引擎在最新的獨立大語言模型（LLM）基準測試中表現卓越，以其驚人的速度和效率重新定義了 AI 解決方案的標準。Groq 是一種即時推理速度的代表，在基於雲的部署中展現了良好的性能。"
   },
+  "huggingface": {
+    "description": "HuggingFace Inference API 提供了一種快速且免費的方式，讓您可以探索成千上萬種模型，適用於各種任務。無論您是在為新應用程式進行原型設計，還是在嘗試機器學習的功能，這個 API 都能讓您即時訪問多個領域的高性能模型。"
+  },
   "hunyuan": {
     "description": "由騰訊研發的大語言模型，具備強大的中文創作能力、複雜語境下的邏輯推理能力，以及可靠的任務執行能力"
   },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lobehub/chat",
-  "version": "1.21.16",
+  "version": "1.22.1",
   "description": "Lobe Chat - an open-source, high-performance chatbot framework that supports speech synthesis, multimodal, and extensible Function Call plugin system. Supports one-click free deployment of your private ChatGPT/LLM web application.",
   "keywords": [
     "framework",
@@ -116,12 +116,13 @@
     "@codesandbox/sandpack-react": "^2.19.8",
     "@cyntler/react-doc-viewer": "^1.16.6",
     "@google/generative-ai": "^0.16.0",
+    "@huggingface/inference": "^2.8.1",
     "@icons-pack/react-simple-icons": "9.6.0",
     "@khmyznikov/pwa-install": "^0.3.9",
     "@langchain/community": "^0.2.31",
     "@lobehub/chat-plugin-sdk": "^1.32.4",
     "@lobehub/chat-plugins-gateway": "^1.9.0",
-    "@lobehub/icons": "^1.34.2",
+    "@lobehub/icons": "^1.35.3",
     "@lobehub/tts": "^1.24.3",
     "@lobehub/ui": "^1.150.3",
     "@neondatabase/serverless": "^0.9.4",
@@ -163,7 +164,7 @@
     "jose": "^5.7.0",
     "js-sha256": "^0.11.0",
     "jsonl-parse-stringify": "^1.0.3",
-    "langchain": "^0.2.17",
+    "langchain": "^0.3.0",
     "langfuse": "^3.19.0",
     "langfuse-core": "^3.19.0",
     "lodash-es": "^4.17.21",
@@ -235,6 +236,7 @@
   "devDependencies": {
     "@commitlint/cli": "^19.4.0",
     "@edge-runtime/vm": "^4.0.2",
+    "@huggingface/tasks": "^0.12.12",
     "@lobehub/i18n-cli": "^1.19.1",
     "@lobehub/lint": "^1.24.4",
     "@lobehub/seo-cli": "^1.4.2",

package/src/app/(main)/settings/llm/ProviderList/HuggingFace/index.tsx

@@ -0,0 +1,53 @@
+'use client';
+
+import { Markdown } from '@lobehub/ui';
+import { Input } from 'antd';
+import { createStyles } from 'antd-style';
+import { useTranslation } from 'react-i18next';
+
+import { HuggingFaceProviderCard } from '@/config/modelProviders';
+import { GlobalLLMProviderKey } from '@/types/user/settings';
+
+import { KeyVaultsConfigKey, LLMProviderApiTokenKey } from '../../const';
+import { ProviderItem } from '../../type';
+
+const useStyles = createStyles(({ css, token }) => ({
+  markdown: css`
+    p {
+      color: ${token.colorTextDescription} !important;
+    }
+  `,
+  tip: css`
+    font-size: 12px;
+    color: ${token.colorTextDescription};
+  `,
+}));
+
+const providerKey: GlobalLLMProviderKey = 'huggingface';
+
+// Same as OpenAIProvider, but replace API Key with HuggingFace Access Token
+export const useHuggingFaceProvider = (): ProviderItem => {
+  const { t } = useTranslation('modelProvider');
+  const { styles } = useStyles();
+
+  return {
+    ...HuggingFaceProviderCard,
+    apiKeyItems: [
+      {
+        children: (
+          <Input.Password
+            autoComplete={'new-password'}
+            placeholder={t(`${providerKey}.accessToken.placeholder`)}
+          />
+        ),
+        desc: (
+          <Markdown className={styles.markdown} fontSize={12} variant={'chat'}>
+            {t(`${providerKey}.accessToken.desc`)}
+          </Markdown>
+        ),
+        label: t(`${providerKey}.accessToken.title`),
+        name: [KeyVaultsConfigKey, providerKey, LLMProviderApiTokenKey],
+      },
+    ],
+  };
+};

package/src/app/(main)/settings/llm/ProviderList/providers.tsx

@@ -31,6 +31,7 @@ import { ProviderItem } from '../type';
 import { useAzureProvider } from './Azure';
 import { useBedrockProvider } from './Bedrock';
 import { useGithubProvider } from './Github';
+import { useHuggingFaceProvider } from './HuggingFace';
 import { useOllamaProvider } from './Ollama';
 import { useOpenAIProvider } from './OpenAI';
 import { useWenxinProvider } from './Wenxin';
@@ -41,6 +42,7 @@ export const useProviderList = (): ProviderItem[] => {
   const OpenAIProvider = useOpenAIProvider();
   const BedrockProvider = useBedrockProvider();
   const GithubProvider = useGithubProvider();
+  const HuggingFaceProvider = useHuggingFaceProvider();
   const WenxinProvider = useWenxinProvider();
 
   return useMemo(
@@ -52,6 +54,7 @@ export const useProviderList = (): ProviderItem[] => {
       BedrockProvider,
       GoogleProviderCard,
       DeepSeekProviderCard,
+      HuggingFaceProvider,
       OpenRouterProviderCard,
       GithubProvider,
       NovitaProviderCard,
@@ -76,6 +79,14 @@ export const useProviderList = (): ProviderItem[] => {
       TaichuProviderCard,
       SiliconCloudProviderCard,
     ],
-    [AzureProvider, OllamaProvider, OpenAIProvider, BedrockProvider, GithubProvider,WenxinProvider],
+    [
+      AzureProvider,
+      OllamaProvider,
+      OpenAIProvider,
+      BedrockProvider,
+      GithubProvider,
+      WenxinProvider,
+      HuggingFaceProvider,
+    ],
   );
 };

package/src/config/llm.ts

@@ -125,6 +125,11 @@ export const getLLMConfig = () => {
       ENABLED_HUNYUAN: z.boolean(),
       HUNYUAN_API_KEY: z.string().optional(),
       HUNYUAN_MODEL_LIST: z.string().optional(),
+
+      ENABLED_HUGGINGFACE: z.boolean(),
+      HUGGINGFACE_API_KEY: z.string().optional(),
+      HUGGINGFACE_PROXY_URL: z.string().optional(),
+      HUGGINGFACE_MODEL_LIST: z.string().optional(),
     },
     runtimeEnv: {
       API_KEY_SELECT_MODE: process.env.API_KEY_SELECT_MODE,
@@ -247,6 +252,11 @@ export const getLLMConfig = () => {
       ENABLED_HUNYUAN: !!process.env.HUNYUAN_API_KEY,
       HUNYUAN_API_KEY: process.env.HUNYUAN_API_KEY,
       HUNYUAN_MODEL_LIST: process.env.HUNYUAN_MODEL_LIST,
+
+      ENABLED_HUGGINGFACE: !!process.env.HUGGINGFACE_API_KEY,
+      HUGGINGFACE_API_KEY: process.env.HUGGINGFACE_API_KEY,
+      HUGGINGFACE_PROXY_URL: process.env.HUGGINGFACE_PROXY_URL,
+      HUGGINGFACE_MODEL_LIST: process.env.HUGGINGFACE_MODEL_LIST,
     },
   });
 };

package/src/config/modelProviders/huggingface.ts

@@ -0,0 +1,50 @@
+import { ModelProviderCard } from '@/types/llm';
+
+const HuggingFace: ModelProviderCard = {
+  chatModels: [
+    {
+      description: 'Mistral AI的指令调优模型',
+      displayName: 'Mistral 7B Instruct v0.2',
+      enabled: true,
+      id: 'mistralai/Mistral-7B-Instruct-v0.2',
+      tokens: 8192,
+    },
+    {
+      description: '高质量多语言聊天模型,具有大型上下文长度',
+      displayName: 'Meta Llama 3.1 70B Instruct',
+      enabled: true,
+      id: 'meta-llama/Meta-Llama-3.1-70B-Instruct',
+      tokens: 32_768,
+    },
+    {
+      description: '高质量多语言聊天模型,具有较大上下文长度',
+      displayName: 'Meta Llama 3.1 8B Instruct',
+      enabled: true,
+      id: 'meta-llama/Meta-Llama-3.1-8B-Instruct',
+      tokens: 8192,
+    },
+    {
+      description: '最佳对话模型之一',
+      displayName: 'Llama 2 7B Chat',
+      id: 'meta-llama/Llama-2-7b-chat-hf',
+      tokens: 4096,
+    },
+    {
+      description: 'Google的轻量级指令调优模型',
+      displayName: 'Gemma 2B Instruct',
+      id: 'google/gemma-2-2b-it',
+      tokens: 8192,
+    },
+  ],
+  checkModel: 'mistralai/Mistral-7B-Instruct-v0.2',
+  description:
+    'HuggingFace Inference API 提供了一种快速且免费的方式，让您可以探索成千上万种模型，适用于各种任务。无论您是在为新应用程序进行原型设计，还是在尝试机器学习的功能，这个 API 都能让您即时访问多个领域的高性能模型。',
+  disableBrowserRequest: true,
+  id: 'huggingface',
+  modelList: { showModelFetcher: true },
+  modelsUrl: 'https://huggingface.co/docs/api-inference/en/supported-models',
+  name: 'HuggingFace',
+  url: 'https://huggingface.co',
+};
+
+export default HuggingFace;

package/src/config/modelProviders/index.ts

@@ -11,6 +11,7 @@ import FireworksAIProvider from './fireworksai';
 import GithubProvider from './github';
 import GoogleProvider from './google';
 import GroqProvider from './groq';
+import HuggingFaceProvider from './huggingface';
 import HunyuanProvider from './hunyuan';
 import MinimaxProvider from './minimax';
 import MistralProvider from './mistral';
@@ -49,6 +50,7 @@ export const LOBE_DEFAULT_MODEL_LIST: ChatModelCard[] = [
   FireworksAIProvider.chatModels,
   PerplexityProvider.chatModels,
   AnthropicProvider.chatModels,
+  HuggingFaceProvider.chatModels,
   ZeroOneProvider.chatModels,
   StepfunProvider.chatModels,
   NovitaProvider.chatModels,
@@ -71,6 +73,7 @@ export const DEFAULT_MODEL_PROVIDER_LIST = [
   BedrockProvider,
   GoogleProvider,
   DeepSeekProvider,
+  HuggingFaceProvider,
   OpenRouterProvider,
   GithubProvider,
   NovitaProvider,
@@ -116,6 +119,7 @@ export { default as FireworksAIProviderCard } from './fireworksai';
 export { default as GithubProviderCard } from './github';
 export { default as GoogleProviderCard } from './google';
 export { default as GroqProviderCard } from './groq';
+export { default as HuggingFaceProviderCard } from './huggingface';
 export { default as HunyuanProviderCard } from './hunyuan';
 export { default as MinimaxProviderCard } from './minimax';
 export { default as MistralProviderCard } from './mistral';

package/src/const/settings/llm.ts

@@ -9,6 +9,7 @@ import {
   GithubProviderCard,
   GoogleProviderCard,
   GroqProviderCard,
+  HuggingFaceProviderCard,
   HunyuanProviderCard,
   MinimaxProviderCard,
   MistralProviderCard,
@@ -77,6 +78,10 @@ export const DEFAULT_LLM_CONFIG: UserModelProviderConfig = {
     enabled: false,
     enabledModels: filterEnabledModels(GroqProviderCard),
   },
+  huggingface: {
+    enabled: false,
+    enabledModels: filterEnabledModels(HuggingFaceProviderCard),
+  },
   hunyuan: {
     enabled: false,
     enabledModels: filterEnabledModels(HunyuanProviderCard),

package/src/features/Conversation/Error/index.tsx

@@ -34,6 +34,7 @@ const getErrorAlertConfig = (
     };
 
   switch (errorType) {
+    case AgentRuntimeErrorType.PermissionDenied:
     case AgentRuntimeErrorType.QuotaLimitReached:
     case AgentRuntimeErrorType.LocationNotSupportError: {
       return {

package/src/libs/agent-runtime/AgentRuntime.ts

@@ -14,6 +14,7 @@ import { LobeFireworksAI } from './fireworksai';
 import { LobeGithubAI } from './github';
 import { LobeGoogleAI } from './google';
 import { LobeGroq } from './groq';
+import { LobeHuggingFaceAI } from './huggingface';
 import { LobeHunyuanAI } from './hunyuan';
 import { LobeMinimaxAI } from './minimax';
 import { LobeMistralAI } from './mistral';
@@ -134,6 +135,7 @@ class AgentRuntime {
       github: Partial<ClientOptions>;
       google: { apiKey?: string; baseURL?: string };
       groq: Partial<ClientOptions>;
+      huggingface: { apiKey?: string; baseURL?: string };
       hunyuan: Partial<ClientOptions>;
       minimax: Partial<ClientOptions>;
       mistral: Partial<ClientOptions>;
@@ -213,6 +215,11 @@ class AgentRuntime {
         break;
       }
 
+      case ModelProvider.HuggingFace: {
+        runtimeModel = new LobeHuggingFaceAI(params.huggingface);
+        break;
+      }
+
       case ModelProvider.Minimax: {
         runtimeModel = new LobeMinimaxAI(params.minimax);
         break;

package/src/libs/agent-runtime/error.ts

@@ -4,6 +4,7 @@ export const AgentRuntimeErrorType = {
   AgentRuntimeError: 'AgentRuntimeError', // Agent Runtime 模块运行时错误
   LocationNotSupportError: 'LocationNotSupportError',
   QuotaLimitReached: 'QuotaLimitReached',
+  PermissionDenied: 'PermissionDenied',
 
   InvalidProviderAPIKey: 'InvalidProviderAPIKey',
   ProviderBizError: 'ProviderBizError',

package/src/libs/agent-runtime/groq/index.ts

@@ -6,7 +6,7 @@ export const LobeGroq = LobeOpenAICompatibleFactory({
   baseURL: 'https://api.groq.com/openai/v1',
   chatCompletion: {
     handleError: (error) => {
-      // 403 means the location is not supporteds
+      // 403 means the location is not supported
       if (error.status === 403)
         return { error, errorType: AgentRuntimeErrorType.LocationNotSupportError };
     },

package/src/libs/agent-runtime/huggingface/index.ts

@@ -0,0 +1,48 @@
+import { HfInference } from '@huggingface/inference';
+
+import { AgentRuntimeErrorType } from '../error';
+import { ModelProvider } from '../types';
+import { LobeOpenAICompatibleFactory } from '../utils/openaiCompatibleFactory';
+import { convertIterableToStream } from '../utils/streams';
+
+export const LobeHuggingFaceAI = LobeOpenAICompatibleFactory({
+  chatCompletion: {
+    handleStreamBizErrorType: (error) => {
+      // e.g.: Server meta-llama/Meta-Llama-3.1-8B-Instruct does not seem to support chat completion. Error: Model requires a Pro subscription; check out hf.co/pricing to learn more. Make sure to include your HF token in your query.
+      if (error.message?.includes('Model requires a Pro subscription')) {
+        return AgentRuntimeErrorType.PermissionDenied;
+      }
+
+      // e.g.: Server meta-llama/Meta-Llama-3.1-8B-Instruct does not seem to support chat completion. Error: Authorization header is correct, but the token seems invalid
+      if (error.message?.includes('the token seems invalid')) {
+        return AgentRuntimeErrorType.InvalidProviderAPIKey;
+      }
+    },
+  },
+  customClient: {
+    createChatCompletionStream: (client: HfInference, payload, instance) => {
+      const hfRes = client.chatCompletionStream({
+        endpointUrl: instance.baseURL,
+        messages: payload.messages,
+        model: payload.model,
+        stream: true,
+        temperature: payload.temperature,
+        //  `top_p` must be > 0.0 and < 1.0
+        top_p: payload?.top_p
+          ? payload?.top_p >= 1
+            ? 0.99
+            : payload?.top_p <= 0
+              ? 0.01
+              : payload?.top_p
+          : undefined,
+      });
+
+      return convertIterableToStream(hfRes);
+    },
+    createClient: (options) => new HfInference(options.apiKey),
+  },
+  debug: {
+    chatCompletion: () => process.env.DEBUG_HUGGINGFACE_CHAT_COMPLETION === '1',
+  },
+  provider: ModelProvider.HuggingFace,
+});

package/src/libs/agent-runtime/siliconcloud/index.ts

@@ -3,6 +3,14 @@ import { LobeOpenAICompatibleFactory } from '../utils/openaiCompatibleFactory';
 
 export const LobeSiliconCloudAI = LobeOpenAICompatibleFactory({
   baseURL: 'https://api.siliconflow.cn/v1',
+  chatCompletion: {
+    handlePayload: (payload) => {
+      return {
+        ...payload,
+        stream: !payload.tools,
+      } as any;
+    },
+  },
   debug: {
     chatCompletion: () => process.env.DEBUG_SILICONCLOUD_CHAT_COMPLETION === '1',
   },

package/src/libs/agent-runtime/types/type.ts

@@ -33,6 +33,7 @@ export enum ModelProvider {
   Github = 'github',
   Google = 'google',
   Groq = 'groq',
+  HuggingFace = 'huggingface',
   Hunyuan = 'hunyuan',
   Minimax = 'minimax',
   Mistral = 'mistral',

package/src/libs/agent-runtime/utils/openaiCompatibleFactory/index.ts

@@ -1,4 +1,5 @@
 import OpenAI, { ClientOptions } from 'openai';
+import { Stream } from 'openai/streaming';
 
 import { LOBE_DEFAULT_MODEL_LIST } from '@/config/modelProviders';
 import { ChatModelCard } from '@/types/llm';
@@ -22,7 +23,7 @@ import { desensitizeUrl } from '../desensitizeUrl';
 import { handleOpenAIError } from '../handleOpenAIError';
 import { convertOpenAIMessages } from '../openaiHelpers';
 import { StreamingResponse } from '../response';
-import { OpenAIStream } from '../streams';
+import { OpenAIStream, OpenAIStreamOptions } from '../streams';
 
 // the model contains the following keywords is not a chat model, so we should filter them out
 const CHAT_MODELS_BLOCK_LIST = [
@@ -39,6 +40,15 @@ const CHAT_MODELS_BLOCK_LIST = [
 
 type ConstructorOptions<T extends Record<string, any> = any> = ClientOptions & T;
 
+export interface CustomClientOptions<T extends Record<string, any> = any> {
+  createChatCompletionStream?: (
+    client: any,
+    payload: ChatStreamPayload,
+    instance: any,
+  ) => ReadableStream<any>;
+  createClient?: (options: ConstructorOptions<T>) => any;
+}
+
 interface OpenAICompatibleFactoryOptions<T extends Record<string, any> = any> {
   baseURL?: string;
   chatCompletion?: {
@@ -50,9 +60,14 @@ interface OpenAICompatibleFactoryOptions<T extends Record<string, any> = any> {
       payload: ChatStreamPayload,
       options: ConstructorOptions<T>,
     ) => OpenAI.ChatCompletionCreateParamsStreaming;
+    handleStreamBizErrorType?: (error: {
+      message: string;
+      name: string;
+    }) => ILobeAgentRuntimeErrorType | undefined;
     noUserId?: boolean;
   };
   constructorOptions?: ConstructorOptions<T>;
+  customClient?: CustomClientOptions<T>;
   debug?: {
     chatCompletion: () => boolean;
   };
@@ -129,6 +144,7 @@ export const LobeOpenAICompatibleFactory = <T extends Record<string, any> = any>
   constructorOptions,
   chatCompletion,
   models,
+  customClient,
 }: OpenAICompatibleFactoryOptions<T>) => {
   const ErrorType = {
     bizError: errorType?.bizError || AgentRuntimeErrorType.ProviderBizError,
@@ -136,9 +152,9 @@ export const LobeOpenAICompatibleFactory = <T extends Record<string, any> = any>
   };
 
   return class LobeOpenAICompatibleAI implements LobeRuntimeAI {
-    client: OpenAI;
+    client!: OpenAI;
 
-    baseURL: string;
+    baseURL!: string;
     private _options: ConstructorOptions<T>;
 
     constructor(options: ClientOptions & Record<string, any> = {}) {
@@ -148,8 +164,16 @@ export const LobeOpenAICompatibleFactory = <T extends Record<string, any> = any>
 
       if (!apiKey) throw AgentRuntimeError.createError(ErrorType?.invalidAPIKey);
 
-      this.client = new OpenAI({ apiKey, baseURL, ...constructorOptions, ...res });
-      this.baseURL = this.client.baseURL;
+      const initOptions = { apiKey, baseURL, ...constructorOptions, ...res };
+
+      // if the custom client is provided, use it as client
+      if (customClient?.createClient) {
+        this.client = customClient.createClient(initOptions as any);
+      } else {
+        this.client = new OpenAI(initOptions);
+      }
+
+      this.baseURL = baseURL || this.client.baseURL;
     }
 
     async chat({ responseMode, ...payload }: ChatStreamPayload, options?: ChatCompetitionOptions) {
@@ -163,27 +187,41 @@ export const LobeOpenAICompatibleFactory = <T extends Record<string, any> = any>
 
         const messages = await convertOpenAIMessages(postPayload.messages);
 
-        const response = await this.client.chat.completions.create(
-          {
-            ...postPayload,
-            messages,
-            ...(chatCompletion?.noUserId ? {} : { user: options?.user }),
-          },
-          {
-            // https://github.com/lobehub/lobe-chat/pull/318
-            headers: { Accept: '*/*' },
-            signal: options?.signal,
-          },
-        );
+        let response: Stream<OpenAI.Chat.Completions.ChatCompletionChunk>;
+
+        const streamOptions: OpenAIStreamOptions = {
+          bizErrorTypeTransformer: chatCompletion?.handleStreamBizErrorType,
+          callbacks: options?.callback,
+          provider,
+        };
+        if (customClient?.createChatCompletionStream) {
+          response = customClient.createChatCompletionStream(this.client, payload, this) as any;
+        } else {
+          response = await this.client.chat.completions.create(
+            {
+              ...postPayload,
+              messages,
+              ...(chatCompletion?.noUserId ? {} : { user: options?.user }),
+            },
+            {
+              // https://github.com/lobehub/lobe-chat/pull/318
+              headers: { Accept: '*/*' },
+              signal: options?.signal,
+            },
+          );
+        }
 
         if (postPayload.stream) {
           const [prod, useForDebug] = response.tee();
 
           if (debug?.chatCompletion?.()) {
-            debugStream(useForDebug.toReadableStream()).catch(console.error);
+            const useForDebugStream =
+              useForDebug instanceof ReadableStream ? useForDebug : useForDebug.toReadableStream();
+
+            debugStream(useForDebugStream).catch(console.error);
           }
 
-          return StreamingResponse(OpenAIStream(prod, options?.callback), {
+          return StreamingResponse(OpenAIStream(prod, streamOptions), {
             headers: options?.headers,
           });
         }
@@ -196,7 +234,7 @@ export const LobeOpenAICompatibleFactory = <T extends Record<string, any> = any>
 
         const stream = transformResponseToStream(response as unknown as OpenAI.ChatCompletion);
 
-        return StreamingResponse(OpenAIStream(stream, options?.callback), {
+        return StreamingResponse(OpenAIStream(stream, streamOptions), {
           headers: options?.headers,
         });
       } catch (error) {