@lobehub/chat 1.21.15 → 1.22.0

package/src/libs/agent-runtime/utils/openaiCompatibleFactory/index.ts

@@ -1,4 +1,5 @@
 import OpenAI, { ClientOptions } from 'openai';
+import { Stream } from 'openai/streaming';
 
 import { LOBE_DEFAULT_MODEL_LIST } from '@/config/modelProviders';
 import { ChatModelCard } from '@/types/llm';
@@ -22,7 +23,7 @@ import { desensitizeUrl } from '../desensitizeUrl';
 import { handleOpenAIError } from '../handleOpenAIError';
 import { convertOpenAIMessages } from '../openaiHelpers';
 import { StreamingResponse } from '../response';
-import { OpenAIStream } from '../streams';
+import { OpenAIStream, OpenAIStreamOptions } from '../streams';
 
 // the model contains the following keywords is not a chat model, so we should filter them out
 const CHAT_MODELS_BLOCK_LIST = [
@@ -39,6 +40,15 @@ const CHAT_MODELS_BLOCK_LIST = [
 
 type ConstructorOptions<T extends Record<string, any> = any> = ClientOptions & T;
 
+export interface CustomClientOptions<T extends Record<string, any> = any> {
+  createChatCompletionStream?: (
+    client: any,
+    payload: ChatStreamPayload,
+    instance: any,
+  ) => ReadableStream<any>;
+  createClient?: (options: ConstructorOptions<T>) => any;
+}
+
 interface OpenAICompatibleFactoryOptions<T extends Record<string, any> = any> {
   baseURL?: string;
   chatCompletion?: {
@@ -50,9 +60,14 @@ interface OpenAICompatibleFactoryOptions<T extends Record<string, any> = any> {
       payload: ChatStreamPayload,
       options: ConstructorOptions<T>,
     ) => OpenAI.ChatCompletionCreateParamsStreaming;
+    handleStreamBizErrorType?: (error: {
+      message: string;
+      name: string;
+    }) => ILobeAgentRuntimeErrorType | undefined;
     noUserId?: boolean;
   };
   constructorOptions?: ConstructorOptions<T>;
+  customClient?: CustomClientOptions<T>;
   debug?: {
     chatCompletion: () => boolean;
   };
@@ -129,6 +144,7 @@ export const LobeOpenAICompatibleFactory = <T extends Record<string, any> = any>
   constructorOptions,
   chatCompletion,
   models,
+  customClient,
 }: OpenAICompatibleFactoryOptions<T>) => {
   const ErrorType = {
     bizError: errorType?.bizError || AgentRuntimeErrorType.ProviderBizError,
@@ -136,9 +152,9 @@ export const LobeOpenAICompatibleFactory = <T extends Record<string, any> = any>
   };
 
   return class LobeOpenAICompatibleAI implements LobeRuntimeAI {
-    client: OpenAI;
+    client!: OpenAI;
 
-    baseURL: string;
+    baseURL!: string;
     private _options: ConstructorOptions<T>;
 
     constructor(options: ClientOptions & Record<string, any> = {}) {
@@ -148,8 +164,16 @@ export const LobeOpenAICompatibleFactory = <T extends Record<string, any> = any>
 
       if (!apiKey) throw AgentRuntimeError.createError(ErrorType?.invalidAPIKey);
 
-      this.client = new OpenAI({ apiKey, baseURL, ...constructorOptions, ...res });
-      this.baseURL = this.client.baseURL;
+      const initOptions = { apiKey, baseURL, ...constructorOptions, ...res };
+
+      // if the custom client is provided, use it as client
+      if (customClient?.createClient) {
+        this.client = customClient.createClient(initOptions as any);
+      } else {
+        this.client = new OpenAI(initOptions);
+      }
+
+      this.baseURL = baseURL || this.client.baseURL;
     }
 
     async chat({ responseMode, ...payload }: ChatStreamPayload, options?: ChatCompetitionOptions) {
@@ -163,27 +187,41 @@ export const LobeOpenAICompatibleFactory = <T extends Record<string, any> = any>
 
         const messages = await convertOpenAIMessages(postPayload.messages);
 
-        const response = await this.client.chat.completions.create(
-          {
-            ...postPayload,
-            messages,
-            ...(chatCompletion?.noUserId ? {} : { user: options?.user }),
-          },
-          {
-            // https://github.com/lobehub/lobe-chat/pull/318
-            headers: { Accept: '*/*' },
-            signal: options?.signal,
-          },
-        );
+        let response: Stream<OpenAI.Chat.Completions.ChatCompletionChunk>;
+
+        const streamOptions: OpenAIStreamOptions = {
+          bizErrorTypeTransformer: chatCompletion?.handleStreamBizErrorType,
+          callbacks: options?.callback,
+          provider,
+        };
+        if (customClient?.createChatCompletionStream) {
+          response = customClient.createChatCompletionStream(this.client, payload, this) as any;
+        } else {
+          response = await this.client.chat.completions.create(
+            {
+              ...postPayload,
+              messages,
+              ...(chatCompletion?.noUserId ? {} : { user: options?.user }),
+            },
+            {
+              // https://github.com/lobehub/lobe-chat/pull/318
+              headers: { Accept: '*/*' },
+              signal: options?.signal,
+            },
+          );
+        }
 
         if (postPayload.stream) {
           const [prod, useForDebug] = response.tee();
 
           if (debug?.chatCompletion?.()) {
-            debugStream(useForDebug.toReadableStream()).catch(console.error);
+            const useForDebugStream =
+              useForDebug instanceof ReadableStream ? useForDebug : useForDebug.toReadableStream();
+
+            debugStream(useForDebugStream).catch(console.error);
           }
 
-          return StreamingResponse(OpenAIStream(prod, options?.callback), {
+          return StreamingResponse(OpenAIStream(prod, streamOptions), {
             headers: options?.headers,
           });
         }
@@ -196,7 +234,7 @@ export const LobeOpenAICompatibleFactory = <T extends Record<string, any> = any>
 
         const stream = transformResponseToStream(response as unknown as OpenAI.ChatCompletion);
 
-        return StreamingResponse(OpenAIStream(stream, options?.callback), {
+        return StreamingResponse(OpenAIStream(stream, streamOptions), {
           headers: options?.headers,
         });
       } catch (error) {

package/src/libs/agent-runtime/utils/streams/openai.test.ts

@@ -1,6 +1,9 @@
 import { describe, expect, it, vi } from 'vitest';
 
+import { AgentRuntimeErrorType } from '@/libs/agent-runtime';
+
 import { OpenAIStream } from './openai';
+import { FIRST_CHUNK_ERROR_KEY } from './protocol';
 
 describe('OpenAIStream', () => {
   it('should transform OpenAI stream to protocol stream', async () => {
@@ -45,10 +48,12 @@ describe('OpenAIStream', () => {
     const onCompletionMock = vi.fn();
 
     const protocolStream = OpenAIStream(mockOpenAIStream, {
-      onStart: onStartMock,
-      onText: onTextMock,
-      onToken: onTokenMock,
-      onCompletion: onCompletionMock,
+      callbacks: {
+        onStart: onStartMock,
+        onText: onTextMock,
+        onToken: onTokenMock,
+        onCompletion: onCompletionMock,
+      },
     });
 
     const decoder = new TextDecoder();
@@ -189,7 +194,9 @@ describe('OpenAIStream', () => {
     const onToolCallMock = vi.fn();
 
     const protocolStream = OpenAIStream(mockOpenAIStream, {
-      onToolCall: onToolCallMock,
+      callbacks: {
+        onToolCall: onToolCallMock,
+      },
     });
 
     const decoder = new TextDecoder();
@@ -281,6 +288,66 @@ describe('OpenAIStream', () => {
     );
   });
 
+  it('should handle FIRST_CHUNK_ERROR_KEY', async () => {
+    const mockOpenAIStream = new ReadableStream({
+      start(controller) {
+        controller.enqueue({
+          [FIRST_CHUNK_ERROR_KEY]: true,
+          errorType: AgentRuntimeErrorType.ProviderBizError,
+          message: 'Test error',
+        });
+        controller.close();
+      },
+    });
+
+    const protocolStream = OpenAIStream(mockOpenAIStream);
+
+    const decoder = new TextDecoder();
+    const chunks = [];
+
+    // @ts-ignore
+    for await (const chunk of protocolStream) {
+      chunks.push(decoder.decode(chunk, { stream: true }));
+    }
+
+    expect(chunks).toEqual([
+      'id: first_chunk_error\n',
+      'event: error\n',
+      `data: {"body":{"errorType":"ProviderBizError","message":"Test error"},"type":"ProviderBizError"}\n\n`,
+    ]);
+  });
+
+  it('should use bizErrorTypeTransformer', async () => {
+    const mockOpenAIStream = new ReadableStream({
+      start(controller) {
+        controller.enqueue(
+          '%FIRST_CHUNK_ERROR%: ' +
+            JSON.stringify({ message: 'Custom error', name: 'CustomError' }),
+        );
+        controller.close();
+      },
+    });
+
+    const protocolStream = OpenAIStream(mockOpenAIStream, {
+      bizErrorTypeTransformer: () => AgentRuntimeErrorType.PermissionDenied,
+      provider: 'grok',
+    });
+
+    const decoder = new TextDecoder();
+    const chunks = [];
+
+    // @ts-ignore
+    for await (const chunk of protocolStream) {
+      chunks.push(decoder.decode(chunk, { stream: true }));
+    }
+
+    expect(chunks).toEqual([
+      'id: first_chunk_error\n',
+      'event: error\n',
+      `data: {"body":{"message":"Custom error","errorType":"PermissionDenied","provider":"grok"},"type":"PermissionDenied"}\n\n`,
+    ]);
+  });
+
   describe('Tools Calling', () => {
     it('should handle OpenAI official tool calls', async () => {
       const mockOpenAIStream = new ReadableStream({
@@ -316,7 +383,9 @@ describe('OpenAIStream', () => {
       const onToolCallMock = vi.fn();
 
       const protocolStream = OpenAIStream(mockOpenAIStream, {
-        onToolCall: onToolCallMock,
+        callbacks: {
+          onToolCall: onToolCallMock,
+        },
       });
 
       const decoder = new TextDecoder();
@@ -447,7 +516,9 @@ describe('OpenAIStream', () => {
       const onToolCallMock = vi.fn();
 
       const protocolStream = OpenAIStream(mockOpenAIStream, {
-        onToolCall: onToolCallMock,
+        callbacks: {
+          onToolCall: onToolCallMock,
+        },
       });
 
       const decoder = new TextDecoder();

package/src/libs/agent-runtime/utils/streams/openai.ts

@@ -3,14 +3,17 @@ import type { Stream } from 'openai/streaming';
 
 import { ChatMessageError } from '@/types/message';
 
+import { AgentRuntimeErrorType, ILobeAgentRuntimeErrorType } from '../../error';
 import { ChatStreamCallbacks } from '../../types';
 import {
+  FIRST_CHUNK_ERROR_KEY,
   StreamProtocolChunk,
   StreamProtocolToolCallChunk,
   StreamStack,
   StreamToolCallChunkData,
   convertIterableToStream,
   createCallbacksTransformer,
+  createFirstErrorHandleTransformer,
   createSSEProtocolTransformer,
   generateToolCallId,
 } from './protocol';
@@ -19,6 +22,21 @@ export const transformOpenAIStream = (
   chunk: OpenAI.ChatCompletionChunk,
   stack?: StreamStack,
 ): StreamProtocolChunk => {
+  // handle the first chunk error
+  if (FIRST_CHUNK_ERROR_KEY in chunk) {
+    delete chunk[FIRST_CHUNK_ERROR_KEY];
+    // @ts-ignore
+    delete chunk['name'];
+    // @ts-ignore
+    delete chunk['stack'];
+
+    const errorData = {
+      body: chunk,
+      type: 'errorType' in chunk ? chunk.errorType : AgentRuntimeErrorType.ProviderBizError,
+    } as ChatMessageError;
+    return { data: errorData, id: 'first_chunk_error', type: 'error' };
+  }
+
   // maybe need another structure to add support for multiple choices
 
   try {
@@ -97,7 +115,7 @@ export const transformOpenAIStream = (
           'chat response streaming chunk parse error, please contact your API Provider to fix it.',
         context: { error: { message: err.message, name: err.name }, chunk },
       },
-      type: 'StreamChunkError',
+      type: errorName,
     } as ChatMessageError;
     /* eslint-enable */
 
@@ -105,16 +123,31 @@ export const transformOpenAIStream = (
   }
 };
 
+export interface OpenAIStreamOptions {
+  bizErrorTypeTransformer?: (error: {
+    message: string;
+    name: string;
+  }) => ILobeAgentRuntimeErrorType | undefined;
+  callbacks?: ChatStreamCallbacks;
+  provider?: string;
+}
+
 export const OpenAIStream = (
   stream: Stream<OpenAI.ChatCompletionChunk> | ReadableStream,
-  callbacks?: ChatStreamCallbacks,
+  { callbacks, provider, bizErrorTypeTransformer }: OpenAIStreamOptions = {},
 ) => {
   const streamStack: StreamStack = { id: '' };
 
   const readableStream =
     stream instanceof ReadableStream ? stream : convertIterableToStream(stream);
 
-  return readableStream
-    .pipeThrough(createSSEProtocolTransformer(transformOpenAIStream, streamStack))
-    .pipeThrough(createCallbacksTransformer(callbacks));
+  return (
+    readableStream
+      // 1. handle the first error if exist
+      // provider like huggingface or minimax will return error in the stream,
+      // so in the first Transformer, we need to handle the error
+      .pipeThrough(createFirstErrorHandleTransformer(bizErrorTypeTransformer, provider))
+      .pipeThrough(createSSEProtocolTransformer(transformOpenAIStream, streamStack))
+      .pipeThrough(createCallbacksTransformer(callbacks))
+  );
 };

package/src/libs/agent-runtime/utils/streams/protocol.ts

@@ -1,7 +1,7 @@
-import { readableFromAsyncIterable } from 'ai';
-
 import { ChatStreamCallbacks } from '@/libs/agent-runtime';
 
+import { AgentRuntimeErrorType } from '../../error';
+
 export interface StreamStack {
   id: string;
   tool?: {
@@ -38,17 +38,52 @@ export interface StreamProtocolToolCallChunk {
 export const generateToolCallId = (index: number, functionName?: string) =>
   `${functionName || 'unknown_tool_call'}_${index}`;
 
-export const chatStreamable = async function* <T>(stream: AsyncIterable<T>) {
+const chatStreamable = async function* <T>(stream: AsyncIterable<T>) {
   for await (const response of stream) {
     yield response;
   }
 };
 
+const ERROR_CHUNK_PREFIX = '%FIRST_CHUNK_ERROR%: ';
 // make the response to the streamable format
 export const convertIterableToStream = <T>(stream: AsyncIterable<T>) => {
-  return readableFromAsyncIterable(chatStreamable(stream));
+  const iterable = chatStreamable(stream);
+
+  // copy from https://github.com/vercel/ai/blob/d3aa5486529e3d1a38b30e3972b4f4c63ea4ae9a/packages/ai/streams/ai-stream.ts#L284
+  // and add an error handle
+  let it = iterable[Symbol.asyncIterator]();
+
+  return new ReadableStream<T>({
+    async cancel(reason) {
+      await it.return?.(reason);
+    },
+    async pull(controller) {
+      const { done, value } = await it.next();
+      if (done) controller.close();
+      else controller.enqueue(value);
+    },
+
+    async start(controller) {
+      try {
+        const { done, value } = await it.next();
+        if (done) controller.close();
+        else controller.enqueue(value);
+      } catch (e) {
+        const error = e as Error;
+
+        controller.enqueue(
+          (ERROR_CHUNK_PREFIX +
+            JSON.stringify({ message: error.message, name: error.name, stack: error.stack })) as T,
+        );
+        controller.close();
+      }
+    },
+  });
 };
 
+/**
+ * Create a transformer to convert the response into an SSE format
+ */
 export const createSSEProtocolTransformer = (
   transformer: (chunk: any, stack: StreamStack) => StreamProtocolChunk,
   streamStack?: StreamStack,
@@ -111,3 +146,27 @@ export function createCallbacksTransformer(cb: ChatStreamCallbacks | undefined)
     },
   });
 }
+
+export const FIRST_CHUNK_ERROR_KEY = '_isFirstChunkError';
+
+export const createFirstErrorHandleTransformer = (
+  errorHandler?: (errorJson: any) => any,
+  provider?: string,
+) => {
+  return new TransformStream({
+    transform(chunk, controller) {
+      if (chunk.toString().startsWith(ERROR_CHUNK_PREFIX)) {
+        const errorData = JSON.parse(chunk.toString().replace(ERROR_CHUNK_PREFIX, ''));
+
+        controller.enqueue({
+          ...errorData,
+          [FIRST_CHUNK_ERROR_KEY]: true,
+          errorType: errorHandler?.(errorData) || AgentRuntimeErrorType.ProviderBizError,
+          provider,
+        });
+      } else {
+        controller.enqueue(chunk);
+      }
+    },
+  });
+};

package/src/locales/default/error.ts

@@ -80,8 +80,8 @@ export default {
     LocationNotSupportError:
       '很抱歉，你的所在地区不支持此模型服务，可能是由于区域限制或服务未开通。请确认当前地区是否支持使用此服务，或尝试使用切换到其他地区后重试。',
     QuotaLimitReached:
-      '很抱歉，当前 Token 用量或请求次数已达该秘钥的配额(quota)上限，请增加该秘钥的配额或稍后再试',
-
+      '很抱歉，当前 Token 用量或请求次数已达该密钥的配额(quota)上限，请增加该密钥的配额或稍后再试',
+    PermissionDenied: '很抱歉，你没有权限访问该服务，请检查你的密钥是否有访问权限',
     InvalidProviderAPIKey: '{{provider}} API Key 不正确或为空，请检查 {{provider}} API Key 后重试',
     ProviderBizError: '请求 {{provider}} 服务出错，请根据以下信息排查或重试',
     /**

package/src/locales/default/modelProvider.ts

@@ -54,11 +54,18 @@ export default {
   },
   github: {
     personalAccessToken: {
-      desc: '填入你的 Github PAT，点击[这里](https://github.com/settings/tokens) 创建',
+      desc: '填入你的 Github PAT，点击 [这里](https://github.com/settings/tokens) 创建',
       placeholder: 'ghp_xxxxxx',
       title: 'Github PAT',
     },
   },
+  huggingface: {
+    accessToken: {
+      desc: '填入你的 HuggingFace Token，点击 [这里](https://huggingface.co/settings/tokens) 创建',
+      placeholder: 'hf_xxxxxxxxx',
+      title: 'HuggingFace Token',
+    },
+  },
   ollama: {
     checker: {
       desc: '测试代理地址是否正确填写',

package/src/server/globalConfig/index.ts

@@ -9,6 +9,7 @@ import {
   GithubProviderCard,
   GoogleProviderCard,
   GroqProviderCard,
+  HuggingFaceProviderCard,
   HunyuanProviderCard,
   NovitaProviderCard,
   OllamaProviderCard,
@@ -98,6 +99,9 @@ export const getServerGlobalConfig = () => {
     FIREWORKSAI_MODEL_LIST,
 
     ENABLED_WENXIN,
+
+    ENABLED_HUGGINGFACE,
+    HUGGINGFACE_MODEL_LIST,
   } = getLLMConfig();
 
   const config: GlobalServerConfig = {
@@ -166,6 +170,14 @@ export const getServerGlobalConfig = () => {
           modelString: GROQ_MODEL_LIST,
         }),
       },
+      huggingface: {
+        enabled: ENABLED_HUGGINGFACE,
+        enabledModels: extractEnabledModels(HUGGINGFACE_MODEL_LIST),
+        serverModelCards: transformToChatModelCards({
+          defaultChatModels: HuggingFaceProviderCard.chatModels,
+          modelString: HUGGINGFACE_MODEL_LIST,
+        }),
+      },
       hunyuan: {
         enabled: ENABLED_HUNYUAN,
         enabledModels: extractEnabledModels(HUNYUAN_MODEL_LIST),
@@ -202,7 +214,6 @@ export const getServerGlobalConfig = () => {
           modelString: OPENAI_MODEL_LIST,
         }),
       },
-
       openrouter: {
         enabled: ENABLED_OPENROUTER,
         enabledModels: extractEnabledModels(OPENROUTER_MODEL_LIST),

package/src/server/modules/AgentRuntime/index.ts

@@ -225,6 +225,16 @@ const getLlmOptionsFromPayload = (provider: string, payload: JWTPayload) => {
 
       return { apiKey, baseURL };
     }
+
+    case ModelProvider.HuggingFace: {
+      const { HUGGINGFACE_PROXY_URL, HUGGINGFACE_API_KEY } = getLLMConfig();
+
+      const apiKey = apiKeyManager.pick(payload?.apiKey || HUGGINGFACE_API_KEY);
+      const baseURL = payload?.endpoint || HUGGINGFACE_PROXY_URL;
+
+      return { apiKey, baseURL };
+    }
+
     case ModelProvider.Upstage: {
       const { UPSTAGE_API_KEY } = getLLMConfig();
 

package/src/services/_url.ts

@@ -1,5 +1,3 @@
-// TODO: 未来所有核心路由需要迁移到 trpc，部分不需要迁移的则走 webapi
-
 /* eslint-disable sort-keys-fix/sort-keys-fix */
 import { transform } from 'lodash-es';
 
@@ -17,9 +15,6 @@ const mapWithBasePath = <T extends object>(apis: T): T => {
 };
 
 export const API_ENDPOINTS = mapWithBasePath({
-  // chat
-  chat: (provider: string) => withBasePath(`/api/chat/${provider}`),
-  chatModels: (provider: string) => withBasePath(`/api/chat/models/${provider}`),
   oauth: '/api/auth',
 
   proxy: '/webapi/proxy',
@@ -35,6 +30,10 @@ export const API_ENDPOINTS = mapWithBasePath({
   // trace
   trace: '/webapi/trace',
 
+  // chat
+  chat: (provider: string) => withBasePath(`/webapi/chat/${provider}`),
+  chatModels: (provider: string) => withBasePath(`/webapi/chat/models/${provider}`),
+
   // image
   images: (provider: string) => `/webapi/text-to-image/${provider}`,
 

package/src/types/user/settings/keyVaults.ts

@@ -33,6 +33,7 @@ export interface UserKeyVaults {
   github?: OpenAICompatibleKeyVault;
   google?: OpenAICompatibleKeyVault;
   groq?: OpenAICompatibleKeyVault;
+  huggingface?: OpenAICompatibleKeyVault;
   hunyuan?: OpenAICompatibleKeyVault;
   lobehub?: any;
   minimax?: OpenAICompatibleKeyVault;