@lobehub/chat 0.122.8 → 0.123.0

package/src/libs/agent-runtime/google/index.ts

@@ -0,0 +1,160 @@
+import { Content, GoogleGenerativeAI, Part } from '@google/generative-ai';
+import { GoogleGenerativeAIStream, StreamingTextResponse } from 'ai';
+
+import { LobeRuntimeAI } from '../BaseAI';
+import { AgentRuntimeErrorType, ILobeAgentRuntimeErrorType } from '../error';
+import { ChatStreamPayload, OpenAIChatMessage, UserMessageContentPart } from '../types';
+import { ModelProvider } from '../types/type';
+import { AgentRuntimeError } from '../utils/createError';
+import { debugStream } from '../utils/debugStream';
+import { DEBUG_CHAT_COMPLETION } from '../utils/env';
+import { parseDataUri } from '../utils/uriParser';
+
+type GoogleChatErrors = GoogleChatError[];
+
+interface GoogleChatError {
+  '@type': string;
+  'domain': string;
+  'metadata': {
+    service: string;
+  };
+  'reason': string;
+}
+
+export class LobeGoogleAI implements LobeRuntimeAI {
+  private client: GoogleGenerativeAI;
+
+  constructor(apiKey: string) {
+    if (!apiKey) throw AgentRuntimeError.createError(AgentRuntimeErrorType.InvalidGoogleAPIKey);
+
+    this.client = new GoogleGenerativeAI(apiKey);
+  }
+
+  async chat(payload: ChatStreamPayload) {
+    try {
+      const { contents, model } = this.buildGoogleMessages(payload.messages, payload.model);
+      const geminiStream = await this.client
+        .getGenerativeModel({
+          generationConfig: {
+            maxOutputTokens: payload.max_tokens,
+            temperature: payload.temperature,
+            topP: payload.top_p,
+          },
+          model,
+        })
+        .generateContentStream({ contents });
+
+      // Convert the response into a friendly text-stream
+      const stream = GoogleGenerativeAIStream(geminiStream);
+
+      const [debug, output] = stream.tee();
+
+      if (DEBUG_CHAT_COMPLETION) {
+        debugStream(debug).catch(console.error);
+      }
+
+      // Respond with the stream
+      return new StreamingTextResponse(output);
+    } catch (e) {
+      const err = e as Error;
+
+      const { errorType, error } = this.parseErrorMessage(err.message);
+
+      throw AgentRuntimeError.chat({ error, errorType, provider: ModelProvider.Google });
+    }
+  }
+
+  private convertContentToGooglePart = (content: UserMessageContentPart): Part => {
+    switch (content.type) {
+      case 'text': {
+        return { text: content.text };
+      }
+      case 'image_url': {
+        const { mimeType, base64 } = parseDataUri(content.image_url.url);
+
+        if (!base64) {
+          throw new TypeError("Image URL doesn't contain base64 data");
+        }
+
+        return {
+          inlineData: {
+            data: base64,
+            mimeType: mimeType || 'image/png',
+          },
+        };
+      }
+    }
+  };
+
+  private convertOAIMessagesToGoogleMessage = (message: OpenAIChatMessage): Content => {
+    const content = message.content as string | UserMessageContentPart[];
+
+    return {
+      parts:
+        typeof content === 'string'
+          ? [{ text: content }]
+          : content.map((c) => this.convertContentToGooglePart(c)),
+      role: message.role === 'user' ? 'user' : 'model',
+    };
+  };
+
+  // convert messages from the Vercel AI SDK Format to the format
+  // that is expected by the Google GenAI SDK
+  private buildGoogleMessages = (
+    messages: OpenAIChatMessage[],
+    model: string,
+  ): { contents: Content[]; model: string } => {
+    const contents = messages
+      .filter((message) => message.role === 'user' || message.role === 'assistant')
+      .map((msg) => this.convertOAIMessagesToGoogleMessage(msg));
+
+    // if message are all text message, use vision will return error
+    // use add an image to use models/gemini-pro-vision, or switch your model to a text model
+    const noImage = messages.every((m) => typeof m.content === 'string');
+
+    return { contents, model: noImage ? 'gemini-pro' : model };
+  };
+
+  private parseErrorMessage(message: string): {
+    error: any;
+    errorType: ILobeAgentRuntimeErrorType;
+  } {
+    const defaultError = {
+      error: { message },
+      errorType: AgentRuntimeErrorType.GoogleBizError,
+    };
+
+    if (message.includes('location is not supported'))
+      return { error: message, errorType: AgentRuntimeErrorType.LocationNotSupportError };
+
+    try {
+      const startIndex = message.lastIndexOf('[');
+      if (startIndex === -1) {
+        return defaultError;
+      }
+
+      // 从开始位置截取字符串到最后
+      const jsonString = message.slice(startIndex);
+
+      // 尝试解析 JSON 字符串
+      const json: GoogleChatErrors = JSON.parse(jsonString);
+
+      const bizError = json[0];
+
+      switch (bizError.reason) {
+        case 'API_KEY_INVALID': {
+          return { ...defaultError, errorType: AgentRuntimeErrorType.InvalidGoogleAPIKey };
+        }
+
+        default: {
+          return { error: json, errorType: AgentRuntimeErrorType.GoogleBizError };
+        }
+      }
+    } catch {
+      // 如果解析失败，则返回原始错误消息
+      return defaultError;
+    }
+  }
+}
+
+export default LobeGoogleAI;

package/src/libs/agent-runtime/index.ts

@@ -0,0 +1,9 @@
+export { LobeAzureOpenAI } from './azureOpenai';
+export * from './BaseAI';
+export { LobeBedrockAI } from './bedrock';
+export * from './error';
+export { LobeGoogleAI } from './google';
+export { LobeOpenAI } from './openai';
+export * from './types';
+export { AgentRuntimeError } from './utils/createError';
+export { LobeZhipuAI } from './zhipu';

package/src/libs/agent-runtime/openai/index.test.ts

@@ -0,0 +1,176 @@
+import OpenAI from 'openai';
+import { Mock, afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { LobeOpenAI } from './index';
+
+// Mock the console.error to avoid polluting test output
+vi.spyOn(console, 'error').mockImplementation(() => {});
+
+describe('LobeOpenAI chat', () => {
+  let openaiInstance: LobeOpenAI;
+
+  beforeEach(() => {
+    openaiInstance = new LobeOpenAI({ apiKey: 'test', dangerouslyAllowBrowser: true });
+
+    // 使用 vi.spyOn 来模拟 chat.completions.create 方法
+    vi.spyOn(openaiInstance['client'].chat.completions, 'create').mockResolvedValue(
+      new ReadableStream() as any,
+    );
+  });
+
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  describe('chat', () => {
+    it('should return a StreamingTextResponse on successful API call', async () => {
+      // Arrange
+      const mockStream = new ReadableStream();
+      const mockResponse = Promise.resolve(mockStream);
+
+      (openaiInstance['client'].chat.completions.create as Mock).mockResolvedValue(mockResponse);
+
+      // Act
+      const result = await openaiInstance.chat({
+        messages: [{ content: 'Hello', role: 'user' }],
+        model: 'text-davinci-003',
+        temperature: 0,
+      });
+
+      // Assert
+      expect(result).toBeInstanceOf(Response);
+    });
+
+    it('should return an openai error response when OpenAI.APIError is thrown', async () => {
+      // Arrange
+      const apiError = new OpenAI.APIError(
+        400,
+        {
+          status: 400,
+          error: {
+            message: 'Bad Request',
+          },
+        },
+        'Error message',
+        {},
+      );
+
+      vi.spyOn(openaiInstance['client'].chat.completions, 'create').mockRejectedValue(apiError);
+
+      // Act
+      try {
+        await openaiInstance.chat({
+          messages: [{ content: 'Hello', role: 'user' }],
+          model: 'text-davinci-003',
+          temperature: 0,
+        });
+      } catch (e) {
+        expect(e).toEqual({
+          endpoint: 'https://api.openai.com/v1',
+          error: {
+            error: { message: 'Bad Request' },
+            status: 400,
+          },
+          errorType: 'OpenAIBizError',
+          provider: 'openai',
+        });
+      }
+    });
+
+    it('should return an cause response when OpenAI.APIError is thrown with cause', async () => {
+      // Arrange
+      const errorInfo = {
+        stack: 'abc',
+        cause: {
+          message: 'api is undefined',
+        },
+      };
+      const apiError = new OpenAI.APIError(400, errorInfo, 'module error', {});
+
+      vi.spyOn(openaiInstance['client'].chat.completions, 'create').mockRejectedValue(apiError);
+
+      // Act
+      try {
+        await openaiInstance.chat({
+          messages: [{ content: 'Hello', role: 'user' }],
+          model: 'text-davinci-003',
+          temperature: 0,
+        });
+      } catch (e) {
+        expect(e).toEqual({
+          endpoint: 'https://api.openai.com/v1',
+          error: {
+            cause: { message: 'api is undefined' },
+            stack: 'abc',
+          },
+          errorType: 'OpenAIBizError',
+          provider: 'openai',
+        });
+      }
+    });
+
+    it('should return an cause response with desensitize Url', async () => {
+      // Arrange
+      const errorInfo = {
+        stack: 'abc',
+        cause: { message: 'api is undefined' },
+      };
+      const apiError = new OpenAI.APIError(400, errorInfo, 'module error', {});
+
+      openaiInstance = new LobeOpenAI({
+        apiKey: 'test',
+        dangerouslyAllowBrowser: true,
+        baseURL: 'https://api.abc.com/v1',
+      });
+
+      vi.spyOn(openaiInstance['client'].chat.completions, 'create').mockRejectedValue(apiError);
+
+      // Act
+      try {
+        await openaiInstance.chat({
+          messages: [{ content: 'Hello', role: 'user' }],
+          model: 'gpt-3.5-turbo',
+          temperature: 0,
+        });
+      } catch (e) {
+        expect(e).toEqual({
+          endpoint: 'https://api.***.com/v1',
+          error: {
+            cause: { message: 'api is undefined' },
+            stack: 'abc',
+          },
+          errorType: 'OpenAIBizError',
+          provider: 'openai',
+        });
+      }
+    });
+
+    it('should return a 500 error response for non-OpenAI errors', async () => {
+      // Arrange
+      const genericError = new Error('Generic Error');
+
+      vi.spyOn(openaiInstance['client'].chat.completions, 'create').mockRejectedValue(genericError);
+
+      // Act
+      try {
+        await openaiInstance.chat({
+          messages: [{ content: 'Hello', role: 'user' }],
+          model: 'text-davinci-003',
+          temperature: 0,
+        });
+      } catch (e) {
+        expect(e).toEqual({
+          endpoint: 'https://api.openai.com/v1',
+          errorType: 'AgentRuntimeError',
+          provider: 'openai',
+          error: {
+            name: genericError.name,
+            cause: genericError.cause,
+            message: genericError.message,
+            stack: genericError.stack,
+          },
+        });
+      }
+    });
+  });
+});

package/src/libs/agent-runtime/openai/index.ts

@@ -0,0 +1,107 @@
+import { OpenAIStream, StreamingTextResponse } from 'ai';
+import OpenAI, { ClientOptions } from 'openai';
+import urlJoin from 'url-join';
+
+import { ChatStreamPayload } from '@/types/openai/chat';
+
+import { LobeRuntimeAI } from '../BaseAI';
+import { AgentRuntimeErrorType } from '../error';
+import { ModelProvider } from '../types';
+import { AgentRuntimeError } from '../utils/createError';
+import { debugStream } from '../utils/debugStream';
+import { desensitizeUrl } from '../utils/desensitizeUrl';
+import { DEBUG_CHAT_COMPLETION } from '../utils/env';
+import { handleOpenAIError } from '../utils/handleOpenAIError';
+
+const DEFAULT_BASE_URL = 'https://api.openai.com/v1';
+
+interface AzureOpenAIOptions extends ClientOptions {
+  azureOptions?: {
+    apiVersion?: string;
+    model?: string;
+  };
+  useAzure?: boolean;
+}
+export class LobeOpenAI implements LobeRuntimeAI {
+  private client: OpenAI;
+
+  constructor(options: AzureOpenAIOptions) {
+    if (!options.apiKey) throw AgentRuntimeError.createError(AgentRuntimeErrorType.NoOpenAIAPIKey);
+
+    if (options.useAzure) {
+      this.client = LobeOpenAI.initWithAzureOpenAI(options);
+    } else {
+      this.client = new OpenAI(options);
+    }
+
+    this.baseURL = this.client.baseURL;
+  }
+
+  baseURL: string;
+
+  async chat(payload: ChatStreamPayload) {
+    // ============  1. preprocess messages   ============ //
+    const { messages, ...params } = payload;
+
+    // ============  2. send api   ============ //
+
+    try {
+      const response = await this.client.chat.completions.create(
+        {
+          messages,
+          ...params,
+          stream: true,
+        } as unknown as OpenAI.ChatCompletionCreateParamsStreaming,
+        { headers: { Accept: '*/*' } },
+      );
+
+      const stream = OpenAIStream(response);
+
+      const [debug, prod] = stream.tee();
+
+      if (DEBUG_CHAT_COMPLETION) {
+        debugStream(debug).catch(console.error);
+      }
+
+      return new StreamingTextResponse(prod);
+    } catch (error) {
+      const { errorResult, RuntimeError } = handleOpenAIError(error);
+
+      const errorType = RuntimeError || AgentRuntimeErrorType.OpenAIBizError;
+
+      let desensitizedEndpoint = this.baseURL;
+
+      // refs: https://github.com/lobehub/lobe-chat/issues/842
+      if (this.baseURL !== DEFAULT_BASE_URL) {
+        desensitizedEndpoint = desensitizeUrl(this.baseURL);
+      }
+
+      throw AgentRuntimeError.chat({
+        endpoint: desensitizedEndpoint,
+        error: errorResult,
+        errorType,
+        provider: ModelProvider.OpenAI,
+      });
+    }
+  }
+
+  static initWithAzureOpenAI(options: AzureOpenAIOptions) {
+    const endpoint = options.baseURL!;
+    const model = options.azureOptions?.model || '';
+
+    // refs: https://test-001.openai.azure.com/openai/deployments/gpt-35-turbo
+    const baseURL = urlJoin(endpoint, `/openai/deployments/${model.replace('.', '')}`);
+
+    const apiVersion = options.azureOptions?.apiVersion || '2023-08-01-preview';
+    const apiKey = options.apiKey!;
+
+    const config: ClientOptions = {
+      apiKey,
+      baseURL,
+      defaultHeaders: { 'api-key': apiKey },
+      defaultQuery: { 'api-version': apiVersion },
+    };
+
+    return new OpenAI(config);
+  }
+}

package/src/libs/agent-runtime/types/chat.ts

@@ -0,0 +1,119 @@
+export type LLMRoleType = 'user' | 'system' | 'assistant' | 'function';
+
+interface UserMessageContentPartText {
+  text: string;
+  type: 'text';
+}
+interface UserMessageContentPartImage {
+  image_url: {
+    detail?: 'auto' | 'low' | 'high';
+    url: string;
+  };
+  type: 'image_url';
+}
+
+export type UserMessageContentPart = UserMessageContentPartText | UserMessageContentPartImage;
+
+export interface OpenAIChatMessage {
+  /**
+   * @title 内容
+   * @description 消息内容
+   */
+  content: string | UserMessageContentPart[];
+
+  name?: string;
+  /**
+   * 角色
+   * @description 消息发送者的角色
+   */
+  role: LLMRoleType;
+}
+
+/**
+ * @title Chat Stream Payload
+ */
+export interface ChatStreamPayload {
+  /**
+   * @title 控制生成文本中的惩罚系数，用于减少重复性
+   * @default 0
+   */
+  frequency_penalty?: number;
+  /**
+   * @title 生成文本的最大长度
+   */
+  max_tokens?: number;
+  /**
+   * @title 聊天信息列表
+   */
+  messages: OpenAIChatMessage[];
+  /**
+   * @title 模型名称
+   */
+  model: string;
+  /**
+   * @title 返回的文本数量
+   */
+  n?: number;
+  /**
+   * 开启的插件列表
+   */
+  plugins?: string[];
+  /**
+   * @title 控制生成文本中的惩罚系数，用于减少主题的变化
+   * @default 0
+   */
+  presence_penalty?: number;
+  /**
+   * @default openai
+   */
+  provider?: string;
+  /**
+   * @title 是否开启流式请求
+   * @default true
+   */
+  stream?: boolean;
+  /**
+   * @title 生成文本的随机度量，用于控制文本的创造性和多样性
+   * @default 0.5
+   */
+  temperature: number;
+  tool_choice?: string;
+  tools?: ChatCompletionTool[];
+  /**
+   * @title 控制生成文本中最高概率的单个令牌
+   * @default 1
+   */
+  top_p?: number;
+}
+
+export interface ChatCompletionFunctions {
+  /**
+   * The description of what the function does.
+   * @type {string}
+   * @memberof ChatCompletionFunctions
+   */
+  description?: string;
+  /**
+   * The name of the function to be called. Must be a-z, A-Z, 0-9, or contain underscores and dashes, with a maximum length of 64.
+   * @type {string}
+   * @memberof ChatCompletionFunctions
+   */
+  name: string;
+  /**
+   * The parameters the functions accepts, described as a JSON Schema object. See the [guide](/docs/guides/gpt/function-calling) for examples, and the [JSON Schema reference](https://json-schema.org/understanding-json-schema/) for documentation about the format.
+   * @type {{ [key: string]: any }}
+   * @memberof ChatCompletionFunctions
+   */
+  parameters?: {
+    [key: string]: any;
+  };
+}
+
+export interface ChatCompletionTool {
+  function: ChatCompletionFunctions;
+
+  /**
+   * The type of the tool. Currently, only `function` is supported.
+   */
+  type: 'function';
+}

package/src/libs/agent-runtime/types/index.ts

@@ -0,0 +1,2 @@
+export * from './chat';
+export * from './type';

package/src/libs/agent-runtime/types/type.ts

@@ -0,0 +1,34 @@
+import OpenAI from 'openai';
+
+import { ILobeAgentRuntimeErrorType } from '../error';
+import { ChatStreamPayload } from './chat';
+
+export interface AgentInitErrorPayload {
+  error: object;
+  type: string | number;
+}
+
+export interface ChatCompletionErrorPayload {
+  [key: string]: any;
+  endpoint?: string;
+  error: object;
+  errorType: ILobeAgentRuntimeErrorType;
+  provider: ModelProvider;
+}
+
+export interface CreateChatCompletionOptions {
+  chatModel: OpenAI;
+  payload: ChatStreamPayload;
+}
+
+export enum ModelProvider {
+  Anthropic = 'anthropic',
+  Azure = 'azure',
+  Bedrock = 'bedrock',
+  ChatGLM = 'chatglm',
+  Google = 'google',
+  Mistral = 'mistral',
+  OpenAI = 'openai',
+  Tongyi = 'tongyi',
+  ZhiPu = 'zhipu',
+}

package/src/libs/agent-runtime/utils/createError.ts

@@ -0,0 +1,10 @@
+import { ILobeAgentRuntimeErrorType } from '../error';
+import { AgentInitErrorPayload, ChatCompletionErrorPayload } from '../types';
+
+export const AgentRuntimeError = {
+  chat: (error: ChatCompletionErrorPayload): ChatCompletionErrorPayload => error,
+  createError: (
+    errorType: ILobeAgentRuntimeErrorType | string | number,
+    error?: any,
+  ): AgentInitErrorPayload => ({ error, type: errorType }),
+};

package/src/libs/agent-runtime/utils/debugStream.ts

@@ -0,0 +1,18 @@
+export const debugStream = async (stream: ReadableStream) => {
+  let done = false;
+  let chunk = 0;
+  const decoder = new TextDecoder();
+
+  const reader = stream.getReader();
+  while (!done) {
+    const { value, done: _done } = await reader.read();
+    const chunkValue = decoder.decode(value, { stream: true });
+    if (!_done) {
+      console.log(`chunk ${chunk}:`);
+      console.log(chunkValue);
+    }
+
+    done = _done;
+    chunk++;
+  }
+};

package/src/libs/agent-runtime/utils/env.ts

@@ -0,0 +1 @@
+export const DEBUG_CHAT_COMPLETION = process.env.DEBUG_CHAT_COMPLETION === '1';

package/src/libs/agent-runtime/utils/handleOpenAIError.ts

@@ -0,0 +1,39 @@
+import OpenAI from 'openai';
+
+import { AgentRuntimeErrorType } from '../error';
+
+export const handleOpenAIError = (
+  error: any,
+): { RuntimeError?: 'AgentRuntimeError'; errorResult: any } => {
+  let errorResult: any;
+
+  // Check if the error is an OpenAI APIError
+  if (error instanceof OpenAI.APIError) {
+    // if error is definitely OpenAI APIError, there will be an error object
+    if (error.error) {
+      errorResult = error.error;
+    }
+    // Or if there is a cause, we use error cause
+    // This often happened when there is a bug of the `openai` package.
+    else if (error.cause) {
+      errorResult = error.cause;
+    }
+    // if there is no other request error, the error object is a Response like object
+    else {
+      errorResult = { headers: error.headers, stack: error.stack, status: error.status };
+    }
+
+    return {
+      errorResult,
+    };
+  } else {
+    const err = error as Error;
+
+    errorResult = { cause: err.cause, message: err.message, name: err.name, stack: err.stack };
+
+    return {
+      RuntimeError: AgentRuntimeErrorType.AgentRuntimeError,
+      errorResult,
+    };
+  }
+};

package/src/libs/agent-runtime/utils/uriParser.ts

@@ -0,0 +1,16 @@
+export const parseDataUri = (
+  dataUri: string,
+): { base64: string | null; mimeType: string | null } => {
+  // 正则表达式匹配整个 Data URI 结构
+  const dataUriMatch = dataUri.match(/^data:([^;]+);base64,(.+)$/);
+
+  // 如果匹配成功，则返回 mimeType 和 base64，否则返回 null
+  if (dataUriMatch) {
+    return {
+      base64: dataUriMatch[2],
+      mimeType: dataUriMatch[1],
+    };
+  }
+
+  return { base64: null, mimeType: null };
+};