@lobehub/chat 1.19.4 → 1.19.6

package/CHANGELOG.md

@@ -2,6 +2,57 @@
 
 # Changelog
 
+### [Version 1.19.6](https://github.com/lobehub/lobe-chat/compare/v1.19.5...v1.19.6)
+
+<sup>Released on **2024-09-19**</sup>
+
+#### ♻ Code Refactoring
+
+- **misc**: Refactor the tts route url.
+
+<br/>
+
+<details>
+<summary><kbd>Improvements and Fixes</kbd></summary>
+
+#### Code refactoring
+
+- **misc**: Refactor the tts route url, closes [#4030](https://github.com/lobehub/lobe-chat/issues/4030) ([60dcf19](https://github.com/lobehub/lobe-chat/commit/60dcf19))
+
+</details>
+
+<div align="right">
+
+[![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top)
+
+</div>
+
+### [Version 1.19.5](https://github.com/lobehub/lobe-chat/compare/v1.19.4...v1.19.5)
+
+<sup>Released on **2024-09-19**</sup>
+
+#### 💄 Styles
+
+- **misc**: Enable functioncall for stepfun models, Update qwen models.
+
+<br/>
+
+<details>
+<summary><kbd>Improvements and Fixes</kbd></summary>
+
+#### Styles
+
+- **misc**: Enable functioncall for stepfun models, closes [#4022](https://github.com/lobehub/lobe-chat/issues/4022) ([afb3509](https://github.com/lobehub/lobe-chat/commit/afb3509))
+- **misc**: Update qwen models, closes [#4026](https://github.com/lobehub/lobe-chat/issues/4026) ([6169e8f](https://github.com/lobehub/lobe-chat/commit/6169e8f))
+
+</details>
+
+<div align="right">
+
+[![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top)
+
+</div>
+
 ### [Version 1.19.4](https://github.com/lobehub/lobe-chat/compare/v1.19.3...v1.19.4)
 
 <sup>Released on **2024-09-19**</sup>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lobehub/chat",
-  "version": "1.19.4",
+  "version": "1.19.6",
   "description": "Lobe Chat - an open-source, high-performance chatbot framework that supports speech synthesis, multimodal, and extensible Function Call plugin system. Supports one-click free deployment of your private ChatGPT/LLM web application.",
   "keywords": [
     "framework",

package/src/app/api/openai/createBizOpenAI/index.ts

@@ -8,6 +8,7 @@ import { checkAuth } from './auth';
 import { createOpenai } from './createOpenai';
 
 /**
+ * @deprecated
  * createOpenAI Instance with Auth and azure openai support
  * if auth not pass ,just return error response
  */

package/src/app/{api/openai/tts → webapi/tts/openai}/route.ts

@@ -28,6 +28,7 @@ export const preferredRegion = [
 export const POST = async (req: Request) => {
   const payload = (await req.json()) as OpenAITTSPayload;
 
+  // need to be refactored with jwt auth mode
   const openaiOrErrResponse = createBizOpenAI(req);
 
   // if resOrOpenAI is a Response, it means there is an error,just return it

package/src/config/modelProviders/qwen.ts

@@ -1,22 +1,10 @@
 import { ModelProviderCard } from '@/types/llm';
 
-// ref :https://help.aliyun.com/zh/dashscope/developer-reference/api-details
+// ref: https://help.aliyun.com/zh/model-studio/getting-started/models
 const Qwen: ModelProviderCard = {
   chatModels: [
     {
-      description:
-        '通义千问超大规模语言模型，支持长文本上下文，以及基于长文档、多文档等多个场景的对话功能。',
-      displayName: 'Qwen Long',
-      id: 'qwen-long',
-      pricing: {
-        currency: 'CNY',
-        input: 0.5,
-        output: 2,
-      },
-      tokens: 1_000_000, // https://help.aliyun.com/zh/dashscope/developer-reference/model-introduction
-    },
-    {
-      description: '通义千问超大规模语言模型，支持中文、英文等不同语言输入',
+      description: '通义千问超大规模语言模型，支持中文、英文等不同语言输入。',
       displayName: 'Qwen Turbo',
       enabled: true,
       functionCall: true,
@@ -26,10 +14,10 @@ const Qwen: ModelProviderCard = {
         input: 0.3,
         output: 0.6,
       },
-      tokens: 131_072, // https://help.aliyun.com/zh/dashscope/developer-reference/model-introduction
+      tokens: 131_072,
     },
     {
-      description: '通义千问超大规模语言模型增强版，支持中文、英文等不同语言输入',
+      description: '通义千问超大规模语言模型增强版，支持中文、英文等不同语言输入。',
       displayName: 'Qwen Plus',
       enabled: true,
       functionCall: true,
@@ -39,11 +27,11 @@ const Qwen: ModelProviderCard = {
         input: 0.8,
         output: 2,
       },
-      tokens: 131_072, // https://help.aliyun.com/zh/dashscope/developer-reference/model-introduction
+      tokens: 131_072,
     },
     {
       description:
-        '通义千问千亿级别超大规模语言模型，支持中文、英文等不同语言输入，当前通义千问2.5产品版本背后的API模型',
+        '通义千问千亿级别超大规模语言模型，支持中文、英文等不同语言输入，当前通义千问2.5产品版本背后的API模型。',
       displayName: 'Qwen Max',
       enabled: true,
       functionCall: true,
@@ -53,7 +41,30 @@ const Qwen: ModelProviderCard = {
         input: 20,
         output: 60,
       },
-      tokens: 32_768, // https://help.aliyun.com/zh/dashscope/developer-reference/model-introduction
+      tokens: 32_768,
+    },
+    {
+      description: '通义千问代码模型。',
+      displayName: 'Qwen Coder',
+      id: 'qwen-coder-turbo-latest',
+      pricing: {
+        currency: 'CNY',
+        input: 2,
+        output: 6,
+      },
+      tokens: 131_072,
+    },
+    {
+      description:
+        '通义千问超大规模语言模型，支持长文本上下文，以及基于长文档、多文档等多个场景的对话功能。',
+      displayName: 'Qwen Long',
+      id: 'qwen-long',
+      pricing: {
+        currency: 'CNY',
+        input: 0.5,
+        output: 2,
+      },
+      tokens: 1_000_000,
     },
     {
       description:
@@ -66,8 +77,8 @@ const Qwen: ModelProviderCard = {
         input: 8,
         output: 8,
       },
-      tokens: 8192,
-      vision: true, // https://help.aliyun.com/zh/dashscope/developer-reference/tongyi-qianwen-vl-plus-api
+      tokens: 8000,
+      vision: true,
     },
     {
       description:
@@ -80,71 +91,120 @@ const Qwen: ModelProviderCard = {
         input: 20,
         output: 20,
       },
-      tokens: 32_768,
-      vision: true, // https://help.aliyun.com/zh/dashscope/developer-reference/tongyi-qianwen-vl-plus-api
+      tokens: 32_000,
+      vision: true,
     },
-    // ref :https://help.aliyun.com/zh/dashscope/developer-reference/tongyi-qianwen-7b-14b-72b-api-detailes
     {
-      description: '通义千问2.5对外开源的7B规模的模型',
+      description:
+        '通义千问数学模型是专门用于数学解题的语言模型。',
+      displayName: 'Qwen Math Turbo',
+      id: 'qwen-math-turbo-latest',
+      pricing: {
+        currency: 'CNY',
+        input: 2,
+        output: 6,
+      },
+      tokens: 4096,
+    },
+    {
+      description:
+        '通义千问数学模型是专门用于数学解题的语言模型。',
+      displayName: 'Qwen Math Plus',
+      id: 'qwen-math-plus-latest',
+      pricing: {
+        currency: 'CNY',
+        input: 4,
+        output: 12,
+      },
+      tokens: 4096,
+    },
+    {
+      description: '通义千问2.5对外开源的7B规模的模型。',
       displayName: 'Qwen2.5 7B',
       functionCall: true,
       id: 'qwen2.5-7b-instruct',
-      tokens: 131_072, // https://huggingface.co/Qwen/Qwen2.5-7B-Instruct
+      tokens: 131_072,
+    },
+    {
+      description: '通义千问2.5对外开源的14B规模的模型。',
+      displayName: 'Qwen2.5 14B',
+      functionCall: true,
+      id: 'qwen2.5-14b-instruct',
+      tokens: 131_072,
     },
     {
-      description: '通义千问2.5对外开源的32B规模的模型',
+      description: '通义千问2.5对外开源的32B规模的模型。',
       displayName: 'Qwen2.5 32B',
       functionCall: true,
       id: 'qwen2.5-32b-instruct',
-      tokens: 131_072, // https://huggingface.co/Qwen/Qwen2.5-32B-Instruct
+      tokens: 131_072,
     },
     {
-      description: '通义千问2.5对外开源的72B规模的模型',
+      description: '通义千问2.5对外开源的72B规模的模型。',
       displayName: 'Qwen2.5 72B',
       functionCall: true,
       id: 'qwen2.5-72b-instruct',
-      tokens: 131_072, // https://huggingface.co/Qwen/Qwen2.5-72B-Instruct
+      tokens: 131_072,
     },
     {
-      description: '通义千问2对外开源的7B规模的模型',
-      displayName: 'Qwen2 7B',
-      functionCall: true,
-      id: 'qwen2-7b-instruct',
-      tokens: 131_072, // https://huggingface.co/Qwen/Qwen2-7B-Instruct
-    },
-    {
-      description: '通义千问2对外开源的57B规模14B激活参数的MOE模型',
+      description: '通义千问2对外开源的57B规模14B激活参数的MOE模型。',
       displayName: 'Qwen2 57B A14B MoE',
       functionCall: true,
       id: 'qwen2-57b-a14b-instruct',
-      tokens: 65_536, // https://huggingface.co/Qwen/Qwen2-57B-A14B-Instruct
+      tokens: 65_536,
     },
     {
-      description: '通义千问2对外开源的72B规模的模型',
-      displayName: 'Qwen2 72B',
-      functionCall: true,
-      id: 'qwen2-72b-instruct',
-      tokens: 131_072, // https://huggingface.co/Qwen/Qwen2-72B-Instruct
+      description: 'Qwen-Math 模型具有强大的数学解题能力。',
+      displayName: 'Qwen2.5 Math 1.5B',
+      id: 'qwen2.5-math-1.5b-instruct',
+      pricing: {
+        currency: 'CNY',
+        input: 0,
+        output: 0,
+      },
+      tokens: 4096,
     },
     {
-      description: 'Qwen2-Math 模型具有强大的数学解题能力',
-      displayName: 'Qwen2 Math 72B',
-      functionCall: true,
-      id: 'qwen2-math-72b-instruct',
-      tokens: 4096, // https://help.aliyun.com/zh/dashscope/developer-reference/use-qwen2-math-by-calling-api
+      description: 'Qwen-Math 模型具有强大的数学解题能力。',
+      displayName: 'Qwen2.5 Math 7B',
+      id: 'qwen2.5-math-7b-instruct',
+      tokens: 4096,
+    },
+    {
+      description: 'Qwen-Math 模型具有强大的数学解题能力。',
+      displayName: 'Qwen2.5 Math 72B',
+      id: 'qwen2.5-math-72b-instruct',
+      tokens: 4096,
+    },
+    {
+      description: '通义千问代码模型开源版。',
+      displayName: 'Qwen2.5 Coder 1.5B',
+      id: 'qwen2.5-coder-1.5b-instruct',
+      pricing: {
+        currency: 'CNY',
+        input: 0,
+        output: 0,
+      },
+      tokens: 131_072,
+    },
+    {
+      description: '通义千问代码模型开源版。',
+      displayName: 'Qwen2.5 Coder 7B',
+      id: 'qwen2.5-coder-7b-instruct',
+      tokens: 131_072,
     },
     {
       description: '以 Qwen-7B 语言模型初始化，添加图像模型，图像输入分辨率为448的预训练模型。',
       displayName: 'Qwen VL',
       id: 'qwen-vl-v1',
-      tokens: 8192, // https://huggingface.co/Qwen/Qwen-VL/blob/main/config.json
+      tokens: 8000,
       vision: true,
     },
     {
       description: '通义千问VL支持灵活的交互方式，包括多图、多轮问答、创作等能力的模型。',
       displayName: 'Qwen VL Chat',
       id: 'qwen-vl-chat-v1',
-      tokens: 8192, // https://huggingface.co/Qwen/Qwen-VL-Chat/blob/main/config.json
+      tokens: 8000,
       vision: true,
     },
   ],
@@ -160,7 +220,7 @@ const Qwen: ModelProviderCard = {
     speed: 2,
     text: true,
   },
-  url: 'https://tongyi.aliyun.com',
+  url: 'https://www.aliyun.com/product/bailian',
 };
 
 export default Qwen;

package/src/config/modelProviders/stepfun.ts

@@ -8,12 +8,14 @@ const Stepfun: ModelProviderCard = {
       description: '支持大规模上下文交互，适合复杂对话场景。',
       displayName: 'Step 2 16K',
       enabled: true,
+      functionCall: true,
       id: 'step-2-16k',
       tokens: 16_000,
     },
     {
       description: '具备超长上下文处理能力，尤其适合长文档分析。',
       displayName: 'Step 1 256K',
+      functionCall: true,
       id: 'step-1-256k',
       tokens: 256_000,
     },
@@ -21,6 +23,7 @@ const Stepfun: ModelProviderCard = {
       description: '平衡性能与成本，适合一般场景。',
       displayName: 'Step 1 128K',
       enabled: true,
+      functionCall: true,
       id: 'step-1-128k',
       tokens: 128_000,
     },
@@ -28,6 +31,7 @@ const Stepfun: ModelProviderCard = {
       description: '支持中等长度的对话，适用于多种应用场景。',
       displayName: 'Step 1 32K',
       enabled: true,
+      functionCall: true,
       id: 'step-1-32k',
       tokens: 32_000,
     },
@@ -35,6 +39,7 @@ const Stepfun: ModelProviderCard = {
       description: '小型模型，适合轻量级任务。',
       displayName: 'Step 1 8K',
       enabled: true,
+      functionCall: true,
       id: 'step-1-8k',
       tokens: 8000,
     },
@@ -42,6 +47,7 @@ const Stepfun: ModelProviderCard = {
       description: '高速模型，适合实时对话。',
       displayName: 'Step 1 Flash',
       enabled: true,
+      functionCall: true,
       id: 'step-1-flash',
       tokens: 8000,
     },
@@ -49,6 +55,7 @@ const Stepfun: ModelProviderCard = {
       description: '支持视觉输入，增强多模态交互体验。',
       displayName: 'Step 1V 32K',
       enabled: true,
+      functionCall: true,
       id: 'step-1v-32k',
       tokens: 32_000,
       vision: true,
@@ -57,6 +64,7 @@ const Stepfun: ModelProviderCard = {
       description: '小型视觉模型，适合基本的图文任务。',
       displayName: 'Step 1V 8K',
       enabled: true,
+      functionCall: true,
       id: 'step-1v-8k',
       tokens: 8000,
       vision: true,

package/src/const/fetch.ts

@@ -1,5 +1,6 @@
 export const OPENAI_END_POINT = 'X-openai-end-point';
 export const OPENAI_API_KEY_HEADER_KEY = 'X-openai-api-key';
+export const LOBE_USER_ID = 'X-lobe-user-id';
 
 export const USE_AZURE_OPENAI = 'X-use-azure-openai';
 
@@ -19,9 +20,10 @@ export const getOpenAIAuthFromRequest = (req: Request) => {
   const useAzureStr = req.headers.get(USE_AZURE_OPENAI);
   const apiVersion = req.headers.get(AZURE_OPENAI_API_VERSION);
   const oauthAuthorizedStr = req.headers.get(OAUTH_AUTHORIZED);
+  const userId = req.headers.get(LOBE_USER_ID);
 
   const oauthAuthorized = !!oauthAuthorizedStr;
   const useAzure = !!useAzureStr;
 
-  return { accessCode, apiKey, apiVersion, endpoint, oauthAuthorized, useAzure };
+  return { accessCode, apiKey, apiVersion, endpoint, oauthAuthorized, useAzure, userId };
 };

package/src/libs/agent-runtime/AgentRuntime.ts

@@ -35,6 +35,7 @@ import {
   EmbeddingsPayload,
   ModelProvider,
   TextToImagePayload,
+  TextToSpeechPayload,
 } from './types';
 import { LobeUpstageAI } from './upstage';
 import { LobeZeroOneAI } from './zeroone';
@@ -97,6 +98,9 @@ class AgentRuntime {
   async embeddings(payload: EmbeddingsPayload, options?: EmbeddingsOptions) {
     return this._runtime.embeddings?.(payload, options);
   }
+  async textToSpeech(payload: TextToSpeechPayload, options?: EmbeddingsOptions) {
+    return this._runtime.textToSpeech?.(payload, options);
+  }
 
   /**
    * @description Initialize the runtime with the provider and the options

package/src/libs/agent-runtime/BaseAI.ts

@@ -1,6 +1,5 @@
 import OpenAI from 'openai';
 
-import { TextToImagePayload } from '@/libs/agent-runtime/types/textToImage';
 import { ChatModelCard } from '@/types/llm';
 
 import {
@@ -9,6 +8,9 @@ import {
   EmbeddingItem,
   EmbeddingsOptions,
   EmbeddingsPayload,
+  TextToImagePayload,
+  TextToSpeechOptions,
+  TextToSpeechPayload,
 } from './types';
 
 export interface LobeRuntimeAI {
@@ -20,6 +22,11 @@ export interface LobeRuntimeAI {
   models?(): Promise<any>;
 
   textToImage?: (payload: TextToImagePayload) => Promise<string[]>;
+
+  textToSpeech?: (
+    payload: TextToSpeechPayload,
+    options?: TextToSpeechOptions,
+  ) => Promise<ArrayBuffer>;
 }
 
 export abstract class LobeOpenAICompatibleRuntime {

package/src/libs/agent-runtime/types/index.ts

@@ -1,4 +1,5 @@
 export * from './chat';
 export * from './embeddings';
 export * from './textToImage';
+export * from './tts';
 export * from './type';

package/src/libs/agent-runtime/types/tts.ts

@@ -0,0 +1,14 @@
+export interface TextToSpeechPayload {
+  input: string;
+  model: string;
+  voice: string;
+}
+
+export interface TextToSpeechOptions {
+  headers?: Record<string, any>;
+  signal?: AbortSignal;
+  /**
+   * userId for the embeddings
+   */
+  user?: string;
+}

package/src/libs/agent-runtime/utils/openaiCompatibleFactory/index.ts

@@ -1,7 +1,6 @@
 import OpenAI, { ClientOptions } from 'openai';
 
 import { LOBE_DEFAULT_MODEL_LIST } from '@/config/modelProviders';
-import { TextToImagePayload } from '@/libs/agent-runtime/types/textToImage';
 import { ChatModelCard } from '@/types/llm';
 
 import { LobeRuntimeAI } from '../../BaseAI';
@@ -13,6 +12,9 @@ import {
   EmbeddingItem,
   EmbeddingsOptions,
   EmbeddingsPayload,
+  TextToImagePayload,
+  TextToSpeechOptions,
+  TextToSpeechPayload,
 } from '../../types';
 import { AgentRuntimeError } from '../createError';
 import { debugResponse, debugStream } from '../debugStream';
@@ -253,6 +255,19 @@ export const LobeOpenAICompatibleFactory = <T extends Record<string, any> = any>
       }
     }
 
+    async textToSpeech(payload: TextToSpeechPayload, options?: TextToSpeechOptions) {
+      try {
+        const mp3 = await this.client.audio.speech.create(payload as any, {
+          headers: options?.headers,
+          signal: options?.signal,
+        });
+
+        return mp3.arrayBuffer();
+      } catch (error) {
+        throw this.handleError(error);
+      }
+    }
+
     private handleError(error: any): ChatCompletionErrorPayload {
       let desensitizedEndpoint = this.baseURL;
 

package/src/services/_header.ts

@@ -1,4 +1,9 @@
-import { LOBE_CHAT_ACCESS_CODE, OPENAI_API_KEY_HEADER_KEY, OPENAI_END_POINT } from '@/const/fetch';
+import {
+  LOBE_CHAT_ACCESS_CODE,
+  LOBE_USER_ID,
+  OPENAI_API_KEY_HEADER_KEY,
+  OPENAI_END_POINT,
+} from '@/const/fetch';
 import { useUserStore } from '@/store/user';
 import { keyVaultsConfigSelectors } from '@/store/user/selectors';
 
@@ -8,12 +13,14 @@ import { keyVaultsConfigSelectors } from '@/store/user/selectors';
  */
 // eslint-disable-next-line no-undef
 export const createHeaderWithOpenAI = (header?: HeadersInit): HeadersInit => {
-  const openAIConfig = keyVaultsConfigSelectors.openAIConfig(useUserStore.getState());
+  const state = useUserStore.getState();
+  const openAIConfig = keyVaultsConfigSelectors.openAIConfig(state);
 
   // eslint-disable-next-line no-undef
   return {
     ...header,
-    [LOBE_CHAT_ACCESS_CODE]: keyVaultsConfigSelectors.password(useUserStore.getState()),
+    [LOBE_CHAT_ACCESS_CODE]: keyVaultsConfigSelectors.password(state),
+    [LOBE_USER_ID]: state.user?.id || '',
     [OPENAI_API_KEY_HEADER_KEY]: openAIConfig.apiKey || '',
     [OPENAI_END_POINT]: openAIConfig.baseURL || '',
   };

package/src/services/_url.ts

@@ -1,4 +1,4 @@
-// TODO: 未来所有路由需要全部迁移到 trpc
+// TODO: 未来路由需要迁移到 trpc or /webapi
 
 /* eslint-disable sort-keys-fix/sort-keys-fix */
 import { transform } from 'lodash-es';
@@ -38,9 +38,11 @@ export const API_ENDPOINTS = mapWithBasePath({
   // image
   images: '/api/text-to-image/openai',
 
-  // TTS & STT
-  stt: '/api/openai/stt',
-  tts: '/api/openai/tts',
-  edge: '/api/tts/edge-speech',
-  microsoft: '/api/tts/microsoft-speech',
+  // STT
+  stt: '/webapi/stt/openai',
+
+  // TTS
+  tts: '/webapi/tts/openai',
+  edge: '/webapi/tts/edge',
+  microsoft: '/webapi/tts/microsoft',
 });

package/src/store/file/slices/tts/action.ts

@@ -39,7 +39,7 @@ export const createTTSFileSlice: StateCreator<
     };
     const file = new File([blob], fileName, fileOptions);
 
-    const res = await get().uploadWithProgress({ file });
+    const res = await get().uploadWithProgress({ file, skipCheckFileType: true });
 
     return res?.id;
   },

package/src/store/file/slices/upload/action.ts

@@ -29,6 +29,12 @@ interface UploadWithProgressParams {
           type: 'removeFile';
         },
   ) => void;
+  /**
+   * Optional flag to indicate whether to skip the file type check.
+   * When set to `true`, any file type checks will be bypassed.
+   * Default is `false`, which means file type checks will be performed.
+   */
+  skipCheckFileType?: boolean;
 }
 
 interface UploadWithProgressResult {
@@ -52,8 +58,8 @@ export const createFileUploadSlice: StateCreator<
   [],
   FileUploadAction
 > = (set, get) => ({
-  internal_uploadToClientDB: async ({ file, onStatusUpdate }) => {
-    if (!file.type.startsWith('image')) {
+  internal_uploadToClientDB: async ({ file, onStatusUpdate, skipCheckFileType }) => {
+    if (!skipCheckFileType && !file.type.startsWith('image')) {
       onStatusUpdate?.({ id: file.name, type: 'removeFile' });
       message.info({
         content: t('upload.fileOnlySupportInServerMode', {
@@ -158,11 +164,11 @@ export const createFileUploadSlice: StateCreator<
     return data;
   },
 
-  uploadWithProgress: async ({ file, onStatusUpdate, knowledgeBaseId }) => {
+  uploadWithProgress: async (payload) => {
     const { internal_uploadToServer, internal_uploadToClientDB } = get();
 
-    if (isServerMode) return internal_uploadToServer({ file, knowledgeBaseId, onStatusUpdate });
+    if (isServerMode) return internal_uploadToServer(payload);
 
-    return internal_uploadToClientDB({ file, onStatusUpdate });
+    return internal_uploadToClientDB(payload);
   },
 });