@lobehub/chat 0.101.7 → 0.102.1

package/locales/ru_RU/setting.json

@@ -207,6 +207,44 @@
     },
     "title": "Настройки системы"
   },
+  "settingTTS": {
+    "showAllLocaleVoice": {
+      "desc": "Если отключено, отображаются только голоса текущего языка",
+      "title": "Показать все голоса локали"
+    },
+    "sttService": {
+      "desc": "где broswer - это встроенная в браузер служба распознавания речи",
+      "title": "Служба распознавания речи"
+    },
+    "title": "Служба речи",
+    "ttsService": {
+      "desc": "Если используется услуга синтеза речи OpenAI, убедитесь, что услуга модели OpenAI включена",
+      "title": "Служба синтеза речи"
+    },
+    "voice": {
+      "title": "Голосовой синтез",
+      "desc": "Выберите голос для текущего помощника, различные службы TTS поддерживают разные источники звука",
+      "preview": "Предварительный просмотр голоса"
+    },
+    "openai": {
+      "sttModel": "Модель распознавания речи OpenAI",
+      "ttsModel": "Модель синтеза речи OpenAI"
+    },
+    "stt": "Настройки распознавания речи",
+    "sttLocale": {
+      "desc": "Язык речи для ввода речи, этот параметр может повысить точность распознавания речи",
+      "title": "Язык распознавания речи"
+    },
+    "sttPersisted": {
+      "desc": "При включении распознавание речи не будет автоматически завершаться, необходимо вручную нажать кнопку завершения",
+      "title": "Вручную завершить распознавание речи"
+    },
+    "tts": "Настройки синтеза речи",
+    "sttAutoStop": {
+      "desc": "После отключения распознавания речи оно не будет автоматически останавливаться, вам нужно будет вручную нажать кнопку завершения",
+      "title": "Автоматическое завершение распознавания речи"
+    }
+  },
   "settingTheme": {
     "avatar": {
       "title": "Аватар"
@@ -245,6 +283,7 @@
   "tab": {
     "agent": "Помощник по умолчанию",
     "common": "Общие настройки",
-    "llm": "Пользовательский API GPT"
+    "llm": "Пользовательский API GPT",
+    "tts": "Сервис речи"
   }
 }

package/locales/zh_CN/chat.json

@@ -35,6 +35,11 @@
     "withSystemRole": "包含助手角色设定"
   },
   "stop": "停止",
+  "stt": {
+    "action": "语音输入",
+    "loading": "识别中...",
+    "prettifying": "润色中..."
+  },
   "temp": "临时",
   "tokenDetail": "角色设定: {{systemRoleToken}} · 历史消息: {{chatsToken}}",
   "tokenTag": {
@@ -55,9 +60,13 @@
     "title": "话题列表"
   },
   "translate": {
+    "action": "翻译",
     "clear": "删除翻译"
   },
-  "translateTo": "翻译",
+  "tts": {
+    "action": "语音朗读",
+    "clear": "删除语音"
+  },
   "updateAgent": "更新助理信息",
   "upload": {
     "actionTooltip": "上传图片",

package/locales/zh_CN/error.json

@@ -28,6 +28,12 @@
     "OpenAIBizError": "请求 OpenAI 服务出错，请根据以下信息排查或重试",
     "NoAPIKey": "OpenAI API Key 为空，请添加自定义 OpenAI API Key"
   },
+  "stt": {
+    "responseError": "服务请求失败，请检查配置或重试"
+  },
+  "tts": {
+    "responseError": "服务请求失败，请检查配置或重试"
+  },
   "unlock": {
     "apikey": {
       "addProxyUrl": "添加 OpenAI 代理地址（可选）",

package/locales/zh_CN/setting.json

@@ -194,6 +194,40 @@
     },
     "title": "系统设置"
   },
+  "settingTTS": {
+    "openai": {
+      "sttModel": "OpenAI 语音识别模型",
+      "ttsModel": "OpenAI 语音合成模型"
+    },
+    "showAllLocaleVoice": {
+      "desc": "关闭则只显示当前语种的声源",
+      "title": "显示所有语种声源"
+    },
+    "stt": "语音识别设置",
+    "sttAutoStop": {
+      "desc": "关闭后，语音识别将不会自动结束，需要手动点击结束按钮",
+      "title": "自动结束语音识别"
+    },
+    "sttLocale": {
+      "desc": "语音输入的语种，此选项可提高语音识别准确率",
+      "title": "语音识别语种"
+    },
+    "sttService": {
+      "desc": "其中 broswer 为浏览器原生的语音识别服务",
+      "title": "语音识别服务"
+    },
+    "title": "语音服务",
+    "tts": "语音合成设置",
+    "ttsService": {
+      "desc": "如使用 OpenAI 语音合成服务，需要保证 OpenAI 模型服务已开启",
+      "title": "语音合成服务"
+    },
+    "voice": {
+      "desc": "为当前助手挑选一个声音，不同 TTS 服务支持的声源不同",
+      "preview": "试听声源",
+      "title": "语音合成声源"
+    }
+  },
   "settingTheme": {
     "avatar": {
       "title": "头像"
@@ -232,6 +266,7 @@
   "tab": {
     "agent": "默认助手",
     "common": "通用设置",
-    "llm": "语言模型"
+    "llm": "语言模型",
+    "tts": "语音服务"
   }
 }

package/locales/zh_TW/chat.json

@@ -35,6 +35,11 @@
     "withSystemRole": "包含助手角色設定"
   },
   "stop": "停止",
+  "stt": {
+    "action": "語音輸入",
+    "loading": "辨識中...",
+    "prettifying": "美化中..."
+  },
   "temp": "臨時",
   "tokenDetail": "角色設定: {{systemRoleToken}} · 歷史訊息: {{chatsToken}}",
   "tokenTag": {
@@ -57,9 +62,14 @@
     "openNewTopic": "開啟新話題"
   },
   "translate": {
-    "clear": "刪除翻譯"
+    "clear": "刪除翻譯",
+    "action": "翻譯"
   },
   "translateTo": "翻譯",
+  "tts": {
+    "action": "語音朗讀",
+    "clear": "清除語音"
+  },
   "updateAgent": "更新助理資訊",
   "upload": {
     "actionTooltip": "上傳圖片",

package/locales/zh_TW/error.json

@@ -28,6 +28,12 @@
     "PluginServerError": "外掛伺服器請求回傳錯誤。請根據下面的錯誤資訊檢查您的外掛描述檔案、外掛設定或伺服器實作",
     "NoAPIKey": "OpenAI API 金鑰為空，請添加自訂 OpenAI API 金鑰"
   },
+  "stt": {
+    "responseError": "服務請求失敗，請檢查配置或重試"
+  },
+  "tts": {
+    "responseError": "服務請求失敗，請檢查配置或重試"
+  },
   "unlock": {
     "apikey": {
       "title": "使用自定義 API Key",

package/locales/zh_TW/setting.json

@@ -207,6 +207,44 @@
     },
     "title": "系統設定"
   },
+  "settingTTS": {
+    "showAllLocaleVoice": {
+      "desc": "關閉則只顯示當前語種的聲源",
+      "title": "顯示所有語種聲源"
+    },
+    "sttService": {
+      "desc": "其中 broswer 為瀏覽器原生的語音識別服務",
+      "title": "語音識別服務"
+    },
+    "title": "語音服務",
+    "ttsService": {
+      "desc": "如使用 OpenAI 語音合成服務，需要保證 OpenAI 模型服務已開啟",
+      "title": "語音合成服務"
+    },
+    "voice": {
+      "title": "語音合成聲源",
+      "desc": "為當前助手挑選一個聲音，不同 TTS 服務支持的聲源不同",
+      "preview": "預覽"
+    },
+    "openai": {
+      "sttModel": "OpenAI 語音識別模型",
+      "ttsModel": "OpenAI 語音合成模型"
+    },
+    "stt": "語音識別設定",
+    "sttLocale": {
+      "desc": "語音輸入的語言，此選項可提高語音識別準確率",
+      "title": "語音識別語言"
+    },
+    "sttPersisted": {
+      "desc": "開啟後，語音識別將不會自動結束，需要手動點擊結束按鈕",
+      "title": "手動結束語音識別"
+    },
+    "tts": "語音合成設定",
+    "sttAutoStop": {
+      "desc": "關閉後，語音識別將不會自動結束，需要手動點擊結束按鈕",
+      "title": "自動結束語音識別"
+    }
+  },
   "settingTheme": {
     "avatar": {
       "title": "頭像"
@@ -245,6 +283,7 @@
   "tab": {
     "agent": "預設助理",
     "common": "通用設定",
-    "llm": "語言模型"
+    "llm": "語言模型",
+    "tts": "語音服務"
   }
 }

package/next.config.mjs

@@ -40,25 +40,7 @@ const nextConfig = {
     remotePatterns: [
       {
         hostname: 'registry.npmmirror.com',
-        pathname: '/@lobehub/assets-emoji/1.3.0/files/assets/**',
-        port: '',
-        protocol: 'https',
-      },
-      {
-        hostname: 'registry.npmmirror.com',
-        pathname: '/@lobehub/assets-emoji-anim/1.0.0/files/assets/**',
-        port: '',
-        protocol: 'https',
-      },
-      {
-        hostname: 'registry.npmmirror.com',
-        pathname: '/@lobehub/assets-logo/1.1.0/files/assets/**',
-        port: '',
-        protocol: 'https',
-      },
-      {
-        hostname: 'registry.npmmirror.com',
-        pathname: '/@lobehub/assets-favicons/latest/files/assets/**',
+        pathname: '/@lobehub/**',
         port: '',
         protocol: 'https',
       },
@@ -69,7 +51,7 @@ const nextConfig = {
 
   reactStrictMode: true,
 
-  transpilePackages: ['antd-style', '@lobehub/ui'],
+  transpilePackages: ['antd-style', '@lobehub/ui', '@lobehub/tts'],
 
   webpack(config) {
     config.experiments = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lobehub/chat",
-  "version": "0.101.7",
+  "version": "0.102.1",
   "description": "An open-source, extensible (Function Calling), high-performance chatbot framework. It supports one-click free deployment of your private ChatGPT/LLM web application",
   "keywords": [
     "chatbot",
@@ -70,6 +70,7 @@
     "@icons-pack/react-simple-icons": "^9",
     "@lobehub/chat-plugin-sdk": "latest",
     "@lobehub/chat-plugins-gateway": "latest",
+    "@lobehub/tts": "latest",
     "@lobehub/ui": "latest",
     "@vercel/analytics": "^1",
     "ahooks": "^3",
@@ -127,8 +128,8 @@
     "@ducanh2912/next-pwa": "^9.7.2",
     "@lobehub/i18n-cli": "latest",
     "@lobehub/lint": "latest",
-    "@next/bundle-analyzer": "14.0.1",
-    "@next/eslint-plugin-next": "14.0.1",
+    "@next/bundle-analyzer": "14.0.3",
+    "@next/eslint-plugin-next": "14.0.3",
     "@testing-library/jest-dom": "^6",
     "@testing-library/react": "^14",
     "@types/chroma-js": "^2",

package/src/app/api/openai/chat/route.ts

@@ -1,14 +1,6 @@
-import OpenAI from 'openai';
-
-import { getServerConfig } from '@/config/server';
-import { getOpenAIAuthFromRequest } from '@/const/fetch';
-import { ChatErrorType, ErrorType } from '@/types/fetch';
 import { OpenAIChatStreamPayload } from '@/types/openai/chat';
 
-import { checkAuth } from '../../auth';
-import { createAzureOpenai } from '../createAzureOpenai';
-import { createOpenai } from '../createOpenai';
-import { createErrorResponse } from '../errorResponse';
+import { createBizOpenAI } from '../createBizOpenAI';
 import { createChatCompletion } from './createChatCompletion';
 
 export const runtime = 'edge';
@@ -16,38 +8,10 @@ export const runtime = 'edge';
 export const POST = async (req: Request) => {
   const payload = (await req.json()) as OpenAIChatStreamPayload;
 
-  const { apiKey, accessCode, endpoint, useAzure, apiVersion } = getOpenAIAuthFromRequest(req);
-
-  const result = checkAuth({ accessCode, apiKey });
-
-  if (!result.auth) {
-    return createErrorResponse(result.error as ErrorType);
-  }
-
-  let openai: OpenAI;
-
-  const { USE_AZURE_OPENAI } = getServerConfig();
-  const useAzureOpenAI = useAzure || USE_AZURE_OPENAI;
-
-  try {
-    if (useAzureOpenAI) {
-      openai = createAzureOpenai({
-        apiVersion,
-        endpoint,
-        model: payload.model,
-        userApiKey: apiKey,
-      });
-    } else {
-      openai = createOpenai(apiKey, endpoint);
-    }
-  } catch (error) {
-    if ((error as Error).cause === ChatErrorType.NoAPIKey) {
-      return createErrorResponse(ChatErrorType.NoAPIKey);
-    }
+  const openaiOrErrResponse = createBizOpenAI(req, payload.model);
 
-    console.error(error); // log error to trace it
-    return createErrorResponse(ChatErrorType.InternalServerError);
-  }
+  // if resOrOpenAI is a Response, it means there is an error,just return it
+  if (openaiOrErrResponse instanceof Response) return openaiOrErrResponse;
 
-  return createChatCompletion({ openai, payload });
+  return createChatCompletion({ openai: openaiOrErrResponse, payload });
 };

package/src/app/api/openai/{createAzureOpenai.ts → createBizOpenAI/createAzureOpenai.ts}

@@ -4,7 +4,7 @@ import urlJoin from 'url-join';
 import { getServerConfig } from '@/config/server';
 import { ChatErrorType } from '@/types/fetch';
 
-// 创建 Azure OpenAI 实例
+// create Azure OpenAI Instance
 export const createAzureOpenai = (params: {
   apiVersion?: string | null;
   endpoint?: string | null;

package/src/app/api/openai/{createOpenai.ts → createBizOpenAI/createOpenai.ts}

@@ -3,7 +3,7 @@ import OpenAI from 'openai';
 import { getServerConfig } from '@/config/server';
 import { ChatErrorType } from '@/types/fetch';
 
-// 创建 OpenAI 实例
+// create OpenAI instance
 export const createOpenai = (userApiKey: string | null, endpoint?: string | null) => {
   const { OPENAI_API_KEY, OPENAI_PROXY_URL } = getServerConfig();
 

package/src/app/api/openai/createBizOpenAI/index.ts

@@ -0,0 +1,46 @@
+import OpenAI from 'openai/index';
+
+import { checkAuth } from '@/app/api/auth';
+import { getServerConfig } from '@/config/server';
+import { getOpenAIAuthFromRequest } from '@/const/fetch';
+import { ChatErrorType, ErrorType } from '@/types/fetch';
+
+import { createErrorResponse } from '../errorResponse';
+import { createAzureOpenai } from './createAzureOpenai';
+import { createOpenai } from './createOpenai';
+
+/**
+ * createOpenAI Instance with Auth and azure openai support
+ * if auth not pass ,just return error response
+ */
+export const createBizOpenAI = (req: Request, model: string): Response | OpenAI => {
+  const { apiKey, accessCode, endpoint, useAzure, apiVersion } = getOpenAIAuthFromRequest(req);
+
+  const result = checkAuth({ accessCode, apiKey });
+
+  if (!result.auth) {
+    return createErrorResponse(result.error as ErrorType);
+  }
+
+  let openai: OpenAI;
+
+  const { USE_AZURE_OPENAI } = getServerConfig();
+  const useAzureOpenAI = useAzure || USE_AZURE_OPENAI;
+
+  try {
+    if (useAzureOpenAI) {
+      openai = createAzureOpenai({ apiVersion, endpoint, model, userApiKey: apiKey });
+    } else {
+      openai = createOpenai(apiKey, endpoint);
+    }
+  } catch (error) {
+    if ((error as Error).cause === ChatErrorType.NoAPIKey) {
+      return createErrorResponse(ChatErrorType.NoAPIKey);
+    }
+
+    console.error(error); // log error to trace it
+    return createErrorResponse(ChatErrorType.InternalServerError);
+  }
+
+  return openai;
+};

package/src/app/api/openai/models/route.ts

@@ -1,6 +1,6 @@
 import { getOpenAIAuthFromRequest } from '@/const/fetch';
 
-import { createOpenai } from '../createOpenai';
+import { createOpenai } from '../createBizOpenAI/createOpenai';
 
 export const runtime = 'edge';
 

package/src/app/api/openai/stt/route.ts

@@ -0,0 +1,29 @@
+import { OpenAISTTPayload } from '@lobehub/tts';
+import { createOpenaiAudioTranscriptions } from '@lobehub/tts/server';
+
+import { createBizOpenAI } from '../createBizOpenAI';
+
+export const runtime = 'edge';
+
+export const POST = async (req: Request) => {
+  const formData = await req.formData();
+  const speechBlob = formData.get('speech') as Blob;
+  const optionsString = formData.get('options') as string;
+  const payload = {
+    options: JSON.parse(optionsString),
+    speech: speechBlob,
+  } as OpenAISTTPayload;
+
+  const openaiOrErrResponse = createBizOpenAI(req, payload.options.model);
+
+  // if resOrOpenAI is a Response, it means there is an error,just return it
+  if (openaiOrErrResponse instanceof Response) return openaiOrErrResponse;
+
+  const res = await createOpenaiAudioTranscriptions({ openai: openaiOrErrResponse, payload });
+
+  return new Response(JSON.stringify(res), {
+    headers: {
+      'content-type': 'application/json;charset=UTF-8',
+    },
+  });
+};

package/src/app/api/openai/tts/route.ts

@@ -0,0 +1,17 @@
+import { OpenAITTSPayload } from '@lobehub/tts';
+import { createOpenaiAudioSpeech } from '@lobehub/tts/server';
+
+import { createBizOpenAI } from '../createBizOpenAI';
+
+export const runtime = 'edge';
+
+export const POST = async (req: Request) => {
+  const payload = (await req.json()) as OpenAITTSPayload;
+
+  const openaiOrErrResponse = createBizOpenAI(req, payload.options.model);
+
+  // if resOrOpenAI is a Response, it means there is an error,just return it
+  if (openaiOrErrResponse instanceof Response) return openaiOrErrResponse;
+
+  return await createOpenaiAudioSpeech({ openai: openaiOrErrResponse, payload });
+};

package/src/app/api/tts/edge-speech/route.ts

@@ -0,0 +1,9 @@
+import { EdgeSpeechPayload, EdgeSpeechTTS } from '@lobehub/tts';
+
+export const runtime = 'edge';
+
+export const POST = async (req: Request) => {
+  const payload = (await req.json()) as EdgeSpeechPayload;
+
+  return await EdgeSpeechTTS.createRequest({ payload });
+};

package/src/app/api/tts/microsoft-speech/route.ts

@@ -0,0 +1,9 @@
+import { MicrosoftSpeechPayload, MicrosoftSpeechTTS } from '@lobehub/tts';
+
+export const runtime = 'edge';
+
+export const POST = async (req: Request) => {
+  const payload = (await req.json()) as MicrosoftSpeechPayload;
+
+  return await MicrosoftSpeechTTS.createRequest({ payload });
+};

package/src/app/chat/(desktop)/features/ChatHeader.tsx

@@ -7,6 +7,7 @@ import { memo } from 'react';
 import { useTranslation } from 'react-i18next';
 import { Flexbox } from 'react-layout-kit';
 
+import { DESKTOP_HEADER_ICON_SIZE } from '@/const/layoutTokens';
 import { useGlobalStore } from '@/store/global';
 import { useSessionChatInit, useSessionStore } from '@/store/session';
 import { agentSelectors, sessionSelectors } from '@/store/session/selectors';
@@ -85,7 +86,7 @@ const Header = memo(() => {
           <ActionIcon
             icon={showAgentSettings ? PanelRightClose : PanelRightOpen}
             onClick={() => toggleConfig()}
-            size={{ fontSize: 24 }}
+            size={DESKTOP_HEADER_ICON_SIZE}
             title={t('roleAndArchive')}
           />
           {!isInbox && (
@@ -94,7 +95,7 @@ const Header = memo(() => {
               onClick={() => {
                 router.push(pathString('/chat/settings', { hash: location.hash }));
               }}
-              size={{ fontSize: 24 }}
+              size={DESKTOP_HEADER_ICON_SIZE}
               title={t('header.session', { ns: 'setting' })}
             />
           )}

package/src/app/chat/(desktop)/features/ChatInput/index.tsx

@@ -47,7 +47,6 @@ const ChatInputDesktopLayout = memo(() => {
         minHeight={CHAT_TEXTAREA_HEIGHT}
         onSizeChange={(_, size) => {
           if (!size) return;
-
           updatePreference({
             inputHeight:
               typeof size.height === 'string' ? Number.parseInt(size.height) : size.height,

package/src/app/chat/(desktop)/features/SessionHeader.tsx

@@ -5,6 +5,7 @@ import { memo } from 'react';
 import { useTranslation } from 'react-i18next';
 import { Flexbox } from 'react-layout-kit';
 
+import { DESKTOP_HEADER_ICON_SIZE } from '@/const/layoutTokens';
 import { useSessionStore } from '@/store/session';
 
 import SessionSearchBar from '../../features/SessionSearchBar';
@@ -28,11 +29,10 @@ const Header = memo(() => {
     <Flexbox className={styles.top} gap={16} padding={16}>
       <Flexbox distribution={'space-between'} horizontal>
         <Logo className={styles.logo} size={36} type={'text'} />
-
         <ActionIcon
           icon={MessageSquarePlus}
           onClick={() => createSession()}
-          size={{ fontSize: 24 }}
+          size={DESKTOP_HEADER_ICON_SIZE}
           style={{ flex: 'none' }}
           title={t('newAgent')}
         />

package/src/app/chat/(mobile)/features/ChatHeader.tsx

@@ -1,7 +1,8 @@
-import { ActionIcon, MobileNavBar, MobileNavBarTitle } from '@lobehub/ui';
-import { Clock3, Settings } from 'lucide-react';
+import { ActionIcon, Icon, MobileNavBar, MobileNavBarTitle } from '@lobehub/ui';
+import { Dropdown, MenuProps } from 'antd';
+import { Clock3, MoreHorizontal, Settings, Share2 } from 'lucide-react';
 import { useRouter } from 'next/navigation';
-import { memo } from 'react';
+import { memo, useState } from 'react';
 import { useTranslation } from 'react-i18next';
 
 import { MOBILE_HEADER_ICON_SIZE } from '@/const/layoutTokens';
@@ -15,6 +16,7 @@ import ShareButton from '../../features/ChatHeader/ShareButton';
 const MobileHeader = memo(() => {
   const { t } = useTranslation('chat');
   const router = useRouter();
+  const [open, setOpen] = useState(false);
 
   const [isInbox, title] = useSessionStore((s) => [
     sessionSelectors.isInboxSession(s),
@@ -25,21 +27,37 @@ const MobileHeader = memo(() => {
 
   const displayTitle = isInbox ? t('inbox.title') : title;
 
+  const items: MenuProps['items'] = [
+    {
+      icon: <Icon icon={Share2} />,
+      key: 'share',
+      label: t('share', { ns: 'common' }),
+      onClick: () => setOpen(true),
+    },
+    !isInbox && {
+      icon: <Icon icon={Settings} />,
+      key: 'settings',
+      label: t('header.session', { ns: 'setting' }),
+      onClick: () => router.push(pathString('/chat/settings', { hash: location.hash })),
+    },
+  ].filter(Boolean) as MenuProps['items'];
+
   return (
     <MobileNavBar
       center={<MobileNavBarTitle title={displayTitle} />}
       onBackClick={() => router.push('/chat')}
       right={
         <>
-          <ShareButton />
           <ActionIcon icon={Clock3} onClick={() => toggleConfig()} size={MOBILE_HEADER_ICON_SIZE} />
-          {!isInbox && (
-            <ActionIcon
-              icon={Settings}
-              onClick={() => router.push(pathString('/chat/settings', { hash: location.hash }))}
-              size={MOBILE_HEADER_ICON_SIZE}
-            />
-          )}
+          <ShareButton mobile open={open} setOpen={setOpen} />
+          <Dropdown
+            menu={{
+              items,
+            }}
+            trigger={['click']}
+          >
+            <ActionIcon icon={MoreHorizontal} />
+          </Dropdown>
         </>
       }
       showBackButton