@lobehub/chat 0.101.7 → 0.102.1

package/src/features/AgentSetting/AgentTTS/index.tsx

@@ -0,0 +1,115 @@
+import { VoiceList } from '@lobehub/tts';
+import { Form, ItemGroup } from '@lobehub/ui';
+import { Form as AFrom, Select, Switch } from 'antd';
+import isEqual from 'fast-deep-equal';
+import { debounce } from 'lodash-es';
+import { Mic } from 'lucide-react';
+import { memo, useEffect } from 'react';
+import { useTranslation } from 'react-i18next';
+
+import { FORM_STYLE } from '@/const/layoutTokens';
+import SelectWithTTSPreview from '@/features/AgentSetting/AgentTTS/SelectWithTTSPreview';
+import { settingsSelectors, useGlobalStore } from '@/store/global';
+
+import { useStore } from '../store';
+import { ttsOptions } from './options';
+
+const TTS_SETTING_KEY = 'tts';
+const { openaiVoiceOptions, localeOptions } = VoiceList;
+
+const AgentTTS = memo(() => {
+  const { t } = useTranslation('setting');
+  const updateConfig = useStore((s) => s.setAgentConfig);
+  const [form] = AFrom.useForm();
+  const voiceList = useGlobalStore((s) => {
+    const locale = settingsSelectors.currentLanguage(s);
+    return (all?: boolean) => new VoiceList(all ? undefined : locale);
+  });
+  const config = useStore((s) => s.config, isEqual);
+
+  useEffect(() => {
+    form.setFieldsValue(config);
+  }, [config]);
+
+  const showAllLocaleVoice = config.tts.showAllLocaleVoice;
+
+  const { edgeVoiceOptions, microsoftVoiceOptions } = voiceList(showAllLocaleVoice);
+
+  const tts: ItemGroup = {
+    children: [
+      {
+        children: <Select options={ttsOptions} />,
+        desc: t('settingTTS.ttsService.desc'),
+        label: t('settingTTS.ttsService.title'),
+        name: [TTS_SETTING_KEY, 'ttsService'],
+      },
+      {
+        children: <Switch />,
+        desc: t('settingTTS.showAllLocaleVoice.desc'),
+        hidden: config.tts.ttsService === 'openai',
+        label: t('settingTTS.showAllLocaleVoice.title'),
+        minWidth: undefined,
+        name: [TTS_SETTING_KEY, 'showAllLocaleVoice'],
+        valuePropName: 'checked',
+      },
+      {
+        children: <SelectWithTTSPreview options={openaiVoiceOptions} server={'openai'} />,
+        desc: t('settingTTS.voice.desc'),
+        hidden: config.tts.ttsService !== 'openai',
+        label: t('settingTTS.voice.title'),
+        name: [TTS_SETTING_KEY, 'voice', 'openai'],
+      },
+      {
+        children: <SelectWithTTSPreview options={edgeVoiceOptions} server={'edge'} />,
+        desc: t('settingTTS.voice.desc'),
+        divider: false,
+        hidden: config.tts.ttsService !== 'edge',
+        label: t('settingTTS.voice.title'),
+        name: [TTS_SETTING_KEY, 'voice', 'edge'],
+      },
+      {
+        children: <SelectWithTTSPreview options={microsoftVoiceOptions} server={'microsoft'} />,
+        desc: t('settingTTS.voice.desc'),
+        divider: false,
+        hidden: config.tts.ttsService !== 'microsoft',
+        label: t('settingTTS.voice.title'),
+        name: [TTS_SETTING_KEY, 'voice', 'microsoft'],
+      },
+      {
+        children: (
+          <Select
+            options={[
+              { label: t('settingTheme.lang.autoMode'), value: 'auto' },
+              ...(localeOptions || []),
+            ]}
+          />
+        ),
+        desc: t('settingTTS.sttLocale.desc'),
+        label: t('settingTTS.sttLocale.title'),
+        name: [TTS_SETTING_KEY, 'sttLocale'],
+      },
+    ],
+    icon: Mic,
+    title: t('settingTTS.title'),
+  };
+
+  return (
+    <Form
+      form={form}
+      initialValues={{
+        [TTS_SETTING_KEY]: {
+          voice: {
+            edge: edgeVoiceOptions?.[0].value,
+            microsoft: microsoftVoiceOptions?.[0].value,
+            openai: openaiVoiceOptions?.[0].value,
+          },
+        },
+      }}
+      items={[tts]}
+      onValuesChange={debounce(updateConfig, 100)}
+      {...FORM_STYLE}
+    />
+  );
+});
+
+export default AgentTTS;

package/src/features/AgentSetting/AgentTTS/options.ts

@@ -0,0 +1,16 @@
+import { SelectProps } from 'antd';
+
+export const ttsOptions: SelectProps['options'] = [
+  {
+    label: 'OpenAI',
+    value: 'openai',
+  },
+  {
+    label: 'Edge Speech',
+    value: 'edge',
+  },
+  {
+    label: 'Microsoft Speech',
+    value: 'microsoft',
+  },
+];

package/src/features/AgentSetting/index.tsx

@@ -4,6 +4,7 @@ import AgentConfig from './AgentConfig';
 import AgentMeta from './AgentMeta';
 import AgentPlugin from './AgentPlugin';
 import AgentPrompt from './AgentPrompt';
+import AgentTTS from './AgentTTS';
 import StoreUpdater, { StoreUpdaterProps } from './StoreUpdater';
 import { Provider, createStore } from './store';
 
@@ -13,10 +14,10 @@ const AgentSettings = memo<AgentSettingsProps>((props) => {
   return (
     <Provider createStore={createStore}>
       <StoreUpdater {...props} />
-
       <AgentPrompt />
       <AgentMeta />
       <AgentConfig />
+      <AgentTTS />
       <AgentPlugin />
     </Provider>
   );

package/src/hooks/useSTT.ts

@@ -0,0 +1,61 @@
+import { getRecordMineType } from '@lobehub/tts';
+import {
+  OpenAISTTOptions,
+  SpeechRecognitionOptions,
+  useOpenAISTT,
+  useSpeechRecognition,
+} from '@lobehub/tts/react';
+import isEqual from 'fast-deep-equal';
+import { SWRConfiguration } from 'swr';
+
+import { createHeaderWithOpenAI } from '@/services/_header';
+import { OPENAI_URLS } from '@/services/_url';
+import { settingsSelectors, useGlobalStore } from '@/store/global';
+import { useSessionStore } from '@/store/session';
+import { agentSelectors } from '@/store/session/slices/agentConfig';
+
+interface STTConfig extends SWRConfiguration {
+  onTextChange: (value: string) => void;
+}
+
+export const useSTT = (config: STTConfig) => {
+  const ttsSettings = useGlobalStore(settingsSelectors.currentTTS, isEqual);
+  const ttsAgentSettings = useSessionStore(agentSelectors.currentAgentTTS, isEqual);
+  const locale = useGlobalStore(settingsSelectors.currentLanguage);
+
+  const autoStop = ttsSettings.sttAutoStop;
+  const sttLocale =
+    ttsAgentSettings?.sttLocale && ttsAgentSettings.sttLocale !== 'auto'
+      ? ttsAgentSettings.sttLocale
+      : locale;
+
+  let useSelectedSTT;
+  let options: any = {};
+
+  switch (ttsSettings.sttServer) {
+    case 'openai': {
+      useSelectedSTT = useOpenAISTT;
+      options = {
+        api: {
+          headers: createHeaderWithOpenAI(),
+          serviceUrl: OPENAI_URLS.stt,
+        },
+        autoStop,
+        options: {
+          mineType: getRecordMineType(),
+          model: ttsSettings.openAI.sttModel,
+        },
+      } as OpenAISTTOptions;
+      break;
+    }
+    case 'browser': {
+      options = {
+        autoStop,
+      } as SpeechRecognitionOptions;
+      useSelectedSTT = useSpeechRecognition;
+      break;
+    }
+  }
+
+  return useSelectedSTT(sttLocale, { ...config, ...options });
+};

package/src/hooks/useTTS.ts

@@ -0,0 +1,87 @@
+import { VoiceList } from '@lobehub/tts';
+import {
+  EdgeSpeechOptions,
+  MicrosoftSpeechOptions,
+  OpenAITTSOptions,
+  useEdgeSpeech,
+  useMicrosoftSpeech,
+  useOpenAITTS,
+} from '@lobehub/tts/react';
+import isEqual from 'fast-deep-equal';
+import { SWRConfiguration } from 'swr';
+
+import { createHeaderWithOpenAI } from '@/services/_header';
+import { OPENAI_URLS, TTS_URL } from '@/services/_url';
+import { settingsSelectors, useGlobalStore } from '@/store/global';
+import { useSessionStore } from '@/store/session';
+import { agentSelectors } from '@/store/session/slices/agentConfig';
+import { TTSServer } from '@/types/session';
+
+interface TTSConfig extends SWRConfiguration {
+  server?: TTSServer;
+  voice?: string;
+}
+
+export const useTTS = (content: string, config?: TTSConfig) => {
+  const ttsSettings = useGlobalStore(settingsSelectors.currentTTS, isEqual);
+  const ttsAgentSettings = useSessionStore(agentSelectors.currentAgentTTS, isEqual);
+  const voiceList = useGlobalStore((s) => new VoiceList(settingsSelectors.currentLanguage(s)));
+
+  let useSelectedTTS;
+  let options: any = {};
+  switch (config?.server || ttsAgentSettings.ttsService) {
+    case 'openai': {
+      useSelectedTTS = useOpenAITTS;
+      options = {
+        api: {
+          headers: createHeaderWithOpenAI(),
+          serviceUrl: OPENAI_URLS.tts,
+        },
+        options: {
+          model: ttsSettings.openAI.ttsModel,
+          voice:
+            config?.voice ||
+            ttsAgentSettings.voice.openai ||
+            VoiceList.openaiVoiceOptions?.[0].value,
+        },
+      } as OpenAITTSOptions;
+      break;
+    }
+    case 'edge': {
+      useSelectedTTS = useEdgeSpeech;
+      options = {
+        api: {
+          /**
+           * @description client fetch
+           * serviceUrl: TTS_URL.edge,
+           */
+        },
+        options: {
+          voice:
+            config?.voice || ttsAgentSettings.voice.edge || voiceList.edgeVoiceOptions?.[0].value,
+        },
+      } as EdgeSpeechOptions;
+      break;
+    }
+    case 'microsoft': {
+      useSelectedTTS = useMicrosoftSpeech;
+      options = {
+        api: {
+          serviceUrl: TTS_URL.microsoft,
+        },
+        options: {
+          voice:
+            config?.voice ||
+            ttsAgentSettings.voice.microsoft ||
+            voiceList.microsoftVoiceOptions?.[0].value,
+        },
+      } as MicrosoftSpeechOptions;
+      break;
+    }
+  }
+
+  return useSelectedTTS(content, {
+    ...config,
+    ...options,
+  });
+};

package/src/locales/default/chat.ts

@@ -38,6 +38,11 @@ export default {
     withSystemRole: '包含助手角色设定',
   },
   stop: '停止',
+  stt: {
+    action: '语音输入',
+    loading: '识别中...',
+    prettifying: '润色中...',
+  },
   temp: '临时',
   tokenDetail: '角色设定: {{systemRoleToken}} · 历史消息: {{chatsToken}}',
   tokenTag: {
@@ -58,9 +63,13 @@ export default {
     title: '话题列表',
   },
   translate: {
+    action: '翻译',
     clear: '删除翻译',
   },
-  translateTo: '翻译',
+  tts: {
+    action: '语音朗读',
+    clear: '删除语音',
+  },
   updateAgent: '更新助理信息',
   upload: {
     actionTooltip: '上传图片',

package/src/locales/default/error.ts

@@ -35,6 +35,12 @@ export default {
     NoAPIKey: 'OpenAI API Key 为空，请添加自定义 OpenAI API Key',
     /* eslint-enable */
   },
+  stt: {
+    responseError: '服务请求失败，请检查配置或重试',
+  },
+  tts: {
+    responseError: '服务请求失败，请检查配置或重试',
+  },
   unlock: {
     apikey: {
       addProxyUrl: '添加 OpenAI 代理地址（可选）',

package/src/locales/default/setting.ts

@@ -109,7 +109,6 @@ export default {
     },
     title: '助手信息',
   },
-
   settingChat: {
     chatStyleType: {
       title: '聊天窗口样式',
@@ -195,6 +194,40 @@ export default {
     },
     title: '系统设置',
   },
+  settingTTS: {
+    openai: {
+      sttModel: 'OpenAI 语音识别模型',
+      ttsModel: 'OpenAI 语音合成模型',
+    },
+    showAllLocaleVoice: {
+      desc: '关闭则只显示当前语种的声源',
+      title: '显示所有语种声源',
+    },
+    stt: '语音识别设置',
+    sttAutoStop: {
+      desc: '关闭后，语音识别将不会自动结束，需要手动点击结束按钮',
+      title: '自动结束语音识别',
+    },
+    sttLocale: {
+      desc: '语音输入的语种，此选项可提高语音识别准确率',
+      title: '语音识别语种',
+    },
+    sttService: {
+      desc: '其中 broswer 为浏览器原生的语音识别服务',
+      title: '语音识别服务',
+    },
+    title: '语音服务',
+    tts: '语音合成设置',
+    ttsService: {
+      desc: '如使用 OpenAI 语音合成服务，需要保证 OpenAI 模型服务已开启',
+      title: '语音合成服务',
+    },
+    voice: {
+      desc: '为当前助手挑选一个声音，不同 TTS 服务支持的声源不同',
+      preview: '试听声源',
+      title: '语音合成声源',
+    },
+  },
   settingTheme: {
     avatar: {
       title: '头像',
@@ -234,5 +267,6 @@ export default {
     agent: '默认助手',
     common: '通用设置',
     llm: '语言模型',
+    tts: '语音服务',
   },
 };

package/src/services/_url.ts

@@ -6,4 +6,11 @@ export const URLS = {
 export const OPENAI_URLS = {
   chat: '/api/openai/chat',
   models: '/api/openai/models',
+  stt: '/api/openai/stt',
+  tts: '/api/openai/tts',
+};
+
+export const TTS_URL = {
+  edge: '/api/tts/edge-speech',
+  microsoft: '/api/tts/microsoft-speech',
 };

package/src/store/global/initialState.ts

@@ -11,6 +11,7 @@ export enum SettingsTabs {
   Agent = 'agent',
   Common = 'common',
   LLM = 'llm',
+  TTS = 'tts',
 }
 
 export interface Guide {

package/src/store/global/selectors/settings.test.ts

@@ -20,12 +20,28 @@ describe('settingsSelectors', () => {
               systemRole: '',
               model: LanguageModel.GPT3_5,
               params: {},
+              tts: {
+                showAllLocaleVoice: false,
+                sttLocale: 'auto',
+                ttsService: 'openai',
+                voice: {
+                  openai: 'alloy',
+                },
+              },
             },
             meta: {
               avatar: 'Default Agent',
               description: 'Default agent for testing',
             },
           },
+          tts: {
+            openAI: {
+              sttModel: 'whisper-1',
+              ttsModel: 'tts-1',
+            },
+            sttAutoStop: true,
+            sttServer: 'openai',
+          },
           languageModel: {
             openAI: {
               OPENAI_API_KEY: 'openai-api-key',
@@ -59,12 +75,28 @@ describe('settingsSelectors', () => {
               top_p: 1,
             },
             plugins: [],
+            tts: {
+              showAllLocaleVoice: false,
+              sttLocale: 'auto',
+              ttsService: 'openai',
+              voice: {
+                openai: 'alloy',
+              },
+            },
           },
           meta: {
             avatar: 'Default Agent',
             description: 'Default agent for testing',
           },
         },
+        tts: {
+          openAI: {
+            sttModel: 'whisper-1',
+            ttsModel: 'tts-1',
+          },
+          sttAutoStop: true,
+          sttServer: 'openai',
+        },
         languageModel: {
           openAI: {
             OPENAI_API_KEY: 'openai-api-key',

package/src/store/global/selectors/settings.ts

@@ -1,7 +1,12 @@
 import { DEFAULT_OPENAI_MODEL_LIST } from '@/const/llm';
 import { DEFAULT_LANG } from '@/const/locale';
 import { DEFAULT_AGENT_META } from '@/const/meta';
-import { DEFAULT_AGENT, DEFAULT_AGENT_CONFIG, DEFAULT_SETTINGS } from '@/const/settings';
+import {
+  DEFAULT_AGENT,
+  DEFAULT_AGENT_CONFIG,
+  DEFAULT_SETTINGS,
+  DEFAULT_TTS_CONFIG,
+} from '@/const/settings';
 import { Locales } from '@/locales/resources';
 import { GlobalSettings } from '@/types/settings';
 import { isOnServerSide } from '@/utils/env';
@@ -11,6 +16,8 @@ import { GlobalStore } from '../store';
 
 const currentSettings = (s: GlobalStore) => merge(DEFAULT_SETTINGS, s.settings);
 
+const currentTTS = (s: GlobalStore) => merge(DEFAULT_TTS_CONFIG, s.settings.tts);
+
 const defaultAgent = (s: GlobalStore) => merge(DEFAULT_AGENT, s.settings.defaultAgent);
 
 const defaultAgentConfig = (s: GlobalStore) => merge(DEFAULT_AGENT_CONFIG, defaultAgent(s).config);
@@ -49,6 +56,7 @@ const currentLanguage = (s: GlobalStore) => {
 export const settingsSelectors = {
   currentLanguage,
   currentSettings,
+  currentTTS,
   defaultAgent,
   defaultAgentConfig,
   defaultAgentMeta,

package/src/store/session/slices/agentConfig/selectors.ts

@@ -2,9 +2,11 @@ import { t } from 'i18next';
 
 import { DEFAULT_OPENAI_MODEL_LIST, VISION_MODEL_WHITE_LIST } from '@/const/llm';
 import { DEFAULT_AVATAR, DEFAULT_BACKGROUND_COLOR } from '@/const/meta';
+import { DEFAUTT_AGENT_TTS_CONFIG } from '@/const/settings';
 import { SessionStore } from '@/store/session';
 import { LanguageModel } from '@/types/llm';
 import { MetaData } from '@/types/meta';
+import { LobeAgentTTSConfig } from '@/types/session';
 import { merge } from '@/utils/merge';
 
 import { sessionSelectors } from '../session/selectors';
@@ -78,6 +80,12 @@ const showTokenTag = (s: SessionStore) => {
   return DEFAULT_OPENAI_MODEL_LIST.includes(model);
 };
 
+const currentAgentTTS = (s: SessionStore): LobeAgentTTSConfig => {
+  const config = currentAgentConfig(s);
+
+  return config?.tts || DEFAUTT_AGENT_TTS_CONFIG;
+};
+
 export const agentSelectors = {
   currentAgentAvatar,
   currentAgentBackgroundColor,
@@ -87,6 +95,7 @@ export const agentSelectors = {
   currentAgentModel,
   currentAgentPlugins,
   currentAgentSystemRole,
+  currentAgentTTS,
   currentAgentTitle,
   getAvatar,
   getDescription,

package/src/store/session/slices/chat/actions/translate.ts

@@ -16,13 +16,15 @@ const t = setNamespace('chat/translate');
  * 翻译事件
  */
 export interface ChatTranslateAction {
-  clearTranslate: (id: string) => void;
+  clearTTS: (id: string) => void;
 
+  clearTranslate: (id: string) => void;
   /**
    * 翻译消息
    * @param id
    */
   translateMessage: (id: string, targetLang: string) => Promise<void>;
+  ttsMessage: (id: string, init?: boolean) => void;
 }
 
 export const chatTranslate: StateCreator<
@@ -31,6 +33,15 @@ export const chatTranslate: StateCreator<
   [],
   ChatTranslateAction
 > = (set, get) => ({
+  clearTTS: (id) => {
+    get().dispatchMessage({
+      id,
+      key: 'tts',
+      type: 'updateMessageExtra',
+      value: null,
+    });
+  },
+
   clearTranslate: (id) => {
     get().dispatchMessage({
       id,
@@ -85,4 +96,16 @@ export const chatTranslate: StateCreator<
 
     toggleChatLoading(false);
   },
+
+  ttsMessage: (id, init) => {
+    const { dispatchMessage } = get();
+    dispatchMessage({
+      id,
+      key: 'tts',
+      type: 'updateMessageExtra',
+      value: {
+        init: Boolean(init),
+      },
+    });
+  },
 });

package/src/store/session/slices/chat/selectors/utils.test.ts

@@ -41,6 +41,14 @@ beforeEach(() => {
         temperature: 0.6,
       },
       systemRole: '',
+      tts: {
+        ttsService: 'openai',
+        sttLocale: 'auto',
+        showAllLocaleVoice: false,
+        voice: {
+          openai: 'alloy',
+        },
+      },
     },
     type: 'agent',
     createAt: 1690110700808,
@@ -215,6 +223,14 @@ describe('organizeChats', () => {
         params: {
           temperature: 0.6,
         },
+        tts: {
+          ttsService: 'openai',
+          sttLocale: 'auto',
+          showAllLocaleVoice: false,
+          voice: {
+            openai: 'alloy',
+          },
+        },
         systemRole: '',
       },
       createAt: 1690110700808,

package/src/store/session/slices/session/reducers/session.test.ts

@@ -312,6 +312,14 @@ describe('sessionsReducer', () => {
             model: 'gpt-3.5-turbo',
             params: {},
             systemRole: 'system-role',
+            tts: {
+              showAllLocaleVoice: false,
+              sttLocale: 'auto',
+              ttsService: 'openai',
+              voice: {
+                openai: 'alloy',
+              },
+            },
           },
           type: 'agent',
           meta: {
@@ -354,6 +362,14 @@ describe('sessionsReducer', () => {
             model: 'gpt-3.5-turbo',
             params: {},
             systemRole: 'system',
+            tts: {
+              showAllLocaleVoice: false,
+              sttLocale: 'auto',
+              ttsService: 'openai',
+              voice: {
+                openai: 'alloy',
+              },
+            },
           },
         } as LobeAgentSession,
         session2: {
@@ -379,6 +395,14 @@ describe('sessionsReducer', () => {
         draft.session1.config = {
           model: LanguageModel.GPT4,
           params: {},
+          tts: {
+            ttsService: 'openai',
+            sttLocale: 'auto',
+            showAllLocaleVoice: false,
+            voice: {
+              openai: 'alloy',
+            },
+          },
           systemRole: 'system',
         };
       });

package/src/types/chatMessage.ts

@@ -22,6 +22,10 @@ export interface OpenAIFunctionCall {
 export interface ChatTranslate extends Translate {
   content?: string;
 }
+
+export interface ChatTTS {
+  init?: boolean;
+}
 export interface ChatMessage extends BaseDataModel {
   /**
    * @title 内容
@@ -34,6 +38,8 @@ export interface ChatMessage extends BaseDataModel {
     fromModel?: string;
     // 翻译
     translate?: ChatTranslate;
+    // TTS
+    tts?: ChatTTS;
   } & Record<string, any>;
 
   files?: string[];