@lobehub/chat 0.101.6 → 0.102.0

package/src/features/AgentSetting/AgentTTS/SelectWithTTSPreview.tsx

@@ -0,0 +1,119 @@
+import { AudioPlayer } from '@lobehub/tts/react';
+import { Alert, Highlighter } from '@lobehub/ui';
+import { Button, RefSelectProps, Select, SelectProps } from 'antd';
+import { useTheme } from 'antd-style';
+import { forwardRef, useCallback, useState } from 'react';
+import { useTranslation } from 'react-i18next';
+import { Flexbox } from 'react-layout-kit';
+
+import { useTTS } from '@/hooks/useTTS';
+import { ChatMessageError } from '@/types/chatMessage';
+import { TTSServer } from '@/types/session';
+import { getMessageError } from '@/utils/fetch';
+
+interface SelectWithTTSPreviewProps extends SelectProps {
+  server: TTSServer;
+}
+
+const SelectWithTTSPreview = forwardRef<RefSelectProps, SelectWithTTSPreviewProps>(
+  ({ value, options, server, onSelect, ...rest }, ref) => {
+    const [error, setError] = useState<ChatMessageError>();
+    const [voice, setVoice] = useState<string>(value);
+    const { t } = useTranslation('welcome');
+    const theme = useTheme();
+    const PREVIEW_TEXT = ['Lobe Chat', t('slogan.title'), t('slogan.desc1')].join('. ');
+
+    const setDefaultError = useCallback(
+      (err?: any) => {
+        setError({ body: err, message: t('tts.responseError', { ns: 'error' }), type: 500 });
+      },
+      [t],
+    );
+
+    const { isGlobalLoading, audio, stop, start, response, setText } = useTTS(PREVIEW_TEXT, {
+      onError: (err) => {
+        stop();
+        setDefaultError(err);
+      },
+      onErrorRetry: (err) => {
+        stop();
+        setDefaultError(err);
+      },
+      onSuccess: async () => {
+        if (!response) return;
+        if (response.status === 200) return;
+        const message = await getMessageError(response);
+        if (message) {
+          setError(message);
+        } else {
+          setDefaultError();
+        }
+        stop();
+      },
+      server,
+      voice,
+    });
+
+    const handleCloseError = useCallback(() => {
+      setError(undefined);
+      stop();
+    }, [stop]);
+
+    const handleRetry = useCallback(() => {
+      setError(undefined);
+      stop();
+      start();
+    }, [stop, start]);
+
+    const handleSelect: SelectProps['onSelect'] = (value, option) => {
+      stop();
+      setVoice(value as string);
+      setText([PREVIEW_TEXT, option?.label].join(' - '));
+      onSelect?.(value, option);
+    };
+    return (
+      <Flexbox gap={8}>
+        <Flexbox align={'center'} gap={8} horizontal style={{ width: '100%' }}>
+          <Select onSelect={handleSelect} options={options} ref={ref} value={value} {...rest} />
+          <AudioPlayer
+            allowPause={false}
+            audio={audio}
+            buttonActive
+            buttonSize={{ blockSize: 36, fontSize: 16 }}
+            buttonStyle={{ border: `1px solid ${theme.colorBorder}` }}
+            isLoading={isGlobalLoading}
+            onInitPlay={start}
+            onLoadingStop={stop}
+            showSlider={false}
+            showTime={false}
+            style={{ flex: 'none', padding: 0, width: 'unset' }}
+            title={t('settingTTS.voice.preview', { ns: 'setting' })}
+          />
+        </Flexbox>
+        {error && (
+          <Alert
+            action={
+              <Button onClick={handleRetry} size={'small'} type={'primary'}>
+                {t('retry', { ns: 'common' })}
+              </Button>
+            }
+            closable
+            extra={
+              error.body && (
+                <Highlighter copyButtonSize={'small'} language={'json'} type={'pure'}>
+                  {JSON.stringify(error.body, null, 2)}
+                </Highlighter>
+              )
+            }
+            message={error.message}
+            onClose={handleCloseError}
+            style={{ alignItems: 'center', width: '100%' }}
+            type="error"
+          />
+        )}
+      </Flexbox>
+    );
+  },
+);
+
+export default SelectWithTTSPreview;

package/src/features/AgentSetting/AgentTTS/index.tsx

@@ -0,0 +1,115 @@
+import { VoiceList } from '@lobehub/tts';
+import { Form, ItemGroup } from '@lobehub/ui';
+import { Form as AFrom, Select, Switch } from 'antd';
+import isEqual from 'fast-deep-equal';
+import { debounce } from 'lodash-es';
+import { Mic } from 'lucide-react';
+import { memo, useEffect } from 'react';
+import { useTranslation } from 'react-i18next';
+
+import { FORM_STYLE } from '@/const/layoutTokens';
+import SelectWithTTSPreview from '@/features/AgentSetting/AgentTTS/SelectWithTTSPreview';
+import { settingsSelectors, useGlobalStore } from '@/store/global';
+
+import { useStore } from '../store';
+import { ttsOptions } from './options';
+
+const TTS_SETTING_KEY = 'tts';
+const { openaiVoiceOptions, localeOptions } = VoiceList;
+
+const AgentTTS = memo(() => {
+  const { t } = useTranslation('setting');
+  const updateConfig = useStore((s) => s.setAgentConfig);
+  const [form] = AFrom.useForm();
+  const voiceList = useGlobalStore((s) => {
+    const locale = settingsSelectors.currentLanguage(s);
+    return (all?: boolean) => new VoiceList(all ? undefined : locale);
+  });
+  const config = useStore((s) => s.config, isEqual);
+
+  useEffect(() => {
+    form.setFieldsValue(config);
+  }, [config]);
+
+  const showAllLocaleVoice = config.tts.showAllLocaleVoice;
+
+  const { edgeVoiceOptions, microsoftVoiceOptions } = voiceList(showAllLocaleVoice);
+
+  const tts: ItemGroup = {
+    children: [
+      {
+        children: <Select options={ttsOptions} />,
+        desc: t('settingTTS.ttsService.desc'),
+        label: t('settingTTS.ttsService.title'),
+        name: [TTS_SETTING_KEY, 'ttsService'],
+      },
+      {
+        children: <Switch />,
+        desc: t('settingTTS.showAllLocaleVoice.desc'),
+        hidden: config.tts.ttsService === 'openai',
+        label: t('settingTTS.showAllLocaleVoice.title'),
+        minWidth: undefined,
+        name: [TTS_SETTING_KEY, 'showAllLocaleVoice'],
+        valuePropName: 'checked',
+      },
+      {
+        children: <SelectWithTTSPreview options={openaiVoiceOptions} server={'openai'} />,
+        desc: t('settingTTS.voice.desc'),
+        hidden: config.tts.ttsService !== 'openai',
+        label: t('settingTTS.voice.title'),
+        name: [TTS_SETTING_KEY, 'voice', 'openai'],
+      },
+      {
+        children: <SelectWithTTSPreview options={edgeVoiceOptions} server={'edge'} />,
+        desc: t('settingTTS.voice.desc'),
+        divider: false,
+        hidden: config.tts.ttsService !== 'edge',
+        label: t('settingTTS.voice.title'),
+        name: [TTS_SETTING_KEY, 'voice', 'edge'],
+      },
+      {
+        children: <SelectWithTTSPreview options={microsoftVoiceOptions} server={'microsoft'} />,
+        desc: t('settingTTS.voice.desc'),
+        divider: false,
+        hidden: config.tts.ttsService !== 'microsoft',
+        label: t('settingTTS.voice.title'),
+        name: [TTS_SETTING_KEY, 'voice', 'microsoft'],
+      },
+      {
+        children: (
+          <Select
+            options={[
+              { label: t('settingTheme.lang.autoMode'), value: 'auto' },
+              ...(localeOptions || []),
+            ]}
+          />
+        ),
+        desc: t('settingTTS.sttLocale.desc'),
+        label: t('settingTTS.sttLocale.title'),
+        name: [TTS_SETTING_KEY, 'sttLocale'],
+      },
+    ],
+    icon: Mic,
+    title: t('settingTTS.title'),
+  };
+
+  return (
+    <Form
+      form={form}
+      initialValues={{
+        [TTS_SETTING_KEY]: {
+          voice: {
+            edge: edgeVoiceOptions?.[0].value,
+            microsoft: microsoftVoiceOptions?.[0].value,
+            openai: openaiVoiceOptions?.[0].value,
+          },
+        },
+      }}
+      items={[tts]}
+      onValuesChange={debounce(updateConfig, 100)}
+      {...FORM_STYLE}
+    />
+  );
+});
+
+export default AgentTTS;

package/src/features/AgentSetting/AgentTTS/options.ts

@@ -0,0 +1,16 @@
+import { SelectProps } from 'antd';
+
+export const ttsOptions: SelectProps['options'] = [
+  {
+    label: 'OpenAI',
+    value: 'openai',
+  },
+  {
+    label: 'Edge Speech',
+    value: 'edge',
+  },
+  {
+    label: 'Microsoft Speech',
+    value: 'microsoft',
+  },
+];

package/src/features/AgentSetting/index.tsx

@@ -4,6 +4,7 @@ import AgentConfig from './AgentConfig';
 import AgentMeta from './AgentMeta';
 import AgentPlugin from './AgentPlugin';
 import AgentPrompt from './AgentPrompt';
+import AgentTTS from './AgentTTS';
 import StoreUpdater, { StoreUpdaterProps } from './StoreUpdater';
 import { Provider, createStore } from './store';
 
@@ -13,10 +14,10 @@ const AgentSettings = memo<AgentSettingsProps>((props) => {
   return (
     <Provider createStore={createStore}>
       <StoreUpdater {...props} />
-
       <AgentPrompt />
       <AgentMeta />
       <AgentConfig />
+      <AgentTTS />
       <AgentPlugin />
     </Provider>
   );

package/src/hooks/useSTT.ts

@@ -0,0 +1,61 @@
+import { getRecordMineType } from '@lobehub/tts';
+import {
+  OpenAISTTOptions,
+  SpeechRecognitionOptions,
+  useOpenAISTT,
+  useSpeechRecognition,
+} from '@lobehub/tts/react';
+import isEqual from 'fast-deep-equal';
+import { SWRConfiguration } from 'swr';
+
+import { createHeaderWithOpenAI } from '@/services/_header';
+import { OPENAI_URLS } from '@/services/_url';
+import { settingsSelectors, useGlobalStore } from '@/store/global';
+import { useSessionStore } from '@/store/session';
+import { agentSelectors } from '@/store/session/slices/agentConfig';
+
+interface STTConfig extends SWRConfiguration {
+  onTextChange: (value: string) => void;
+}
+
+export const useSTT = (config: STTConfig) => {
+  const ttsSettings = useGlobalStore(settingsSelectors.currentTTS, isEqual);
+  const ttsAgentSettings = useSessionStore(agentSelectors.currentAgentTTS, isEqual);
+  const locale = useGlobalStore(settingsSelectors.currentLanguage);
+
+  const autoStop = ttsSettings.sttAutoStop;
+  const sttLocale =
+    ttsAgentSettings?.sttLocale && ttsAgentSettings.sttLocale !== 'auto'
+      ? ttsAgentSettings.sttLocale
+      : locale;
+
+  let useSelectedSTT;
+  let options: any = {};
+
+  switch (ttsSettings.sttServer) {
+    case 'openai': {
+      useSelectedSTT = useOpenAISTT;
+      options = {
+        api: {
+          headers: createHeaderWithOpenAI(),
+          serviceUrl: OPENAI_URLS.stt,
+        },
+        autoStop,
+        options: {
+          mineType: getRecordMineType(),
+          model: ttsSettings.openAI.sttModel,
+        },
+      } as OpenAISTTOptions;
+      break;
+    }
+    case 'browser': {
+      options = {
+        autoStop,
+      } as SpeechRecognitionOptions;
+      useSelectedSTT = useSpeechRecognition;
+      break;
+    }
+  }
+
+  return useSelectedSTT(sttLocale, { ...config, ...options });
+};

package/src/hooks/useTTS.ts

@@ -0,0 +1,87 @@
+import { VoiceList } from '@lobehub/tts';
+import {
+  EdgeSpeechOptions,
+  MicrosoftSpeechOptions,
+  OpenAITTSOptions,
+  useEdgeSpeech,
+  useMicrosoftSpeech,
+  useOpenAITTS,
+} from '@lobehub/tts/react';
+import isEqual from 'fast-deep-equal';
+import { SWRConfiguration } from 'swr';
+
+import { createHeaderWithOpenAI } from '@/services/_header';
+import { OPENAI_URLS, TTS_URL } from '@/services/_url';
+import { settingsSelectors, useGlobalStore } from '@/store/global';
+import { useSessionStore } from '@/store/session';
+import { agentSelectors } from '@/store/session/slices/agentConfig';
+import { TTSServer } from '@/types/session';
+
+interface TTSConfig extends SWRConfiguration {
+  server?: TTSServer;
+  voice?: string;
+}
+
+export const useTTS = (content: string, config?: TTSConfig) => {
+  const ttsSettings = useGlobalStore(settingsSelectors.currentTTS, isEqual);
+  const ttsAgentSettings = useSessionStore(agentSelectors.currentAgentTTS, isEqual);
+  const voiceList = useGlobalStore((s) => new VoiceList(settingsSelectors.currentLanguage(s)));
+
+  let useSelectedTTS;
+  let options: any = {};
+  switch (config?.server || ttsAgentSettings.ttsService) {
+    case 'openai': {
+      useSelectedTTS = useOpenAITTS;
+      options = {
+        api: {
+          headers: createHeaderWithOpenAI(),
+          serviceUrl: OPENAI_URLS.tts,
+        },
+        options: {
+          model: ttsSettings.openAI.ttsModel,
+          voice:
+            config?.voice ||
+            ttsAgentSettings.voice.openai ||
+            VoiceList.openaiVoiceOptions?.[0].value,
+        },
+      } as OpenAITTSOptions;
+      break;
+    }
+    case 'edge': {
+      useSelectedTTS = useEdgeSpeech;
+      options = {
+        api: {
+          /**
+           * @description client fetch
+           * serviceUrl: TTS_URL.edge,
+           */
+        },
+        options: {
+          voice:
+            config?.voice || ttsAgentSettings.voice.edge || voiceList.edgeVoiceOptions?.[0].value,
+        },
+      } as EdgeSpeechOptions;
+      break;
+    }
+    case 'microsoft': {
+      useSelectedTTS = useMicrosoftSpeech;
+      options = {
+        api: {
+          serviceUrl: TTS_URL.microsoft,
+        },
+        options: {
+          voice:
+            config?.voice ||
+            ttsAgentSettings.voice.microsoft ||
+            voiceList.microsoftVoiceOptions?.[0].value,
+        },
+      } as MicrosoftSpeechOptions;
+      break;
+    }
+  }
+
+  return useSelectedTTS(content, {
+    ...config,
+    ...options,
+  });
+};

package/src/locales/default/chat.ts

@@ -38,6 +38,11 @@ export default {
     withSystemRole: '包含助手角色设定',
   },
   stop: '停止',
+  stt: {
+    action: '语音输入',
+    loading: '识别中...',
+    prettifying: '润色中...',
+  },
   temp: '临时',
   tokenDetail: '角色设定: {{systemRoleToken}} · 历史消息: {{chatsToken}}',
   tokenTag: {
@@ -58,9 +63,13 @@ export default {
     title: '话题列表',
   },
   translate: {
+    action: '翻译',
     clear: '删除翻译',
   },
-  translateTo: '翻译',
+  tts: {
+    action: '语音朗读',
+    clear: '删除语音',
+  },
   updateAgent: '更新助理信息',
   upload: {
     actionTooltip: '上传图片',

package/src/locales/default/error.ts

@@ -35,6 +35,12 @@ export default {
     NoAPIKey: 'OpenAI API Key 为空，请添加自定义 OpenAI API Key',
     /* eslint-enable */
   },
+  stt: {
+    responseError: '服务请求失败，请检查配置或重试',
+  },
+  tts: {
+    responseError: '服务请求失败，请检查配置或重试',
+  },
   unlock: {
     apikey: {
       addProxyUrl: '添加 OpenAI 代理地址（可选）',

package/src/locales/default/setting.ts

@@ -109,7 +109,6 @@ export default {
     },
     title: '助手信息',
   },
-
   settingChat: {
     chatStyleType: {
       title: '聊天窗口样式',
@@ -195,6 +194,40 @@ export default {
     },
     title: '系统设置',
   },
+  settingTTS: {
+    openai: {
+      sttModel: 'OpenAI 语音识别模型',
+      ttsModel: 'OpenAI 语音合成模型',
+    },
+    showAllLocaleVoice: {
+      desc: '关闭则只显示当前语种的声源',
+      title: '显示所有语种声源',
+    },
+    stt: '语音识别设置',
+    sttAutoStop: {
+      desc: '关闭后，语音识别将不会自动结束，需要手动点击结束按钮',
+      title: '自动结束语音识别',
+    },
+    sttLocale: {
+      desc: '语音输入的语种，此选项可提高语音识别准确率',
+      title: '语音识别语种',
+    },
+    sttService: {
+      desc: '其中 broswer 为浏览器原生的语音识别服务',
+      title: '语音识别服务',
+    },
+    title: '语音服务',
+    tts: '语音合成设置',
+    ttsService: {
+      desc: '如使用 OpenAI 语音合成服务，需要保证 OpenAI 模型服务已开启',
+      title: '语音合成服务',
+    },
+    voice: {
+      desc: '为当前助手挑选一个声音，不同 TTS 服务支持的声源不同',
+      preview: '试听声源',
+      title: '语音合成声源',
+    },
+  },
   settingTheme: {
     avatar: {
       title: '头像',
@@ -234,5 +267,6 @@ export default {
     agent: '默认助手',
     common: '通用设置',
     llm: '语言模型',
+    tts: '语音服务',
   },
 };

package/src/services/_url.ts

@@ -6,4 +6,11 @@ export const URLS = {
 export const OPENAI_URLS = {
   chat: '/api/openai/chat',
   models: '/api/openai/models',
+  stt: '/api/openai/stt',
+  tts: '/api/openai/tts',
+};
+
+export const TTS_URL = {
+  edge: '/api/tts/edge-speech',
+  microsoft: '/api/tts/microsoft-speech',
 };

package/src/store/global/initialState.ts

@@ -11,6 +11,7 @@ export enum SettingsTabs {
   Agent = 'agent',
   Common = 'common',
   LLM = 'llm',
+  TTS = 'tts',
 }
 
 export interface Guide {

package/src/store/global/selectors/settings.test.ts

@@ -20,12 +20,28 @@ describe('settingsSelectors', () => {
               systemRole: '',
               model: LanguageModel.GPT3_5,
               params: {},
+              tts: {
+                showAllLocaleVoice: false,
+                sttLocale: 'auto',
+                ttsService: 'openai',
+                voice: {
+                  openai: 'alloy',
+                },
+              },
             },
             meta: {
               avatar: 'Default Agent',
               description: 'Default agent for testing',
             },
           },
+          tts: {
+            openAI: {
+              sttModel: 'whisper-1',
+              ttsModel: 'tts-1',
+            },
+            sttAutoStop: true,
+            sttServer: 'openai',
+          },
           languageModel: {
             openAI: {
               OPENAI_API_KEY: 'openai-api-key',
@@ -59,12 +75,28 @@ describe('settingsSelectors', () => {
               top_p: 1,
             },
             plugins: [],
+            tts: {
+              showAllLocaleVoice: false,
+              sttLocale: 'auto',
+              ttsService: 'openai',
+              voice: {
+                openai: 'alloy',
+              },
+            },
           },
           meta: {
             avatar: 'Default Agent',
             description: 'Default agent for testing',
           },
         },
+        tts: {
+          openAI: {
+            sttModel: 'whisper-1',
+            ttsModel: 'tts-1',
+          },
+          sttAutoStop: true,
+          sttServer: 'openai',
+        },
         languageModel: {
           openAI: {
             OPENAI_API_KEY: 'openai-api-key',

package/src/store/global/selectors/settings.ts

@@ -1,7 +1,12 @@
 import { DEFAULT_OPENAI_MODEL_LIST } from '@/const/llm';
 import { DEFAULT_LANG } from '@/const/locale';
 import { DEFAULT_AGENT_META } from '@/const/meta';
-import { DEFAULT_AGENT, DEFAULT_AGENT_CONFIG, DEFAULT_SETTINGS } from '@/const/settings';
+import {
+  DEFAULT_AGENT,
+  DEFAULT_AGENT_CONFIG,
+  DEFAULT_SETTINGS,
+  DEFAULT_TTS_CONFIG,
+} from '@/const/settings';
 import { Locales } from '@/locales/resources';
 import { GlobalSettings } from '@/types/settings';
 import { isOnServerSide } from '@/utils/env';
@@ -11,6 +16,8 @@ import { GlobalStore } from '../store';
 
 const currentSettings = (s: GlobalStore) => merge(DEFAULT_SETTINGS, s.settings);
 
+const currentTTS = (s: GlobalStore) => merge(DEFAULT_TTS_CONFIG, s.settings.tts);
+
 const defaultAgent = (s: GlobalStore) => merge(DEFAULT_AGENT, s.settings.defaultAgent);
 
 const defaultAgentConfig = (s: GlobalStore) => merge(DEFAULT_AGENT_CONFIG, defaultAgent(s).config);
@@ -49,6 +56,7 @@ const currentLanguage = (s: GlobalStore) => {
 export const settingsSelectors = {
   currentLanguage,
   currentSettings,
+  currentTTS,
   defaultAgent,
   defaultAgentConfig,
   defaultAgentMeta,

package/src/store/session/slices/agentConfig/selectors.ts

@@ -2,9 +2,11 @@ import { t } from 'i18next';
 
 import { DEFAULT_OPENAI_MODEL_LIST, VISION_MODEL_WHITE_LIST } from '@/const/llm';
 import { DEFAULT_AVATAR, DEFAULT_BACKGROUND_COLOR } from '@/const/meta';
+import { DEFAUTT_AGENT_TTS_CONFIG } from '@/const/settings';
 import { SessionStore } from '@/store/session';
 import { LanguageModel } from '@/types/llm';
 import { MetaData } from '@/types/meta';
+import { LobeAgentTTSConfig } from '@/types/session';
 import { merge } from '@/utils/merge';
 
 import { sessionSelectors } from '../session/selectors';
@@ -78,6 +80,12 @@ const showTokenTag = (s: SessionStore) => {
   return DEFAULT_OPENAI_MODEL_LIST.includes(model);
 };
 
+const currentAgentTTS = (s: SessionStore): LobeAgentTTSConfig => {
+  const config = currentAgentConfig(s);
+
+  return config?.tts || DEFAUTT_AGENT_TTS_CONFIG;
+};
+
 export const agentSelectors = {
   currentAgentAvatar,
   currentAgentBackgroundColor,
@@ -87,6 +95,7 @@ export const agentSelectors = {
   currentAgentModel,
   currentAgentPlugins,
   currentAgentSystemRole,
+  currentAgentTTS,
   currentAgentTitle,
   getAvatar,
   getDescription,