@lobehub/chat 0.109.1 → 0.110.0

package/CHANGELOG.md

@@ -2,6 +2,31 @@
 
 # Changelog
 
+## [Version 0.110.0](https://github.com/lobehub/lobe-chat/compare/v0.109.1...v0.110.0)
+
+<sup>Released on **2023-12-07**</sup>
+
+#### ✨ Features
+
+- **misc**: Local TTS Player.
+
+<br/>
+
+<details>
+<summary><kbd>Improvements and Fixes</kbd></summary>
+
+#### What's improved
+
+- **misc**: Local TTS Player, closes [#587](https://github.com/lobehub/lobe-chat/issues/587) ([87b51bd](https://github.com/lobehub/lobe-chat/commit/87b51bd))
+
+</details>
+
+<div align="right">
+
+[![](https://img.shields.io/badge/-BACK_TO_TOP-151515?style=flat-square)](#readme-top)
+
+</div>
+
 ### [Version 0.109.1](https://github.com/lobehub/lobe-chat/compare/v0.109.0...v0.109.1)
 
 <sup>Released on **2023-12-07**</sup>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lobehub/chat",
-  "version": "0.109.1",
+  "version": "0.110.0",
   "description": "An open-source, extensible (Function Calling), high-performance chatbot framework. It supports one-click free deployment of your private ChatGPT/LLM web application",
   "keywords": [
     "chatbot",
@@ -106,12 +106,12 @@
     "react-intersection-observer": "^9",
     "react-layout-kit": "^1",
     "react-lazy-load": "^4",
-    "react-wrap-balancer": "^1.1.0",
+    "react-wrap-balancer": "^1",
     "remark": "^14",
     "remark-gfm": "^3",
     "remark-html": "^15",
     "semver": "^7",
-    "sharp": "^0.33.0",
+    "sharp": "^0.33",
     "swr": "^2",
     "systemjs": "^6",
     "ts-md5": "^1",
@@ -126,11 +126,11 @@
   },
   "devDependencies": {
     "@commitlint/cli": "^18",
-    "@ducanh2912/next-pwa": "^9.7.2",
+    "@ducanh2912/next-pwa": "^9",
     "@lobehub/i18n-cli": "latest",
     "@lobehub/lint": "latest",
-    "@next/bundle-analyzer": "^14.0.3",
-    "@next/eslint-plugin-next": "^14.0.3",
+    "@next/bundle-analyzer": "^14",
+    "@next/eslint-plugin-next": "^14",
     "@testing-library/jest-dom": "^6",
     "@testing-library/react": "^14",
     "@types/chroma-js": "^2",

package/src/app/chat/features/Conversation/ChatList/Extras/TTS/FilePlayer.tsx

@@ -0,0 +1,25 @@
+import { useAudioPlayer } from '@lobehub/tts/react';
+import { memo, useCallback } from 'react';
+
+import { useChatStore } from '@/store/chat';
+import { useFileStore } from '@/store/file';
+
+import { TTSProps } from './InitPlayer';
+import Player from './Player';
+
+const FilePlayer = memo<TTSProps>(({ file, id }) => {
+  const useFetchTTSFile = useFileStore((s) => s.useFetchTTSFile);
+  const [clearTTS] = useChatStore((s) => [s.clearTTS]);
+  const { data, isLoading: isFileLoading } = useFetchTTSFile(file || null);
+  const { isLoading, ...audio } = useAudioPlayer({ src: data ? data.url : '' });
+
+  const handleDelete = useCallback(() => {
+    clearTTS(id);
+  }, [id]);
+
+  if (!audio || isFileLoading) return;
+
+  return <Player audio={audio} isLoading={isLoading} onDelete={handleDelete} />;
+});
+
+export default FilePlayer;

package/src/app/chat/features/Conversation/ChatList/Extras/TTS/InitPlayer.tsx

@@ -0,0 +1,93 @@
+import { memo, useCallback, useEffect, useState } from 'react';
+import { useTranslation } from 'react-i18next';
+
+import { useTTS } from '@/hooks/useTTS';
+import { useChatStore } from '@/store/chat';
+import { useFileStore } from '@/store/file';
+import { ChatMessageError, ChatTTS } from '@/types/chatMessage';
+import { getMessageError } from '@/utils/fetch';
+
+import Player from './Player';
+
+export interface TTSProps extends ChatTTS {
+  content: string;
+  id: string;
+  loading?: boolean;
+}
+
+const InitPlayer = memo<TTSProps>(({ id, content, contentMd5, file }) => {
+  const [isStart, setIsStart] = useState(false);
+  const [error, setError] = useState<ChatMessageError>();
+  const uploadTTS = useFileStore((s) => s.uploadTTSByArrayBuffers);
+  const { t } = useTranslation('chat');
+
+  const [ttsMessage, clearTTS] = useChatStore((s) => [s.ttsMessage, s.clearTTS]);
+
+  const setDefaultError = useCallback(
+    (err?: any) => {
+      setError({ body: err, message: t('tts.responseError', { ns: 'error' }), type: 500 });
+    },
+    [t],
+  );
+
+  const { isGlobalLoading, audio, start, stop, response } = useTTS(content, {
+    onError: (err) => {
+      stop();
+      setDefaultError(err);
+    },
+    onErrorRetry: (err) => {
+      stop();
+      setDefaultError(err);
+    },
+    onSuccess: async () => {
+      if (!response || response.ok) return;
+      const message = await getMessageError(response);
+      if (message) {
+        setError(message);
+      } else {
+        setDefaultError();
+      }
+      stop();
+    },
+    onUpload: async (currentVoice, arrayBuffers) => {
+      const fileID = await uploadTTS(id, arrayBuffers);
+      ttsMessage(id, { contentMd5, file: fileID, voice: currentVoice });
+    },
+  });
+
+  const handleInitStart = useCallback(() => {
+    if (isStart) return;
+    start();
+    setIsStart(true);
+  }, [isStart, start]);
+
+  const handleDelete = useCallback(() => {
+    stop();
+    clearTTS(id);
+  }, [stop, id]);
+
+  const handleRetry = useCallback(() => {
+    setError(undefined);
+    start();
+  }, [start]);
+
+  useEffect(() => {
+    if (file) return;
+    setTimeout(() => {
+      handleInitStart();
+    }, 100);
+  }, [file]);
+
+  return (
+    <Player
+      audio={audio}
+      error={error}
+      isLoading={isGlobalLoading}
+      onDelete={handleDelete}
+      onInitPlay={handleInitStart}
+      onRetry={handleRetry}
+    />
+  );
+});
+
+export default InitPlayer;

package/src/app/chat/features/Conversation/ChatList/Extras/TTS/Player.tsx

@@ -0,0 +1,61 @@
+import { AudioPlayer, AudioPlayerProps } from '@lobehub/tts/react';
+import { ActionIcon, Alert, Highlighter } from '@lobehub/ui';
+import { Button } from 'antd';
+import { TrashIcon } from 'lucide-react';
+import { memo } from 'react';
+import { useTranslation } from 'react-i18next';
+import { Flexbox } from 'react-layout-kit';
+
+import { ChatMessageError } from '@/types/chatMessage';
+
+interface PlayerProps extends AudioPlayerProps {
+  error?: ChatMessageError;
+  onDelete: () => void;
+  onRetry?: () => void;
+}
+
+const Player = memo<PlayerProps>(({ onRetry, error, onDelete, audio, isLoading, onInitPlay }) => {
+  const { t } = useTranslation('chat');
+
+  return (
+    <Flexbox align={'center'} horizontal style={{ minWidth: 160, width: '100%' }}>
+      {error ? (
+        <Alert
+          action={
+            <Button onClick={onRetry} size={'small'} type={'primary'}>
+              {t('retry', { ns: 'common' })}
+            </Button>
+          }
+          closable
+          extra={
+            error.body && (
+              <Highlighter copyButtonSize={'small'} language={'json'} type={'pure'}>
+                {JSON.stringify(error.body, null, 2)}
+              </Highlighter>
+            )
+          }
+          message={error.message}
+          onClose={onDelete}
+          style={{ alignItems: 'center', width: '100%' }}
+          type="error"
+        />
+      ) : (
+        <>
+          <AudioPlayer
+            allowPause={false}
+            audio={audio}
+            buttonSize={'small'}
+            isLoading={isLoading}
+            onInitPlay={onInitPlay}
+            onLoadingStop={stop}
+            timeRender={'tag'}
+            timeStyle={{ margin: 0 }}
+          />
+          <ActionIcon icon={TrashIcon} onClick={onDelete} size={'small'} title={t('tts.clear')} />
+        </>
+      )}
+    </Flexbox>
+  );
+});
+
+export default Player;

package/src/app/chat/features/Conversation/ChatList/Extras/TTS/index.tsx

@@ -0,0 +1,31 @@
+import { memo, useMemo } from 'react';
+import { Md5 } from 'ts-md5';
+
+import { settingsSelectors, useGlobalStore } from '@/store/global';
+import { useSessionStore } from '@/store/session';
+import { agentSelectors } from '@/store/session/slices/agent';
+
+import FilePlayer from './FilePlayer';
+import InitPlayer, { TTSProps } from './InitPlayer';
+
+const TTS = memo<TTSProps>(
+  (props) => {
+    const { file, voice, content, contentMd5 } = props;
+    const lang = useGlobalStore(settingsSelectors.currentLanguage);
+    const currentVoice = useSessionStore(agentSelectors.currentAgentTTSVoice(lang));
+
+    const md5 = useMemo(() => Md5.hashStr(content).toString(), [content]);
+
+    const isContentEqual = contentMd5 === md5;
+    const isVoiceEqual = currentVoice === voice;
+    const isEqual = isVoiceEqual && isContentEqual;
+
+    const PlayerRender = file && isEqual ? FilePlayer : InitPlayer;
+
+    return <PlayerRender {...props} contentMd5={md5} />;
+  },
+  (prevProps, nextProps) => {
+    return prevProps.id === nextProps.id && prevProps.content === nextProps.content;
+  },
+);
+export default TTS;

package/src/hooks/useTTS.ts

@@ -1,14 +1,13 @@
-import { VoiceList } from '@lobehub/tts';
 import {
   EdgeSpeechOptions,
   MicrosoftSpeechOptions,
   OpenAITTSOptions,
+  TTSOptions,
   useEdgeSpeech,
   useMicrosoftSpeech,
   useOpenAITTS,
 } from '@lobehub/tts/react';
 import isEqual from 'fast-deep-equal';
-import { SWRConfiguration } from 'swr';
 
 import { createHeaderWithOpenAI } from '@/services/_header';
 import { OPENAI_URLS, TTS_URL } from '@/services/_url';
@@ -17,7 +16,8 @@ import { useSessionStore } from '@/store/session';
 import { agentSelectors } from '@/store/session/selectors';
 import { TTSServer } from '@/types/agent';
 
-interface TTSConfig extends SWRConfiguration {
+interface TTSConfig extends TTSOptions {
+  onUpload?: (currentVoice: string, arraybuffers: ArrayBuffer[]) => void;
   server?: TTSServer;
   voice?: string;
 }
@@ -25,8 +25,8 @@ interface TTSConfig extends SWRConfiguration {
 export const useTTS = (content: string, config?: TTSConfig) => {
   const ttsSettings = useGlobalStore(settingsSelectors.currentTTS, isEqual);
   const ttsAgentSettings = useSessionStore(agentSelectors.currentAgentTTS, isEqual);
-  const voiceList = useGlobalStore((s) => new VoiceList(settingsSelectors.currentLanguage(s)));
-
+  const lang = useGlobalStore(settingsSelectors.currentLanguage);
+  const voice = useSessionStore(agentSelectors.currentAgentTTSVoice(lang));
   let useSelectedTTS;
   let options: any = {};
   switch (config?.server || ttsAgentSettings.ttsService) {
@@ -39,10 +39,7 @@ export const useTTS = (content: string, config?: TTSConfig) => {
         },
         options: {
           model: ttsSettings.openAI.ttsModel,
-          voice:
-            config?.voice ||
-            ttsAgentSettings.voice.openai ||
-            VoiceList.openaiVoiceOptions?.[0].value,
+          voice: config?.voice || voice,
         },
       } as OpenAITTSOptions;
       break;
@@ -57,8 +54,7 @@ export const useTTS = (content: string, config?: TTSConfig) => {
            */
         },
         options: {
-          voice:
-            config?.voice || ttsAgentSettings.voice.edge || voiceList.edgeVoiceOptions?.[0].value,
+          voice: config?.voice || voice,
         },
       } as EdgeSpeechOptions;
       break;
@@ -70,10 +66,7 @@ export const useTTS = (content: string, config?: TTSConfig) => {
           serviceUrl: TTS_URL.microsoft,
         },
         options: {
-          voice:
-            config?.voice ||
-            ttsAgentSettings.voice.microsoft ||
-            voiceList.microsoftVoiceOptions?.[0].value,
+          voice: config?.voice || voice,
         },
       } as MicrosoftSpeechOptions;
       break;
@@ -83,5 +76,8 @@ export const useTTS = (content: string, config?: TTSConfig) => {
   return useSelectedTTS(content, {
     ...config,
     ...options,
+    onFinish: (arraybuffers) => {
+      config?.onUpload?.(options.voice || 'alloy', arraybuffers);
+    },
   });
 };

package/src/store/chat/actions/enhance.test.ts

@@ -0,0 +1,139 @@
+import { act, renderHook, waitFor } from '@testing-library/react';
+import { Mock, afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { chainLangDetect } from '@/chains/langDetect';
+import { chainTranslate } from '@/chains/translate';
+import { chatService } from '@/services/chat';
+import { messageService } from '@/services/message';
+
+import { useChatStore } from '../store';
+
+// Mock messageService 和 chatService
+vi.mock('@/services/message', () => ({
+  messageService: {
+    updateMessageTTS: vi.fn(),
+    updateMessageTranslate: vi.fn(),
+  },
+}));
+
+vi.mock('@/services/chat', () => ({
+  chatService: {
+    fetchPresetTaskResult: vi.fn(),
+  },
+}));
+
+vi.mock('@/chains/langDetect', () => ({
+  chainLangDetect: vi.fn(),
+}));
+
+vi.mock('@/chains/translate', () => ({
+  chainTranslate: vi.fn(),
+}));
+
+// Mock supportLocales
+vi.mock('@/locales/options', () => ({
+  supportLocales: ['en-US', 'zh-CN'],
+}));
+
+beforeEach(() => {
+  vi.clearAllMocks();
+  useChatStore.setState(
+    {
+      // ... 初始状态
+    },
+    false,
+  );
+});
+
+afterEach(() => {
+  vi.restoreAllMocks();
+});
+
+describe('ChatEnhanceAction', () => {
+  describe('clearTTS', () => {
+    it('should clear TTS for a message and refresh messages', async () => {
+      const { result } = renderHook(() => useChatStore());
+      const messageId = 'message-id';
+
+      await act(async () => {
+        await result.current.clearTTS(messageId);
+      });
+
+      expect(messageService.updateMessageTTS).toHaveBeenCalledWith(messageId, null);
+    });
+  });
+
+  describe('translateMessage', () => {
+    it('should translate a message to the target language and refresh messages', async () => {
+      const { result } = renderHook(() => useChatStore());
+      const messageId = 'message-id';
+      const targetLang = 'zh-CN';
+      const messageContent = 'Hello World';
+      const detectedLang = 'en-US';
+
+      // 设置初始消息状态
+      useChatStore.setState({
+        messages: [
+          {
+            id: messageId,
+            content: messageContent,
+            createdAt: Date.now(),
+            updatedAt: Date.now(),
+            role: 'user',
+            sessionId: 'test',
+            topicId: 'test',
+            meta: {},
+          },
+        ],
+      });
+
+      // 模拟语言检测和翻译结果
+      (chatService.fetchPresetTaskResult as Mock).mockImplementation(({ params }) => {
+        if (params === chainLangDetect(messageContent)) {
+          return Promise.resolve(detectedLang);
+        }
+        if (params === chainTranslate(messageContent, targetLang)) {
+          return Promise.resolve('Hola Mundo');
+        }
+        return Promise.resolve(undefined);
+      });
+
+      await act(async () => {
+        await result.current.translateMessage(messageId, targetLang);
+      });
+
+      expect(messageService.updateMessageTranslate).toHaveBeenCalled();
+    });
+  });
+
+  describe('clearTranslate', () => {
+    it('should clear translation for a message and refresh messages', async () => {
+      const { result } = renderHook(() => useChatStore());
+      const messageId = 'message-id';
+
+      await act(async () => {
+        await result.current.clearTranslate(messageId);
+      });
+
+      expect(messageService.updateMessageTranslate).toHaveBeenCalledWith(messageId, null);
+    });
+  });
+
+  describe('ttsMessage', () => {
+    it('should update TTS state for a message and refresh messages', async () => {
+      const { result } = renderHook(() => useChatStore());
+      const messageId = 'message-id';
+      const ttsState = {
+        contentMd5: 'some-md5',
+        file: 'path-to-tts-file',
+        voice: 'voice-type',
+      };
+
+      await act(async () => {
+        await result.current.ttsMessage(messageId, ttsState);
+      });
+
+      expect(messageService.updateMessageTTS).toHaveBeenCalledWith(messageId, ttsState);
+    });
+  });
+});

package/src/store/chat/actions/enhance.ts

@@ -20,7 +20,10 @@ export interface ChatEnhanceAction {
 
   clearTranslate: (id: string) => Promise<void>;
   translateMessage: (id: string, targetLang: string) => Promise<void>;
-  ttsMessage: (id: string, init?: boolean) => Promise<void>;
+  ttsMessage: (
+    id: string,
+    state?: { contentMd5?: string; file?: string; voice?: string },
+  ) => Promise<void>;
 }
 
 export const chatEnhance: StateCreator<
@@ -88,8 +91,8 @@ export const chatEnhance: StateCreator<
     toggleChatLoading(false);
   },
 
-  ttsMessage: async (id, init) => {
-    await messageService.updateMessageTTS(id, { init: Boolean(init) });
+  ttsMessage: async (id, state = {}) => {
+    await messageService.updateMessageTTS(id, state);
     await get().refreshMessages();
   },
 });

package/src/store/file/selectors.ts

@@ -1,34 +1,7 @@
-import { FilesStoreState } from './initialState';
-
-const getImageDetailByList = (list: string[]) => (s: FilesStoreState) =>
-  list.map((i) => s.imagesMap[i]).filter(Boolean);
-
-const imageDetailList = (s: FilesStoreState) => getImageDetailByList(s.inputFilesList)(s);
-
-const getImageUrlOrBase64ById =
-  (id: string) =>
-  (s: FilesStoreState): { id: string; url: string } | undefined => {
-    const preview = s.imagesMap[id];
-
-    if (!preview) return undefined;
-
-    const url = preview.saveMode === 'local' ? (preview.base64Url as string) : preview.url;
-
-    return { id, url: url };
-  };
-
-const getImageUrlOrBase64ByList = (idList: string[]) => (s: FilesStoreState) =>
-  idList.map((i) => getImageUrlOrBase64ById(i)(s)).filter(Boolean) as {
-    id: string;
-    url: string;
-  }[];
-
-const imageUrlOrBase64List = (s: FilesStoreState) => getImageUrlOrBase64ByList(s.inputFilesList)(s);
+import { filesSelectors as imageFilesSelectors } from './slices/images';
+import { ttsFilesSelectors } from './slices/tts';
 
 export const filesSelectors = {
-  getImageDetailByList,
-  getImageUrlOrBase64ById,
-  getImageUrlOrBase64ByList,
-  imageDetailList,
-  imageUrlOrBase64List,
+  ...imageFilesSelectors,
+  ...ttsFilesSelectors,
 };

package/src/store/file/slices/images/index.ts

@@ -1,3 +1,3 @@
 export * from './action';
 export * from './initialState';
-// export * from './selectors';
+export * from './selectors';

package/src/store/file/slices/images/selectors.ts

@@ -0,0 +1,34 @@
+import { FilesStoreState } from '../../initialState';
+
+const getImageDetailByList = (list: string[]) => (s: FilesStoreState) =>
+  list.map((i) => s.imagesMap[i]).filter(Boolean);
+
+const imageDetailList = (s: FilesStoreState) => getImageDetailByList(s.inputFilesList)(s);
+
+const getImageUrlOrBase64ById =
+  (id: string) =>
+  (s: FilesStoreState): { id: string; url: string } | undefined => {
+    const preview = s.imagesMap[id];
+
+    if (!preview) return undefined;
+
+    const url = preview.saveMode === 'local' ? (preview.base64Url as string) : preview.url;
+
+    return { id, url: url };
+  };
+
+const getImageUrlOrBase64ByList = (idList: string[]) => (s: FilesStoreState) =>
+  idList.map((i) => getImageUrlOrBase64ById(i)(s)).filter(Boolean) as {
+    id: string;
+    url: string;
+  }[];
+
+const imageUrlOrBase64List = (s: FilesStoreState) => getImageUrlOrBase64ByList(s.inputFilesList)(s);
+
+export const filesSelectors = {
+  getImageDetailByList,
+  getImageUrlOrBase64ById,
+  getImageUrlOrBase64ByList,
+  imageDetailList,
+  imageUrlOrBase64List,
+};

package/src/store/file/slices/tts/action.test.ts

@@ -0,0 +1,147 @@
+import { act, renderHook } from '@testing-library/react';
+import useSWR from 'swr';
+import { Mock, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { fileService } from '@/services/file';
+
+import { useFileStore as useStore } from '../../store';
+
+vi.mock('zustand/traditional');
+
+// Mocks for fileService
+vi.mock('@/services/file', () => ({
+  fileService: {
+    removeFile: vi.fn(),
+    uploadFile: vi.fn(),
+    getFile: vi.fn(),
+  },
+}));
+
+// Mock for useSWR
+vi.mock('swr', () => ({
+  default: vi.fn(),
+}));
+
+//  mock the arrayBuffer
+beforeAll(() => {
+  Object.defineProperty(File.prototype, 'arrayBuffer', {
+    writable: true,
+    value: function () {
+      return new Promise((resolve) => {
+        const reader = new FileReader();
+        reader.onloadend = () => {
+          resolve(reader.result);
+        };
+        reader.readAsArrayBuffer(this);
+      });
+    },
+  });
+});
+
+beforeEach(() => {
+  // Reset all mocks before each test
+  vi.resetAllMocks();
+});
+
+describe('TTSFileAction', () => {
+  // Test for removeTTSFile
+  it('removeTTSFile should call fileService.removeFile', async () => {
+    const fileId = 'tts-file-id';
+
+    // Mock the fileService.removeFile to resolve
+    (fileService.removeFile as Mock).mockResolvedValue(undefined);
+
+    await act(async () => {
+      await useStore.getState().removeTTSFile(fileId);
+    });
+
+    expect(fileService.removeFile).toHaveBeenCalledWith(fileId);
+  });
+
+  // Test for uploadTTSFile
+  it('uploadTTSFile should upload the file and return the file id', async () => {
+    const testFile = new File(['content'], 'test.mp3', { type: 'audio/mp3' });
+    const uploadedFileData = {
+      id: 'new-tts-file-id',
+      createdAt: testFile.lastModified,
+      data: await testFile.arrayBuffer(),
+      fileType: testFile.type,
+      name: testFile.name,
+      saveMode: 'local',
+      size: testFile.size,
+    };
+
+    // Mock the fileService.uploadFile to resolve with uploadedFileData
+    (fileService.uploadFile as Mock).mockResolvedValue(uploadedFileData);
+
+    let fileId;
+    await act(async () => {
+      fileId = await useStore.getState().uploadTTSFile(testFile);
+    });
+
+    expect(fileService.uploadFile).toHaveBeenCalledWith({
+      createdAt: testFile.lastModified,
+      data: await testFile.arrayBuffer(),
+      fileType: testFile.type,
+      name: testFile.name,
+      saveMode: 'local',
+      size: testFile.size,
+    });
+    expect(fileId).toBe(uploadedFileData.id);
+  });
+
+  // Test for uploadTTSByArrayBuffers
+  it('uploadTTSByArrayBuffers should create a file and call uploadTTSFile', async () => {
+    const messageId = 'message-id';
+    const arrayBuffers = [new ArrayBuffer(10)];
+    const fileType = 'audio/mp3';
+    const fileName = `${messageId}.mp3`;
+
+    // Spy on uploadTTSFile to simulate a successful upload
+    const uploadTTSFileSpy = vi
+      .spyOn(useStore.getState(), 'uploadTTSFile')
+      .mockResolvedValue('new-tts-file-id');
+
+    let fileId;
+    await act(async () => {
+      fileId = await useStore.getState().uploadTTSByArrayBuffers(messageId, arrayBuffers);
+    });
+
+    expect(uploadTTSFileSpy).toHaveBeenCalled();
+    expect(fileId).toBe('new-tts-file-id');
+
+    // Cleanup spy
+    uploadTTSFileSpy.mockRestore();
+  });
+
+  // Test for useFetchTTSFile
+  it('useFetchTTSFile should call useSWR and return file data', async () => {
+    const fileId = 'tts-file-id';
+    const fileData = {
+      id: fileId,
+      name: 'test',
+      url: 'blob:test',
+      fileType: 'audio/mp3',
+      base64Url: '',
+      saveMode: 'local',
+    };
+
+    // Mock the fileService.getFile to resolve with fileData
+    (fileService.getFile as Mock).mockResolvedValue(fileData);
+
+    // Mock useSWR to call the fetcher function immediately
+    const useSWRMock = vi.mocked(useSWR);
+    useSWRMock.mockImplementation(((key: string, fetcher: any) => {
+      const data = fetcher(key);
+      return { data, error: undefined, isValidating: false, mutate: vi.fn() };
+    }) as any);
+
+    const { result } = renderHook(() => useStore.getState().useFetchTTSFile(fileId));
+
+    await act(async () => {
+      await result.current.data;
+    });
+
+    expect(fileService.getFile).toHaveBeenCalledWith(fileId);
+  });
+});

package/src/store/file/slices/tts/action.ts

@@ -0,0 +1,60 @@
+import useSWR, { SWRResponse } from 'swr';
+import { StateCreator } from 'zustand/vanilla';
+
+import { fileService } from '@/services/file';
+import { FilePreview } from '@/types/files';
+
+import { FileStore } from '../../store';
+
+export interface TTSFileAction {
+  removeTTSFile: (id: string) => Promise<void>;
+
+  uploadTTSByArrayBuffers: (
+    messageId: string,
+    arrayBuffers: ArrayBuffer[],
+  ) => Promise<string | undefined>;
+
+  uploadTTSFile: (file: File) => Promise<string | undefined>;
+
+  useFetchTTSFile: (id: string | null) => SWRResponse<FilePreview>;
+}
+
+export const createTTSFileSlice: StateCreator<
+  FileStore,
+  [['zustand/devtools', never]],
+  [],
+  TTSFileAction
+> = (_, get) => ({
+  removeTTSFile: async (id) => {
+    await fileService.removeFile(id);
+  },
+  uploadTTSByArrayBuffers: async (messageId, arrayBuffers) => {
+    const fileType = 'audio/mp3';
+    const blob = new Blob(arrayBuffers, { type: fileType });
+    const fileName = `${messageId}.mp3`;
+    const fileOptions = {
+      lastModified: Date.now(),
+      type: fileType,
+    };
+    const file = new File([blob], fileName, fileOptions);
+    return get().uploadTTSFile(file);
+  },
+  uploadTTSFile: async (file) => {
+    try {
+      const data = await fileService.uploadFile({
+        createdAt: file.lastModified,
+        data: await file.arrayBuffer(),
+        fileType: file.type,
+        name: file.name,
+        saveMode: 'local',
+        size: file.size,
+      });
+
+      return data.id;
+    } catch (error) {
+      // 提示用户上传失败
+      console.error('upload error:', error);
+    }
+  },
+  useFetchTTSFile: (id) => useSWR(id, fileService.getFile),
+});

package/src/store/file/slices/tts/index.ts

@@ -0,0 +1,2 @@
+export * from './action';
+export * from './selectors';

package/src/store/file/slices/tts/selectors.ts

@@ -0,0 +1,3 @@
+// import { FileStore } from '../../store';
+
+export const ttsFilesSelectors = {};

package/src/store/file/store.ts

@@ -7,14 +7,16 @@ import { isDev } from '@/utils/env';
 
 import { FilesStoreState, initialState } from './initialState';
 import { FileAction, createFileSlice } from './slices/images';
+import { TTSFileAction, createTTSFileSlice } from './slices/tts';
 
 //  ===============  聚合 createStoreFn ============ //
 
-export type FileStore = FilesStoreState & FileAction;
+export type FileStore = FilesStoreState & FileAction & TTSFileAction;
 
 const createStore: StateCreator<FileStore, [['zustand/devtools', never]]> = (...parameters) => ({
   ...initialState,
   ...createFileSlice(...parameters),
+  ...createTTSFileSlice(...parameters),
 });
 
 //  ===============  实装 useStore ============ //

package/src/store/session/slices/agent/selectors.test.ts

@@ -0,0 +1,228 @@
+import { describe, expect, it } from 'vitest';
+
+import { DEFAULT_AVATAR } from '@/const/meta';
+import { DEFAULT_AGENT_CONFIG, DEFAUTT_AGENT_TTS_CONFIG } from '@/const/settings';
+import { SessionStore } from '@/store/session';
+import { MetaData } from '@/types/meta';
+import { LobeAgentSession, LobeSessionType } from '@/types/session';
+
+import { agentSelectors } from '../agent';
+
+vi.mock('i18next', () => ({
+  t: vi.fn((key) => key), // Simplified mock return value
+}));
+
+const mockSessionStore = {
+  activeId: '1',
+  sessions: [
+    {
+      id: '1',
+      config: DEFAULT_AGENT_CONFIG,
+      meta: {
+        title: 'title1',
+        description: 'description1',
+      },
+      type: LobeSessionType.Agent,
+    } as LobeAgentSession,
+    {
+      id: '2',
+      meta: {
+        title: 'title2',
+        description: 'description2',
+      },
+      config: DEFAULT_AGENT_CONFIG,
+      type: LobeSessionType.Agent,
+    } as LobeAgentSession,
+  ],
+} as unknown as SessionStore;
+
+describe('agentSelectors', () => {
+  describe('currentAgentConfig', () => {
+    it('should return the merged default and session-specific agent config', () => {
+      const config = agentSelectors.currentAgentConfig(mockSessionStore);
+      expect(config).toEqual(expect.objectContaining(mockSessionStore.sessions[0].config));
+    });
+  });
+
+  describe('currentAgentModel', () => {
+    it('should return the model from the agent config', () => {
+      const model = agentSelectors.currentAgentModel(mockSessionStore);
+      expect(model).toBe(mockSessionStore.sessions[0].config.model);
+    });
+  });
+
+  describe('currentAgentMeta', () => {
+    it('should return the merged default and session-specific meta data', () => {
+      const meta = agentSelectors.currentAgentMeta(mockSessionStore);
+      expect(meta).toEqual(expect.objectContaining(mockSessionStore.sessions[0].meta));
+    });
+
+    it('should return inbox defaults if it is an inbox session', () => {
+      // Assume sessionSelectors.isInboxSession() is mocked to return true for this test
+      const meta = agentSelectors.currentAgentMeta(mockSessionStore);
+      expect(meta.avatar).toBe(DEFAULT_AVATAR);
+    });
+  });
+
+  describe('currentAgentTitle', () => {
+    it('should return the title from the session meta data', () => {
+      const title = agentSelectors.currentAgentTitle(mockSessionStore);
+      expect(title).toBe(mockSessionStore.sessions[0].meta.title);
+    });
+  });
+
+  describe('currentAgentDescription', () => {
+    it('should return the description from the session meta data', () => {
+      const description = agentSelectors.currentAgentDescription(mockSessionStore);
+      expect(description).toBe(mockSessionStore.sessions[0].meta.description);
+    });
+  });
+
+  // ... More tests for other selectors
+
+  describe('hasSystemRole', () => {
+    it('should return true if the system role is defined in the agent config', () => {
+      const hasRole = agentSelectors.hasSystemRole(mockSessionStore);
+      expect(hasRole).toBe(false);
+    });
+
+    it('should return false if the system role is not defined in the agent config', () => {
+      const modifiedSessionStore = {
+        ...mockSessionStore,
+        sessions: [
+          {
+            ...mockSessionStore.sessions[0],
+            config: {
+              ...mockSessionStore.sessions[0].config,
+              systemRole: 'test',
+            },
+          },
+        ],
+      };
+      const hasRole = agentSelectors.hasSystemRole(modifiedSessionStore);
+      expect(hasRole).toBe(true);
+    });
+  });
+
+  describe('currentAgentTTS', () => {
+    it('should return the TTS config from the agent config', () => {
+      const ttsConfig = agentSelectors.currentAgentTTS(mockSessionStore);
+      expect(ttsConfig).toEqual(mockSessionStore.sessions[0].config.tts);
+    });
+
+    it('should return the default TTS config if none is defined in the agent config', () => {
+      const modifiedSessionStore = {
+        ...mockSessionStore,
+        sessions: [
+          {
+            ...mockSessionStore.sessions[0],
+            config: {
+              ...mockSessionStore.sessions[0].config,
+              tts: DEFAUTT_AGENT_TTS_CONFIG,
+            },
+          },
+        ],
+      };
+      const ttsConfig = agentSelectors.currentAgentTTS(modifiedSessionStore);
+      expect(ttsConfig).toEqual(DEFAUTT_AGENT_TTS_CONFIG);
+    });
+  });
+
+  describe('currentAgentTTSVoice', () => {
+    it('should return the appropriate TTS voice based on the service and language', () => {
+      const lang = 'en';
+      const ttsVoice = agentSelectors.currentAgentTTSVoice(lang)(mockSessionStore);
+      expect(ttsVoice).toBe(mockSessionStore.sessions[0].config.tts.voice.openai);
+    });
+  });
+
+  describe('getAvatar', () => {
+    it('should return the avatar from the meta data', () => {
+      const meta: MetaData = { avatar: 'custom-avatar.png' };
+      const avatar = agentSelectors.getAvatar(meta);
+      expect(avatar).toBe(meta.avatar);
+    });
+
+    it('should return the default avatar if none is defined in the meta data', () => {
+      const meta: MetaData = {};
+      const avatar = agentSelectors.getAvatar(meta);
+      expect(avatar).toBe(DEFAULT_AVATAR);
+    });
+  });
+
+  describe('getTitle', () => {
+    it('should return the title from the meta data', () => {
+      const meta: MetaData = { title: 'Custom Title' };
+      const title = agentSelectors.getTitle(meta);
+      expect(title).toBe(meta.title);
+    });
+
+    it('should return the default title if none is defined in the meta data', () => {
+      const meta: MetaData = {};
+      const title = agentSelectors.getTitle(meta);
+      expect(title).toBe('defaultSession'); // Assuming translation returns this key
+    });
+  });
+
+  describe('getDescription', () => {
+    it('should return the description from the meta data', () => {
+      const meta: MetaData = { description: 'Custom Description' };
+      const description = agentSelectors.getDescription(meta);
+      expect(description).toBe(meta.description);
+    });
+
+    it('should return the default description if none is defined in the meta data', () => {
+      const meta: MetaData = {};
+      const description = agentSelectors.getDescription(meta);
+      expect(description).toBe('noDescription'); // Assuming translation returns this key
+    });
+  });
+
+  describe('showTokenTag', () => {
+    it('should return true if the model is in the list of models that show tokens', () => {
+      const show = agentSelectors.showTokenTag(mockSessionStore);
+      expect(show).toBe(true);
+    });
+
+    it('should return false if the model is not in the list of models that show tokens', () => {
+      const modifiedSessionStore = {
+        ...mockSessionStore,
+        sessions: [
+          {
+            ...mockSessionStore.sessions[0],
+            config: {
+              ...mockSessionStore.sessions[0].config,
+              model: 'some-other-model',
+            },
+          },
+        ],
+      };
+      const show = agentSelectors.showTokenTag(modifiedSessionStore);
+      expect(show).toBe(false);
+    });
+  });
+
+  describe('modelHasVisionAbility', () => {
+    it('should return true if the model has vision ability', () => {
+      const hasAbility = agentSelectors.modelHasVisionAbility(mockSessionStore);
+      expect(hasAbility).toBe(false);
+    });
+
+    it('should return false if the model does not have vision ability', () => {
+      const modifiedSessionStore = {
+        ...mockSessionStore,
+        sessions: [
+          {
+            ...mockSessionStore.sessions[0],
+            config: {
+              ...mockSessionStore.sessions[0].config,
+              model: 'some-other-model',
+            },
+          },
+        ],
+      };
+      const hasAbility = agentSelectors.modelHasVisionAbility(modifiedSessionStore);
+      expect(hasAbility).toBe(false);
+    });
+  });
+});

package/src/store/session/slices/agent/selectors.ts

@@ -1,3 +1,4 @@
+import { VoiceList } from '@lobehub/tts';
 import { t } from 'i18next';
 
 import { DEFAULT_OPENAI_MODEL_LIST, VISION_MODEL_WHITE_LIST } from '@/const/llm';
@@ -46,6 +47,29 @@ const currentAgentTTS = (s: SessionStore): LobeAgentTTSConfig => {
   return config?.tts || DEFAUTT_AGENT_TTS_CONFIG;
 };
 
+const currentAgentTTSVoice =
+  (lang: string) =>
+  (s: SessionStore): string => {
+    const { voice, ttsService } = currentAgentTTS(s);
+    const voiceList = new VoiceList(lang);
+    let currentVoice;
+    switch (ttsService) {
+      case 'openai': {
+        currentVoice = voice.openai || (VoiceList.openaiVoiceOptions?.[0].value as string);
+        break;
+      }
+      case 'edge': {
+        currentVoice = voice.edge || (voiceList.edgeVoiceOptions?.[0].value as string);
+        break;
+      }
+      case 'microsoft': {
+        currentVoice = voice.microsoft || (voiceList.microsoftVoiceOptions?.[0].value as string);
+        break;
+      }
+    }
+    return currentVoice || 'alloy';
+  };
+
 // ==========   Meta   ============== //
 const currentAgentMeta = (s: SessionStore): MetaData => {
   const isInbox = sessionSelectors.isInboxSession(s);
@@ -97,6 +121,7 @@ export const agentSelectors = {
   currentAgentPlugins,
   currentAgentSystemRole,
   currentAgentTTS,
+  currentAgentTTSVoice,
   currentAgentTitle,
   getAvatar,
   getDescription,

package/src/types/chatMessage.ts

@@ -18,7 +18,9 @@ export interface ChatTranslate extends Translate {
 }
 
 export interface ChatTTS {
-  init?: boolean;
+  contentMd5?: string;
+  file?: string;
+  voice?: string;
 }
 
 export interface ChatPluginPayload {

package/vitest.config.ts

@@ -2,6 +2,10 @@ import { resolve } from 'node:path';
 import { defineConfig } from 'vitest/config';
 
 export default defineConfig({
+  optimizeDeps: {
+    exclude: ['crypto', 'util', 'tty'],
+    include: ['@lobehub/tts'],
+  },
   test: {
     alias: {
       '@': resolve(__dirname, './src'),

package/src/app/chat/features/Conversation/ChatList/Extras/TTS.tsx

@@ -1,122 +0,0 @@
-import { AudioPlayer } from '@lobehub/tts/react';
-import { ActionIcon, Alert, Highlighter } from '@lobehub/ui';
-import { Button } from 'antd';
-import { TrashIcon } from 'lucide-react';
-import { memo, useCallback, useEffect, useState } from 'react';
-import { useTranslation } from 'react-i18next';
-import { Flexbox } from 'react-layout-kit';
-
-import { useTTS } from '@/hooks/useTTS';
-import { useChatStore } from '@/store/chat';
-import { ChatMessageError, ChatTTS } from '@/types/chatMessage';
-import { getMessageError } from '@/utils/fetch';
-
-interface TTSProps extends ChatTTS {
-  content: string;
-  id: string;
-  loading?: boolean;
-}
-
-const TTS = memo<TTSProps>(({ id, init, content }) => {
-  const [isStart, setIsStart] = useState(false);
-  const [error, setError] = useState<ChatMessageError>();
-  const { t } = useTranslation('chat');
-
-  const [ttsMessage, clearTTS] = useChatStore((s) => [s.ttsMessage, s.clearTTS]);
-
-  const setDefaultError = useCallback(
-    (err?: any) => {
-      setError({ body: err, message: t('tts.responseError', { ns: 'error' }), type: 500 });
-    },
-    [t],
-  );
-
-  const { isGlobalLoading, audio, start, stop, response } = useTTS(content, {
-    onError: (err) => {
-      stop();
-      setDefaultError(err);
-    },
-    onErrorRetry: (err) => {
-      stop();
-      setDefaultError(err);
-    },
-    onSuccess: async () => {
-      if (!response) return;
-      if (response.status === 200) return ttsMessage(id, true);
-      const message = await getMessageError(response);
-      if (message) {
-        setError(message);
-      } else {
-        setDefaultError();
-      }
-      stop();
-    },
-  });
-
-  const handleInitStart = useCallback(() => {
-    if (isStart) return;
-    start();
-    setIsStart(true);
-  }, [isStart]);
-
-  const handleDelete = useCallback(() => {
-    stop();
-    clearTTS(id);
-  }, [stop, id]);
-
-  const handleRetry = useCallback(() => {
-    setError(undefined);
-    start();
-  }, [start]);
-
-  useEffect(() => {
-    if (init) return;
-    handleInitStart();
-  }, [init]);
-
-  return (
-    <Flexbox align={'center'} horizontal style={{ minWidth: 160, width: '100%' }}>
-      {error ? (
-        <Alert
-          action={
-            <Button onClick={handleRetry} size={'small'} type={'primary'}>
-              {t('retry', { ns: 'common' })}
-            </Button>
-          }
-          closable
-          extra={
-            error.body && (
-              <Highlighter copyButtonSize={'small'} language={'json'} type={'pure'}>
-                {JSON.stringify(error.body, null, 2)}
-              </Highlighter>
-            )
-          }
-          message={error.message}
-          onClose={handleDelete}
-          style={{ alignItems: 'center', width: '100%' }}
-          type="error"
-        />
-      ) : (
-        <>
-          <AudioPlayer
-            audio={audio}
-            buttonSize={'small'}
-            isLoading={isGlobalLoading}
-            onInitPlay={handleInitStart}
-            onLoadingStop={stop}
-            timeRender={'tag'}
-            timeStyle={{ margin: 0 }}
-          />
-          <ActionIcon
-            icon={TrashIcon}
-            onClick={handleDelete}
-            size={'small'}
-            title={t('tts.clear')}
-          />
-        </>
-      )}
-    </Flexbox>
-  );
-});
-
-export default TTS;