@rimori/client 1.4.9 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -49
- package/dist/react-client/src/plugin/ThemeSetter.d.ts +2 -0
- package/dist/react-client/src/plugin/ThemeSetter.js +19 -0
- package/dist/react-client/src/utils/PluginUtils.d.ts +2 -0
- package/dist/react-client/src/utils/PluginUtils.js +23 -0
- package/dist/{cli → rimori-client/src/cli}/scripts/init/main.js +0 -0
- package/dist/{cli → rimori-client/src/cli}/scripts/release/release.js +0 -0
- package/dist/{core → rimori-client/src}/controller/AIController.d.ts +1 -1
- package/dist/{core → rimori-client/src}/controller/AIController.js +2 -2
- package/dist/{plugin/AccomplishmentHandler.d.ts → rimori-client/src/controller/AccomplishmentController.d.ts} +1 -1
- package/dist/{plugin/AccomplishmentHandler.js → rimori-client/src/controller/AccomplishmentController.js} +1 -1
- package/dist/{core → rimori-client/src}/controller/ExerciseController.d.ts +8 -6
- package/dist/{core → rimori-client/src}/controller/ExerciseController.js +10 -9
- package/dist/{core → rimori-client/src}/controller/ObjectController.d.ts +1 -1
- package/dist/{core → rimori-client/src}/controller/ObjectController.js +1 -1
- package/dist/{core → rimori-client/src}/controller/SettingsController.d.ts +2 -2
- package/dist/{core → rimori-client/src}/controller/SharedContentController.d.ts +1 -1
- package/dist/{plugin → rimori-client/src/controller}/TranslationController.js +5 -4
- package/dist/{core → rimori-client/src}/controller/VoiceController.d.ts +1 -1
- package/dist/rimori-client/src/index.d.ts +11 -0
- package/dist/rimori-client/src/index.js +10 -0
- package/dist/{plugin/PluginController.d.ts → rimori-client/src/plugin/CommunicationHandler.d.ts} +3 -7
- package/dist/{plugin/PluginController.js → rimori-client/src/plugin/CommunicationHandler.js} +3 -28
- package/dist/{plugin → rimori-client/src/plugin}/RimoriClient.d.ts +67 -68
- package/dist/{plugin → rimori-client/src/plugin}/RimoriClient.js +101 -43
- package/dist/{worker → rimori-client/src/worker}/WorkerSetup.js +2 -2
- package/example/docs/devdocs.md +1 -1
- package/package.json +4 -26
- package/src/{core/controller → controller}/AIController.ts +3 -3
- package/src/{plugin/AccomplishmentHandler.ts → controller/AccomplishmentController.ts} +1 -1
- package/src/{core/controller → controller}/ExerciseController.ts +14 -11
- package/src/{core/controller → controller}/ObjectController.ts +2 -2
- package/src/{core/controller → controller}/SettingsController.ts +2 -2
- package/src/{core/controller → controller}/SharedContentController.ts +1 -1
- package/src/{plugin → controller}/TranslationController.ts +6 -4
- package/src/{core/controller → controller}/VoiceController.ts +2 -2
- package/src/fromRimori/readme.md +1 -1
- package/src/index.ts +8 -8
- package/src/plugin/{PluginController.ts → CommunicationHandler.ts} +8 -36
- package/src/plugin/RimoriClient.ts +125 -118
- package/src/worker/WorkerSetup.ts +5 -3
- package/tsconfig.json +4 -2
- package/dist/cli/scripts/release/release-translation-upload.d.ts +0 -6
- package/dist/cli/scripts/release/release-translation-upload.js +0 -87
- package/dist/components/CRUDModal.d.ts +0 -17
- package/dist/components/CRUDModal.js +0 -24
- package/dist/components/MarkdownEditor.d.ts +0 -8
- package/dist/components/MarkdownEditor.js +0 -48
- package/dist/components/Spinner.d.ts +0 -8
- package/dist/components/Spinner.js +0 -4
- package/dist/components/ai/Assistant.d.ts +0 -9
- package/dist/components/ai/Assistant.js +0 -58
- package/dist/components/ai/Avatar.d.ts +0 -14
- package/dist/components/ai/Avatar.js +0 -59
- package/dist/components/ai/EmbeddedAssistent/AudioInputField.d.ts +0 -7
- package/dist/components/ai/EmbeddedAssistent/AudioInputField.js +0 -37
- package/dist/components/ai/EmbeddedAssistent/CircleAudioAvatar.d.ts +0 -8
- package/dist/components/ai/EmbeddedAssistent/CircleAudioAvatar.js +0 -79
- package/dist/components/ai/EmbeddedAssistent/TTS/MessageSender.d.ts +0 -19
- package/dist/components/ai/EmbeddedAssistent/TTS/MessageSender.js +0 -91
- package/dist/components/ai/EmbeddedAssistent/TTS/Player.d.ts +0 -27
- package/dist/components/ai/EmbeddedAssistent/TTS/Player.js +0 -185
- package/dist/components/ai/EmbeddedAssistent/VoiceRecoder.d.ts +0 -11
- package/dist/components/ai/EmbeddedAssistent/VoiceRecoder.js +0 -95
- package/dist/components/ai/utils.d.ts +0 -6
- package/dist/components/ai/utils.js +0 -13
- package/dist/components/audio/Playbutton.d.ts +0 -15
- package/dist/components/audio/Playbutton.js +0 -80
- package/dist/components/components/ContextMenu.d.ts +0 -10
- package/dist/components/components/ContextMenu.js +0 -135
- package/dist/components.d.ts +0 -10
- package/dist/components.js +0 -11
- package/dist/core/controller/EnhancedUserInfo.d.ts +0 -0
- package/dist/core/controller/EnhancedUserInfo.js +0 -1
- package/dist/core/core.d.ts +0 -14
- package/dist/core/core.js +0 -7
- package/dist/hooks/I18nHooks.d.ts +0 -11
- package/dist/hooks/I18nHooks.js +0 -25
- package/dist/hooks/UseChatHook.d.ts +0 -10
- package/dist/hooks/UseChatHook.js +0 -29
- package/dist/i18n/I18nHooks.d.ts +0 -11
- package/dist/i18n/I18nHooks.js +0 -25
- package/dist/i18n/Translator.d.ts +0 -43
- package/dist/i18n/Translator.js +0 -118
- package/dist/i18n/config.d.ts +0 -7
- package/dist/i18n/config.js +0 -20
- package/dist/i18n/createI18nInstance.d.ts +0 -7
- package/dist/i18n/createI18nInstance.js +0 -31
- package/dist/i18n/hooks.d.ts +0 -11
- package/dist/i18n/hooks.js +0 -25
- package/dist/i18n/index.d.ts +0 -4
- package/dist/i18n/index.js +0 -4
- package/dist/i18n/types.d.ts +0 -7
- package/dist/i18n/types.js +0 -1
- package/dist/i18n/useRimoriI18n.d.ts +0 -11
- package/dist/i18n/useRimoriI18n.js +0 -41
- package/dist/index.d.ts +0 -11
- package/dist/index.js +0 -11
- package/dist/plugin/Translator.d.ts +0 -38
- package/dist/plugin/Translator.js +0 -101
- package/dist/providers/PluginProvider.d.ts +0 -12
- package/dist/providers/PluginProvider.js +0 -152
- package/dist/style.css +0 -110
- package/dist/style.css.map +0 -1
- package/dist/utils/Language.d.ts +0 -67
- package/dist/utils/Language.js +0 -69
- package/dist/utils/LanguageClass.d.ts +0 -36
- package/dist/utils/LanguageClass.example.d.ts +0 -0
- package/dist/utils/LanguageClass.example.js +0 -1
- package/dist/utils/LanguageClass.js +0 -50
- package/dist/utils/LanguageClass.test.d.ts +0 -0
- package/dist/utils/LanguageClass.test.js +0 -1
- package/src/components/CRUDModal.tsx +0 -75
- package/src/components/MarkdownEditor.tsx +0 -144
- package/src/components/Spinner.tsx +0 -29
- package/src/components/ai/Assistant.tsx +0 -96
- package/src/components/ai/Avatar.tsx +0 -99
- package/src/components/ai/EmbeddedAssistent/AudioInputField.tsx +0 -73
- package/src/components/ai/EmbeddedAssistent/CircleAudioAvatar.tsx +0 -107
- package/src/components/ai/EmbeddedAssistent/TTS/MessageSender.ts +0 -96
- package/src/components/ai/EmbeddedAssistent/TTS/Player.ts +0 -197
- package/src/components/ai/EmbeddedAssistent/VoiceRecoder.tsx +0 -129
- package/src/components/ai/utils.ts +0 -21
- package/src/components/audio/Playbutton.tsx +0 -126
- package/src/components/components/ContextMenu.tsx +0 -179
- package/src/components.ts +0 -11
- package/src/core/core.ts +0 -15
- package/src/hooks/I18nHooks.ts +0 -33
- package/src/hooks/UseChatHook.ts +0 -38
- package/src/plugin/ThemeSetter.ts +0 -23
- package/src/providers/PluginProvider.tsx +0 -209
- package/src/style.scss +0 -136
- package/src/utils/PluginUtils.ts +0 -22
- /package/dist/{plugin → react-client/plugin}/ThemeSetter.d.ts +0 -0
- /package/dist/{plugin → react-client/plugin}/ThemeSetter.js +0 -0
- /package/dist/{utils/PluginUtils.d.ts → react-client/src/utils/FullscreenUtils.d.ts} +0 -0
- /package/dist/{utils/PluginUtils.js → react-client/src/utils/FullscreenUtils.js} +0 -0
- /package/dist/{cli → rimori-client/src/cli}/scripts/init/dev-registration.d.ts +0 -0
- /package/dist/{cli → rimori-client/src/cli}/scripts/init/dev-registration.js +0 -0
- /package/dist/{cli → rimori-client/src/cli}/scripts/init/env-setup.d.ts +0 -0
- /package/dist/{cli → rimori-client/src/cli}/scripts/init/env-setup.js +0 -0
- /package/dist/{cli → rimori-client/src/cli}/scripts/init/file-operations.d.ts +0 -0
- /package/dist/{cli → rimori-client/src/cli}/scripts/init/file-operations.js +0 -0
- /package/dist/{cli → rimori-client/src/cli}/scripts/init/html-cleaner.d.ts +0 -0
- /package/dist/{cli → rimori-client/src/cli}/scripts/init/html-cleaner.js +0 -0
- /package/dist/{cli → rimori-client/src/cli}/scripts/init/main.d.ts +0 -0
- /package/dist/{cli → rimori-client/src/cli}/scripts/init/package-setup.d.ts +0 -0
- /package/dist/{cli → rimori-client/src/cli}/scripts/init/package-setup.js +0 -0
- /package/dist/{cli → rimori-client/src/cli}/scripts/init/router-transformer.d.ts +0 -0
- /package/dist/{cli → rimori-client/src/cli}/scripts/init/router-transformer.js +0 -0
- /package/dist/{cli → rimori-client/src/cli}/scripts/init/tailwind-config.d.ts +0 -0
- /package/dist/{cli → rimori-client/src/cli}/scripts/init/tailwind-config.js +0 -0
- /package/dist/{cli → rimori-client/src/cli}/scripts/init/vite-config.d.ts +0 -0
- /package/dist/{cli → rimori-client/src/cli}/scripts/init/vite-config.js +0 -0
- /package/dist/{cli → rimori-client/src/cli}/scripts/release/detect-translation-languages.d.ts +0 -0
- /package/dist/{cli → rimori-client/src/cli}/scripts/release/detect-translation-languages.js +0 -0
- /package/dist/{cli → rimori-client/src/cli}/scripts/release/release-config-upload.d.ts +0 -0
- /package/dist/{cli → rimori-client/src/cli}/scripts/release/release-config-upload.js +0 -0
- /package/dist/{cli → rimori-client/src/cli}/scripts/release/release-db-update.d.ts +0 -0
- /package/dist/{cli → rimori-client/src/cli}/scripts/release/release-db-update.js +0 -0
- /package/dist/{cli → rimori-client/src/cli}/scripts/release/release-file-upload.d.ts +0 -0
- /package/dist/{cli → rimori-client/src/cli}/scripts/release/release-file-upload.js +0 -0
- /package/dist/{cli → rimori-client/src/cli}/scripts/release/release.d.ts +0 -0
- /package/dist/{cli → rimori-client/src/cli}/types/DatabaseTypes.d.ts +0 -0
- /package/dist/{cli → rimori-client/src/cli}/types/DatabaseTypes.js +0 -0
- /package/dist/{plugin → rimori-client/src/controller}/AudioController.d.ts +0 -0
- /package/dist/{plugin → rimori-client/src/controller}/AudioController.js +0 -0
- /package/dist/{core → rimori-client/src}/controller/SettingsController.js +0 -0
- /package/dist/{core → rimori-client/src}/controller/SharedContentController.js +0 -0
- /package/dist/{plugin → rimori-client/src/controller}/TranslationController.d.ts +0 -0
- /package/dist/{core → rimori-client/src}/controller/VoiceController.js +0 -0
- /package/dist/{fromRimori → rimori-client/src/fromRimori}/EventBus.d.ts +0 -0
- /package/dist/{fromRimori → rimori-client/src/fromRimori}/EventBus.js +0 -0
- /package/dist/{fromRimori → rimori-client/src/fromRimori}/PluginTypes.d.ts +0 -0
- /package/dist/{fromRimori → rimori-client/src/fromRimori}/PluginTypes.js +0 -0
- /package/dist/{plugin → rimori-client/src/plugin}/Logger.d.ts +0 -0
- /package/dist/{plugin → rimori-client/src/plugin}/Logger.js +0 -0
- /package/dist/{plugin → rimori-client/src/plugin}/StandaloneClient.d.ts +0 -0
- /package/dist/{plugin → rimori-client/src/plugin}/StandaloneClient.js +0 -0
- /package/dist/{utils → rimori-client/src/utils}/difficultyConverter.d.ts +0 -0
- /package/dist/{utils → rimori-client/src/utils}/difficultyConverter.js +0 -0
- /package/dist/{utils → rimori-client/src/utils}/endpoint.d.ts +0 -0
- /package/dist/{utils → rimori-client/src/utils}/endpoint.js +0 -0
- /package/dist/{worker → rimori-client/src/worker}/WorkerSetup.d.ts +0 -0
- /package/src/{plugin → controller}/AudioController.ts +0 -0
|
@@ -1,99 +0,0 @@
|
|
|
1
|
-
import { useEffect, useMemo, useState } from 'react';
|
|
2
|
-
import { VoiceRecorder } from './EmbeddedAssistent/VoiceRecoder';
|
|
3
|
-
import { MessageSender } from './EmbeddedAssistent/TTS/MessageSender';
|
|
4
|
-
import { CircleAudioAvatar } from './EmbeddedAssistent/CircleAudioAvatar';
|
|
5
|
-
import { Tool } from '../../fromRimori/PluginTypes';
|
|
6
|
-
import { useChat } from '../../hooks/UseChatHook';
|
|
7
|
-
import { useRimori } from '../../components';
|
|
8
|
-
import { getFirstMessages } from './utils';
|
|
9
|
-
import { FirstMessages } from './utils';
|
|
10
|
-
|
|
11
|
-
interface Props {
|
|
12
|
-
voiceId: string;
|
|
13
|
-
agentTools: Tool[];
|
|
14
|
-
avatarImageUrl: string;
|
|
15
|
-
circleSize?: string;
|
|
16
|
-
isDarkTheme?: boolean;
|
|
17
|
-
children?: React.ReactNode;
|
|
18
|
-
autoStartConversation?: FirstMessages;
|
|
19
|
-
className?: string;
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
export function Avatar({
|
|
23
|
-
avatarImageUrl,
|
|
24
|
-
voiceId,
|
|
25
|
-
agentTools,
|
|
26
|
-
autoStartConversation,
|
|
27
|
-
children,
|
|
28
|
-
isDarkTheme = false,
|
|
29
|
-
circleSize = '300px',
|
|
30
|
-
className,
|
|
31
|
-
}: Props) {
|
|
32
|
-
const { ai, event } = useRimori();
|
|
33
|
-
const [agentReplying, setAgentReplying] = useState(false);
|
|
34
|
-
const [isProcessingMessage, setIsProcessingMessage] = useState(false);
|
|
35
|
-
const sender = useMemo(() => new MessageSender(ai.getVoice, voiceId), [voiceId]);
|
|
36
|
-
const { messages, append, isLoading, lastMessage, setMessages } = useChat(agentTools);
|
|
37
|
-
|
|
38
|
-
useEffect(() => {
|
|
39
|
-
console.log('messages', messages);
|
|
40
|
-
}, [messages]);
|
|
41
|
-
|
|
42
|
-
useEffect(() => {
|
|
43
|
-
if (!isLoading) setIsProcessingMessage(false);
|
|
44
|
-
}, [isLoading]);
|
|
45
|
-
|
|
46
|
-
useEffect(() => {
|
|
47
|
-
if (!voiceId) return; //at the beginning when being mounted the voiceId is undefined
|
|
48
|
-
sender.setOnLoudnessChange((value) => event.emit('self.avatar.triggerLoudness', { loudness: value }));
|
|
49
|
-
sender.setOnEndOfSpeech(() => setAgentReplying(false));
|
|
50
|
-
|
|
51
|
-
if (!autoStartConversation) return;
|
|
52
|
-
|
|
53
|
-
setMessages(getFirstMessages(autoStartConversation));
|
|
54
|
-
// append([{ role: 'user', content: autoStartConversation.userMessage }]);
|
|
55
|
-
|
|
56
|
-
if (autoStartConversation.assistantMessage) {
|
|
57
|
-
// console.log("autostartmessages", { autoStartConversation, isLoading });
|
|
58
|
-
sender.handleNewText(autoStartConversation.assistantMessage, isLoading);
|
|
59
|
-
} else if (autoStartConversation.userMessage) {
|
|
60
|
-
append([{ role: 'user', content: autoStartConversation.userMessage, id: messages.length.toString() }]);
|
|
61
|
-
}
|
|
62
|
-
}, [autoStartConversation, voiceId]);
|
|
63
|
-
|
|
64
|
-
useEffect(() => {
|
|
65
|
-
if (lastMessage?.role === 'assistant') {
|
|
66
|
-
sender.handleNewText(lastMessage.content, isLoading);
|
|
67
|
-
if (lastMessage.toolCalls) {
|
|
68
|
-
// console.log("unlocking mic", lastMessage)
|
|
69
|
-
setAgentReplying(false);
|
|
70
|
-
setIsProcessingMessage(false);
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
}, [lastMessage, isLoading]);
|
|
74
|
-
|
|
75
|
-
return (
|
|
76
|
-
<div className={`md:pb-8 ${className || ''}`}>
|
|
77
|
-
<CircleAudioAvatar width={circleSize} className="mx-auto" imageUrl={avatarImageUrl} isDarkTheme={isDarkTheme} />
|
|
78
|
-
{children}
|
|
79
|
-
<VoiceRecorder
|
|
80
|
-
iconSize="30"
|
|
81
|
-
className="w-16 h-16 shadow-lg rounded-full bg-gray-400 dark:bg-gray-800"
|
|
82
|
-
disabled={agentReplying}
|
|
83
|
-
loading={isProcessingMessage}
|
|
84
|
-
enablePushToTalk={true}
|
|
85
|
-
onVoiceRecorded={(message) => {
|
|
86
|
-
setAgentReplying(true);
|
|
87
|
-
append([
|
|
88
|
-
{
|
|
89
|
-
role: 'user',
|
|
90
|
-
content: 'Message(' + Math.floor((messages.length + 1) / 2) + '): ' + message,
|
|
91
|
-
id: messages.length.toString(),
|
|
92
|
-
},
|
|
93
|
-
]);
|
|
94
|
-
}}
|
|
95
|
-
onRecordingStatusChange={(running) => !running && setIsProcessingMessage(true)}
|
|
96
|
-
/>
|
|
97
|
-
</div>
|
|
98
|
-
);
|
|
99
|
-
}
|
|
@@ -1,73 +0,0 @@
|
|
|
1
|
-
import React, { useState } from 'react';
|
|
2
|
-
import { VoiceRecorder } from './VoiceRecoder';
|
|
3
|
-
import { BiSolidRightArrow } from 'react-icons/bi';
|
|
4
|
-
import { HiMiniSpeakerXMark, HiMiniSpeakerWave } from 'react-icons/hi2';
|
|
5
|
-
|
|
6
|
-
interface AudioInputFieldProps {
|
|
7
|
-
onSubmit: (text: string) => void;
|
|
8
|
-
onAudioControl?: (voice: boolean) => void;
|
|
9
|
-
blockSubmission?: boolean;
|
|
10
|
-
}
|
|
11
|
-
|
|
12
|
-
export function AudioInputField({ onSubmit, onAudioControl, blockSubmission = false }: AudioInputFieldProps) {
|
|
13
|
-
const [text, setText] = useState('');
|
|
14
|
-
const [audioEnabled, setAudioEnabled] = useState(true);
|
|
15
|
-
|
|
16
|
-
const handleSubmit = (manualText?: string) => {
|
|
17
|
-
if (blockSubmission) return;
|
|
18
|
-
const sendableText = manualText || text;
|
|
19
|
-
if (sendableText.trim()) {
|
|
20
|
-
onSubmit(sendableText);
|
|
21
|
-
setTimeout(() => {
|
|
22
|
-
setText('');
|
|
23
|
-
}, 100);
|
|
24
|
-
}
|
|
25
|
-
};
|
|
26
|
-
|
|
27
|
-
const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
|
|
28
|
-
if (blockSubmission) return;
|
|
29
|
-
if (e.key === 'Enter' && e.ctrlKey) {
|
|
30
|
-
setText(text + '\n');
|
|
31
|
-
} else if (e.key === 'Enter') {
|
|
32
|
-
handleSubmit();
|
|
33
|
-
}
|
|
34
|
-
};
|
|
35
|
-
|
|
36
|
-
return (
|
|
37
|
-
<div className="flex items-center bg-gray-600 pt-2 pb-2 p-2">
|
|
38
|
-
{onAudioControl && (
|
|
39
|
-
<button
|
|
40
|
-
onClick={() => {
|
|
41
|
-
onAudioControl(!audioEnabled);
|
|
42
|
-
setAudioEnabled(!audioEnabled);
|
|
43
|
-
}}
|
|
44
|
-
className="cursor-default"
|
|
45
|
-
>
|
|
46
|
-
{audioEnabled ? (
|
|
47
|
-
<HiMiniSpeakerWave className="w-9 h-9 cursor-pointer" />
|
|
48
|
-
) : (
|
|
49
|
-
<HiMiniSpeakerXMark className="w-9 h-9 cursor-pointer" />
|
|
50
|
-
)}
|
|
51
|
-
</button>
|
|
52
|
-
)}
|
|
53
|
-
<VoiceRecorder
|
|
54
|
-
onRecordingStatusChange={() => {}}
|
|
55
|
-
onVoiceRecorded={(m: string) => {
|
|
56
|
-
console.log('onVoiceRecorded', m);
|
|
57
|
-
handleSubmit(m);
|
|
58
|
-
}}
|
|
59
|
-
/>
|
|
60
|
-
<textarea
|
|
61
|
-
value={text}
|
|
62
|
-
onChange={(e) => setText(e.target.value)}
|
|
63
|
-
onKeyDown={handleKeyDown}
|
|
64
|
-
className="flex-1 border-none rounded-lg p-2 text-gray-800 focus::outline-none"
|
|
65
|
-
placeholder="Type a message..."
|
|
66
|
-
disabled={blockSubmission}
|
|
67
|
-
/>
|
|
68
|
-
<button onClick={() => handleSubmit()} className="cursor-default" disabled={blockSubmission}>
|
|
69
|
-
<BiSolidRightArrow className="w-9 h-10 cursor-pointer" />
|
|
70
|
-
</button>
|
|
71
|
-
</div>
|
|
72
|
-
);
|
|
73
|
-
}
|
|
@@ -1,107 +0,0 @@
|
|
|
1
|
-
import { useEffect, useRef } from 'react';
|
|
2
|
-
import { EventBus, EventBusMessage } from '../../../fromRimori/EventBus';
|
|
3
|
-
|
|
4
|
-
interface CircleAudioAvatarProps {
|
|
5
|
-
width?: string;
|
|
6
|
-
imageUrl: string;
|
|
7
|
-
className?: string;
|
|
8
|
-
isDarkTheme?: boolean;
|
|
9
|
-
}
|
|
10
|
-
|
|
11
|
-
export function CircleAudioAvatar({
|
|
12
|
-
imageUrl,
|
|
13
|
-
className,
|
|
14
|
-
isDarkTheme = false,
|
|
15
|
-
width = '150px',
|
|
16
|
-
}: CircleAudioAvatarProps) {
|
|
17
|
-
const canvasRef = useRef<HTMLCanvasElement>(null);
|
|
18
|
-
const currentLoudnessRef = useRef(0);
|
|
19
|
-
const targetLoudnessRef = useRef(0);
|
|
20
|
-
const animationFrameRef = useRef<number | null>(null);
|
|
21
|
-
|
|
22
|
-
useEffect(() => {
|
|
23
|
-
const canvas = canvasRef.current;
|
|
24
|
-
if (canvas) {
|
|
25
|
-
const ctx = canvas.getContext('2d');
|
|
26
|
-
if (ctx) {
|
|
27
|
-
const image = new Image();
|
|
28
|
-
image.src = imageUrl;
|
|
29
|
-
let isMounted = true;
|
|
30
|
-
|
|
31
|
-
image.onload = () => {
|
|
32
|
-
if (!isMounted) return;
|
|
33
|
-
draw(ctx, canvas, image, 0);
|
|
34
|
-
const animate = () => {
|
|
35
|
-
const decayRate = 0.06;
|
|
36
|
-
if (currentLoudnessRef.current > targetLoudnessRef.current) {
|
|
37
|
-
currentLoudnessRef.current = Math.max(
|
|
38
|
-
targetLoudnessRef.current,
|
|
39
|
-
currentLoudnessRef.current - decayRate * currentLoudnessRef.current,
|
|
40
|
-
);
|
|
41
|
-
} else {
|
|
42
|
-
currentLoudnessRef.current = targetLoudnessRef.current;
|
|
43
|
-
}
|
|
44
|
-
draw(ctx, canvas, image, currentLoudnessRef.current);
|
|
45
|
-
animationFrameRef.current = requestAnimationFrame(animate);
|
|
46
|
-
};
|
|
47
|
-
animationFrameRef.current = requestAnimationFrame(animate);
|
|
48
|
-
};
|
|
49
|
-
|
|
50
|
-
const handleLoudness = ({ data }: EventBusMessage) => {
|
|
51
|
-
const newLoudness = data.loudness;
|
|
52
|
-
if (newLoudness > currentLoudnessRef.current) {
|
|
53
|
-
currentLoudnessRef.current = newLoudness;
|
|
54
|
-
}
|
|
55
|
-
targetLoudnessRef.current = newLoudness;
|
|
56
|
-
};
|
|
57
|
-
|
|
58
|
-
const listener = EventBus.on('self.avatar.triggerLoudness', handleLoudness);
|
|
59
|
-
|
|
60
|
-
return () => {
|
|
61
|
-
isMounted = false;
|
|
62
|
-
listener.off();
|
|
63
|
-
if (animationFrameRef.current) {
|
|
64
|
-
cancelAnimationFrame(animationFrameRef.current);
|
|
65
|
-
}
|
|
66
|
-
};
|
|
67
|
-
}
|
|
68
|
-
}
|
|
69
|
-
}, [imageUrl]);
|
|
70
|
-
|
|
71
|
-
const draw = (
|
|
72
|
-
ctx: CanvasRenderingContext2D,
|
|
73
|
-
canvas: HTMLCanvasElement,
|
|
74
|
-
image: HTMLImageElement,
|
|
75
|
-
loudness: number,
|
|
76
|
-
) => {
|
|
77
|
-
if (canvas && ctx) {
|
|
78
|
-
ctx.clearRect(0, 0, canvas.width, canvas.height);
|
|
79
|
-
|
|
80
|
-
const radius = Math.min(canvas.width, canvas.height) / 3;
|
|
81
|
-
const centerX = canvas.width / 2;
|
|
82
|
-
const centerY = canvas.height / 2;
|
|
83
|
-
const pulseRadius = radius + loudness / 2.5;
|
|
84
|
-
ctx.beginPath();
|
|
85
|
-
ctx.arc(centerX, centerY, pulseRadius, 0, Math.PI * 2, true);
|
|
86
|
-
ctx.strokeStyle = isDarkTheme ? 'rgba(255, 255, 255, 0.5)' : 'rgba(0, 0, 0, 0.5)';
|
|
87
|
-
ctx.lineWidth = 5;
|
|
88
|
-
ctx.stroke();
|
|
89
|
-
|
|
90
|
-
ctx.save();
|
|
91
|
-
ctx.beginPath();
|
|
92
|
-
ctx.arc(centerX, centerY, radius, 0, Math.PI * 2, true);
|
|
93
|
-
ctx.closePath();
|
|
94
|
-
ctx.clip();
|
|
95
|
-
ctx.drawImage(image, centerX - radius, centerY - radius, radius * 2, radius * 2);
|
|
96
|
-
ctx.restore();
|
|
97
|
-
|
|
98
|
-
ctx.beginPath();
|
|
99
|
-
ctx.arc(centerX, centerY, radius, 0, Math.PI * 2, true);
|
|
100
|
-
ctx.strokeStyle = isDarkTheme ? 'rgba(255, 255, 255, 0.9)' : 'rgba(0, 0, 0, 0.9)';
|
|
101
|
-
ctx.lineWidth = 5;
|
|
102
|
-
ctx.stroke();
|
|
103
|
-
}
|
|
104
|
-
};
|
|
105
|
-
|
|
106
|
-
return <canvas ref={canvasRef} className={className} width={500} height={500} style={{ width }} />;
|
|
107
|
-
}
|
|
@@ -1,96 +0,0 @@
|
|
|
1
|
-
import { ChunkedAudioPlayer } from './Player';
|
|
2
|
-
|
|
3
|
-
type VoiceBackend = (text: string, voice?: string, speed?: number) => Promise<Blob>;
|
|
4
|
-
|
|
5
|
-
export class MessageSender {
|
|
6
|
-
private player = new ChunkedAudioPlayer();
|
|
7
|
-
private fetchedSentences = new Set<string>();
|
|
8
|
-
private lastLoading = false;
|
|
9
|
-
private voice: string;
|
|
10
|
-
private voiceBackend: VoiceBackend;
|
|
11
|
-
|
|
12
|
-
constructor(voiceBackend: VoiceBackend, voice: string) {
|
|
13
|
-
if (voice?.split('_').length !== 2) {
|
|
14
|
-
throw new Error("Invalid voice id format '" + voice + "'. Voice id needs to look like <provider>_<voice_id>");
|
|
15
|
-
}
|
|
16
|
-
this.voiceBackend = voiceBackend;
|
|
17
|
-
this.voice = voice;
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
private getCompletedSentences(currentText: string, isLoading: boolean): string[] {
|
|
21
|
-
// Split the text based on the following characters: .?!
|
|
22
|
-
// Only split on : when followed by a space
|
|
23
|
-
const pattern = /(.+?[.?!]|.+?:\s+|.+?\n+)/g;
|
|
24
|
-
const result: string[] = [];
|
|
25
|
-
let match;
|
|
26
|
-
while ((match = pattern.exec(currentText)) !== null) {
|
|
27
|
-
const sentence = match[0].trim();
|
|
28
|
-
if (sentence.length > 0) {
|
|
29
|
-
result.push(sentence);
|
|
30
|
-
}
|
|
31
|
-
}
|
|
32
|
-
if (!isLoading) {
|
|
33
|
-
const lastFullSentence = result[result.length - 1];
|
|
34
|
-
const leftoverIndex = currentText.lastIndexOf(lastFullSentence) + lastFullSentence.length;
|
|
35
|
-
if (leftoverIndex < currentText.length) {
|
|
36
|
-
result.push(currentText.slice(leftoverIndex).trim());
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
return result;
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
public async handleNewText(currentText: string | undefined, isLoading: boolean) {
|
|
43
|
-
if (!this.lastLoading && isLoading) {
|
|
44
|
-
this.reset();
|
|
45
|
-
}
|
|
46
|
-
this.lastLoading = isLoading;
|
|
47
|
-
|
|
48
|
-
if (!currentText) {
|
|
49
|
-
return;
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
const sentences = this.getCompletedSentences(currentText, isLoading);
|
|
53
|
-
|
|
54
|
-
for (let i = 0; i < sentences.length; i++) {
|
|
55
|
-
const sentence = sentences[i];
|
|
56
|
-
if (!this.fetchedSentences.has(sentence)) {
|
|
57
|
-
this.fetchedSentences.add(sentence);
|
|
58
|
-
const audioData = await this.generateSpeech(sentence);
|
|
59
|
-
await this.player.addChunk(audioData, i);
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
private async generateSpeech(sentence: string): Promise<ArrayBuffer> {
|
|
65
|
-
const blob = await this.voiceBackend(sentence, this.voice, 1.0);
|
|
66
|
-
return await blob.arrayBuffer();
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
public play() {
|
|
70
|
-
this.player.playAgain();
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
public stop() {
|
|
74
|
-
this.player.stopPlayback();
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
private reset() {
|
|
78
|
-
this.stop();
|
|
79
|
-
this.fetchedSentences.clear();
|
|
80
|
-
this.player.reset();
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
public setVolume(volume: number) {
|
|
84
|
-
this.player.setVolume(volume);
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
public setOnLoudnessChange(callback: (value: number) => void) {
|
|
88
|
-
this.player.setOnLoudnessChange((loudness) => {
|
|
89
|
-
callback(loudness);
|
|
90
|
-
});
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
public setOnEndOfSpeech(callback: () => void) {
|
|
94
|
-
this.player.setOnEndOfSpeech(callback);
|
|
95
|
-
}
|
|
96
|
-
}
|
|
@@ -1,197 +0,0 @@
|
|
|
1
|
-
export class ChunkedAudioPlayer {
|
|
2
|
-
private audioContext!: AudioContext;
|
|
3
|
-
private chunkQueue: ArrayBuffer[] = [];
|
|
4
|
-
private isPlaying = false;
|
|
5
|
-
private analyser!: AnalyserNode;
|
|
6
|
-
private dataArray!: Uint8Array<ArrayBuffer>;
|
|
7
|
-
private shouldMonitorLoudness = true;
|
|
8
|
-
private isMonitoring = false;
|
|
9
|
-
private handle = 0;
|
|
10
|
-
private volume = 1.0;
|
|
11
|
-
private loudnessCallback: (value: number) => void = () => {};
|
|
12
|
-
private currentIndex = 0;
|
|
13
|
-
private startedPlaying = false;
|
|
14
|
-
private onEndOfSpeech: () => void = () => {};
|
|
15
|
-
|
|
16
|
-
constructor() {
|
|
17
|
-
this.init();
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
private init(): void {
|
|
21
|
-
this.audioContext = new AudioContext();
|
|
22
|
-
this.analyser = this.audioContext.createAnalyser();
|
|
23
|
-
this.analyser.fftSize = 256; // Set the FFT size (smaller values provide faster updates, larger ones give better resolution)
|
|
24
|
-
const bufferLength = this.analyser.frequencyBinCount;
|
|
25
|
-
this.dataArray = new Uint8Array(bufferLength); // Array to hold frequency data
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
public setOnLoudnessChange(callback: (value: number) => void) {
|
|
29
|
-
this.loudnessCallback = callback;
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
public setVolume(volume: number) {
|
|
33
|
-
this.volume = volume;
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
public async addChunk(chunk: ArrayBuffer, position: number): Promise<void> {
|
|
37
|
-
console.log('Adding chunk', position, chunk);
|
|
38
|
-
this.chunkQueue[position] = chunk;
|
|
39
|
-
// console.log("received chunk", {
|
|
40
|
-
// chunkQueue: this.chunkQueue.length,
|
|
41
|
-
// isPlaying: this.isPlaying,
|
|
42
|
-
// })
|
|
43
|
-
|
|
44
|
-
if (position === 0 && !this.startedPlaying) {
|
|
45
|
-
this.startedPlaying = true;
|
|
46
|
-
this.playChunks();
|
|
47
|
-
}
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
private playChunks(): void {
|
|
51
|
-
// console.log({ isPlaying: this.isPlaying });
|
|
52
|
-
if (this.isPlaying) return;
|
|
53
|
-
if (!this.chunkQueue[this.currentIndex]) {
|
|
54
|
-
// wait until the correct chunk arrives
|
|
55
|
-
setTimeout(() => this.playChunks(), 10);
|
|
56
|
-
}
|
|
57
|
-
this.isPlaying = true;
|
|
58
|
-
|
|
59
|
-
this.playChunk(this.chunkQueue[this.currentIndex]).then(() => {
|
|
60
|
-
this.isPlaying = false;
|
|
61
|
-
this.currentIndex++;
|
|
62
|
-
if (this.chunkQueue[this.currentIndex]) {
|
|
63
|
-
this.shouldMonitorLoudness = true;
|
|
64
|
-
this.playChunks();
|
|
65
|
-
} else {
|
|
66
|
-
// console.log('Playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
|
|
67
|
-
setTimeout(() => {
|
|
68
|
-
// console.log('Check again if really playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
|
|
69
|
-
if (this.chunkQueue.length > this.currentIndex) {
|
|
70
|
-
this.playChunks();
|
|
71
|
-
} else {
|
|
72
|
-
this.startedPlaying = false;
|
|
73
|
-
this.shouldMonitorLoudness = false;
|
|
74
|
-
}
|
|
75
|
-
}, 1000);
|
|
76
|
-
}
|
|
77
|
-
});
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
public stopPlayback(): void {
|
|
81
|
-
// console.log('Stopping playback');
|
|
82
|
-
// Implement logic to stop the current playback
|
|
83
|
-
this.isPlaying = false;
|
|
84
|
-
this.chunkQueue = [];
|
|
85
|
-
this.startedPlaying = false;
|
|
86
|
-
this.shouldMonitorLoudness = false;
|
|
87
|
-
cancelAnimationFrame(this.handle);
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
private playChunk(chunk: ArrayBuffer): Promise<void> {
|
|
91
|
-
// console.log({queue: this.chunkQueue})
|
|
92
|
-
if (!chunk) {
|
|
93
|
-
return Promise.resolve();
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
// console.log('Playing chunk', chunk);
|
|
97
|
-
return new Promise((resolve) => {
|
|
98
|
-
const source = this.audioContext.createBufferSource();
|
|
99
|
-
this.audioContext.decodeAudioData(chunk.slice(0)).then((audioBuffer) => {
|
|
100
|
-
source.buffer = audioBuffer;
|
|
101
|
-
|
|
102
|
-
// Create a GainNode for volume control
|
|
103
|
-
const gainNode = this.audioContext.createGain();
|
|
104
|
-
gainNode.gain.value = this.volume;
|
|
105
|
-
|
|
106
|
-
// Connect the source to the GainNode, then to the analyser node, then to the destination (speakers)
|
|
107
|
-
source.connect(gainNode);
|
|
108
|
-
gainNode.connect(this.analyser);
|
|
109
|
-
this.analyser.connect(this.audioContext.destination);
|
|
110
|
-
|
|
111
|
-
source.start(0);
|
|
112
|
-
// console.log('Playing chunk', this.currentIndex);
|
|
113
|
-
gainNode.gain.value = this.volume;
|
|
114
|
-
source.onended = () => {
|
|
115
|
-
// console.log('Chunk ended');
|
|
116
|
-
resolve();
|
|
117
|
-
};
|
|
118
|
-
|
|
119
|
-
// Start monitoring loudness only once
|
|
120
|
-
if (!this.isMonitoring) {
|
|
121
|
-
this.isMonitoring = true;
|
|
122
|
-
this.shouldMonitorLoudness = true;
|
|
123
|
-
this.monitorLoudness();
|
|
124
|
-
}
|
|
125
|
-
});
|
|
126
|
-
});
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
async playAgain(): Promise<void> {
|
|
130
|
-
console.log('Playing again');
|
|
131
|
-
if (this.chunkQueue.length > 0 && !this.isPlaying) {
|
|
132
|
-
this.playChunks();
|
|
133
|
-
}
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
private monitorLoudness(): void {
|
|
137
|
-
// Stop monitoring when the flag is false
|
|
138
|
-
if (!this.shouldMonitorLoudness) {
|
|
139
|
-
// console.log('Loudness monitoring stopped.');
|
|
140
|
-
cancelAnimationFrame(this.handle);
|
|
141
|
-
this.loudnessCallback(0);
|
|
142
|
-
this.onEndOfSpeech();
|
|
143
|
-
return;
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
// Get the time domain data from the analyser (this is a snapshot of the waveform)
|
|
147
|
-
this.analyser.getByteTimeDomainData(this.dataArray);
|
|
148
|
-
|
|
149
|
-
// Calculate the RMS (root mean square) of the waveform values to get the perceived loudness
|
|
150
|
-
let sum = 0;
|
|
151
|
-
for (let i = 0; i < this.dataArray.length; i++) {
|
|
152
|
-
const value = this.dataArray[i] / 128.0 - 1.0; // Normalize between -1 and 1
|
|
153
|
-
sum += value * value;
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
const rms = Math.sqrt(sum / this.dataArray.length);
|
|
157
|
-
|
|
158
|
-
// Handle the case where RMS is 0 to avoid log10(0)
|
|
159
|
-
if (rms === 0) {
|
|
160
|
-
// console.log('Current loudness: Silent');
|
|
161
|
-
} else {
|
|
162
|
-
let loudnessInDb = 20 * Math.log10(rms); // Convert to dB
|
|
163
|
-
// console.log('Current loudness:' + loudnessInDb);
|
|
164
|
-
const minDb = -57;
|
|
165
|
-
const maxDb = -15;
|
|
166
|
-
|
|
167
|
-
if (loudnessInDb < minDb) {
|
|
168
|
-
loudnessInDb = minDb;
|
|
169
|
-
}
|
|
170
|
-
if (loudnessInDb > maxDb) {
|
|
171
|
-
loudnessInDb = maxDb;
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
const loudnessScale = ((loudnessInDb - minDb) / (maxDb - minDb)) * 100;
|
|
175
|
-
// console.log("root:corrent loudness", loudnessScale);
|
|
176
|
-
|
|
177
|
-
this.loudnessCallback(loudnessScale);
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
// Call this method again at regular intervals if you want continuous loudness monitoring
|
|
181
|
-
this.handle = requestAnimationFrame(() => this.monitorLoudness());
|
|
182
|
-
}
|
|
183
|
-
public reset() {
|
|
184
|
-
// console.log('Resetting player');
|
|
185
|
-
this.stopPlayback();
|
|
186
|
-
this.currentIndex = 0;
|
|
187
|
-
this.shouldMonitorLoudness = true;
|
|
188
|
-
//reset to the beginning when the class gets initialized
|
|
189
|
-
this.isMonitoring = false;
|
|
190
|
-
this.isPlaying = false;
|
|
191
|
-
this.init();
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
public setOnEndOfSpeech(callback: () => void) {
|
|
195
|
-
this.onEndOfSpeech = callback;
|
|
196
|
-
}
|
|
197
|
-
}
|
|
@@ -1,129 +0,0 @@
|
|
|
1
|
-
import { useRimori } from '../../../components';
|
|
2
|
-
import { FaMicrophone, FaSpinner } from 'react-icons/fa6';
|
|
3
|
-
import { AudioController } from '../../../plugin/AudioController';
|
|
4
|
-
import { useState, useRef, forwardRef, useImperativeHandle, useEffect } from 'react';
|
|
5
|
-
|
|
6
|
-
interface Props {
|
|
7
|
-
iconSize?: string;
|
|
8
|
-
className?: string;
|
|
9
|
-
disabled?: boolean;
|
|
10
|
-
loading?: boolean;
|
|
11
|
-
enablePushToTalk?: boolean;
|
|
12
|
-
onRecordingStatusChange: (running: boolean) => void;
|
|
13
|
-
onVoiceRecorded: (message: string) => void;
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
export const VoiceRecorder = forwardRef(
|
|
17
|
-
(
|
|
18
|
-
{
|
|
19
|
-
onVoiceRecorded,
|
|
20
|
-
iconSize,
|
|
21
|
-
className,
|
|
22
|
-
disabled,
|
|
23
|
-
loading,
|
|
24
|
-
onRecordingStatusChange,
|
|
25
|
-
enablePushToTalk = false,
|
|
26
|
-
}: Props,
|
|
27
|
-
ref,
|
|
28
|
-
) => {
|
|
29
|
-
const [isRecording, setIsRecording] = useState(false);
|
|
30
|
-
const [internalIsProcessing, setInternalIsProcessing] = useState(false);
|
|
31
|
-
const audioControllerRef = useRef<AudioController | null>(null);
|
|
32
|
-
const { ai, plugin } = useRimori();
|
|
33
|
-
|
|
34
|
-
// Ref for latest onVoiceRecorded callback
|
|
35
|
-
const onVoiceRecordedRef = useRef(onVoiceRecorded);
|
|
36
|
-
useEffect(() => {
|
|
37
|
-
onVoiceRecordedRef.current = onVoiceRecorded;
|
|
38
|
-
}, [onVoiceRecorded]);
|
|
39
|
-
|
|
40
|
-
const startRecording = async () => {
|
|
41
|
-
try {
|
|
42
|
-
if (!audioControllerRef.current) {
|
|
43
|
-
audioControllerRef.current = new AudioController(plugin.pluginId);
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
await audioControllerRef.current.startRecording();
|
|
47
|
-
setIsRecording(true);
|
|
48
|
-
onRecordingStatusChange(true);
|
|
49
|
-
} catch (error) {
|
|
50
|
-
console.error('Failed to start recording:', error);
|
|
51
|
-
// Handle permission denied or other errors
|
|
52
|
-
}
|
|
53
|
-
};
|
|
54
|
-
|
|
55
|
-
const stopRecording = async () => {
|
|
56
|
-
try {
|
|
57
|
-
if (audioControllerRef.current && isRecording) {
|
|
58
|
-
const audioResult = await audioControllerRef.current.stopRecording();
|
|
59
|
-
// console.log("audioResult: ", audioResult);
|
|
60
|
-
|
|
61
|
-
setInternalIsProcessing(true);
|
|
62
|
-
|
|
63
|
-
// Play the recorded audio from the Blob
|
|
64
|
-
// const blobUrl = URL.createObjectURL(audioResult.recording);
|
|
65
|
-
// const audioRef = new Audio(blobUrl);
|
|
66
|
-
// audioRef.onended = () => URL.revokeObjectURL(blobUrl);
|
|
67
|
-
// audioRef.play().catch((e) => console.error('Playback error:', e));
|
|
68
|
-
|
|
69
|
-
// console.log("audioBlob: ", audioResult.recording);
|
|
70
|
-
const text = await ai.getTextFromVoice(audioResult.recording);
|
|
71
|
-
// console.log("stt result", text);
|
|
72
|
-
// throw new Error("test");
|
|
73
|
-
setInternalIsProcessing(false);
|
|
74
|
-
onVoiceRecordedRef.current(text);
|
|
75
|
-
}
|
|
76
|
-
} catch (error) {
|
|
77
|
-
console.error('Failed to stop recording:', error);
|
|
78
|
-
} finally {
|
|
79
|
-
setIsRecording(false);
|
|
80
|
-
onRecordingStatusChange(false);
|
|
81
|
-
}
|
|
82
|
-
};
|
|
83
|
-
|
|
84
|
-
useImperativeHandle(ref, () => ({
|
|
85
|
-
startRecording,
|
|
86
|
-
stopRecording,
|
|
87
|
-
}));
|
|
88
|
-
|
|
89
|
-
// push to talk feature
|
|
90
|
-
const spacePressedRef = useRef(false);
|
|
91
|
-
|
|
92
|
-
useEffect(() => {
|
|
93
|
-
if (!enablePushToTalk) return;
|
|
94
|
-
|
|
95
|
-
const handleKeyDown = async (event: KeyboardEvent) => {
|
|
96
|
-
if (event.code === 'Space' && !spacePressedRef.current) {
|
|
97
|
-
spacePressedRef.current = true;
|
|
98
|
-
await startRecording();
|
|
99
|
-
}
|
|
100
|
-
};
|
|
101
|
-
const handleKeyUp = (event: KeyboardEvent) => {
|
|
102
|
-
if (event.code === 'Space' && spacePressedRef.current) {
|
|
103
|
-
spacePressedRef.current = false;
|
|
104
|
-
stopRecording();
|
|
105
|
-
}
|
|
106
|
-
};
|
|
107
|
-
window.addEventListener('keydown', handleKeyDown);
|
|
108
|
-
window.addEventListener('keyup', handleKeyUp);
|
|
109
|
-
return () => {
|
|
110
|
-
window.removeEventListener('keydown', handleKeyDown);
|
|
111
|
-
window.removeEventListener('keyup', handleKeyUp);
|
|
112
|
-
};
|
|
113
|
-
}, [enablePushToTalk]);
|
|
114
|
-
|
|
115
|
-
return (
|
|
116
|
-
<button
|
|
117
|
-
className={'flex flex-row justify-center items-center rounded-full mx-auto disabled:opacity-50 ' + className}
|
|
118
|
-
onClick={isRecording ? stopRecording : startRecording}
|
|
119
|
-
disabled={disabled || loading || internalIsProcessing}
|
|
120
|
-
>
|
|
121
|
-
{loading || internalIsProcessing ? (
|
|
122
|
-
<FaSpinner className="animate-spin" />
|
|
123
|
-
) : (
|
|
124
|
-
<FaMicrophone size={iconSize} className={isRecording ? 'text-red-600' : ''} />
|
|
125
|
-
)}
|
|
126
|
-
</button>
|
|
127
|
-
);
|
|
128
|
-
},
|
|
129
|
-
);
|