@rimori/client 1.4.10 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (185) hide show
  1. package/README.md +0 -49
  2. package/dist/react-client/src/plugin/ThemeSetter.d.ts +2 -0
  3. package/dist/react-client/src/plugin/ThemeSetter.js +19 -0
  4. package/dist/react-client/src/utils/PluginUtils.d.ts +2 -0
  5. package/dist/react-client/src/utils/PluginUtils.js +23 -0
  6. package/dist/{cli → rimori-client/src/cli}/scripts/init/main.js +0 -0
  7. package/dist/{cli → rimori-client/src/cli}/scripts/release/release.js +0 -0
  8. package/dist/{core → rimori-client/src}/controller/AIController.d.ts +1 -1
  9. package/dist/{core → rimori-client/src}/controller/AIController.js +2 -2
  10. package/dist/{plugin/AccomplishmentHandler.d.ts → rimori-client/src/controller/AccomplishmentController.d.ts} +1 -1
  11. package/dist/{plugin/AccomplishmentHandler.js → rimori-client/src/controller/AccomplishmentController.js} +1 -1
  12. package/dist/{core → rimori-client/src}/controller/ExerciseController.d.ts +8 -6
  13. package/dist/{core → rimori-client/src}/controller/ExerciseController.js +10 -9
  14. package/dist/{core → rimori-client/src}/controller/ObjectController.d.ts +1 -1
  15. package/dist/{core → rimori-client/src}/controller/ObjectController.js +1 -1
  16. package/dist/{core → rimori-client/src}/controller/SettingsController.d.ts +2 -2
  17. package/dist/{core → rimori-client/src}/controller/SharedContentController.d.ts +1 -1
  18. package/dist/{core → rimori-client/src}/controller/VoiceController.d.ts +1 -1
  19. package/dist/rimori-client/src/index.d.ts +11 -0
  20. package/dist/rimori-client/src/index.js +10 -0
  21. package/dist/{plugin/PluginController.d.ts → rimori-client/src/plugin/CommunicationHandler.d.ts} +3 -7
  22. package/dist/{plugin/PluginController.js → rimori-client/src/plugin/CommunicationHandler.js} +3 -28
  23. package/dist/{plugin → rimori-client/src/plugin}/RimoriClient.d.ts +67 -68
  24. package/dist/{plugin → rimori-client/src/plugin}/RimoriClient.js +101 -43
  25. package/dist/{worker → rimori-client/src/worker}/WorkerSetup.js +2 -2
  26. package/example/docs/devdocs.md +1 -1
  27. package/package.json +4 -26
  28. package/src/{core/controller → controller}/AIController.ts +3 -3
  29. package/src/{plugin/AccomplishmentHandler.ts → controller/AccomplishmentController.ts} +1 -1
  30. package/src/{core/controller → controller}/ExerciseController.ts +14 -11
  31. package/src/{core/controller → controller}/ObjectController.ts +2 -2
  32. package/src/{core/controller → controller}/SettingsController.ts +2 -2
  33. package/src/{core/controller → controller}/SharedContentController.ts +1 -1
  34. package/src/{core/controller → controller}/VoiceController.ts +2 -2
  35. package/src/fromRimori/readme.md +1 -1
  36. package/src/index.ts +8 -8
  37. package/src/plugin/{PluginController.ts → CommunicationHandler.ts} +8 -36
  38. package/src/plugin/RimoriClient.ts +125 -118
  39. package/src/worker/WorkerSetup.ts +5 -3
  40. package/tsconfig.json +4 -2
  41. package/dist/cli/scripts/release/release-translation-upload.d.ts +0 -6
  42. package/dist/cli/scripts/release/release-translation-upload.js +0 -87
  43. package/dist/components/CRUDModal.d.ts +0 -17
  44. package/dist/components/CRUDModal.js +0 -24
  45. package/dist/components/MarkdownEditor.d.ts +0 -8
  46. package/dist/components/MarkdownEditor.js +0 -48
  47. package/dist/components/Spinner.d.ts +0 -8
  48. package/dist/components/Spinner.js +0 -4
  49. package/dist/components/ai/Assistant.d.ts +0 -9
  50. package/dist/components/ai/Assistant.js +0 -58
  51. package/dist/components/ai/Avatar.d.ts +0 -14
  52. package/dist/components/ai/Avatar.js +0 -59
  53. package/dist/components/ai/EmbeddedAssistent/AudioInputField.d.ts +0 -7
  54. package/dist/components/ai/EmbeddedAssistent/AudioInputField.js +0 -37
  55. package/dist/components/ai/EmbeddedAssistent/CircleAudioAvatar.d.ts +0 -8
  56. package/dist/components/ai/EmbeddedAssistent/CircleAudioAvatar.js +0 -79
  57. package/dist/components/ai/EmbeddedAssistent/TTS/MessageSender.d.ts +0 -19
  58. package/dist/components/ai/EmbeddedAssistent/TTS/MessageSender.js +0 -91
  59. package/dist/components/ai/EmbeddedAssistent/TTS/Player.d.ts +0 -27
  60. package/dist/components/ai/EmbeddedAssistent/TTS/Player.js +0 -185
  61. package/dist/components/ai/EmbeddedAssistent/VoiceRecoder.d.ts +0 -11
  62. package/dist/components/ai/EmbeddedAssistent/VoiceRecoder.js +0 -95
  63. package/dist/components/ai/utils.d.ts +0 -6
  64. package/dist/components/ai/utils.js +0 -13
  65. package/dist/components/audio/Playbutton.d.ts +0 -15
  66. package/dist/components/audio/Playbutton.js +0 -80
  67. package/dist/components/components/ContextMenu.d.ts +0 -10
  68. package/dist/components/components/ContextMenu.js +0 -135
  69. package/dist/components.d.ts +0 -10
  70. package/dist/components.js +0 -11
  71. package/dist/core/controller/EnhancedUserInfo.d.ts +0 -0
  72. package/dist/core/controller/EnhancedUserInfo.js +0 -1
  73. package/dist/core/core.d.ts +0 -14
  74. package/dist/core/core.js +0 -7
  75. package/dist/hooks/I18nHooks.d.ts +0 -11
  76. package/dist/hooks/I18nHooks.js +0 -25
  77. package/dist/hooks/UseChatHook.d.ts +0 -10
  78. package/dist/hooks/UseChatHook.js +0 -29
  79. package/dist/i18n/I18nHooks.d.ts +0 -11
  80. package/dist/i18n/I18nHooks.js +0 -25
  81. package/dist/i18n/Translator.d.ts +0 -43
  82. package/dist/i18n/Translator.js +0 -118
  83. package/dist/i18n/config.d.ts +0 -7
  84. package/dist/i18n/config.js +0 -20
  85. package/dist/i18n/createI18nInstance.d.ts +0 -7
  86. package/dist/i18n/createI18nInstance.js +0 -31
  87. package/dist/i18n/hooks.d.ts +0 -11
  88. package/dist/i18n/hooks.js +0 -25
  89. package/dist/i18n/index.d.ts +0 -4
  90. package/dist/i18n/index.js +0 -4
  91. package/dist/i18n/types.d.ts +0 -7
  92. package/dist/i18n/types.js +0 -1
  93. package/dist/i18n/useRimoriI18n.d.ts +0 -11
  94. package/dist/i18n/useRimoriI18n.js +0 -41
  95. package/dist/index.d.ts +0 -11
  96. package/dist/index.js +0 -11
  97. package/dist/plugin/Translator.d.ts +0 -38
  98. package/dist/plugin/Translator.js +0 -101
  99. package/dist/providers/PluginProvider.d.ts +0 -12
  100. package/dist/providers/PluginProvider.js +0 -152
  101. package/dist/style.css +0 -110
  102. package/dist/style.css.map +0 -1
  103. package/dist/utils/Language.d.ts +0 -67
  104. package/dist/utils/Language.js +0 -69
  105. package/dist/utils/LanguageClass.d.ts +0 -36
  106. package/dist/utils/LanguageClass.example.d.ts +0 -0
  107. package/dist/utils/LanguageClass.example.js +0 -1
  108. package/dist/utils/LanguageClass.js +0 -50
  109. package/dist/utils/LanguageClass.test.d.ts +0 -0
  110. package/dist/utils/LanguageClass.test.js +0 -1
  111. package/src/components/CRUDModal.tsx +0 -75
  112. package/src/components/MarkdownEditor.tsx +0 -144
  113. package/src/components/Spinner.tsx +0 -29
  114. package/src/components/ai/Assistant.tsx +0 -96
  115. package/src/components/ai/Avatar.tsx +0 -99
  116. package/src/components/ai/EmbeddedAssistent/AudioInputField.tsx +0 -73
  117. package/src/components/ai/EmbeddedAssistent/CircleAudioAvatar.tsx +0 -107
  118. package/src/components/ai/EmbeddedAssistent/TTS/MessageSender.ts +0 -96
  119. package/src/components/ai/EmbeddedAssistent/TTS/Player.ts +0 -197
  120. package/src/components/ai/EmbeddedAssistent/VoiceRecoder.tsx +0 -129
  121. package/src/components/ai/utils.ts +0 -21
  122. package/src/components/audio/Playbutton.tsx +0 -126
  123. package/src/components/components/ContextMenu.tsx +0 -179
  124. package/src/components.ts +0 -11
  125. package/src/core/core.ts +0 -15
  126. package/src/hooks/I18nHooks.ts +0 -33
  127. package/src/hooks/UseChatHook.ts +0 -38
  128. package/src/plugin/ThemeSetter.ts +0 -23
  129. package/src/providers/PluginProvider.tsx +0 -209
  130. package/src/style.scss +0 -136
  131. package/src/utils/PluginUtils.ts +0 -22
  132. /package/dist/{plugin → react-client/plugin}/ThemeSetter.d.ts +0 -0
  133. /package/dist/{plugin → react-client/plugin}/ThemeSetter.js +0 -0
  134. /package/dist/{utils/PluginUtils.d.ts → react-client/src/utils/FullscreenUtils.d.ts} +0 -0
  135. /package/dist/{utils/PluginUtils.js → react-client/src/utils/FullscreenUtils.js} +0 -0
  136. /package/dist/{cli → rimori-client/src/cli}/scripts/init/dev-registration.d.ts +0 -0
  137. /package/dist/{cli → rimori-client/src/cli}/scripts/init/dev-registration.js +0 -0
  138. /package/dist/{cli → rimori-client/src/cli}/scripts/init/env-setup.d.ts +0 -0
  139. /package/dist/{cli → rimori-client/src/cli}/scripts/init/env-setup.js +0 -0
  140. /package/dist/{cli → rimori-client/src/cli}/scripts/init/file-operations.d.ts +0 -0
  141. /package/dist/{cli → rimori-client/src/cli}/scripts/init/file-operations.js +0 -0
  142. /package/dist/{cli → rimori-client/src/cli}/scripts/init/html-cleaner.d.ts +0 -0
  143. /package/dist/{cli → rimori-client/src/cli}/scripts/init/html-cleaner.js +0 -0
  144. /package/dist/{cli → rimori-client/src/cli}/scripts/init/main.d.ts +0 -0
  145. /package/dist/{cli → rimori-client/src/cli}/scripts/init/package-setup.d.ts +0 -0
  146. /package/dist/{cli → rimori-client/src/cli}/scripts/init/package-setup.js +0 -0
  147. /package/dist/{cli → rimori-client/src/cli}/scripts/init/router-transformer.d.ts +0 -0
  148. /package/dist/{cli → rimori-client/src/cli}/scripts/init/router-transformer.js +0 -0
  149. /package/dist/{cli → rimori-client/src/cli}/scripts/init/tailwind-config.d.ts +0 -0
  150. /package/dist/{cli → rimori-client/src/cli}/scripts/init/tailwind-config.js +0 -0
  151. /package/dist/{cli → rimori-client/src/cli}/scripts/init/vite-config.d.ts +0 -0
  152. /package/dist/{cli → rimori-client/src/cli}/scripts/init/vite-config.js +0 -0
  153. /package/dist/{cli → rimori-client/src/cli}/scripts/release/detect-translation-languages.d.ts +0 -0
  154. /package/dist/{cli → rimori-client/src/cli}/scripts/release/detect-translation-languages.js +0 -0
  155. /package/dist/{cli → rimori-client/src/cli}/scripts/release/release-config-upload.d.ts +0 -0
  156. /package/dist/{cli → rimori-client/src/cli}/scripts/release/release-config-upload.js +0 -0
  157. /package/dist/{cli → rimori-client/src/cli}/scripts/release/release-db-update.d.ts +0 -0
  158. /package/dist/{cli → rimori-client/src/cli}/scripts/release/release-db-update.js +0 -0
  159. /package/dist/{cli → rimori-client/src/cli}/scripts/release/release-file-upload.d.ts +0 -0
  160. /package/dist/{cli → rimori-client/src/cli}/scripts/release/release-file-upload.js +0 -0
  161. /package/dist/{cli → rimori-client/src/cli}/scripts/release/release.d.ts +0 -0
  162. /package/dist/{cli → rimori-client/src/cli}/types/DatabaseTypes.d.ts +0 -0
  163. /package/dist/{cli → rimori-client/src/cli}/types/DatabaseTypes.js +0 -0
  164. /package/dist/{plugin → rimori-client/src/controller}/AudioController.d.ts +0 -0
  165. /package/dist/{plugin → rimori-client/src/controller}/AudioController.js +0 -0
  166. /package/dist/{core → rimori-client/src}/controller/SettingsController.js +0 -0
  167. /package/dist/{core → rimori-client/src}/controller/SharedContentController.js +0 -0
  168. /package/dist/{plugin → rimori-client/src/controller}/TranslationController.d.ts +0 -0
  169. /package/dist/{plugin → rimori-client/src/controller}/TranslationController.js +0 -0
  170. /package/dist/{core → rimori-client/src}/controller/VoiceController.js +0 -0
  171. /package/dist/{fromRimori → rimori-client/src/fromRimori}/EventBus.d.ts +0 -0
  172. /package/dist/{fromRimori → rimori-client/src/fromRimori}/EventBus.js +0 -0
  173. /package/dist/{fromRimori → rimori-client/src/fromRimori}/PluginTypes.d.ts +0 -0
  174. /package/dist/{fromRimori → rimori-client/src/fromRimori}/PluginTypes.js +0 -0
  175. /package/dist/{plugin → rimori-client/src/plugin}/Logger.d.ts +0 -0
  176. /package/dist/{plugin → rimori-client/src/plugin}/Logger.js +0 -0
  177. /package/dist/{plugin → rimori-client/src/plugin}/StandaloneClient.d.ts +0 -0
  178. /package/dist/{plugin → rimori-client/src/plugin}/StandaloneClient.js +0 -0
  179. /package/dist/{utils → rimori-client/src/utils}/difficultyConverter.d.ts +0 -0
  180. /package/dist/{utils → rimori-client/src/utils}/difficultyConverter.js +0 -0
  181. /package/dist/{utils → rimori-client/src/utils}/endpoint.d.ts +0 -0
  182. /package/dist/{utils → rimori-client/src/utils}/endpoint.js +0 -0
  183. /package/dist/{worker → rimori-client/src/worker}/WorkerSetup.d.ts +0 -0
  184. /package/src/{plugin → controller}/AudioController.ts +0 -0
  185. /package/src/{plugin → controller}/TranslationController.ts +0 -0
@@ -1,99 +0,0 @@
1
- import { useEffect, useMemo, useState } from 'react';
2
- import { VoiceRecorder } from './EmbeddedAssistent/VoiceRecoder';
3
- import { MessageSender } from './EmbeddedAssistent/TTS/MessageSender';
4
- import { CircleAudioAvatar } from './EmbeddedAssistent/CircleAudioAvatar';
5
- import { Tool } from '../../fromRimori/PluginTypes';
6
- import { useChat } from '../../hooks/UseChatHook';
7
- import { useRimori } from '../../components';
8
- import { getFirstMessages } from './utils';
9
- import { FirstMessages } from './utils';
10
-
11
- interface Props {
12
- voiceId: string;
13
- agentTools: Tool[];
14
- avatarImageUrl: string;
15
- circleSize?: string;
16
- isDarkTheme?: boolean;
17
- children?: React.ReactNode;
18
- autoStartConversation?: FirstMessages;
19
- className?: string;
20
- }
21
-
22
- export function Avatar({
23
- avatarImageUrl,
24
- voiceId,
25
- agentTools,
26
- autoStartConversation,
27
- children,
28
- isDarkTheme = false,
29
- circleSize = '300px',
30
- className,
31
- }: Props) {
32
- const { ai, event } = useRimori();
33
- const [agentReplying, setAgentReplying] = useState(false);
34
- const [isProcessingMessage, setIsProcessingMessage] = useState(false);
35
- const sender = useMemo(() => new MessageSender(ai.getVoice, voiceId), [voiceId]);
36
- const { messages, append, isLoading, lastMessage, setMessages } = useChat(agentTools);
37
-
38
- useEffect(() => {
39
- console.log('messages', messages);
40
- }, [messages]);
41
-
42
- useEffect(() => {
43
- if (!isLoading) setIsProcessingMessage(false);
44
- }, [isLoading]);
45
-
46
- useEffect(() => {
47
- if (!voiceId) return; //at the beginning when being mounted the voiceId is undefined
48
- sender.setOnLoudnessChange((value) => event.emit('self.avatar.triggerLoudness', { loudness: value }));
49
- sender.setOnEndOfSpeech(() => setAgentReplying(false));
50
-
51
- if (!autoStartConversation) return;
52
-
53
- setMessages(getFirstMessages(autoStartConversation));
54
- // append([{ role: 'user', content: autoStartConversation.userMessage }]);
55
-
56
- if (autoStartConversation.assistantMessage) {
57
- // console.log("autostartmessages", { autoStartConversation, isLoading });
58
- sender.handleNewText(autoStartConversation.assistantMessage, isLoading);
59
- } else if (autoStartConversation.userMessage) {
60
- append([{ role: 'user', content: autoStartConversation.userMessage, id: messages.length.toString() }]);
61
- }
62
- }, [autoStartConversation, voiceId]);
63
-
64
- useEffect(() => {
65
- if (lastMessage?.role === 'assistant') {
66
- sender.handleNewText(lastMessage.content, isLoading);
67
- if (lastMessage.toolCalls) {
68
- // console.log("unlocking mic", lastMessage)
69
- setAgentReplying(false);
70
- setIsProcessingMessage(false);
71
- }
72
- }
73
- }, [lastMessage, isLoading]);
74
-
75
- return (
76
- <div className={`md:pb-8 ${className || ''}`}>
77
- <CircleAudioAvatar width={circleSize} className="mx-auto" imageUrl={avatarImageUrl} isDarkTheme={isDarkTheme} />
78
- {children}
79
- <VoiceRecorder
80
- iconSize="30"
81
- className="w-16 h-16 shadow-lg rounded-full bg-gray-400 dark:bg-gray-800"
82
- disabled={agentReplying}
83
- loading={isProcessingMessage}
84
- enablePushToTalk={true}
85
- onVoiceRecorded={(message) => {
86
- setAgentReplying(true);
87
- append([
88
- {
89
- role: 'user',
90
- content: 'Message(' + Math.floor((messages.length + 1) / 2) + '): ' + message,
91
- id: messages.length.toString(),
92
- },
93
- ]);
94
- }}
95
- onRecordingStatusChange={(running) => !running && setIsProcessingMessage(true)}
96
- />
97
- </div>
98
- );
99
- }
@@ -1,73 +0,0 @@
1
- import React, { useState } from 'react';
2
- import { VoiceRecorder } from './VoiceRecoder';
3
- import { BiSolidRightArrow } from 'react-icons/bi';
4
- import { HiMiniSpeakerXMark, HiMiniSpeakerWave } from 'react-icons/hi2';
5
-
6
- interface AudioInputFieldProps {
7
- onSubmit: (text: string) => void;
8
- onAudioControl?: (voice: boolean) => void;
9
- blockSubmission?: boolean;
10
- }
11
-
12
- export function AudioInputField({ onSubmit, onAudioControl, blockSubmission = false }: AudioInputFieldProps) {
13
- const [text, setText] = useState('');
14
- const [audioEnabled, setAudioEnabled] = useState(true);
15
-
16
- const handleSubmit = (manualText?: string) => {
17
- if (blockSubmission) return;
18
- const sendableText = manualText || text;
19
- if (sendableText.trim()) {
20
- onSubmit(sendableText);
21
- setTimeout(() => {
22
- setText('');
23
- }, 100);
24
- }
25
- };
26
-
27
- const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
28
- if (blockSubmission) return;
29
- if (e.key === 'Enter' && e.ctrlKey) {
30
- setText(text + '\n');
31
- } else if (e.key === 'Enter') {
32
- handleSubmit();
33
- }
34
- };
35
-
36
- return (
37
- <div className="flex items-center bg-gray-600 pt-2 pb-2 p-2">
38
- {onAudioControl && (
39
- <button
40
- onClick={() => {
41
- onAudioControl(!audioEnabled);
42
- setAudioEnabled(!audioEnabled);
43
- }}
44
- className="cursor-default"
45
- >
46
- {audioEnabled ? (
47
- <HiMiniSpeakerWave className="w-9 h-9 cursor-pointer" />
48
- ) : (
49
- <HiMiniSpeakerXMark className="w-9 h-9 cursor-pointer" />
50
- )}
51
- </button>
52
- )}
53
- <VoiceRecorder
54
- onRecordingStatusChange={() => {}}
55
- onVoiceRecorded={(m: string) => {
56
- console.log('onVoiceRecorded', m);
57
- handleSubmit(m);
58
- }}
59
- />
60
- <textarea
61
- value={text}
62
- onChange={(e) => setText(e.target.value)}
63
- onKeyDown={handleKeyDown}
64
- className="flex-1 border-none rounded-lg p-2 text-gray-800 focus::outline-none"
65
- placeholder="Type a message..."
66
- disabled={blockSubmission}
67
- />
68
- <button onClick={() => handleSubmit()} className="cursor-default" disabled={blockSubmission}>
69
- <BiSolidRightArrow className="w-9 h-10 cursor-pointer" />
70
- </button>
71
- </div>
72
- );
73
- }
@@ -1,107 +0,0 @@
1
- import { useEffect, useRef } from 'react';
2
- import { EventBus, EventBusMessage } from '../../../fromRimori/EventBus';
3
-
4
- interface CircleAudioAvatarProps {
5
- width?: string;
6
- imageUrl: string;
7
- className?: string;
8
- isDarkTheme?: boolean;
9
- }
10
-
11
- export function CircleAudioAvatar({
12
- imageUrl,
13
- className,
14
- isDarkTheme = false,
15
- width = '150px',
16
- }: CircleAudioAvatarProps) {
17
- const canvasRef = useRef<HTMLCanvasElement>(null);
18
- const currentLoudnessRef = useRef(0);
19
- const targetLoudnessRef = useRef(0);
20
- const animationFrameRef = useRef<number | null>(null);
21
-
22
- useEffect(() => {
23
- const canvas = canvasRef.current;
24
- if (canvas) {
25
- const ctx = canvas.getContext('2d');
26
- if (ctx) {
27
- const image = new Image();
28
- image.src = imageUrl;
29
- let isMounted = true;
30
-
31
- image.onload = () => {
32
- if (!isMounted) return;
33
- draw(ctx, canvas, image, 0);
34
- const animate = () => {
35
- const decayRate = 0.06;
36
- if (currentLoudnessRef.current > targetLoudnessRef.current) {
37
- currentLoudnessRef.current = Math.max(
38
- targetLoudnessRef.current,
39
- currentLoudnessRef.current - decayRate * currentLoudnessRef.current,
40
- );
41
- } else {
42
- currentLoudnessRef.current = targetLoudnessRef.current;
43
- }
44
- draw(ctx, canvas, image, currentLoudnessRef.current);
45
- animationFrameRef.current = requestAnimationFrame(animate);
46
- };
47
- animationFrameRef.current = requestAnimationFrame(animate);
48
- };
49
-
50
- const handleLoudness = ({ data }: EventBusMessage) => {
51
- const newLoudness = data.loudness;
52
- if (newLoudness > currentLoudnessRef.current) {
53
- currentLoudnessRef.current = newLoudness;
54
- }
55
- targetLoudnessRef.current = newLoudness;
56
- };
57
-
58
- const listener = EventBus.on('self.avatar.triggerLoudness', handleLoudness);
59
-
60
- return () => {
61
- isMounted = false;
62
- listener.off();
63
- if (animationFrameRef.current) {
64
- cancelAnimationFrame(animationFrameRef.current);
65
- }
66
- };
67
- }
68
- }
69
- }, [imageUrl]);
70
-
71
- const draw = (
72
- ctx: CanvasRenderingContext2D,
73
- canvas: HTMLCanvasElement,
74
- image: HTMLImageElement,
75
- loudness: number,
76
- ) => {
77
- if (canvas && ctx) {
78
- ctx.clearRect(0, 0, canvas.width, canvas.height);
79
-
80
- const radius = Math.min(canvas.width, canvas.height) / 3;
81
- const centerX = canvas.width / 2;
82
- const centerY = canvas.height / 2;
83
- const pulseRadius = radius + loudness / 2.5;
84
- ctx.beginPath();
85
- ctx.arc(centerX, centerY, pulseRadius, 0, Math.PI * 2, true);
86
- ctx.strokeStyle = isDarkTheme ? 'rgba(255, 255, 255, 0.5)' : 'rgba(0, 0, 0, 0.5)';
87
- ctx.lineWidth = 5;
88
- ctx.stroke();
89
-
90
- ctx.save();
91
- ctx.beginPath();
92
- ctx.arc(centerX, centerY, radius, 0, Math.PI * 2, true);
93
- ctx.closePath();
94
- ctx.clip();
95
- ctx.drawImage(image, centerX - radius, centerY - radius, radius * 2, radius * 2);
96
- ctx.restore();
97
-
98
- ctx.beginPath();
99
- ctx.arc(centerX, centerY, radius, 0, Math.PI * 2, true);
100
- ctx.strokeStyle = isDarkTheme ? 'rgba(255, 255, 255, 0.9)' : 'rgba(0, 0, 0, 0.9)';
101
- ctx.lineWidth = 5;
102
- ctx.stroke();
103
- }
104
- };
105
-
106
- return <canvas ref={canvasRef} className={className} width={500} height={500} style={{ width }} />;
107
- }
@@ -1,96 +0,0 @@
1
- import { ChunkedAudioPlayer } from './Player';
2
-
3
- type VoiceBackend = (text: string, voice?: string, speed?: number) => Promise<Blob>;
4
-
5
- export class MessageSender {
6
- private player = new ChunkedAudioPlayer();
7
- private fetchedSentences = new Set<string>();
8
- private lastLoading = false;
9
- private voice: string;
10
- private voiceBackend: VoiceBackend;
11
-
12
- constructor(voiceBackend: VoiceBackend, voice: string) {
13
- if (voice?.split('_').length !== 2) {
14
- throw new Error("Invalid voice id format '" + voice + "'. Voice id needs to look like <provider>_<voice_id>");
15
- }
16
- this.voiceBackend = voiceBackend;
17
- this.voice = voice;
18
- }
19
-
20
- private getCompletedSentences(currentText: string, isLoading: boolean): string[] {
21
- // Split the text based on the following characters: .?!
22
- // Only split on : when followed by a space
23
- const pattern = /(.+?[.?!]|.+?:\s+|.+?\n+)/g;
24
- const result: string[] = [];
25
- let match;
26
- while ((match = pattern.exec(currentText)) !== null) {
27
- const sentence = match[0].trim();
28
- if (sentence.length > 0) {
29
- result.push(sentence);
30
- }
31
- }
32
- if (!isLoading) {
33
- const lastFullSentence = result[result.length - 1];
34
- const leftoverIndex = currentText.lastIndexOf(lastFullSentence) + lastFullSentence.length;
35
- if (leftoverIndex < currentText.length) {
36
- result.push(currentText.slice(leftoverIndex).trim());
37
- }
38
- }
39
- return result;
40
- }
41
-
42
- public async handleNewText(currentText: string | undefined, isLoading: boolean) {
43
- if (!this.lastLoading && isLoading) {
44
- this.reset();
45
- }
46
- this.lastLoading = isLoading;
47
-
48
- if (!currentText) {
49
- return;
50
- }
51
-
52
- const sentences = this.getCompletedSentences(currentText, isLoading);
53
-
54
- for (let i = 0; i < sentences.length; i++) {
55
- const sentence = sentences[i];
56
- if (!this.fetchedSentences.has(sentence)) {
57
- this.fetchedSentences.add(sentence);
58
- const audioData = await this.generateSpeech(sentence);
59
- await this.player.addChunk(audioData, i);
60
- }
61
- }
62
- }
63
-
64
- private async generateSpeech(sentence: string): Promise<ArrayBuffer> {
65
- const blob = await this.voiceBackend(sentence, this.voice, 1.0);
66
- return await blob.arrayBuffer();
67
- }
68
-
69
- public play() {
70
- this.player.playAgain();
71
- }
72
-
73
- public stop() {
74
- this.player.stopPlayback();
75
- }
76
-
77
- private reset() {
78
- this.stop();
79
- this.fetchedSentences.clear();
80
- this.player.reset();
81
- }
82
-
83
- public setVolume(volume: number) {
84
- this.player.setVolume(volume);
85
- }
86
-
87
- public setOnLoudnessChange(callback: (value: number) => void) {
88
- this.player.setOnLoudnessChange((loudness) => {
89
- callback(loudness);
90
- });
91
- }
92
-
93
- public setOnEndOfSpeech(callback: () => void) {
94
- this.player.setOnEndOfSpeech(callback);
95
- }
96
- }
@@ -1,197 +0,0 @@
1
- export class ChunkedAudioPlayer {
2
- private audioContext!: AudioContext;
3
- private chunkQueue: ArrayBuffer[] = [];
4
- private isPlaying = false;
5
- private analyser!: AnalyserNode;
6
- private dataArray!: Uint8Array<ArrayBuffer>;
7
- private shouldMonitorLoudness = true;
8
- private isMonitoring = false;
9
- private handle = 0;
10
- private volume = 1.0;
11
- private loudnessCallback: (value: number) => void = () => {};
12
- private currentIndex = 0;
13
- private startedPlaying = false;
14
- private onEndOfSpeech: () => void = () => {};
15
-
16
- constructor() {
17
- this.init();
18
- }
19
-
20
- private init(): void {
21
- this.audioContext = new AudioContext();
22
- this.analyser = this.audioContext.createAnalyser();
23
- this.analyser.fftSize = 256; // Set the FFT size (smaller values provide faster updates, larger ones give better resolution)
24
- const bufferLength = this.analyser.frequencyBinCount;
25
- this.dataArray = new Uint8Array(bufferLength); // Array to hold frequency data
26
- }
27
-
28
- public setOnLoudnessChange(callback: (value: number) => void) {
29
- this.loudnessCallback = callback;
30
- }
31
-
32
- public setVolume(volume: number) {
33
- this.volume = volume;
34
- }
35
-
36
- public async addChunk(chunk: ArrayBuffer, position: number): Promise<void> {
37
- console.log('Adding chunk', position, chunk);
38
- this.chunkQueue[position] = chunk;
39
- // console.log("received chunk", {
40
- // chunkQueue: this.chunkQueue.length,
41
- // isPlaying: this.isPlaying,
42
- // })
43
-
44
- if (position === 0 && !this.startedPlaying) {
45
- this.startedPlaying = true;
46
- this.playChunks();
47
- }
48
- }
49
-
50
- private playChunks(): void {
51
- // console.log({ isPlaying: this.isPlaying });
52
- if (this.isPlaying) return;
53
- if (!this.chunkQueue[this.currentIndex]) {
54
- // wait until the correct chunk arrives
55
- setTimeout(() => this.playChunks(), 10);
56
- }
57
- this.isPlaying = true;
58
-
59
- this.playChunk(this.chunkQueue[this.currentIndex]).then(() => {
60
- this.isPlaying = false;
61
- this.currentIndex++;
62
- if (this.chunkQueue[this.currentIndex]) {
63
- this.shouldMonitorLoudness = true;
64
- this.playChunks();
65
- } else {
66
- // console.log('Playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
67
- setTimeout(() => {
68
- // console.log('Check again if really playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
69
- if (this.chunkQueue.length > this.currentIndex) {
70
- this.playChunks();
71
- } else {
72
- this.startedPlaying = false;
73
- this.shouldMonitorLoudness = false;
74
- }
75
- }, 1000);
76
- }
77
- });
78
- }
79
-
80
- public stopPlayback(): void {
81
- // console.log('Stopping playback');
82
- // Implement logic to stop the current playback
83
- this.isPlaying = false;
84
- this.chunkQueue = [];
85
- this.startedPlaying = false;
86
- this.shouldMonitorLoudness = false;
87
- cancelAnimationFrame(this.handle);
88
- }
89
-
90
- private playChunk(chunk: ArrayBuffer): Promise<void> {
91
- // console.log({queue: this.chunkQueue})
92
- if (!chunk) {
93
- return Promise.resolve();
94
- }
95
-
96
- // console.log('Playing chunk', chunk);
97
- return new Promise((resolve) => {
98
- const source = this.audioContext.createBufferSource();
99
- this.audioContext.decodeAudioData(chunk.slice(0)).then((audioBuffer) => {
100
- source.buffer = audioBuffer;
101
-
102
- // Create a GainNode for volume control
103
- const gainNode = this.audioContext.createGain();
104
- gainNode.gain.value = this.volume;
105
-
106
- // Connect the source to the GainNode, then to the analyser node, then to the destination (speakers)
107
- source.connect(gainNode);
108
- gainNode.connect(this.analyser);
109
- this.analyser.connect(this.audioContext.destination);
110
-
111
- source.start(0);
112
- // console.log('Playing chunk', this.currentIndex);
113
- gainNode.gain.value = this.volume;
114
- source.onended = () => {
115
- // console.log('Chunk ended');
116
- resolve();
117
- };
118
-
119
- // Start monitoring loudness only once
120
- if (!this.isMonitoring) {
121
- this.isMonitoring = true;
122
- this.shouldMonitorLoudness = true;
123
- this.monitorLoudness();
124
- }
125
- });
126
- });
127
- }
128
-
129
- async playAgain(): Promise<void> {
130
- console.log('Playing again');
131
- if (this.chunkQueue.length > 0 && !this.isPlaying) {
132
- this.playChunks();
133
- }
134
- }
135
-
136
- private monitorLoudness(): void {
137
- // Stop monitoring when the flag is false
138
- if (!this.shouldMonitorLoudness) {
139
- // console.log('Loudness monitoring stopped.');
140
- cancelAnimationFrame(this.handle);
141
- this.loudnessCallback(0);
142
- this.onEndOfSpeech();
143
- return;
144
- }
145
-
146
- // Get the time domain data from the analyser (this is a snapshot of the waveform)
147
- this.analyser.getByteTimeDomainData(this.dataArray);
148
-
149
- // Calculate the RMS (root mean square) of the waveform values to get the perceived loudness
150
- let sum = 0;
151
- for (let i = 0; i < this.dataArray.length; i++) {
152
- const value = this.dataArray[i] / 128.0 - 1.0; // Normalize between -1 and 1
153
- sum += value * value;
154
- }
155
-
156
- const rms = Math.sqrt(sum / this.dataArray.length);
157
-
158
- // Handle the case where RMS is 0 to avoid log10(0)
159
- if (rms === 0) {
160
- // console.log('Current loudness: Silent');
161
- } else {
162
- let loudnessInDb = 20 * Math.log10(rms); // Convert to dB
163
- // console.log('Current loudness:' + loudnessInDb);
164
- const minDb = -57;
165
- const maxDb = -15;
166
-
167
- if (loudnessInDb < minDb) {
168
- loudnessInDb = minDb;
169
- }
170
- if (loudnessInDb > maxDb) {
171
- loudnessInDb = maxDb;
172
- }
173
-
174
- const loudnessScale = ((loudnessInDb - minDb) / (maxDb - minDb)) * 100;
175
- // console.log("root:corrent loudness", loudnessScale);
176
-
177
- this.loudnessCallback(loudnessScale);
178
- }
179
-
180
- // Call this method again at regular intervals if you want continuous loudness monitoring
181
- this.handle = requestAnimationFrame(() => this.monitorLoudness());
182
- }
183
- public reset() {
184
- // console.log('Resetting player');
185
- this.stopPlayback();
186
- this.currentIndex = 0;
187
- this.shouldMonitorLoudness = true;
188
- //reset to the beginning when the class gets initialized
189
- this.isMonitoring = false;
190
- this.isPlaying = false;
191
- this.init();
192
- }
193
-
194
- public setOnEndOfSpeech(callback: () => void) {
195
- this.onEndOfSpeech = callback;
196
- }
197
- }
@@ -1,129 +0,0 @@
1
- import { useRimori } from '../../../components';
2
- import { FaMicrophone, FaSpinner } from 'react-icons/fa6';
3
- import { AudioController } from '../../../plugin/AudioController';
4
- import { useState, useRef, forwardRef, useImperativeHandle, useEffect } from 'react';
5
-
6
- interface Props {
7
- iconSize?: string;
8
- className?: string;
9
- disabled?: boolean;
10
- loading?: boolean;
11
- enablePushToTalk?: boolean;
12
- onRecordingStatusChange: (running: boolean) => void;
13
- onVoiceRecorded: (message: string) => void;
14
- }
15
-
16
- export const VoiceRecorder = forwardRef(
17
- (
18
- {
19
- onVoiceRecorded,
20
- iconSize,
21
- className,
22
- disabled,
23
- loading,
24
- onRecordingStatusChange,
25
- enablePushToTalk = false,
26
- }: Props,
27
- ref,
28
- ) => {
29
- const [isRecording, setIsRecording] = useState(false);
30
- const [internalIsProcessing, setInternalIsProcessing] = useState(false);
31
- const audioControllerRef = useRef<AudioController | null>(null);
32
- const { ai, plugin } = useRimori();
33
-
34
- // Ref for latest onVoiceRecorded callback
35
- const onVoiceRecordedRef = useRef(onVoiceRecorded);
36
- useEffect(() => {
37
- onVoiceRecordedRef.current = onVoiceRecorded;
38
- }, [onVoiceRecorded]);
39
-
40
- const startRecording = async () => {
41
- try {
42
- if (!audioControllerRef.current) {
43
- audioControllerRef.current = new AudioController(plugin.pluginId);
44
- }
45
-
46
- await audioControllerRef.current.startRecording();
47
- setIsRecording(true);
48
- onRecordingStatusChange(true);
49
- } catch (error) {
50
- console.error('Failed to start recording:', error);
51
- // Handle permission denied or other errors
52
- }
53
- };
54
-
55
- const stopRecording = async () => {
56
- try {
57
- if (audioControllerRef.current && isRecording) {
58
- const audioResult = await audioControllerRef.current.stopRecording();
59
- // console.log("audioResult: ", audioResult);
60
-
61
- setInternalIsProcessing(true);
62
-
63
- // Play the recorded audio from the Blob
64
- // const blobUrl = URL.createObjectURL(audioResult.recording);
65
- // const audioRef = new Audio(blobUrl);
66
- // audioRef.onended = () => URL.revokeObjectURL(blobUrl);
67
- // audioRef.play().catch((e) => console.error('Playback error:', e));
68
-
69
- // console.log("audioBlob: ", audioResult.recording);
70
- const text = await ai.getTextFromVoice(audioResult.recording);
71
- // console.log("stt result", text);
72
- // throw new Error("test");
73
- setInternalIsProcessing(false);
74
- onVoiceRecordedRef.current(text);
75
- }
76
- } catch (error) {
77
- console.error('Failed to stop recording:', error);
78
- } finally {
79
- setIsRecording(false);
80
- onRecordingStatusChange(false);
81
- }
82
- };
83
-
84
- useImperativeHandle(ref, () => ({
85
- startRecording,
86
- stopRecording,
87
- }));
88
-
89
- // push to talk feature
90
- const spacePressedRef = useRef(false);
91
-
92
- useEffect(() => {
93
- if (!enablePushToTalk) return;
94
-
95
- const handleKeyDown = async (event: KeyboardEvent) => {
96
- if (event.code === 'Space' && !spacePressedRef.current) {
97
- spacePressedRef.current = true;
98
- await startRecording();
99
- }
100
- };
101
- const handleKeyUp = (event: KeyboardEvent) => {
102
- if (event.code === 'Space' && spacePressedRef.current) {
103
- spacePressedRef.current = false;
104
- stopRecording();
105
- }
106
- };
107
- window.addEventListener('keydown', handleKeyDown);
108
- window.addEventListener('keyup', handleKeyUp);
109
- return () => {
110
- window.removeEventListener('keydown', handleKeyDown);
111
- window.removeEventListener('keyup', handleKeyUp);
112
- };
113
- }, [enablePushToTalk]);
114
-
115
- return (
116
- <button
117
- className={'flex flex-row justify-center items-center rounded-full mx-auto disabled:opacity-50 ' + className}
118
- onClick={isRecording ? stopRecording : startRecording}
119
- disabled={disabled || loading || internalIsProcessing}
120
- >
121
- {loading || internalIsProcessing ? (
122
- <FaSpinner className="animate-spin" />
123
- ) : (
124
- <FaMicrophone size={iconSize} className={isRecording ? 'text-red-600' : ''} />
125
- )}
126
- </button>
127
- );
128
- },
129
- );