@rimori/client 1.4.10 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. package/README.md +140 -1171
  2. package/dist/{core/controller → controller}/AIController.d.ts +1 -1
  3. package/dist/{core/controller → controller}/AIController.js +2 -2
  4. package/dist/{plugin/AccomplishmentHandler.d.ts → controller/AccomplishmentController.d.ts} +1 -1
  5. package/dist/{plugin/AccomplishmentHandler.js → controller/AccomplishmentController.js} +1 -1
  6. package/dist/{core/controller → controller}/ExerciseController.d.ts +8 -6
  7. package/dist/{core/controller → controller}/ExerciseController.js +10 -9
  8. package/dist/{core/controller → controller}/ObjectController.d.ts +1 -1
  9. package/dist/{core/controller → controller}/ObjectController.js +1 -1
  10. package/dist/{core/controller → controller}/SettingsController.d.ts +2 -2
  11. package/dist/{core/controller → controller}/SharedContentController.d.ts +1 -1
  12. package/dist/{core/controller → controller}/VoiceController.d.ts +1 -1
  13. package/dist/index.d.ts +14 -8
  14. package/dist/index.js +7 -7
  15. package/dist/plugin/{PluginController.d.ts → CommunicationHandler.d.ts} +4 -7
  16. package/dist/plugin/{PluginController.js → CommunicationHandler.js} +19 -27
  17. package/dist/plugin/RimoriClient.d.ts +67 -68
  18. package/dist/plugin/RimoriClient.js +101 -43
  19. package/dist/worker/WorkerSetup.js +3 -3
  20. package/example/docs/devdocs.md +1 -1
  21. package/package.json +5 -26
  22. package/src/{core/controller → controller}/AIController.ts +3 -3
  23. package/src/{plugin/AccomplishmentHandler.ts → controller/AccomplishmentController.ts} +1 -1
  24. package/src/{core/controller → controller}/ExerciseController.ts +14 -11
  25. package/src/{core/controller → controller}/ObjectController.ts +2 -2
  26. package/src/{core/controller → controller}/SettingsController.ts +2 -2
  27. package/src/{core/controller → controller}/SharedContentController.ts +1 -1
  28. package/src/{core/controller → controller}/VoiceController.ts +2 -2
  29. package/src/fromRimori/readme.md +1 -1
  30. package/src/index.ts +14 -8
  31. package/src/plugin/{PluginController.ts → CommunicationHandler.ts} +24 -36
  32. package/src/plugin/RimoriClient.ts +127 -118
  33. package/src/worker/WorkerSetup.ts +6 -4
  34. package/tsconfig.json +5 -2
  35. package/dist/cli/scripts/release/release-translation-upload.d.ts +0 -6
  36. package/dist/cli/scripts/release/release-translation-upload.js +0 -87
  37. package/dist/components/CRUDModal.d.ts +0 -17
  38. package/dist/components/CRUDModal.js +0 -24
  39. package/dist/components/MarkdownEditor.d.ts +0 -8
  40. package/dist/components/MarkdownEditor.js +0 -48
  41. package/dist/components/Spinner.d.ts +0 -8
  42. package/dist/components/Spinner.js +0 -4
  43. package/dist/components/ai/Assistant.d.ts +0 -9
  44. package/dist/components/ai/Assistant.js +0 -58
  45. package/dist/components/ai/Avatar.d.ts +0 -14
  46. package/dist/components/ai/Avatar.js +0 -59
  47. package/dist/components/ai/EmbeddedAssistent/AudioInputField.d.ts +0 -7
  48. package/dist/components/ai/EmbeddedAssistent/AudioInputField.js +0 -37
  49. package/dist/components/ai/EmbeddedAssistent/CircleAudioAvatar.d.ts +0 -8
  50. package/dist/components/ai/EmbeddedAssistent/CircleAudioAvatar.js +0 -79
  51. package/dist/components/ai/EmbeddedAssistent/TTS/MessageSender.d.ts +0 -19
  52. package/dist/components/ai/EmbeddedAssistent/TTS/MessageSender.js +0 -91
  53. package/dist/components/ai/EmbeddedAssistent/TTS/Player.d.ts +0 -27
  54. package/dist/components/ai/EmbeddedAssistent/TTS/Player.js +0 -185
  55. package/dist/components/ai/EmbeddedAssistent/VoiceRecoder.d.ts +0 -11
  56. package/dist/components/ai/EmbeddedAssistent/VoiceRecoder.js +0 -95
  57. package/dist/components/ai/utils.d.ts +0 -6
  58. package/dist/components/ai/utils.js +0 -13
  59. package/dist/components/audio/Playbutton.d.ts +0 -15
  60. package/dist/components/audio/Playbutton.js +0 -80
  61. package/dist/components/components/ContextMenu.d.ts +0 -10
  62. package/dist/components/components/ContextMenu.js +0 -135
  63. package/dist/components.d.ts +0 -10
  64. package/dist/components.js +0 -11
  65. package/dist/core/controller/EnhancedUserInfo.d.ts +0 -0
  66. package/dist/core/controller/EnhancedUserInfo.js +0 -1
  67. package/dist/core/core.d.ts +0 -14
  68. package/dist/core/core.js +0 -7
  69. package/dist/hooks/I18nHooks.d.ts +0 -11
  70. package/dist/hooks/I18nHooks.js +0 -25
  71. package/dist/hooks/UseChatHook.d.ts +0 -10
  72. package/dist/hooks/UseChatHook.js +0 -29
  73. package/dist/i18n/I18nHooks.d.ts +0 -11
  74. package/dist/i18n/I18nHooks.js +0 -25
  75. package/dist/i18n/Translator.d.ts +0 -43
  76. package/dist/i18n/Translator.js +0 -118
  77. package/dist/i18n/config.d.ts +0 -7
  78. package/dist/i18n/config.js +0 -20
  79. package/dist/i18n/createI18nInstance.d.ts +0 -7
  80. package/dist/i18n/createI18nInstance.js +0 -31
  81. package/dist/i18n/hooks.d.ts +0 -11
  82. package/dist/i18n/hooks.js +0 -25
  83. package/dist/i18n/index.d.ts +0 -4
  84. package/dist/i18n/index.js +0 -4
  85. package/dist/i18n/types.d.ts +0 -7
  86. package/dist/i18n/types.js +0 -1
  87. package/dist/i18n/useRimoriI18n.d.ts +0 -11
  88. package/dist/i18n/useRimoriI18n.js +0 -41
  89. package/dist/plugin/ThemeSetter.d.ts +0 -2
  90. package/dist/plugin/ThemeSetter.js +0 -19
  91. package/dist/plugin/Translator.d.ts +0 -38
  92. package/dist/plugin/Translator.js +0 -101
  93. package/dist/providers/PluginProvider.d.ts +0 -12
  94. package/dist/providers/PluginProvider.js +0 -152
  95. package/dist/style.css +0 -110
  96. package/dist/style.css.map +0 -1
  97. package/dist/utils/Language.d.ts +0 -67
  98. package/dist/utils/Language.js +0 -69
  99. package/dist/utils/LanguageClass.d.ts +0 -36
  100. package/dist/utils/LanguageClass.example.d.ts +0 -0
  101. package/dist/utils/LanguageClass.example.js +0 -1
  102. package/dist/utils/LanguageClass.js +0 -50
  103. package/dist/utils/LanguageClass.test.d.ts +0 -0
  104. package/dist/utils/LanguageClass.test.js +0 -1
  105. package/dist/utils/PluginUtils.d.ts +0 -2
  106. package/dist/utils/PluginUtils.js +0 -23
  107. package/src/components/CRUDModal.tsx +0 -75
  108. package/src/components/MarkdownEditor.tsx +0 -144
  109. package/src/components/Spinner.tsx +0 -29
  110. package/src/components/ai/Assistant.tsx +0 -96
  111. package/src/components/ai/Avatar.tsx +0 -99
  112. package/src/components/ai/EmbeddedAssistent/AudioInputField.tsx +0 -73
  113. package/src/components/ai/EmbeddedAssistent/CircleAudioAvatar.tsx +0 -107
  114. package/src/components/ai/EmbeddedAssistent/TTS/MessageSender.ts +0 -96
  115. package/src/components/ai/EmbeddedAssistent/TTS/Player.ts +0 -197
  116. package/src/components/ai/EmbeddedAssistent/VoiceRecoder.tsx +0 -129
  117. package/src/components/ai/utils.ts +0 -21
  118. package/src/components/audio/Playbutton.tsx +0 -126
  119. package/src/components/components/ContextMenu.tsx +0 -179
  120. package/src/components.ts +0 -11
  121. package/src/core/core.ts +0 -15
  122. package/src/hooks/I18nHooks.ts +0 -33
  123. package/src/hooks/UseChatHook.ts +0 -38
  124. package/src/plugin/ThemeSetter.ts +0 -23
  125. package/src/providers/PluginProvider.tsx +0 -209
  126. package/src/style.scss +0 -136
  127. package/src/utils/PluginUtils.ts +0 -22
  128. /package/dist/{plugin → controller}/AudioController.d.ts +0 -0
  129. /package/dist/{plugin → controller}/AudioController.js +0 -0
  130. /package/dist/{core/controller → controller}/SettingsController.js +0 -0
  131. /package/dist/{core/controller → controller}/SharedContentController.js +0 -0
  132. /package/dist/{plugin → controller}/TranslationController.d.ts +0 -0
  133. /package/dist/{plugin → controller}/TranslationController.js +0 -0
  134. /package/dist/{core/controller → controller}/VoiceController.js +0 -0
  135. /package/src/{plugin → controller}/AudioController.ts +0 -0
  136. /package/src/{plugin → controller}/TranslationController.ts +0 -0
@@ -1,58 +0,0 @@
1
- import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
2
- import React, { useEffect, useMemo } from 'react';
3
- import { CircleAudioAvatar } from './EmbeddedAssistent/CircleAudioAvatar';
4
- import { AudioInputField } from './EmbeddedAssistent/AudioInputField';
5
- import { MessageSender } from './EmbeddedAssistent/TTS/MessageSender';
6
- import Markdown from 'react-markdown';
7
- import { useChat } from '../../hooks/UseChatHook';
8
- import { useRimori } from '../../components';
9
- import { getFirstMessages } from './utils';
10
- export function AssistantChat({ avatarImageUrl, voiceId, onComplete, autoStartConversation }) {
11
- var _a;
12
- const [oralCommunication, setOralCommunication] = React.useState(true);
13
- const { ai: llm, event } = useRimori();
14
- const sender = useMemo(() => new MessageSender(llm.getVoice, voiceId), []);
15
- const { messages, append, isLoading, setMessages } = useChat();
16
- const lastAssistantMessage = (_a = [...messages].filter((m) => m.role === 'assistant').pop()) === null || _a === void 0 ? void 0 : _a.content;
17
- useEffect(() => {
18
- sender.setOnLoudnessChange((value) => event.emit('self.avatar.triggerLoudness', { loudness: value }));
19
- if (!autoStartConversation) {
20
- return;
21
- }
22
- setMessages(getFirstMessages(autoStartConversation));
23
- // append([{ role: 'user', content: autoStartConversation.userMessage }]);
24
- if (autoStartConversation.assistantMessage) {
25
- // console.log("autostartmessages", { autoStartConversation, isLoading });
26
- sender.handleNewText(autoStartConversation.assistantMessage, isLoading);
27
- }
28
- }, []);
29
- useEffect(() => {
30
- var _a;
31
- let message = lastAssistantMessage;
32
- if (message !== ((_a = messages[messages.length - 1]) === null || _a === void 0 ? void 0 : _a.content)) {
33
- message = undefined;
34
- }
35
- sender.handleNewText(message, isLoading);
36
- }, [messages, isLoading]);
37
- const lastMessage = messages[messages.length - 1];
38
- useEffect(() => {
39
- console.log('lastMessage', lastMessage);
40
- const toolInvocations = lastMessage === null || lastMessage === void 0 ? void 0 : lastMessage.toolCalls;
41
- if (toolInvocations && toolInvocations.length > 0) {
42
- console.log('toolInvocations', toolInvocations);
43
- onComplete(toolInvocations[0].args);
44
- }
45
- }, [lastMessage]);
46
- if ((lastMessage === null || lastMessage === void 0 ? void 0 : lastMessage.toolCalls) && lastMessage.toolCalls.length > 0) {
47
- console.log('lastMessage test2', lastMessage);
48
- const args = lastMessage.toolCalls[0].args;
49
- const success = args.explanationUnderstood === 'TRUE' || args.studentKnowsTopic === 'TRUE';
50
- return (_jsxs("div", { className: "px-5 pt-5 overflow-y-auto text-center", style: { height: '478px' }, children: [_jsx("h1", { className: "text-center mt-5 mb-5", children: success ? 'Great job!' : 'You failed' }), _jsx("p", { children: args.improvementHints })] }));
51
- }
52
- return (_jsxs("div", { children: [oralCommunication && _jsx(CircleAudioAvatar, { imageUrl: avatarImageUrl, className: "mx-auto my-10" }), _jsx("div", { className: "w-full", children: lastAssistantMessage && (_jsx("div", { className: "px-5 pt-5 overflow-y-auto remirror-theme", style: { height: '4k78px' }, children: _jsx(Markdown, { children: lastAssistantMessage }) })) }), _jsx(AudioInputField, { blockSubmission: isLoading, onSubmit: (message) => {
53
- append([{ role: 'user', content: message, id: messages.length.toString() }]);
54
- }, onAudioControl: (voice) => {
55
- setOralCommunication(voice);
56
- sender.setVolume(voice ? 1 : 0);
57
- } })] }));
58
- }
@@ -1,14 +0,0 @@
1
- import { Tool } from '../../fromRimori/PluginTypes';
2
- import { FirstMessages } from './utils';
3
- interface Props {
4
- voiceId: string;
5
- agentTools: Tool[];
6
- avatarImageUrl: string;
7
- circleSize?: string;
8
- isDarkTheme?: boolean;
9
- children?: React.ReactNode;
10
- autoStartConversation?: FirstMessages;
11
- className?: string;
12
- }
13
- export declare function Avatar({ avatarImageUrl, voiceId, agentTools, autoStartConversation, children, isDarkTheme, circleSize, className, }: Props): import("react/jsx-runtime").JSX.Element;
14
- export {};
@@ -1,59 +0,0 @@
1
- import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
2
- import { useEffect, useMemo, useState } from 'react';
3
- import { VoiceRecorder } from './EmbeddedAssistent/VoiceRecoder';
4
- import { MessageSender } from './EmbeddedAssistent/TTS/MessageSender';
5
- import { CircleAudioAvatar } from './EmbeddedAssistent/CircleAudioAvatar';
6
- import { useChat } from '../../hooks/UseChatHook';
7
- import { useRimori } from '../../components';
8
- import { getFirstMessages } from './utils';
9
- export function Avatar({ avatarImageUrl, voiceId, agentTools, autoStartConversation, children, isDarkTheme = false, circleSize = '300px', className, }) {
10
- const { ai, event } = useRimori();
11
- const [agentReplying, setAgentReplying] = useState(false);
12
- const [isProcessingMessage, setIsProcessingMessage] = useState(false);
13
- const sender = useMemo(() => new MessageSender(ai.getVoice, voiceId), [voiceId]);
14
- const { messages, append, isLoading, lastMessage, setMessages } = useChat(agentTools);
15
- useEffect(() => {
16
- console.log('messages', messages);
17
- }, [messages]);
18
- useEffect(() => {
19
- if (!isLoading)
20
- setIsProcessingMessage(false);
21
- }, [isLoading]);
22
- useEffect(() => {
23
- if (!voiceId)
24
- return; //at the beginning when being mounted the voiceId is undefined
25
- sender.setOnLoudnessChange((value) => event.emit('self.avatar.triggerLoudness', { loudness: value }));
26
- sender.setOnEndOfSpeech(() => setAgentReplying(false));
27
- if (!autoStartConversation)
28
- return;
29
- setMessages(getFirstMessages(autoStartConversation));
30
- // append([{ role: 'user', content: autoStartConversation.userMessage }]);
31
- if (autoStartConversation.assistantMessage) {
32
- // console.log("autostartmessages", { autoStartConversation, isLoading });
33
- sender.handleNewText(autoStartConversation.assistantMessage, isLoading);
34
- }
35
- else if (autoStartConversation.userMessage) {
36
- append([{ role: 'user', content: autoStartConversation.userMessage, id: messages.length.toString() }]);
37
- }
38
- }, [autoStartConversation, voiceId]);
39
- useEffect(() => {
40
- if ((lastMessage === null || lastMessage === void 0 ? void 0 : lastMessage.role) === 'assistant') {
41
- sender.handleNewText(lastMessage.content, isLoading);
42
- if (lastMessage.toolCalls) {
43
- // console.log("unlocking mic", lastMessage)
44
- setAgentReplying(false);
45
- setIsProcessingMessage(false);
46
- }
47
- }
48
- }, [lastMessage, isLoading]);
49
- return (_jsxs("div", { className: `md:pb-8 ${className || ''}`, children: [_jsx(CircleAudioAvatar, { width: circleSize, className: "mx-auto", imageUrl: avatarImageUrl, isDarkTheme: isDarkTheme }), children, _jsx(VoiceRecorder, { iconSize: "30", className: "w-16 h-16 shadow-lg rounded-full bg-gray-400 dark:bg-gray-800", disabled: agentReplying, loading: isProcessingMessage, enablePushToTalk: true, onVoiceRecorded: (message) => {
50
- setAgentReplying(true);
51
- append([
52
- {
53
- role: 'user',
54
- content: 'Message(' + Math.floor((messages.length + 1) / 2) + '): ' + message,
55
- id: messages.length.toString(),
56
- },
57
- ]);
58
- }, onRecordingStatusChange: (running) => !running && setIsProcessingMessage(true) })] }));
59
- }
@@ -1,7 +0,0 @@
1
- interface AudioInputFieldProps {
2
- onSubmit: (text: string) => void;
3
- onAudioControl?: (voice: boolean) => void;
4
- blockSubmission?: boolean;
5
- }
6
- export declare function AudioInputField({ onSubmit, onAudioControl, blockSubmission }: AudioInputFieldProps): import("react/jsx-runtime").JSX.Element;
7
- export {};
@@ -1,37 +0,0 @@
1
- import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
2
- import { useState } from 'react';
3
- import { VoiceRecorder } from './VoiceRecoder';
4
- import { BiSolidRightArrow } from 'react-icons/bi';
5
- import { HiMiniSpeakerXMark, HiMiniSpeakerWave } from 'react-icons/hi2';
6
- export function AudioInputField({ onSubmit, onAudioControl, blockSubmission = false }) {
7
- const [text, setText] = useState('');
8
- const [audioEnabled, setAudioEnabled] = useState(true);
9
- const handleSubmit = (manualText) => {
10
- if (blockSubmission)
11
- return;
12
- const sendableText = manualText || text;
13
- if (sendableText.trim()) {
14
- onSubmit(sendableText);
15
- setTimeout(() => {
16
- setText('');
17
- }, 100);
18
- }
19
- };
20
- const handleKeyDown = (e) => {
21
- if (blockSubmission)
22
- return;
23
- if (e.key === 'Enter' && e.ctrlKey) {
24
- setText(text + '\n');
25
- }
26
- else if (e.key === 'Enter') {
27
- handleSubmit();
28
- }
29
- };
30
- return (_jsxs("div", { className: "flex items-center bg-gray-600 pt-2 pb-2 p-2", children: [onAudioControl && (_jsx("button", { onClick: () => {
31
- onAudioControl(!audioEnabled);
32
- setAudioEnabled(!audioEnabled);
33
- }, className: "cursor-default", children: audioEnabled ? (_jsx(HiMiniSpeakerWave, { className: "w-9 h-9 cursor-pointer" })) : (_jsx(HiMiniSpeakerXMark, { className: "w-9 h-9 cursor-pointer" })) })), _jsx(VoiceRecorder, { onRecordingStatusChange: () => { }, onVoiceRecorded: (m) => {
34
- console.log('onVoiceRecorded', m);
35
- handleSubmit(m);
36
- } }), _jsx("textarea", { value: text, onChange: (e) => setText(e.target.value), onKeyDown: handleKeyDown, className: "flex-1 border-none rounded-lg p-2 text-gray-800 focus::outline-none", placeholder: "Type a message...", disabled: blockSubmission }), _jsx("button", { onClick: () => handleSubmit(), className: "cursor-default", disabled: blockSubmission, children: _jsx(BiSolidRightArrow, { className: "w-9 h-10 cursor-pointer" }) })] }));
37
- }
@@ -1,8 +0,0 @@
1
- interface CircleAudioAvatarProps {
2
- width?: string;
3
- imageUrl: string;
4
- className?: string;
5
- isDarkTheme?: boolean;
6
- }
7
- export declare function CircleAudioAvatar({ imageUrl, className, isDarkTheme, width, }: CircleAudioAvatarProps): import("react/jsx-runtime").JSX.Element;
8
- export {};
@@ -1,79 +0,0 @@
1
- import { jsx as _jsx } from "react/jsx-runtime";
2
- import { useEffect, useRef } from 'react';
3
- import { EventBus } from '../../../fromRimori/EventBus';
4
- export function CircleAudioAvatar({ imageUrl, className, isDarkTheme = false, width = '150px', }) {
5
- const canvasRef = useRef(null);
6
- const currentLoudnessRef = useRef(0);
7
- const targetLoudnessRef = useRef(0);
8
- const animationFrameRef = useRef(null);
9
- useEffect(() => {
10
- const canvas = canvasRef.current;
11
- if (canvas) {
12
- const ctx = canvas.getContext('2d');
13
- if (ctx) {
14
- const image = new Image();
15
- image.src = imageUrl;
16
- let isMounted = true;
17
- image.onload = () => {
18
- if (!isMounted)
19
- return;
20
- draw(ctx, canvas, image, 0);
21
- const animate = () => {
22
- const decayRate = 0.06;
23
- if (currentLoudnessRef.current > targetLoudnessRef.current) {
24
- currentLoudnessRef.current = Math.max(targetLoudnessRef.current, currentLoudnessRef.current - decayRate * currentLoudnessRef.current);
25
- }
26
- else {
27
- currentLoudnessRef.current = targetLoudnessRef.current;
28
- }
29
- draw(ctx, canvas, image, currentLoudnessRef.current);
30
- animationFrameRef.current = requestAnimationFrame(animate);
31
- };
32
- animationFrameRef.current = requestAnimationFrame(animate);
33
- };
34
- const handleLoudness = ({ data }) => {
35
- const newLoudness = data.loudness;
36
- if (newLoudness > currentLoudnessRef.current) {
37
- currentLoudnessRef.current = newLoudness;
38
- }
39
- targetLoudnessRef.current = newLoudness;
40
- };
41
- const listener = EventBus.on('self.avatar.triggerLoudness', handleLoudness);
42
- return () => {
43
- isMounted = false;
44
- listener.off();
45
- if (animationFrameRef.current) {
46
- cancelAnimationFrame(animationFrameRef.current);
47
- }
48
- };
49
- }
50
- }
51
- }, [imageUrl]);
52
- const draw = (ctx, canvas, image, loudness) => {
53
- if (canvas && ctx) {
54
- ctx.clearRect(0, 0, canvas.width, canvas.height);
55
- const radius = Math.min(canvas.width, canvas.height) / 3;
56
- const centerX = canvas.width / 2;
57
- const centerY = canvas.height / 2;
58
- const pulseRadius = radius + loudness / 2.5;
59
- ctx.beginPath();
60
- ctx.arc(centerX, centerY, pulseRadius, 0, Math.PI * 2, true);
61
- ctx.strokeStyle = isDarkTheme ? 'rgba(255, 255, 255, 0.5)' : 'rgba(0, 0, 0, 0.5)';
62
- ctx.lineWidth = 5;
63
- ctx.stroke();
64
- ctx.save();
65
- ctx.beginPath();
66
- ctx.arc(centerX, centerY, radius, 0, Math.PI * 2, true);
67
- ctx.closePath();
68
- ctx.clip();
69
- ctx.drawImage(image, centerX - radius, centerY - radius, radius * 2, radius * 2);
70
- ctx.restore();
71
- ctx.beginPath();
72
- ctx.arc(centerX, centerY, radius, 0, Math.PI * 2, true);
73
- ctx.strokeStyle = isDarkTheme ? 'rgba(255, 255, 255, 0.9)' : 'rgba(0, 0, 0, 0.9)';
74
- ctx.lineWidth = 5;
75
- ctx.stroke();
76
- }
77
- };
78
- return _jsx("canvas", { ref: canvasRef, className: className, width: 500, height: 500, style: { width } });
79
- }
@@ -1,19 +0,0 @@
1
- type VoiceBackend = (text: string, voice?: string, speed?: number) => Promise<Blob>;
2
- export declare class MessageSender {
3
- private player;
4
- private fetchedSentences;
5
- private lastLoading;
6
- private voice;
7
- private voiceBackend;
8
- constructor(voiceBackend: VoiceBackend, voice: string);
9
- private getCompletedSentences;
10
- handleNewText(currentText: string | undefined, isLoading: boolean): Promise<void>;
11
- private generateSpeech;
12
- play(): void;
13
- stop(): void;
14
- private reset;
15
- setVolume(volume: number): void;
16
- setOnLoudnessChange(callback: (value: number) => void): void;
17
- setOnEndOfSpeech(callback: () => void): void;
18
- }
19
- export {};
@@ -1,91 +0,0 @@
1
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
2
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
3
- return new (P || (P = Promise))(function (resolve, reject) {
4
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
5
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
6
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
7
- step((generator = generator.apply(thisArg, _arguments || [])).next());
8
- });
9
- };
10
- import { ChunkedAudioPlayer } from './Player';
11
- export class MessageSender {
12
- constructor(voiceBackend, voice) {
13
- this.player = new ChunkedAudioPlayer();
14
- this.fetchedSentences = new Set();
15
- this.lastLoading = false;
16
- if ((voice === null || voice === void 0 ? void 0 : voice.split('_').length) !== 2) {
17
- throw new Error("Invalid voice id format '" + voice + "'. Voice id needs to look like <provider>_<voice_id>");
18
- }
19
- this.voiceBackend = voiceBackend;
20
- this.voice = voice;
21
- }
22
- getCompletedSentences(currentText, isLoading) {
23
- // Split the text based on the following characters: .?!
24
- // Only split on : when followed by a space
25
- const pattern = /(.+?[.?!]|.+?:\s+|.+?\n+)/g;
26
- const result = [];
27
- let match;
28
- while ((match = pattern.exec(currentText)) !== null) {
29
- const sentence = match[0].trim();
30
- if (sentence.length > 0) {
31
- result.push(sentence);
32
- }
33
- }
34
- if (!isLoading) {
35
- const lastFullSentence = result[result.length - 1];
36
- const leftoverIndex = currentText.lastIndexOf(lastFullSentence) + lastFullSentence.length;
37
- if (leftoverIndex < currentText.length) {
38
- result.push(currentText.slice(leftoverIndex).trim());
39
- }
40
- }
41
- return result;
42
- }
43
- handleNewText(currentText, isLoading) {
44
- return __awaiter(this, void 0, void 0, function* () {
45
- if (!this.lastLoading && isLoading) {
46
- this.reset();
47
- }
48
- this.lastLoading = isLoading;
49
- if (!currentText) {
50
- return;
51
- }
52
- const sentences = this.getCompletedSentences(currentText, isLoading);
53
- for (let i = 0; i < sentences.length; i++) {
54
- const sentence = sentences[i];
55
- if (!this.fetchedSentences.has(sentence)) {
56
- this.fetchedSentences.add(sentence);
57
- const audioData = yield this.generateSpeech(sentence);
58
- yield this.player.addChunk(audioData, i);
59
- }
60
- }
61
- });
62
- }
63
- generateSpeech(sentence) {
64
- return __awaiter(this, void 0, void 0, function* () {
65
- const blob = yield this.voiceBackend(sentence, this.voice, 1.0);
66
- return yield blob.arrayBuffer();
67
- });
68
- }
69
- play() {
70
- this.player.playAgain();
71
- }
72
- stop() {
73
- this.player.stopPlayback();
74
- }
75
- reset() {
76
- this.stop();
77
- this.fetchedSentences.clear();
78
- this.player.reset();
79
- }
80
- setVolume(volume) {
81
- this.player.setVolume(volume);
82
- }
83
- setOnLoudnessChange(callback) {
84
- this.player.setOnLoudnessChange((loudness) => {
85
- callback(loudness);
86
- });
87
- }
88
- setOnEndOfSpeech(callback) {
89
- this.player.setOnEndOfSpeech(callback);
90
- }
91
- }
@@ -1,27 +0,0 @@
1
- export declare class ChunkedAudioPlayer {
2
- private audioContext;
3
- private chunkQueue;
4
- private isPlaying;
5
- private analyser;
6
- private dataArray;
7
- private shouldMonitorLoudness;
8
- private isMonitoring;
9
- private handle;
10
- private volume;
11
- private loudnessCallback;
12
- private currentIndex;
13
- private startedPlaying;
14
- private onEndOfSpeech;
15
- constructor();
16
- private init;
17
- setOnLoudnessChange(callback: (value: number) => void): void;
18
- setVolume(volume: number): void;
19
- addChunk(chunk: ArrayBuffer, position: number): Promise<void>;
20
- private playChunks;
21
- stopPlayback(): void;
22
- private playChunk;
23
- playAgain(): Promise<void>;
24
- private monitorLoudness;
25
- reset(): void;
26
- setOnEndOfSpeech(callback: () => void): void;
27
- }
@@ -1,185 +0,0 @@
1
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
2
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
3
- return new (P || (P = Promise))(function (resolve, reject) {
4
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
5
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
6
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
7
- step((generator = generator.apply(thisArg, _arguments || [])).next());
8
- });
9
- };
10
- export class ChunkedAudioPlayer {
11
- constructor() {
12
- this.chunkQueue = [];
13
- this.isPlaying = false;
14
- this.shouldMonitorLoudness = true;
15
- this.isMonitoring = false;
16
- this.handle = 0;
17
- this.volume = 1.0;
18
- this.loudnessCallback = () => { };
19
- this.currentIndex = 0;
20
- this.startedPlaying = false;
21
- this.onEndOfSpeech = () => { };
22
- this.init();
23
- }
24
- init() {
25
- this.audioContext = new AudioContext();
26
- this.analyser = this.audioContext.createAnalyser();
27
- this.analyser.fftSize = 256; // Set the FFT size (smaller values provide faster updates, larger ones give better resolution)
28
- const bufferLength = this.analyser.frequencyBinCount;
29
- this.dataArray = new Uint8Array(bufferLength); // Array to hold frequency data
30
- }
31
- setOnLoudnessChange(callback) {
32
- this.loudnessCallback = callback;
33
- }
34
- setVolume(volume) {
35
- this.volume = volume;
36
- }
37
- addChunk(chunk, position) {
38
- return __awaiter(this, void 0, void 0, function* () {
39
- console.log('Adding chunk', position, chunk);
40
- this.chunkQueue[position] = chunk;
41
- // console.log("received chunk", {
42
- // chunkQueue: this.chunkQueue.length,
43
- // isPlaying: this.isPlaying,
44
- // })
45
- if (position === 0 && !this.startedPlaying) {
46
- this.startedPlaying = true;
47
- this.playChunks();
48
- }
49
- });
50
- }
51
- playChunks() {
52
- // console.log({ isPlaying: this.isPlaying });
53
- if (this.isPlaying)
54
- return;
55
- if (!this.chunkQueue[this.currentIndex]) {
56
- // wait until the correct chunk arrives
57
- setTimeout(() => this.playChunks(), 10);
58
- }
59
- this.isPlaying = true;
60
- this.playChunk(this.chunkQueue[this.currentIndex]).then(() => {
61
- this.isPlaying = false;
62
- this.currentIndex++;
63
- if (this.chunkQueue[this.currentIndex]) {
64
- this.shouldMonitorLoudness = true;
65
- this.playChunks();
66
- }
67
- else {
68
- // console.log('Playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
69
- setTimeout(() => {
70
- // console.log('Check again if really playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
71
- if (this.chunkQueue.length > this.currentIndex) {
72
- this.playChunks();
73
- }
74
- else {
75
- this.startedPlaying = false;
76
- this.shouldMonitorLoudness = false;
77
- }
78
- }, 1000);
79
- }
80
- });
81
- }
82
- stopPlayback() {
83
- // console.log('Stopping playback');
84
- // Implement logic to stop the current playback
85
- this.isPlaying = false;
86
- this.chunkQueue = [];
87
- this.startedPlaying = false;
88
- this.shouldMonitorLoudness = false;
89
- cancelAnimationFrame(this.handle);
90
- }
91
- playChunk(chunk) {
92
- // console.log({queue: this.chunkQueue})
93
- if (!chunk) {
94
- return Promise.resolve();
95
- }
96
- // console.log('Playing chunk', chunk);
97
- return new Promise((resolve) => {
98
- const source = this.audioContext.createBufferSource();
99
- this.audioContext.decodeAudioData(chunk.slice(0)).then((audioBuffer) => {
100
- source.buffer = audioBuffer;
101
- // Create a GainNode for volume control
102
- const gainNode = this.audioContext.createGain();
103
- gainNode.gain.value = this.volume;
104
- // Connect the source to the GainNode, then to the analyser node, then to the destination (speakers)
105
- source.connect(gainNode);
106
- gainNode.connect(this.analyser);
107
- this.analyser.connect(this.audioContext.destination);
108
- source.start(0);
109
- // console.log('Playing chunk', this.currentIndex);
110
- gainNode.gain.value = this.volume;
111
- source.onended = () => {
112
- // console.log('Chunk ended');
113
- resolve();
114
- };
115
- // Start monitoring loudness only once
116
- if (!this.isMonitoring) {
117
- this.isMonitoring = true;
118
- this.shouldMonitorLoudness = true;
119
- this.monitorLoudness();
120
- }
121
- });
122
- });
123
- }
124
- playAgain() {
125
- return __awaiter(this, void 0, void 0, function* () {
126
- console.log('Playing again');
127
- if (this.chunkQueue.length > 0 && !this.isPlaying) {
128
- this.playChunks();
129
- }
130
- });
131
- }
132
- monitorLoudness() {
133
- // Stop monitoring when the flag is false
134
- if (!this.shouldMonitorLoudness) {
135
- // console.log('Loudness monitoring stopped.');
136
- cancelAnimationFrame(this.handle);
137
- this.loudnessCallback(0);
138
- this.onEndOfSpeech();
139
- return;
140
- }
141
- // Get the time domain data from the analyser (this is a snapshot of the waveform)
142
- this.analyser.getByteTimeDomainData(this.dataArray);
143
- // Calculate the RMS (root mean square) of the waveform values to get the perceived loudness
144
- let sum = 0;
145
- for (let i = 0; i < this.dataArray.length; i++) {
146
- const value = this.dataArray[i] / 128.0 - 1.0; // Normalize between -1 and 1
147
- sum += value * value;
148
- }
149
- const rms = Math.sqrt(sum / this.dataArray.length);
150
- // Handle the case where RMS is 0 to avoid log10(0)
151
- if (rms === 0) {
152
- // console.log('Current loudness: Silent');
153
- }
154
- else {
155
- let loudnessInDb = 20 * Math.log10(rms); // Convert to dB
156
- // console.log('Current loudness:' + loudnessInDb);
157
- const minDb = -57;
158
- const maxDb = -15;
159
- if (loudnessInDb < minDb) {
160
- loudnessInDb = minDb;
161
- }
162
- if (loudnessInDb > maxDb) {
163
- loudnessInDb = maxDb;
164
- }
165
- const loudnessScale = ((loudnessInDb - minDb) / (maxDb - minDb)) * 100;
166
- // console.log("root:corrent loudness", loudnessScale);
167
- this.loudnessCallback(loudnessScale);
168
- }
169
- // Call this method again at regular intervals if you want continuous loudness monitoring
170
- this.handle = requestAnimationFrame(() => this.monitorLoudness());
171
- }
172
- reset() {
173
- // console.log('Resetting player');
174
- this.stopPlayback();
175
- this.currentIndex = 0;
176
- this.shouldMonitorLoudness = true;
177
- //reset to the beginning when the class gets initialized
178
- this.isMonitoring = false;
179
- this.isPlaying = false;
180
- this.init();
181
- }
182
- setOnEndOfSpeech(callback) {
183
- this.onEndOfSpeech = callback;
184
- }
185
- }
@@ -1,11 +0,0 @@
1
- interface Props {
2
- iconSize?: string;
3
- className?: string;
4
- disabled?: boolean;
5
- loading?: boolean;
6
- enablePushToTalk?: boolean;
7
- onRecordingStatusChange: (running: boolean) => void;
8
- onVoiceRecorded: (message: string) => void;
9
- }
10
- export declare const VoiceRecorder: import("react").ForwardRefExoticComponent<Props & import("react").RefAttributes<unknown>>;
11
- export {};