@rimori/client 1.3.1 → 1.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.prettierignore +35 -0
- package/README.md +77 -71
- package/dist/cli/scripts/init/dev-registration.d.ts +1 -1
- package/dist/cli/scripts/init/dev-registration.js +4 -4
- package/dist/cli/scripts/init/main.js +1 -1
- package/dist/cli/scripts/init/package-setup.d.ts +1 -1
- package/dist/cli/scripts/init/package-setup.js +3 -3
- package/dist/cli/scripts/init/router-transformer.js +19 -12
- package/dist/cli/scripts/init/vite-config.d.ts +2 -2
- package/dist/cli/scripts/init/vite-config.js +2 -2
- package/dist/cli/scripts/release/release-config-upload.js +9 -9
- package/dist/cli/scripts/release/release-db-update.d.ts +1 -1
- package/dist/cli/scripts/release/release-db-update.js +9 -9
- package/dist/cli/scripts/release/release-file-upload.js +2 -2
- package/dist/cli/scripts/release/release.js +2 -2
- package/dist/cli/types/DatabaseTypes.d.ts +2 -2
- package/dist/components/CRUDModal.d.ts +1 -1
- package/dist/components/CRUDModal.js +3 -3
- package/dist/components/MarkdownEditor.js +16 -16
- package/dist/components/Spinner.js +2 -2
- package/dist/components/ai/Assistant.js +7 -8
- package/dist/components/ai/Avatar.d.ts +2 -2
- package/dist/components/ai/Avatar.js +14 -7
- package/dist/components/ai/EmbeddedAssistent/AudioInputField.js +5 -6
- package/dist/components/ai/EmbeddedAssistent/CircleAudioAvatar.d.ts +1 -1
- package/dist/components/ai/EmbeddedAssistent/CircleAudioAvatar.js +1 -2
- package/dist/components/ai/EmbeddedAssistent/TTS/MessageSender.d.ts +1 -2
- package/dist/components/ai/EmbeddedAssistent/TTS/MessageSender.js +4 -2
- package/dist/components/ai/EmbeddedAssistent/TTS/Player.js +1 -1
- package/dist/components/ai/EmbeddedAssistent/VoiceRecoder.js +2 -3
- package/dist/components/audio/Playbutton.js +10 -7
- package/dist/components/components/ContextMenu.d.ts +1 -1
- package/dist/components/components/ContextMenu.js +19 -16
- package/dist/components.d.ts +10 -10
- package/dist/components.js +10 -10
- package/dist/core/controller/AIController.d.ts +2 -2
- package/dist/core/controller/AIController.js +20 -18
- package/dist/core/controller/ExerciseController.d.ts +52 -0
- package/dist/core/controller/ExerciseController.js +73 -0
- package/dist/core/controller/ObjectController.js +5 -5
- package/dist/core/controller/SettingsController.d.ts +22 -7
- package/dist/core/controller/SettingsController.js +73 -8
- package/dist/core/controller/SharedContentController.d.ts +3 -3
- package/dist/core/controller/SharedContentController.js +38 -20
- package/dist/core/controller/VoiceController.js +6 -4
- package/dist/core/core.d.ts +15 -14
- package/dist/core/core.js +7 -7
- package/dist/fromRimori/EventBus.js +23 -23
- package/dist/fromRimori/PluginTypes.d.ts +4 -4
- package/dist/hooks/UseChatHook.d.ts +3 -3
- package/dist/hooks/UseChatHook.js +9 -3
- package/dist/index.d.ts +10 -10
- package/dist/index.js +9 -9
- package/dist/plugin/AccomplishmentHandler.d.ts +5 -5
- package/dist/plugin/AccomplishmentHandler.js +31 -27
- package/dist/plugin/AudioController.d.ts +1 -1
- package/dist/plugin/AudioController.js +6 -6
- package/dist/plugin/Logger.d.ts +5 -0
- package/dist/plugin/Logger.js +65 -13
- package/dist/plugin/PluginController.d.ts +7 -1
- package/dist/plugin/PluginController.js +32 -27
- package/dist/plugin/RimoriClient.d.ts +39 -14
- package/dist/plugin/RimoriClient.js +60 -31
- package/dist/plugin/StandaloneClient.d.ts +1 -1
- package/dist/plugin/StandaloneClient.js +35 -16
- package/dist/plugin/ThemeSetter.js +4 -4
- package/dist/providers/PluginProvider.js +44 -14
- package/dist/utils/Language.js +57 -57
- package/dist/utils/PluginUtils.js +3 -3
- package/dist/utils/difficultyConverter.d.ts +1 -1
- package/dist/utils/difficultyConverter.js +1 -1
- package/dist/utils/endpoint.js +2 -2
- package/dist/worker/WorkerSetup.d.ts +1 -1
- package/dist/worker/WorkerSetup.js +6 -6
- package/eslint.config.js +53 -0
- package/example/docs/devdocs.md +50 -40
- package/example/docs/overview.md +1 -1
- package/example/docs/userdocs.md +4 -1
- package/example/rimori.config.ts +51 -49
- package/example/worker/vite.config.ts +3 -3
- package/example/worker/worker.ts +2 -2
- package/package.json +17 -4
- package/prettier.config.js +8 -0
- package/src/cli/scripts/init/dev-registration.ts +5 -8
- package/src/cli/scripts/init/env-setup.ts +1 -1
- package/src/cli/scripts/init/file-operations.ts +1 -1
- package/src/cli/scripts/init/html-cleaner.ts +2 -5
- package/src/cli/scripts/init/main.ts +16 -13
- package/src/cli/scripts/init/package-setup.ts +11 -15
- package/src/cli/scripts/init/router-transformer.ts +40 -37
- package/src/cli/scripts/init/tailwind-config.ts +17 -26
- package/src/cli/scripts/init/vite-config.ts +3 -3
- package/src/cli/scripts/release/release-config-upload.ts +11 -11
- package/src/cli/scripts/release/release-db-update.ts +12 -12
- package/src/cli/scripts/release/release-file-upload.ts +3 -3
- package/src/cli/scripts/release/release.ts +4 -4
- package/src/cli/types/DatabaseTypes.ts +7 -8
- package/src/components/CRUDModal.tsx +64 -48
- package/src/components/MarkdownEditor.tsx +58 -27
- package/src/components/Spinner.tsx +24 -17
- package/src/components/ai/Assistant.tsx +70 -70
- package/src/components/ai/Avatar.tsx +20 -16
- package/src/components/ai/EmbeddedAssistent/AudioInputField.tsx +63 -54
- package/src/components/ai/EmbeddedAssistent/CircleAudioAvatar.tsx +14 -5
- package/src/components/ai/EmbeddedAssistent/TTS/MessageSender.ts +75 -74
- package/src/components/ai/EmbeddedAssistent/TTS/Player.ts +177 -178
- package/src/components/ai/EmbeddedAssistent/VoiceRecoder.tsx +109 -94
- package/src/components/ai/utils.ts +4 -4
- package/src/components/audio/Playbutton.tsx +101 -93
- package/src/components/components/ContextMenu.tsx +47 -35
- package/src/components.ts +10 -10
- package/src/core/controller/AIController.ts +62 -50
- package/src/core/controller/ExerciseController.ts +98 -0
- package/src/core/controller/ObjectController.ts +15 -10
- package/src/core/controller/SettingsController.ts +89 -16
- package/src/core/controller/SharedContentController.ts +80 -44
- package/src/core/controller/VoiceController.ts +10 -8
- package/src/core/core.ts +15 -15
- package/src/fromRimori/EventBus.ts +76 -47
- package/src/fromRimori/PluginTypes.ts +26 -17
- package/src/fromRimori/readme.md +2 -2
- package/src/hooks/UseChatHook.ts +25 -15
- package/src/index.ts +10 -10
- package/src/plugin/AccomplishmentHandler.ts +53 -35
- package/src/plugin/AudioController.ts +18 -12
- package/src/plugin/Logger.ts +77 -19
- package/src/plugin/PluginController.ts +60 -44
- package/src/plugin/RimoriClient.ts +133 -69
- package/src/plugin/StandaloneClient.ts +51 -24
- package/src/plugin/ThemeSetter.ts +5 -5
- package/src/providers/PluginProvider.tsx +90 -36
- package/src/style.scss +3 -3
- package/src/utils/Language.ts +58 -58
- package/src/utils/PluginUtils.ts +16 -20
- package/src/utils/difficultyConverter.ts +2 -2
- package/src/utils/endpoint.ts +3 -2
- package/src/worker/WorkerSetup.ts +8 -9
- package/tsconfig.json +2 -4
- package/dist/components/LoggerExample.d.ts +0 -6
- package/dist/components/LoggerExample.js +0 -79
- package/dist/core/controller/AudioController.d.ts +0 -0
- package/dist/core/controller/AudioController.js +0 -1
- package/dist/hooks/UseLogger.d.ts +0 -30
- package/dist/hooks/UseLogger.js +0 -122
- package/dist/plugin/LoggerExample.d.ts +0 -16
- package/dist/plugin/LoggerExample.js +0 -140
- package/dist/utils/audioFormats.d.ts +0 -26
- package/dist/utils/audioFormats.js +0 -67
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import { Markdown } from 'tiptap-markdown';
|
|
2
|
-
import StarterKit from
|
|
3
|
-
import { PiCodeBlock } from
|
|
4
|
-
import { TbBlockquote } from
|
|
5
|
-
import { GoListOrdered } from
|
|
6
|
-
import { AiOutlineUnorderedList } from
|
|
7
|
-
import { EditorProvider, useCurrentEditor } from
|
|
8
|
-
import { LuHeading1, LuHeading2, LuHeading3 } from
|
|
9
|
-
import { FaBold, FaCode, FaItalic, FaParagraph, FaStrikethrough } from
|
|
2
|
+
import StarterKit from '@tiptap/starter-kit';
|
|
3
|
+
import { PiCodeBlock } from 'react-icons/pi';
|
|
4
|
+
import { TbBlockquote } from 'react-icons/tb';
|
|
5
|
+
import { GoListOrdered } from 'react-icons/go';
|
|
6
|
+
import { AiOutlineUnorderedList } from 'react-icons/ai';
|
|
7
|
+
import { EditorProvider, useCurrentEditor } from '@tiptap/react';
|
|
8
|
+
import { LuHeading1, LuHeading2, LuHeading3 } from 'react-icons/lu';
|
|
9
|
+
import { FaBold, FaCode, FaItalic, FaParagraph, FaStrikethrough } from 'react-icons/fa';
|
|
10
10
|
|
|
11
11
|
// This inplementation is rooted in the Tiptap editor basic example https://codesandbox.io/p/devbox/editor-9x9dkd
|
|
12
12
|
|
|
@@ -24,12 +24,12 @@ const EditorButton = ({ action, isActive, label, disabled }: EditorButtonProps)
|
|
|
24
24
|
return null;
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
-
if (action.includes(
|
|
27
|
+
if (action.includes('heading')) {
|
|
28
28
|
const level = parseInt(action[action.length - 1]);
|
|
29
29
|
return (
|
|
30
30
|
<button
|
|
31
31
|
onClick={() => editor.chain().focus().toggleHeading({ level: level }).run()}
|
|
32
|
-
className={`pl-2 ${isActive ?
|
|
32
|
+
className={`pl-2 ${isActive ? 'is-active' : ''}`}
|
|
33
33
|
>
|
|
34
34
|
{label}
|
|
35
35
|
</button>
|
|
@@ -40,7 +40,7 @@ const EditorButton = ({ action, isActive, label, disabled }: EditorButtonProps)
|
|
|
40
40
|
<button
|
|
41
41
|
onClick={() => editor.chain().focus()[action]().run()}
|
|
42
42
|
disabled={disabled ? !editor.can().chain().focus()[action]().run() : false}
|
|
43
|
-
className={`pl-2 ${isActive ?
|
|
43
|
+
className={`pl-2 ${isActive ? 'is-active' : ''}`}
|
|
44
44
|
>
|
|
45
45
|
{label}
|
|
46
46
|
</button>
|
|
@@ -56,18 +56,46 @@ const MenuBar = () => {
|
|
|
56
56
|
|
|
57
57
|
return (
|
|
58
58
|
<div className="bg-gray-400 dark:bg-gray-800 dark:text-white text-lg flex flex-row flex-wrap items-center p-1">
|
|
59
|
-
<EditorButton action="toggleBold" isActive={editor.isActive(
|
|
60
|
-
<EditorButton action="toggleItalic" isActive={editor.isActive(
|
|
61
|
-
<EditorButton action="toggleStrike" isActive={editor.isActive(
|
|
62
|
-
<EditorButton action="toggleCode" isActive={editor.isActive(
|
|
63
|
-
<EditorButton action="setParagraph" isActive={editor.isActive(
|
|
64
|
-
<EditorButton
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
<EditorButton
|
|
70
|
-
|
|
59
|
+
<EditorButton action="toggleBold" isActive={editor.isActive('bold')} label={<FaBold />} disabled />
|
|
60
|
+
<EditorButton action="toggleItalic" isActive={editor.isActive('italic')} label={<FaItalic />} disabled />
|
|
61
|
+
<EditorButton action="toggleStrike" isActive={editor.isActive('strike')} label={<FaStrikethrough />} disabled />
|
|
62
|
+
<EditorButton action="toggleCode" isActive={editor.isActive('code')} label={<FaCode />} disabled />
|
|
63
|
+
<EditorButton action="setParagraph" isActive={editor.isActive('paragraph')} label={<FaParagraph />} />
|
|
64
|
+
<EditorButton
|
|
65
|
+
action="setHeading1"
|
|
66
|
+
isActive={editor.isActive('heading', { level: 1 })}
|
|
67
|
+
label={<LuHeading1 size={'24px'} />}
|
|
68
|
+
/>
|
|
69
|
+
<EditorButton
|
|
70
|
+
action="setHeading2"
|
|
71
|
+
isActive={editor.isActive('heading', { level: 2 })}
|
|
72
|
+
label={<LuHeading2 size={'24px'} />}
|
|
73
|
+
/>
|
|
74
|
+
<EditorButton
|
|
75
|
+
action="setHeading3"
|
|
76
|
+
isActive={editor.isActive('heading', { level: 3 })}
|
|
77
|
+
label={<LuHeading3 size={'24px'} />}
|
|
78
|
+
/>
|
|
79
|
+
<EditorButton
|
|
80
|
+
action="toggleBulletList"
|
|
81
|
+
isActive={editor.isActive('bulletList')}
|
|
82
|
+
label={<AiOutlineUnorderedList size={'24px'} />}
|
|
83
|
+
/>
|
|
84
|
+
<EditorButton
|
|
85
|
+
action="toggleOrderedList"
|
|
86
|
+
isActive={editor.isActive('orderedList')}
|
|
87
|
+
label={<GoListOrdered size={'24px'} />}
|
|
88
|
+
/>
|
|
89
|
+
<EditorButton
|
|
90
|
+
action="toggleCodeBlock"
|
|
91
|
+
isActive={editor.isActive('codeBlock')}
|
|
92
|
+
label={<PiCodeBlock size={'24px'} />}
|
|
93
|
+
/>
|
|
94
|
+
<EditorButton
|
|
95
|
+
action="toggleBlockquote"
|
|
96
|
+
isActive={editor.isActive('blockquote')}
|
|
97
|
+
label={<TbBlockquote size={'24px'} />}
|
|
98
|
+
/>
|
|
71
99
|
</div>
|
|
72
100
|
);
|
|
73
101
|
};
|
|
@@ -76,12 +104,12 @@ const extensions = [
|
|
|
76
104
|
StarterKit.configure({
|
|
77
105
|
bulletList: {
|
|
78
106
|
HTMLAttributes: {
|
|
79
|
-
class:
|
|
107
|
+
class: 'list-disc list-inside dark:text-white p-1 mt-1 [&_li]:mb-1 [&_p]:inline m-0',
|
|
80
108
|
},
|
|
81
109
|
},
|
|
82
110
|
orderedList: {
|
|
83
111
|
HTMLAttributes: {
|
|
84
|
-
className:
|
|
112
|
+
className: 'list-decimal list-inside dark:text-white p-1 mt-1 [&_li]:mb-1 [&_p]:inline m-0',
|
|
85
113
|
},
|
|
86
114
|
},
|
|
87
115
|
}),
|
|
@@ -97,9 +125,12 @@ interface Props {
|
|
|
97
125
|
|
|
98
126
|
export const MarkdownEditor = (props: Props) => {
|
|
99
127
|
return (
|
|
100
|
-
<div
|
|
128
|
+
<div
|
|
129
|
+
className={'text-md border border-gray-800 overflow-hidden ' + props.className}
|
|
130
|
+
style={{ borderWidth: props.editable ? 1 : 0 }}
|
|
131
|
+
>
|
|
101
132
|
<EditorProvider
|
|
102
|
-
key={(props.editable ?
|
|
133
|
+
key={(props.editable ? 'editable' : 'readonly') + props.content}
|
|
103
134
|
slotBefore={props.editable ? <MenuBar /> : null}
|
|
104
135
|
extensions={extensions}
|
|
105
136
|
content={props.content}
|
|
@@ -1,22 +1,29 @@
|
|
|
1
1
|
import React from 'react';
|
|
2
2
|
|
|
3
3
|
interface SpinnerProps {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
4
|
+
text?: string;
|
|
5
|
+
size?: string;
|
|
6
|
+
className?: string;
|
|
7
7
|
}
|
|
8
8
|
|
|
9
|
-
export const Spinner: React.FC<SpinnerProps> = ({ text, className, size =
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
9
|
+
export const Spinner: React.FC<SpinnerProps> = ({ text, className, size = '30px' }) => {
|
|
10
|
+
return (
|
|
11
|
+
<div className={'flex items-center space-x-2 ' + className}>
|
|
12
|
+
<svg
|
|
13
|
+
style={{ width: size, height: size }}
|
|
14
|
+
className="animate-spin -ml-1 mr-3 h-5 w-5 text-white"
|
|
15
|
+
xmlns="http://www.w3.org/2000/svg"
|
|
16
|
+
fill="none"
|
|
17
|
+
viewBox="0 0 24 24"
|
|
18
|
+
>
|
|
19
|
+
<circle className="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" strokeWidth="4"></circle>
|
|
20
|
+
<path
|
|
21
|
+
className="opacity-75"
|
|
22
|
+
fill="currentColor"
|
|
23
|
+
d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"
|
|
24
|
+
></path>
|
|
25
|
+
</svg>
|
|
26
|
+
{text && <span className="">{text}</span>}
|
|
27
|
+
</div>
|
|
28
|
+
);
|
|
29
|
+
};
|
|
@@ -8,89 +8,89 @@ import { useRimori } from '../../components';
|
|
|
8
8
|
import { FirstMessages, getFirstMessages } from './utils';
|
|
9
9
|
|
|
10
10
|
interface Props {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
11
|
+
voiceId: any;
|
|
12
|
+
avatarImageUrl: string;
|
|
13
|
+
onComplete: (result: any) => void;
|
|
14
|
+
autoStartConversation?: FirstMessages;
|
|
15
15
|
}
|
|
16
16
|
|
|
17
17
|
export function AssistantChat({ avatarImageUrl, voiceId, onComplete, autoStartConversation }: Props) {
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
18
|
+
const [oralCommunication, setOralCommunication] = React.useState(true);
|
|
19
|
+
const { ai: llm, event } = useRimori();
|
|
20
|
+
const sender = useMemo(() => new MessageSender(llm.getVoice, voiceId), []);
|
|
21
|
+
const { messages, append, isLoading, setMessages } = useChat();
|
|
22
22
|
|
|
23
|
-
|
|
23
|
+
const lastAssistantMessage = [...messages].filter((m) => m.role === 'assistant').pop()?.content;
|
|
24
24
|
|
|
25
|
-
|
|
26
|
-
|
|
25
|
+
useEffect(() => {
|
|
26
|
+
sender.setOnLoudnessChange((value: number) => event.emit('self.avatar.triggerLoudness', { loudness: value }));
|
|
27
27
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
setMessages(getFirstMessages(autoStartConversation));
|
|
33
|
-
// append([{ role: 'user', content: autoStartConversation.userMessage }]);
|
|
28
|
+
if (!autoStartConversation) {
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
34
31
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
sender.handleNewText(autoStartConversation.assistantMessage, isLoading);
|
|
38
|
-
}
|
|
39
|
-
}, []);
|
|
32
|
+
setMessages(getFirstMessages(autoStartConversation));
|
|
33
|
+
// append([{ role: 'user', content: autoStartConversation.userMessage }]);
|
|
40
34
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
sender.handleNewText(message, isLoading);
|
|
47
|
-
}, [messages, isLoading]);
|
|
35
|
+
if (autoStartConversation.assistantMessage) {
|
|
36
|
+
// console.log("autostartmessages", { autoStartConversation, isLoading });
|
|
37
|
+
sender.handleNewText(autoStartConversation.assistantMessage, isLoading);
|
|
38
|
+
}
|
|
39
|
+
}, []);
|
|
48
40
|
|
|
49
|
-
|
|
41
|
+
useEffect(() => {
|
|
42
|
+
let message = lastAssistantMessage;
|
|
43
|
+
if (message !== messages[messages.length - 1]?.content) {
|
|
44
|
+
message = undefined;
|
|
45
|
+
}
|
|
46
|
+
sender.handleNewText(message, isLoading);
|
|
47
|
+
}, [messages, isLoading]);
|
|
50
48
|
|
|
51
|
-
|
|
52
|
-
console.log("lastMessage", lastMessage);
|
|
53
|
-
const toolInvocations = lastMessage?.toolCalls;
|
|
54
|
-
if (toolInvocations && toolInvocations.length > 0) {
|
|
55
|
-
console.log("toolInvocations", toolInvocations);
|
|
56
|
-
onComplete(toolInvocations[0].args);
|
|
57
|
-
}
|
|
58
|
-
}, [lastMessage]);
|
|
49
|
+
const lastMessage = messages[messages.length - 1];
|
|
59
50
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
51
|
+
useEffect(() => {
|
|
52
|
+
console.log('lastMessage', lastMessage);
|
|
53
|
+
const toolInvocations = lastMessage?.toolCalls;
|
|
54
|
+
if (toolInvocations && toolInvocations.length > 0) {
|
|
55
|
+
console.log('toolInvocations', toolInvocations);
|
|
56
|
+
onComplete(toolInvocations[0].args);
|
|
57
|
+
}
|
|
58
|
+
}, [lastMessage]);
|
|
63
59
|
|
|
64
|
-
|
|
60
|
+
if (lastMessage?.toolCalls && lastMessage.toolCalls.length > 0) {
|
|
61
|
+
console.log('lastMessage test2', lastMessage);
|
|
62
|
+
const args = lastMessage.toolCalls[0].args;
|
|
65
63
|
|
|
66
|
-
|
|
67
|
-
<h1 className='text-center mt-5 mb-5'>
|
|
68
|
-
{success ? "Great job!" : "You failed"}
|
|
69
|
-
</h1>
|
|
70
|
-
<p>{args.improvementHints}</p>
|
|
71
|
-
</div>
|
|
72
|
-
}
|
|
64
|
+
const success = args.explanationUnderstood === 'TRUE' || args.studentKnowsTopic === 'TRUE';
|
|
73
65
|
|
|
74
66
|
return (
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
<Markdown>{lastAssistantMessage}</Markdown>
|
|
80
|
-
</div>}
|
|
81
|
-
</div>
|
|
82
|
-
<AudioInputField
|
|
83
|
-
blockSubmission={isLoading}
|
|
84
|
-
onSubmit={message => {
|
|
85
|
-
append([{ role: 'user', content: message, id: messages.length.toString() }]);
|
|
86
|
-
}}
|
|
87
|
-
onAudioControl={voice => {
|
|
88
|
-
setOralCommunication(voice);
|
|
89
|
-
sender.setVolume(voice ? 1 : 0);
|
|
90
|
-
}} />
|
|
91
|
-
</div>
|
|
67
|
+
<div className="px-5 pt-5 overflow-y-auto text-center" style={{ height: '478px' }}>
|
|
68
|
+
<h1 className="text-center mt-5 mb-5">{success ? 'Great job!' : 'You failed'}</h1>
|
|
69
|
+
<p>{args.improvementHints}</p>
|
|
70
|
+
</div>
|
|
92
71
|
);
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
return (
|
|
75
|
+
<div>
|
|
76
|
+
{oralCommunication && <CircleAudioAvatar imageUrl={avatarImageUrl} className="mx-auto my-10" />}
|
|
77
|
+
<div className="w-full">
|
|
78
|
+
{lastAssistantMessage && (
|
|
79
|
+
<div className="px-5 pt-5 overflow-y-auto remirror-theme" style={{ height: '4k78px' }}>
|
|
80
|
+
<Markdown>{lastAssistantMessage}</Markdown>
|
|
81
|
+
</div>
|
|
82
|
+
)}
|
|
83
|
+
</div>
|
|
84
|
+
<AudioInputField
|
|
85
|
+
blockSubmission={isLoading}
|
|
86
|
+
onSubmit={(message) => {
|
|
87
|
+
append([{ role: 'user', content: message, id: messages.length.toString() }]);
|
|
88
|
+
}}
|
|
89
|
+
onAudioControl={(voice) => {
|
|
90
|
+
setOralCommunication(voice);
|
|
91
|
+
sender.setVolume(voice ? 1 : 0);
|
|
92
|
+
}}
|
|
93
|
+
/>
|
|
94
|
+
</div>
|
|
95
|
+
);
|
|
96
|
+
}
|
|
@@ -9,7 +9,7 @@ import { getFirstMessages } from './utils';
|
|
|
9
9
|
import { FirstMessages } from './utils';
|
|
10
10
|
|
|
11
11
|
interface Props {
|
|
12
|
-
voiceId:
|
|
12
|
+
voiceId: string;
|
|
13
13
|
agentTools: Tool[];
|
|
14
14
|
avatarImageUrl: string;
|
|
15
15
|
circleSize?: string;
|
|
@@ -26,8 +26,8 @@ export function Avatar({
|
|
|
26
26
|
autoStartConversation,
|
|
27
27
|
children,
|
|
28
28
|
isDarkTheme = false,
|
|
29
|
-
circleSize =
|
|
30
|
-
className
|
|
29
|
+
circleSize = '300px',
|
|
30
|
+
className,
|
|
31
31
|
}: Props) {
|
|
32
32
|
const { ai, event } = useRimori();
|
|
33
33
|
const [agentReplying, setAgentReplying] = useState(false);
|
|
@@ -36,7 +36,7 @@ export function Avatar({
|
|
|
36
36
|
const { messages, append, isLoading, lastMessage, setMessages } = useChat(agentTools);
|
|
37
37
|
|
|
38
38
|
useEffect(() => {
|
|
39
|
-
console.log(
|
|
39
|
+
console.log('messages', messages);
|
|
40
40
|
}, [messages]);
|
|
41
41
|
|
|
42
42
|
useEffect(() => {
|
|
@@ -44,6 +44,7 @@ export function Avatar({
|
|
|
44
44
|
}, [isLoading]);
|
|
45
45
|
|
|
46
46
|
useEffect(() => {
|
|
47
|
+
if (!voiceId) return; //at the beginning when being mounted the voiceId is undefined
|
|
47
48
|
sender.setOnLoudnessChange((value) => event.emit('self.avatar.triggerLoudness', { loudness: value }));
|
|
48
49
|
sender.setOnEndOfSpeech(() => setAgentReplying(false));
|
|
49
50
|
|
|
@@ -58,13 +59,13 @@ export function Avatar({
|
|
|
58
59
|
} else if (autoStartConversation.userMessage) {
|
|
59
60
|
append([{ role: 'user', content: autoStartConversation.userMessage, id: messages.length.toString() }]);
|
|
60
61
|
}
|
|
61
|
-
}, [autoStartConversation]);
|
|
62
|
+
}, [autoStartConversation, voiceId]);
|
|
62
63
|
|
|
63
64
|
useEffect(() => {
|
|
64
65
|
if (lastMessage?.role === 'assistant') {
|
|
65
66
|
sender.handleNewText(lastMessage.content, isLoading);
|
|
66
67
|
if (lastMessage.toolCalls) {
|
|
67
|
-
console.log("unlocking mic",lastMessage)
|
|
68
|
+
// console.log("unlocking mic", lastMessage)
|
|
68
69
|
setAgentReplying(false);
|
|
69
70
|
setIsProcessingMessage(false);
|
|
70
71
|
}
|
|
@@ -73,23 +74,26 @@ export function Avatar({
|
|
|
73
74
|
|
|
74
75
|
return (
|
|
75
76
|
<div className={`md:pb-8 ${className || ''}`}>
|
|
76
|
-
<CircleAudioAvatar
|
|
77
|
-
width={circleSize}
|
|
78
|
-
className='mx-auto'
|
|
79
|
-
imageUrl={avatarImageUrl}
|
|
80
|
-
isDarkTheme={isDarkTheme} />
|
|
77
|
+
<CircleAudioAvatar width={circleSize} className="mx-auto" imageUrl={avatarImageUrl} isDarkTheme={isDarkTheme} />
|
|
81
78
|
{children}
|
|
82
79
|
<VoiceRecorder
|
|
83
|
-
iconSize=
|
|
84
|
-
className=
|
|
80
|
+
iconSize="30"
|
|
81
|
+
className="w-16 h-16 shadow-lg rounded-full bg-gray-400 dark:bg-gray-800"
|
|
85
82
|
disabled={agentReplying}
|
|
86
83
|
loading={isProcessingMessage}
|
|
87
84
|
enablePushToTalk={true}
|
|
88
85
|
onVoiceRecorded={(message) => {
|
|
89
86
|
setAgentReplying(true);
|
|
90
|
-
append([
|
|
87
|
+
append([
|
|
88
|
+
{
|
|
89
|
+
role: 'user',
|
|
90
|
+
content: 'Message(' + Math.floor((messages.length + 1) / 2) + '): ' + message,
|
|
91
|
+
id: messages.length.toString(),
|
|
92
|
+
},
|
|
93
|
+
]);
|
|
91
94
|
}}
|
|
92
|
-
onRecordingStatusChange={(running) => !running && setIsProcessingMessage(true)}
|
|
95
|
+
onRecordingStatusChange={(running) => !running && setIsProcessingMessage(true)}
|
|
96
|
+
/>
|
|
93
97
|
</div>
|
|
94
98
|
);
|
|
95
|
-
}
|
|
99
|
+
}
|
|
@@ -1,64 +1,73 @@
|
|
|
1
1
|
import React, { useState } from 'react';
|
|
2
2
|
import { VoiceRecorder } from './VoiceRecoder';
|
|
3
|
-
import { BiSolidRightArrow } from
|
|
4
|
-
import { HiMiniSpeakerXMark, HiMiniSpeakerWave } from
|
|
3
|
+
import { BiSolidRightArrow } from 'react-icons/bi';
|
|
4
|
+
import { HiMiniSpeakerXMark, HiMiniSpeakerWave } from 'react-icons/hi2';
|
|
5
5
|
|
|
6
6
|
interface AudioInputFieldProps {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
7
|
+
onSubmit: (text: string) => void;
|
|
8
|
+
onAudioControl?: (voice: boolean) => void;
|
|
9
|
+
blockSubmission?: boolean;
|
|
10
10
|
}
|
|
11
11
|
|
|
12
12
|
export function AudioInputField({ onSubmit, onAudioControl, blockSubmission = false }: AudioInputFieldProps) {
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
const [text, setText] = useState('');
|
|
14
|
+
const [audioEnabled, setAudioEnabled] = useState(true);
|
|
15
15
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
16
|
+
const handleSubmit = (manualText?: string) => {
|
|
17
|
+
if (blockSubmission) return;
|
|
18
|
+
const sendableText = manualText || text;
|
|
19
|
+
if (sendableText.trim()) {
|
|
20
|
+
onSubmit(sendableText);
|
|
21
|
+
setTimeout(() => {
|
|
22
|
+
setText('');
|
|
23
|
+
}, 100);
|
|
24
|
+
}
|
|
25
|
+
};
|
|
26
26
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
27
|
+
const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
|
|
28
|
+
if (blockSubmission) return;
|
|
29
|
+
if (e.key === 'Enter' && e.ctrlKey) {
|
|
30
|
+
setText(text + '\n');
|
|
31
|
+
} else if (e.key === 'Enter') {
|
|
32
|
+
handleSubmit();
|
|
33
|
+
}
|
|
34
|
+
};
|
|
35
35
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
36
|
+
return (
|
|
37
|
+
<div className="flex items-center bg-gray-600 pt-2 pb-2 p-2">
|
|
38
|
+
{onAudioControl && (
|
|
39
|
+
<button
|
|
40
|
+
onClick={() => {
|
|
41
|
+
onAudioControl(!audioEnabled);
|
|
42
|
+
setAudioEnabled(!audioEnabled);
|
|
43
|
+
}}
|
|
44
|
+
className="cursor-default"
|
|
45
|
+
>
|
|
46
|
+
{audioEnabled ? (
|
|
47
|
+
<HiMiniSpeakerWave className="w-9 h-9 cursor-pointer" />
|
|
48
|
+
) : (
|
|
49
|
+
<HiMiniSpeakerXMark className="w-9 h-9 cursor-pointer" />
|
|
50
|
+
)}
|
|
51
|
+
</button>
|
|
52
|
+
)}
|
|
53
|
+
<VoiceRecorder
|
|
54
|
+
onRecordingStatusChange={() => {}}
|
|
55
|
+
onVoiceRecorded={(m: string) => {
|
|
56
|
+
console.log('onVoiceRecorded', m);
|
|
57
|
+
handleSubmit(m);
|
|
58
|
+
}}
|
|
59
|
+
/>
|
|
60
|
+
<textarea
|
|
61
|
+
value={text}
|
|
62
|
+
onChange={(e) => setText(e.target.value)}
|
|
63
|
+
onKeyDown={handleKeyDown}
|
|
64
|
+
className="flex-1 border-none rounded-lg p-2 text-gray-800 focus::outline-none"
|
|
65
|
+
placeholder="Type a message..."
|
|
66
|
+
disabled={blockSubmission}
|
|
67
|
+
/>
|
|
68
|
+
<button onClick={() => handleSubmit()} className="cursor-default" disabled={blockSubmission}>
|
|
69
|
+
<BiSolidRightArrow className="w-9 h-10 cursor-pointer" />
|
|
70
|
+
</button>
|
|
71
|
+
</div>
|
|
72
|
+
);
|
|
73
|
+
}
|
|
@@ -8,7 +8,12 @@ interface CircleAudioAvatarProps {
|
|
|
8
8
|
isDarkTheme?: boolean;
|
|
9
9
|
}
|
|
10
10
|
|
|
11
|
-
export function CircleAudioAvatar({
|
|
11
|
+
export function CircleAudioAvatar({
|
|
12
|
+
imageUrl,
|
|
13
|
+
className,
|
|
14
|
+
isDarkTheme = false,
|
|
15
|
+
width = '150px',
|
|
16
|
+
}: CircleAudioAvatarProps) {
|
|
12
17
|
const canvasRef = useRef<HTMLCanvasElement>(null);
|
|
13
18
|
const currentLoudnessRef = useRef(0);
|
|
14
19
|
const targetLoudnessRef = useRef(0);
|
|
@@ -31,7 +36,7 @@ export function CircleAudioAvatar({ imageUrl, className, isDarkTheme = false, wi
|
|
|
31
36
|
if (currentLoudnessRef.current > targetLoudnessRef.current) {
|
|
32
37
|
currentLoudnessRef.current = Math.max(
|
|
33
38
|
targetLoudnessRef.current,
|
|
34
|
-
currentLoudnessRef.current - decayRate * currentLoudnessRef.current
|
|
39
|
+
currentLoudnessRef.current - decayRate * currentLoudnessRef.current,
|
|
35
40
|
);
|
|
36
41
|
} else {
|
|
37
42
|
currentLoudnessRef.current = targetLoudnessRef.current;
|
|
@@ -63,7 +68,12 @@ export function CircleAudioAvatar({ imageUrl, className, isDarkTheme = false, wi
|
|
|
63
68
|
}
|
|
64
69
|
}, [imageUrl]);
|
|
65
70
|
|
|
66
|
-
const draw = (
|
|
71
|
+
const draw = (
|
|
72
|
+
ctx: CanvasRenderingContext2D,
|
|
73
|
+
canvas: HTMLCanvasElement,
|
|
74
|
+
image: HTMLImageElement,
|
|
75
|
+
loudness: number,
|
|
76
|
+
) => {
|
|
67
77
|
if (canvas && ctx) {
|
|
68
78
|
ctx.clearRect(0, 0, canvas.width, canvas.height);
|
|
69
79
|
|
|
@@ -94,5 +104,4 @@ export function CircleAudioAvatar({ imageUrl, className, isDarkTheme = false, wi
|
|
|
94
104
|
};
|
|
95
105
|
|
|
96
106
|
return <canvas ref={canvasRef} className={className} width={500} height={500} style={{ width }} />;
|
|
97
|
-
}
|
|
98
|
-
|
|
107
|
+
}
|