@rimori/client 1.0.5 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -0
- package/dist/components/MarkdownEditor.js +6 -4
- package/dist/components/PluginController.d.ts +21 -0
- package/dist/components/PluginController.js +116 -0
- package/dist/components/ai/Assistant.js +1 -1
- package/dist/components/ai/Avatar.d.ts +5 -3
- package/dist/components/ai/Avatar.js +14 -6
- package/dist/components/ai/EmbeddedAssistent/CircleAudioAvatar.d.ts +2 -1
- package/dist/components/ai/EmbeddedAssistent/CircleAudioAvatar.js +35 -14
- package/dist/components/ai/EmbeddedAssistent/TTS/MessageSender.d.ts +1 -0
- package/dist/components/ai/EmbeddedAssistent/TTS/MessageSender.js +3 -0
- package/dist/components/ai/EmbeddedAssistent/TTS/Player.d.ts +2 -0
- package/dist/components/ai/EmbeddedAssistent/TTS/Player.js +5 -0
- package/dist/components/ai/EmbeddedAssistent/VoiceRecoder.d.ts +3 -0
- package/dist/components/ai/EmbeddedAssistent/VoiceRecoder.js +41 -5
- package/dist/components/ai/utils.d.ts +1 -1
- package/dist/components.d.ts +1 -0
- package/dist/components.js +1 -0
- package/dist/controller/AIController.js +2 -1
- package/dist/controller/SettingsController.d.ts +15 -15
- package/dist/controller/SettingsController.js +15 -16
- package/dist/controller/SharedContentController.d.ts +58 -11
- package/dist/controller/SharedContentController.js +161 -26
- package/dist/controller/SidePluginController.d.ts +1 -12
- package/dist/core/components/ContextMenu.d.ts +10 -0
- package/dist/core/components/ContextMenu.js +93 -0
- package/dist/core.d.ts +2 -0
- package/dist/core.js +2 -0
- package/dist/hooks/UseChatHook.d.ts +1 -1
- package/dist/plugin/AccomplishmentHandler.d.ts +38 -0
- package/dist/plugin/AccomplishmentHandler.js +108 -0
- package/dist/plugin/ContextMenu.d.ts +17 -0
- package/dist/plugin/ContextMenu.js +45 -0
- package/dist/plugin/PluginController.js +6 -3
- package/dist/plugin/RimoriClient.d.ts +92 -65
- package/dist/plugin/RimoriClient.js +105 -75
- package/dist/plugin/ThemeSetter.js +2 -2
- package/dist/plugin/fromRimori/EventBus.d.ts +6 -3
- package/dist/plugin/fromRimori/EventBus.js +15 -9
- package/dist/plugin/fromRimori/PluginTypes.d.ts +51 -0
- package/dist/plugin/fromRimori/PluginTypes.js +1 -0
- package/dist/providers/PluginController.d.ts +21 -0
- package/dist/providers/PluginController.js +116 -0
- package/dist/providers/PluginProvider.js +26 -73
- package/dist/types/Actions.d.ts +4 -0
- package/dist/types/Actions.js +1 -0
- package/dist/utils/Language.d.ts +66 -0
- package/dist/utils/Language.js +67 -0
- package/dist/utils/difficultyConverter.d.ts +1 -0
- package/dist/utils/difficultyConverter.js +3 -0
- package/dist/worker/WorkerSetup.js +4 -4
- package/package.json +2 -3
- package/src/components/MarkdownEditor.tsx +78 -76
- package/src/components/ai/Assistant.tsx +1 -1
- package/src/components/ai/Avatar.tsx +65 -48
- package/src/components/ai/EmbeddedAssistent/CircleAudioAvatar.tsx +81 -58
- package/src/components/ai/EmbeddedAssistent/TTS/MessageSender.ts +4 -0
- package/src/components/ai/EmbeddedAssistent/TTS/Player.ts +6 -0
- package/src/components/ai/EmbeddedAssistent/VoiceRecoder.tsx +51 -8
- package/src/components/ai/utils.ts +1 -1
- package/src/components.ts +2 -1
- package/src/controller/AIController.ts +2 -1
- package/src/controller/SettingsController.ts +83 -84
- package/src/controller/SharedContentController.ts +214 -53
- package/src/controller/SidePluginController.ts +1 -13
- package/src/core/components/ContextMenu.tsx +123 -0
- package/src/core.ts +3 -1
- package/src/hooks/UseChatHook.ts +17 -17
- package/src/plugin/AccomplishmentHandler.ts +165 -0
- package/src/plugin/PluginController.ts +105 -103
- package/src/plugin/RimoriClient.ts +267 -250
- package/src/plugin/ThemeSetter.ts +2 -2
- package/src/plugin/fromRimori/EventBus.ts +23 -12
- package/src/plugin/fromRimori/PluginTypes.ts +67 -0
- package/src/providers/PluginProvider.tsx +63 -110
- package/src/types/Actions.ts +6 -0
- package/src/utils/Language.ts +70 -0
- package/src/utils/difficultyConverter.ts +4 -0
- package/src/worker/WorkerSetup.ts +4 -4
- package/dist/components/avatar/Assistant.d.ts +0 -9
- package/dist/components/avatar/Assistant.js +0 -59
- package/dist/components/avatar/Avatar.d.ts +0 -12
- package/dist/components/avatar/Avatar.js +0 -42
- package/dist/components/avatar/EmbeddedAssistent/AudioInputField.d.ts +0 -7
- package/dist/components/avatar/EmbeddedAssistent/AudioInputField.js +0 -38
- package/dist/components/avatar/EmbeddedAssistent/CircleAudioAvatar.d.ts +0 -7
- package/dist/components/avatar/EmbeddedAssistent/CircleAudioAvatar.js +0 -59
- package/dist/components/avatar/EmbeddedAssistent/TTS/MessageSender.d.ts +0 -19
- package/dist/components/avatar/EmbeddedAssistent/TTS/MessageSender.js +0 -84
- package/dist/components/avatar/EmbeddedAssistent/TTS/Player.d.ts +0 -25
- package/dist/components/avatar/EmbeddedAssistent/TTS/Player.js +0 -180
- package/dist/components/avatar/EmbeddedAssistent/VoiceRecoder.d.ts +0 -7
- package/dist/components/avatar/EmbeddedAssistent/VoiceRecoder.js +0 -45
- package/dist/components/avatar/utils.d.ts +0 -6
- package/dist/components/avatar/utils.js +0 -14
|
@@ -23,7 +23,7 @@ export function AssistantChat({ avatarImageUrl, voiceId, onComplete, autoStartCo
|
|
|
23
23
|
const lastAssistantMessage = [...messages].filter((m) => m.role === 'assistant').pop()?.content;
|
|
24
24
|
|
|
25
25
|
useEffect(() => {
|
|
26
|
-
sender.setOnLoudnessChange((value: number) => event.emit('self.avatar.triggerLoudness', value));
|
|
26
|
+
sender.setOnLoudnessChange((value: number) => event.emit('self.avatar.triggerLoudness', { loudness: value }));
|
|
27
27
|
|
|
28
28
|
if (!autoStartConversation) {
|
|
29
29
|
return;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Tool } from '../../core';
|
|
2
|
-
import { useEffect, useMemo } from 'react';
|
|
2
|
+
import { useEffect, useMemo, useState } from 'react';
|
|
3
3
|
import { VoiceRecorder } from './EmbeddedAssistent/VoiceRecoder';
|
|
4
4
|
import { MessageSender } from './EmbeddedAssistent/TTS/MessageSender';
|
|
5
5
|
import { CircleAudioAvatar } from './EmbeddedAssistent/CircleAudioAvatar';
|
|
@@ -9,53 +9,70 @@ import { getFirstMessages } from './utils';
|
|
|
9
9
|
import { FirstMessages } from './utils';
|
|
10
10
|
|
|
11
11
|
interface Props {
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
12
|
+
voiceId: any;
|
|
13
|
+
agentTools: Tool[];
|
|
14
|
+
avatarImageUrl: string;
|
|
15
|
+
circleSize?: string;
|
|
16
|
+
isDarkTheme?: boolean;
|
|
17
|
+
children?: React.ReactNode;
|
|
18
|
+
autoStartConversation?: FirstMessages;
|
|
17
19
|
}
|
|
18
20
|
|
|
19
|
-
export function Avatar({ avatarImageUrl, voiceId,
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
21
|
+
export function Avatar({ avatarImageUrl, voiceId, agentTools, autoStartConversation, children, isDarkTheme = false, circleSize = "300px" }: Props) {
|
|
22
|
+
const { llm, event } = usePlugin();
|
|
23
|
+
const [agentReplying, setAgentReplying] = useState(false);
|
|
24
|
+
const [isProcessingMessage, setIsProcessingMessage] = useState(false);
|
|
25
|
+
const sender = useMemo(() => new MessageSender(llm.getVoice, voiceId), []);
|
|
26
|
+
const { messages, append, isLoading, lastMessage, setMessages } = useChat(agentTools);
|
|
27
|
+
|
|
28
|
+
useEffect(() => {
|
|
29
|
+
console.log("messages", messages);
|
|
30
|
+
}, [messages]);
|
|
31
|
+
|
|
32
|
+
useEffect(() => {
|
|
33
|
+
if (!isLoading) setIsProcessingMessage(false);
|
|
34
|
+
}, [isLoading]);
|
|
35
|
+
|
|
36
|
+
useEffect(() => {
|
|
37
|
+
sender.setOnLoudnessChange((value) => event.emit('self.avatar.triggerLoudness', { loudness: value }));
|
|
38
|
+
sender.setOnEndOfSpeech(() => setAgentReplying(false));
|
|
39
|
+
|
|
40
|
+
if (!autoStartConversation) return;
|
|
41
|
+
|
|
42
|
+
setMessages(getFirstMessages(autoStartConversation));
|
|
43
|
+
// append([{ role: 'user', content: autoStartConversation.userMessage }]);
|
|
44
|
+
|
|
45
|
+
if (autoStartConversation.assistantMessage) {
|
|
46
|
+
// console.log("autostartmessages", { autoStartConversation, isLoading });
|
|
47
|
+
sender.handleNewText(autoStartConversation.assistantMessage, isLoading);
|
|
48
|
+
} else if (autoStartConversation.userMessage) {
|
|
49
|
+
append([{ role: 'user', content: autoStartConversation.userMessage, id: messages.length.toString() }]);
|
|
50
|
+
}
|
|
51
|
+
}, []);
|
|
52
|
+
|
|
53
|
+
useEffect(() => {
|
|
54
|
+
if (lastMessage?.role === 'assistant') {
|
|
55
|
+
sender.handleNewText(lastMessage.content, isLoading);
|
|
56
|
+
}
|
|
57
|
+
}, [lastMessage, isLoading]);
|
|
58
|
+
|
|
59
|
+
return (
|
|
60
|
+
<div className='pb-8'>
|
|
61
|
+
<CircleAudioAvatar
|
|
62
|
+
width={circleSize}
|
|
63
|
+
className='mx-auto'
|
|
64
|
+
imageUrl={avatarImageUrl}
|
|
65
|
+
isDarkTheme={isDarkTheme} />
|
|
66
|
+
{children}
|
|
67
|
+
<VoiceRecorder
|
|
68
|
+
iconSize='300'
|
|
69
|
+
disabled={agentReplying}
|
|
70
|
+
loading={isProcessingMessage}
|
|
71
|
+
onVoiceRecorded={(message) => {
|
|
72
|
+
setAgentReplying(true);
|
|
73
|
+
append([{ role: 'user', content: "Message(" + Math.floor((messages.length + 1) / 2) + "): " + message, id: messages.length.toString() }]);
|
|
74
|
+
}}
|
|
75
|
+
onRecordingStatusChange={(running) => !running && setIsProcessingMessage(true)} />
|
|
76
|
+
</div>
|
|
77
|
+
);
|
|
61
78
|
};
|
|
@@ -1,75 +1,98 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { useEffect, useRef } from 'react';
|
|
2
2
|
import { EventBus, EventBusMessage } from '../../../core';
|
|
3
3
|
|
|
4
4
|
interface CircleAudioAvatarProps {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
5
|
+
width?: string;
|
|
6
|
+
imageUrl: string;
|
|
7
|
+
className?: string;
|
|
8
|
+
isDarkTheme?: boolean;
|
|
8
9
|
}
|
|
9
10
|
|
|
11
|
+
export function CircleAudioAvatar({ imageUrl, className, isDarkTheme = false, width = "150px" }: CircleAudioAvatarProps) {
|
|
12
|
+
const canvasRef = useRef<HTMLCanvasElement>(null);
|
|
13
|
+
const currentLoudnessRef = useRef(0);
|
|
14
|
+
const targetLoudnessRef = useRef(0);
|
|
15
|
+
const animationFrameRef = useRef<number | null>(null);
|
|
10
16
|
|
|
11
|
-
|
|
12
|
-
const
|
|
17
|
+
useEffect(() => {
|
|
18
|
+
const canvas = canvasRef.current;
|
|
19
|
+
if (canvas) {
|
|
20
|
+
const ctx = canvas.getContext('2d');
|
|
21
|
+
if (ctx) {
|
|
22
|
+
const image = new Image();
|
|
23
|
+
image.src = imageUrl;
|
|
24
|
+
let isMounted = true;
|
|
13
25
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
26
|
+
image.onload = () => {
|
|
27
|
+
if (!isMounted) return;
|
|
28
|
+
draw(ctx, canvas, image, 0);
|
|
29
|
+
const animate = () => {
|
|
30
|
+
const decayRate = 0.06;
|
|
31
|
+
if (currentLoudnessRef.current > targetLoudnessRef.current) {
|
|
32
|
+
currentLoudnessRef.current = Math.max(
|
|
33
|
+
targetLoudnessRef.current,
|
|
34
|
+
currentLoudnessRef.current - decayRate * currentLoudnessRef.current
|
|
35
|
+
);
|
|
36
|
+
} else {
|
|
37
|
+
currentLoudnessRef.current = targetLoudnessRef.current;
|
|
38
|
+
}
|
|
39
|
+
draw(ctx, canvas, image, currentLoudnessRef.current);
|
|
40
|
+
animationFrameRef.current = requestAnimationFrame(animate);
|
|
41
|
+
};
|
|
42
|
+
animationFrameRef.current = requestAnimationFrame(animate);
|
|
43
|
+
};
|
|
24
44
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
45
|
+
const handleLoudness = ({ data }: EventBusMessage) => {
|
|
46
|
+
const newLoudness = data.loudness;
|
|
47
|
+
if (newLoudness > currentLoudnessRef.current) {
|
|
48
|
+
currentLoudnessRef.current = newLoudness;
|
|
49
|
+
}
|
|
50
|
+
targetLoudnessRef.current = newLoudness;
|
|
51
|
+
};
|
|
28
52
|
|
|
29
|
-
|
|
30
|
-
const listenerId = EventBus.on('self.avatar.triggerLoudness', handleLoudness);
|
|
53
|
+
const listener = EventBus.on('self.avatar.triggerLoudness', handleLoudness);
|
|
31
54
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
55
|
+
return () => {
|
|
56
|
+
isMounted = false;
|
|
57
|
+
listener.off();
|
|
58
|
+
if (animationFrameRef.current) {
|
|
59
|
+
cancelAnimationFrame(animationFrameRef.current);
|
|
60
|
+
}
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}, [imageUrl]);
|
|
38
65
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
ctx.clearRect(0, 0, canvas.width, canvas.height);
|
|
66
|
+
const draw = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, image: HTMLImageElement, loudness: number) => {
|
|
67
|
+
if (canvas && ctx) {
|
|
68
|
+
ctx.clearRect(0, 0, canvas.width, canvas.height);
|
|
43
69
|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
ctx.stroke();
|
|
70
|
+
const radius = Math.min(canvas.width, canvas.height) / 3;
|
|
71
|
+
const centerX = canvas.width / 2;
|
|
72
|
+
const centerY = canvas.height / 2;
|
|
73
|
+
const pulseRadius = radius + loudness / 2.5;
|
|
74
|
+
ctx.beginPath();
|
|
75
|
+
ctx.arc(centerX, centerY, pulseRadius, 0, Math.PI * 2, true);
|
|
76
|
+
ctx.strokeStyle = isDarkTheme ? 'rgba(255, 255, 255, 0.5)' : 'rgba(0, 0, 0, 0.5)';
|
|
77
|
+
ctx.lineWidth = 5;
|
|
78
|
+
ctx.stroke();
|
|
54
79
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
ctx.restore();
|
|
80
|
+
ctx.save();
|
|
81
|
+
ctx.beginPath();
|
|
82
|
+
ctx.arc(centerX, centerY, radius, 0, Math.PI * 2, true);
|
|
83
|
+
ctx.closePath();
|
|
84
|
+
ctx.clip();
|
|
85
|
+
ctx.drawImage(image, centerX - radius, centerY - radius, radius * 2, radius * 2);
|
|
86
|
+
ctx.restore();
|
|
63
87
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
};
|
|
88
|
+
ctx.beginPath();
|
|
89
|
+
ctx.arc(centerX, centerY, radius, 0, Math.PI * 2, true);
|
|
90
|
+
ctx.strokeStyle = isDarkTheme ? 'rgba(255, 255, 255, 0.9)' : 'rgba(0, 0, 0, 0.9)';
|
|
91
|
+
ctx.lineWidth = 5;
|
|
92
|
+
ctx.stroke();
|
|
93
|
+
}
|
|
94
|
+
};
|
|
72
95
|
|
|
73
|
-
|
|
96
|
+
return <canvas ref={canvasRef} className={className} width={500} height={500} style={{ width }} />;
|
|
74
97
|
};
|
|
75
98
|
|
|
@@ -12,6 +12,7 @@ export class ChunkedAudioPlayer {
|
|
|
12
12
|
private loudnessCallback: (value: number) => void = () => { };
|
|
13
13
|
private currentIndex = 0;
|
|
14
14
|
private startedPlaying = false;
|
|
15
|
+
private onEndOfSpeech: () => void = () => { };
|
|
15
16
|
|
|
16
17
|
constructor() {
|
|
17
18
|
this.init();
|
|
@@ -139,6 +140,7 @@ export class ChunkedAudioPlayer {
|
|
|
139
140
|
// console.log('Loudness monitoring stopped.');
|
|
140
141
|
cancelAnimationFrame(this.handle);
|
|
141
142
|
this.loudnessCallback(0);
|
|
143
|
+
this.onEndOfSpeech();
|
|
142
144
|
return;
|
|
143
145
|
}
|
|
144
146
|
|
|
@@ -189,4 +191,8 @@ export class ChunkedAudioPlayer {
|
|
|
189
191
|
this.isPlaying = false;
|
|
190
192
|
this.init();
|
|
191
193
|
}
|
|
194
|
+
|
|
195
|
+
public setOnEndOfSpeech(callback: () => void) {
|
|
196
|
+
this.onEndOfSpeech = callback;
|
|
197
|
+
}
|
|
192
198
|
}
|
|
@@ -1,21 +1,32 @@
|
|
|
1
|
-
import { useState, useRef, forwardRef, useImperativeHandle } from 'react';
|
|
2
|
-
import { FaMicrophone } from 'react-icons/fa6';
|
|
1
|
+
import { useState, useRef, forwardRef, useImperativeHandle, useEffect } from 'react';
|
|
2
|
+
import { FaMicrophone, FaSpinner } from 'react-icons/fa6';
|
|
3
3
|
import { usePlugin } from '../../../components';
|
|
4
4
|
|
|
5
5
|
interface Props {
|
|
6
6
|
iconSize?: string;
|
|
7
7
|
className?: string;
|
|
8
|
+
disabled?: boolean;
|
|
9
|
+
loading?: boolean;
|
|
10
|
+
onRecordingStatusChange: (running: boolean) => void;
|
|
8
11
|
onVoiceRecorded: (message: string) => void;
|
|
9
12
|
}
|
|
10
13
|
|
|
11
|
-
export const VoiceRecorder = forwardRef(({ onVoiceRecorded, iconSize, className }: Props, ref) => {
|
|
14
|
+
export const VoiceRecorder = forwardRef(({ onVoiceRecorded, iconSize, className, disabled, loading, onRecordingStatusChange }: Props, ref) => {
|
|
12
15
|
const [isRecording, setIsRecording] = useState(false);
|
|
13
16
|
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
|
|
14
17
|
const audioChunksRef = useRef<Blob[]>([]);
|
|
18
|
+
const mediaStreamRef = useRef<MediaStream | null>(null);
|
|
15
19
|
const { llm } = usePlugin();
|
|
16
20
|
|
|
21
|
+
// Ref for latest onVoiceRecorded callback
|
|
22
|
+
const onVoiceRecordedRef = useRef(onVoiceRecorded);
|
|
23
|
+
useEffect(() => {
|
|
24
|
+
onVoiceRecordedRef.current = onVoiceRecorded;
|
|
25
|
+
}, [onVoiceRecorded]);
|
|
26
|
+
|
|
17
27
|
const startRecording = async () => {
|
|
18
28
|
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
29
|
+
mediaStreamRef.current = stream;
|
|
19
30
|
const mediaRecorder = new MediaRecorder(stream);
|
|
20
31
|
mediaRecorderRef.current = mediaRecorder;
|
|
21
32
|
|
|
@@ -27,17 +38,23 @@ export const VoiceRecorder = forwardRef(({ onVoiceRecorded, iconSize, className
|
|
|
27
38
|
const audioBlob = new Blob(audioChunksRef.current);
|
|
28
39
|
audioChunksRef.current = [];
|
|
29
40
|
|
|
30
|
-
|
|
41
|
+
onVoiceRecordedRef.current(await llm.getTextFromVoice(audioBlob));
|
|
31
42
|
};
|
|
32
43
|
|
|
33
44
|
mediaRecorder.start();
|
|
34
45
|
setIsRecording(true);
|
|
46
|
+
onRecordingStatusChange(true);
|
|
35
47
|
};
|
|
36
48
|
|
|
37
49
|
const stopRecording = () => {
|
|
38
50
|
if (mediaRecorderRef.current) {
|
|
39
51
|
mediaRecorderRef.current.stop();
|
|
40
52
|
setIsRecording(false);
|
|
53
|
+
onRecordingStatusChange(false);
|
|
54
|
+
}
|
|
55
|
+
if (mediaStreamRef.current) {
|
|
56
|
+
mediaStreamRef.current.getTracks().forEach(track => track.stop());
|
|
57
|
+
mediaStreamRef.current = null;
|
|
41
58
|
}
|
|
42
59
|
};
|
|
43
60
|
|
|
@@ -46,11 +63,37 @@ export const VoiceRecorder = forwardRef(({ onVoiceRecorded, iconSize, className
|
|
|
46
63
|
stopRecording,
|
|
47
64
|
}));
|
|
48
65
|
|
|
66
|
+
// push to talk feature
|
|
67
|
+
const spacePressedRef = useRef(false);
|
|
68
|
+
|
|
69
|
+
useEffect(() => {
|
|
70
|
+
const handleKeyDown = async (event: KeyboardEvent) => {
|
|
71
|
+
if (event.code === 'Space' && !spacePressedRef.current) {
|
|
72
|
+
spacePressedRef.current = true;
|
|
73
|
+
await startRecording();
|
|
74
|
+
}
|
|
75
|
+
};
|
|
76
|
+
const handleKeyUp = (event: KeyboardEvent) => {
|
|
77
|
+
if (event.code === 'Space' && spacePressedRef.current) {
|
|
78
|
+
spacePressedRef.current = false;
|
|
79
|
+
stopRecording();
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
window.addEventListener('keydown', handleKeyDown);
|
|
83
|
+
window.addEventListener('keyup', handleKeyUp);
|
|
84
|
+
return () => {
|
|
85
|
+
window.removeEventListener('keydown', handleKeyDown);
|
|
86
|
+
window.removeEventListener('keyup', handleKeyUp);
|
|
87
|
+
};
|
|
88
|
+
}, []);
|
|
89
|
+
|
|
49
90
|
return (
|
|
50
|
-
<
|
|
51
|
-
|
|
91
|
+
<button className={"w-16 h-16 flex text-4xl shadow-lg flex-row justify-center items-center rounded-full mx-auto bg-gray-400 dark:bg-gray-800 pl-[6px] disabled:opacity-50 " + className}
|
|
92
|
+
onClick={isRecording ? stopRecording : startRecording}
|
|
93
|
+
disabled={disabled || loading}>
|
|
94
|
+
{loading ? <FaSpinner className="animate-spin mr-[6px]" /> :
|
|
52
95
|
<FaMicrophone size={iconSize} className={"h-7 w-7 mr-2 " + (isRecording ? "text-red-600" : "")} />
|
|
53
|
-
|
|
54
|
-
</
|
|
96
|
+
}
|
|
97
|
+
</button>
|
|
55
98
|
);
|
|
56
99
|
});
|
package/src/components.ts
CHANGED
|
@@ -7,4 +7,5 @@ export * from "./hooks/UseChatHook";
|
|
|
7
7
|
export * from "./plugin/ThemeSetter";
|
|
8
8
|
export * from "./providers/PluginProvider";
|
|
9
9
|
export * from "./components/ai/Avatar";
|
|
10
|
-
export * from "./components/ai/Assistant";
|
|
10
|
+
export * from "./components/ai/Assistant";
|
|
11
|
+
export * from "./types/Actions";
|
|
@@ -69,7 +69,8 @@ export async function streamChatGPT(supabaseUrl: string, messages: Message[], to
|
|
|
69
69
|
// console.log("AI response:", content);
|
|
70
70
|
|
|
71
71
|
//content \n\n should be real line break when message is displayed
|
|
72
|
-
|
|
72
|
+
//content \"\" should be real double quote when message is displayed
|
|
73
|
+
onResponse(messageId, content.replace(/\\n/g, '\n').replace(/\\+"|"\\+/g, '"'), false);
|
|
73
74
|
} else if (command === 'd') {
|
|
74
75
|
// console.log("AI usage:", JSON.parse(line.substring(2)));
|
|
75
76
|
done = true;
|
|
@@ -1,102 +1,101 @@
|
|
|
1
1
|
import { SupabaseClient } from "@supabase/supabase-js";
|
|
2
2
|
import { LanguageLevel } from "../utils/difficultyConverter";
|
|
3
|
+
import { Language } from "../utils/Language";
|
|
3
4
|
|
|
4
5
|
export interface UserInfo {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
}
|
|
6
|
+
skill_level_reading: LanguageLevel;
|
|
7
|
+
skill_level_writing: LanguageLevel;
|
|
8
|
+
skill_level_grammar: LanguageLevel;
|
|
9
|
+
skill_level_speaking: LanguageLevel;
|
|
10
|
+
skill_level_listening: LanguageLevel;
|
|
11
|
+
skill_level_understanding: LanguageLevel;
|
|
12
|
+
goal_longterm: string;
|
|
13
|
+
goal_weekly: string;
|
|
14
|
+
study_buddy: string;
|
|
15
|
+
story_genre: string;
|
|
16
|
+
study_duration: number;
|
|
17
|
+
mother_tongue: Language;
|
|
18
|
+
motivation_type: string;
|
|
19
|
+
onboarding_completed: boolean;
|
|
20
|
+
context_menu_on_select: boolean;
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
export class SettingsController {
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
constructor(supabase: SupabaseClient, pluginId: string) {
|
|
28
|
-
this.supabase = supabase;
|
|
29
|
-
this.pluginId = pluginId;
|
|
30
|
-
}
|
|
24
|
+
private pluginId: string;
|
|
25
|
+
private supabase: SupabaseClient;
|
|
31
26
|
|
|
32
|
-
|
|
33
|
-
|
|
27
|
+
constructor(supabase: SupabaseClient, pluginId: string) {
|
|
28
|
+
this.supabase = supabase;
|
|
29
|
+
this.pluginId = pluginId;
|
|
30
|
+
}
|
|
34
31
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
}
|
|
32
|
+
private async fetchSettings(): Promise<any | null> {
|
|
33
|
+
const { data } = await this.supabase.from("plugin_settings").select("*").eq("plugin_id", this.pluginId)
|
|
38
34
|
|
|
39
|
-
|
|
35
|
+
if (!data || data.length === 0) {
|
|
36
|
+
return null;
|
|
40
37
|
}
|
|
41
38
|
|
|
42
|
-
|
|
43
|
-
|
|
39
|
+
return data[0].settings;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
public async setSettings(settings: any): Promise<void> {
|
|
43
|
+
await this.supabase.from("plugin_settings").upsert({ plugin_id: this.pluginId, settings });
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
public async getUserInfo(): Promise<UserInfo> {
|
|
47
|
+
const { data } = await this.supabase.from("profiles").select("*");
|
|
48
|
+
|
|
49
|
+
if (!data || data.length === 0) {
|
|
50
|
+
return {
|
|
51
|
+
mother_tongue: "en",
|
|
52
|
+
skill_level_listening: "Pre-A1",
|
|
53
|
+
skill_level_reading: "Pre-A1",
|
|
54
|
+
skill_level_speaking: "Pre-A1",
|
|
55
|
+
skill_level_writing: "Pre-A1",
|
|
56
|
+
skill_level_understanding: "Pre-A1",
|
|
57
|
+
skill_level_grammar: "Pre-A1",
|
|
58
|
+
goal_longterm: "",
|
|
59
|
+
goal_weekly: "",
|
|
60
|
+
study_buddy: "clarence",
|
|
61
|
+
story_genre: "adventure",
|
|
62
|
+
study_duration: 30,
|
|
63
|
+
motivation_type: "self-motivated",
|
|
64
|
+
onboarding_completed: false,
|
|
65
|
+
context_menu_on_select: false,
|
|
66
|
+
}
|
|
44
67
|
}
|
|
45
68
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
motivation_type: "self-motivated",
|
|
61
|
-
study_buddy: "clarence",
|
|
62
|
-
preferred_genre: "adventure",
|
|
63
|
-
milestone: "",
|
|
64
|
-
settings: {
|
|
65
|
-
contextMenuOnSelect: false,
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
return data[0];
|
|
69
|
+
return data[0].settings;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Get the settings for the plugin. T can be any type of settings, UserSettings or SystemSettings.
|
|
74
|
+
* @param defaultSettings The default settings to use if no settings are found.
|
|
75
|
+
* @returns The settings for the plugin.
|
|
76
|
+
*/
|
|
77
|
+
public async getSettings<T extends object>(defaultSettings: T): Promise<T> {
|
|
78
|
+
const storedSettings = await this.fetchSettings() as T | null;
|
|
79
|
+
|
|
80
|
+
if (!storedSettings) {
|
|
81
|
+
await this.setSettings(defaultSettings);
|
|
82
|
+
return defaultSettings;
|
|
71
83
|
}
|
|
72
84
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
// Handle settings migration
|
|
87
|
-
const storedKeys = Object.keys(storedSettings);
|
|
88
|
-
const defaultKeys = Object.keys(defaultSettings);
|
|
89
|
-
|
|
90
|
-
if (storedKeys.length !== defaultKeys.length) {
|
|
91
|
-
const validStoredSettings = Object.fromEntries(
|
|
92
|
-
Object.entries(storedSettings).filter(([key]) => defaultKeys.includes(key))
|
|
93
|
-
);
|
|
94
|
-
const mergedSettings = { ...defaultSettings, ...validStoredSettings } as T;
|
|
95
|
-
|
|
96
|
-
await this.setSettings(mergedSettings);
|
|
97
|
-
return mergedSettings;
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
return storedSettings;
|
|
85
|
+
// Handle settings migration
|
|
86
|
+
const storedKeys = Object.keys(storedSettings);
|
|
87
|
+
const defaultKeys = Object.keys(defaultSettings);
|
|
88
|
+
|
|
89
|
+
if (storedKeys.length !== defaultKeys.length) {
|
|
90
|
+
const validStoredSettings = Object.fromEntries(
|
|
91
|
+
Object.entries(storedSettings).filter(([key]) => defaultKeys.includes(key))
|
|
92
|
+
);
|
|
93
|
+
const mergedSettings = { ...defaultSettings, ...validStoredSettings } as T;
|
|
94
|
+
|
|
95
|
+
await this.setSettings(mergedSettings);
|
|
96
|
+
return mergedSettings;
|
|
101
97
|
}
|
|
98
|
+
|
|
99
|
+
return storedSettings;
|
|
100
|
+
}
|
|
102
101
|
}
|