@rimori/client 1.0.5 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/README.md +13 -0
  2. package/dist/components/MarkdownEditor.js +6 -4
  3. package/dist/components/PluginController.d.ts +21 -0
  4. package/dist/components/PluginController.js +116 -0
  5. package/dist/components/ai/Assistant.js +1 -1
  6. package/dist/components/ai/Avatar.d.ts +5 -3
  7. package/dist/components/ai/Avatar.js +14 -6
  8. package/dist/components/ai/EmbeddedAssistent/CircleAudioAvatar.d.ts +2 -1
  9. package/dist/components/ai/EmbeddedAssistent/CircleAudioAvatar.js +35 -14
  10. package/dist/components/ai/EmbeddedAssistent/TTS/MessageSender.d.ts +1 -0
  11. package/dist/components/ai/EmbeddedAssistent/TTS/MessageSender.js +3 -0
  12. package/dist/components/ai/EmbeddedAssistent/TTS/Player.d.ts +2 -0
  13. package/dist/components/ai/EmbeddedAssistent/TTS/Player.js +5 -0
  14. package/dist/components/ai/EmbeddedAssistent/VoiceRecoder.d.ts +3 -0
  15. package/dist/components/ai/EmbeddedAssistent/VoiceRecoder.js +41 -5
  16. package/dist/components/ai/utils.d.ts +1 -1
  17. package/dist/components.d.ts +1 -0
  18. package/dist/components.js +1 -0
  19. package/dist/controller/AIController.js +2 -1
  20. package/dist/controller/SettingsController.d.ts +15 -15
  21. package/dist/controller/SettingsController.js +15 -16
  22. package/dist/controller/SharedContentController.d.ts +58 -11
  23. package/dist/controller/SharedContentController.js +161 -26
  24. package/dist/controller/SidePluginController.d.ts +1 -12
  25. package/dist/core/components/ContextMenu.d.ts +10 -0
  26. package/dist/core/components/ContextMenu.js +93 -0
  27. package/dist/core.d.ts +2 -0
  28. package/dist/core.js +2 -0
  29. package/dist/hooks/UseChatHook.d.ts +1 -1
  30. package/dist/plugin/AccomplishmentHandler.d.ts +38 -0
  31. package/dist/plugin/AccomplishmentHandler.js +108 -0
  32. package/dist/plugin/ContextMenu.d.ts +17 -0
  33. package/dist/plugin/ContextMenu.js +45 -0
  34. package/dist/plugin/PluginController.js +6 -3
  35. package/dist/plugin/RimoriClient.d.ts +92 -65
  36. package/dist/plugin/RimoriClient.js +105 -75
  37. package/dist/plugin/ThemeSetter.js +2 -2
  38. package/dist/plugin/fromRimori/EventBus.d.ts +6 -3
  39. package/dist/plugin/fromRimori/EventBus.js +15 -9
  40. package/dist/plugin/fromRimori/PluginTypes.d.ts +51 -0
  41. package/dist/plugin/fromRimori/PluginTypes.js +1 -0
  42. package/dist/providers/PluginController.d.ts +21 -0
  43. package/dist/providers/PluginController.js +116 -0
  44. package/dist/providers/PluginProvider.js +26 -73
  45. package/dist/types/Actions.d.ts +4 -0
  46. package/dist/types/Actions.js +1 -0
  47. package/dist/utils/Language.d.ts +66 -0
  48. package/dist/utils/Language.js +67 -0
  49. package/dist/utils/difficultyConverter.d.ts +1 -0
  50. package/dist/utils/difficultyConverter.js +3 -0
  51. package/dist/worker/WorkerSetup.js +4 -4
  52. package/package.json +2 -3
  53. package/src/components/MarkdownEditor.tsx +78 -76
  54. package/src/components/ai/Assistant.tsx +1 -1
  55. package/src/components/ai/Avatar.tsx +65 -48
  56. package/src/components/ai/EmbeddedAssistent/CircleAudioAvatar.tsx +81 -58
  57. package/src/components/ai/EmbeddedAssistent/TTS/MessageSender.ts +4 -0
  58. package/src/components/ai/EmbeddedAssistent/TTS/Player.ts +6 -0
  59. package/src/components/ai/EmbeddedAssistent/VoiceRecoder.tsx +51 -8
  60. package/src/components/ai/utils.ts +1 -1
  61. package/src/components.ts +2 -1
  62. package/src/controller/AIController.ts +2 -1
  63. package/src/controller/SettingsController.ts +83 -84
  64. package/src/controller/SharedContentController.ts +214 -53
  65. package/src/controller/SidePluginController.ts +1 -13
  66. package/src/core/components/ContextMenu.tsx +123 -0
  67. package/src/core.ts +3 -1
  68. package/src/hooks/UseChatHook.ts +17 -17
  69. package/src/plugin/AccomplishmentHandler.ts +165 -0
  70. package/src/plugin/PluginController.ts +105 -103
  71. package/src/plugin/RimoriClient.ts +267 -250
  72. package/src/plugin/ThemeSetter.ts +2 -2
  73. package/src/plugin/fromRimori/EventBus.ts +23 -12
  74. package/src/plugin/fromRimori/PluginTypes.ts +67 -0
  75. package/src/providers/PluginProvider.tsx +63 -110
  76. package/src/types/Actions.ts +6 -0
  77. package/src/utils/Language.ts +70 -0
  78. package/src/utils/difficultyConverter.ts +4 -0
  79. package/src/worker/WorkerSetup.ts +4 -4
  80. package/dist/components/avatar/Assistant.d.ts +0 -9
  81. package/dist/components/avatar/Assistant.js +0 -59
  82. package/dist/components/avatar/Avatar.d.ts +0 -12
  83. package/dist/components/avatar/Avatar.js +0 -42
  84. package/dist/components/avatar/EmbeddedAssistent/AudioInputField.d.ts +0 -7
  85. package/dist/components/avatar/EmbeddedAssistent/AudioInputField.js +0 -38
  86. package/dist/components/avatar/EmbeddedAssistent/CircleAudioAvatar.d.ts +0 -7
  87. package/dist/components/avatar/EmbeddedAssistent/CircleAudioAvatar.js +0 -59
  88. package/dist/components/avatar/EmbeddedAssistent/TTS/MessageSender.d.ts +0 -19
  89. package/dist/components/avatar/EmbeddedAssistent/TTS/MessageSender.js +0 -84
  90. package/dist/components/avatar/EmbeddedAssistent/TTS/Player.d.ts +0 -25
  91. package/dist/components/avatar/EmbeddedAssistent/TTS/Player.js +0 -180
  92. package/dist/components/avatar/EmbeddedAssistent/VoiceRecoder.d.ts +0 -7
  93. package/dist/components/avatar/EmbeddedAssistent/VoiceRecoder.js +0 -45
  94. package/dist/components/avatar/utils.d.ts +0 -6
  95. package/dist/components/avatar/utils.js +0 -14
@@ -23,7 +23,7 @@ export function AssistantChat({ avatarImageUrl, voiceId, onComplete, autoStartCo
23
23
  const lastAssistantMessage = [...messages].filter((m) => m.role === 'assistant').pop()?.content;
24
24
 
25
25
  useEffect(() => {
26
- sender.setOnLoudnessChange((value: number) => event.emit('self.avatar.triggerLoudness', value));
26
+ sender.setOnLoudnessChange((value: number) => event.emit('self.avatar.triggerLoudness', { loudness: value }));
27
27
 
28
28
  if (!autoStartConversation) {
29
29
  return;
@@ -1,5 +1,5 @@
1
1
  import { Tool } from '../../core';
2
- import { useEffect, useMemo } from 'react';
2
+ import { useEffect, useMemo, useState } from 'react';
3
3
  import { VoiceRecorder } from './EmbeddedAssistent/VoiceRecoder';
4
4
  import { MessageSender } from './EmbeddedAssistent/TTS/MessageSender';
5
5
  import { CircleAudioAvatar } from './EmbeddedAssistent/CircleAudioAvatar';
@@ -9,53 +9,70 @@ import { getFirstMessages } from './utils';
9
9
  import { FirstMessages } from './utils';
10
10
 
11
11
  interface Props {
12
- title?: string;
13
- voiceId: any;
14
- avatarImageUrl: string;
15
- agentTools: Tool[];
16
- autoStartConversation?: FirstMessages;
12
+ voiceId: any;
13
+ agentTools: Tool[];
14
+ avatarImageUrl: string;
15
+ circleSize?: string;
16
+ isDarkTheme?: boolean;
17
+ children?: React.ReactNode;
18
+ autoStartConversation?: FirstMessages;
17
19
  }
18
20
 
19
- export function Avatar({ avatarImageUrl, voiceId, title, agentTools, autoStartConversation }: Props) {
20
- const { llm, event } = usePlugin();
21
- const sender = useMemo(() => new MessageSender(llm.getVoice, voiceId), []);
22
- const { messages, append, isLoading, lastMessage, setMessages } = useChat(agentTools);
23
-
24
- useEffect(() => {
25
- console.log("messages", messages);
26
- }, [messages]);
27
-
28
- useEffect(() => {
29
- sender.setOnLoudnessChange((value: number) => event.emit('self.avatar.triggerLoudness', value));
30
-
31
- if (!autoStartConversation) return;
32
-
33
- setMessages(getFirstMessages(autoStartConversation));
34
- // append([{ role: 'user', content: autoStartConversation.userMessage }]);
35
-
36
- if (autoStartConversation.assistantMessage) {
37
- // console.log("autostartmessages", { autoStartConversation, isLoading });
38
- sender.handleNewText(autoStartConversation.assistantMessage, isLoading);
39
- } else if (autoStartConversation.userMessage) {
40
- append([{ role: 'user', content: autoStartConversation.userMessage, id: messages.length.toString() }]);
41
- }
42
- }, []);
43
-
44
- useEffect(() => {
45
- if (lastMessage?.role === 'assistant') {
46
- sender.handleNewText(lastMessage.content, isLoading);
47
- }
48
- }, [lastMessage, isLoading]);
49
-
50
- return (
51
- <div className='pb-8'>
52
- {title && <p className="text-center mt-5 w-3/4 mx-auto rounded-lg dark:text-gray-100">{title}</p>}
53
- <CircleAudioAvatar imageUrl={avatarImageUrl} width={"250px"} className='mx-auto' />
54
- <div className='w-16 h-16 flex text-4xl shadow-lg flex-row justify-center items-center rounded-full mx-auto bg-gray-400 dark:bg-gray-800'>
55
- <VoiceRecorder className='w-7' iconSize='300' onVoiceRecorded={(message) => {
56
- append([{ role: 'user', content: "Message(" + Math.floor((messages.length + 1) / 2) + "): " + message, id: messages.length.toString() }]);
57
- }} />
58
- </div>
59
- </div>
60
- );
21
+ export function Avatar({ avatarImageUrl, voiceId, agentTools, autoStartConversation, children, isDarkTheme = false, circleSize = "300px" }: Props) {
22
+ const { llm, event } = usePlugin();
23
+ const [agentReplying, setAgentReplying] = useState(false);
24
+ const [isProcessingMessage, setIsProcessingMessage] = useState(false);
25
+ const sender = useMemo(() => new MessageSender(llm.getVoice, voiceId), []);
26
+ const { messages, append, isLoading, lastMessage, setMessages } = useChat(agentTools);
27
+
28
+ useEffect(() => {
29
+ console.log("messages", messages);
30
+ }, [messages]);
31
+
32
+ useEffect(() => {
33
+ if (!isLoading) setIsProcessingMessage(false);
34
+ }, [isLoading]);
35
+
36
+ useEffect(() => {
37
+ sender.setOnLoudnessChange((value) => event.emit('self.avatar.triggerLoudness', { loudness: value }));
38
+ sender.setOnEndOfSpeech(() => setAgentReplying(false));
39
+
40
+ if (!autoStartConversation) return;
41
+
42
+ setMessages(getFirstMessages(autoStartConversation));
43
+ // append([{ role: 'user', content: autoStartConversation.userMessage }]);
44
+
45
+ if (autoStartConversation.assistantMessage) {
46
+ // console.log("autostartmessages", { autoStartConversation, isLoading });
47
+ sender.handleNewText(autoStartConversation.assistantMessage, isLoading);
48
+ } else if (autoStartConversation.userMessage) {
49
+ append([{ role: 'user', content: autoStartConversation.userMessage, id: messages.length.toString() }]);
50
+ }
51
+ }, []);
52
+
53
+ useEffect(() => {
54
+ if (lastMessage?.role === 'assistant') {
55
+ sender.handleNewText(lastMessage.content, isLoading);
56
+ }
57
+ }, [lastMessage, isLoading]);
58
+
59
+ return (
60
+ <div className='pb-8'>
61
+ <CircleAudioAvatar
62
+ width={circleSize}
63
+ className='mx-auto'
64
+ imageUrl={avatarImageUrl}
65
+ isDarkTheme={isDarkTheme} />
66
+ {children}
67
+ <VoiceRecorder
68
+ iconSize='300'
69
+ disabled={agentReplying}
70
+ loading={isProcessingMessage}
71
+ onVoiceRecorded={(message) => {
72
+ setAgentReplying(true);
73
+ append([{ role: 'user', content: "Message(" + Math.floor((messages.length + 1) / 2) + "): " + message, id: messages.length.toString() }]);
74
+ }}
75
+ onRecordingStatusChange={(running) => !running && setIsProcessingMessage(true)} />
76
+ </div>
77
+ );
61
78
  };
@@ -1,75 +1,98 @@
1
- import React, { useEffect, useRef } from 'react';
1
+ import { useEffect, useRef } from 'react';
2
2
  import { EventBus, EventBusMessage } from '../../../core';
3
3
 
4
4
  interface CircleAudioAvatarProps {
5
- width?: string;
6
- imageUrl: string;
7
- className?: string;
5
+ width?: string;
6
+ imageUrl: string;
7
+ className?: string;
8
+ isDarkTheme?: boolean;
8
9
  }
9
10
 
11
+ export function CircleAudioAvatar({ imageUrl, className, isDarkTheme = false, width = "150px" }: CircleAudioAvatarProps) {
12
+ const canvasRef = useRef<HTMLCanvasElement>(null);
13
+ const currentLoudnessRef = useRef(0);
14
+ const targetLoudnessRef = useRef(0);
15
+ const animationFrameRef = useRef<number | null>(null);
10
16
 
11
- export function CircleAudioAvatar({ imageUrl, className, width = "150px" }: CircleAudioAvatarProps) {
12
- const canvasRef = useRef<HTMLCanvasElement>(null);
17
+ useEffect(() => {
18
+ const canvas = canvasRef.current;
19
+ if (canvas) {
20
+ const ctx = canvas.getContext('2d');
21
+ if (ctx) {
22
+ const image = new Image();
23
+ image.src = imageUrl;
24
+ let isMounted = true;
13
25
 
14
- useEffect(() => {
15
- const canvas = canvasRef.current;
16
- if (canvas) {
17
- const ctx = canvas.getContext('2d');
18
- if (ctx) {
19
- const image = new Image();
20
- image.src = imageUrl;
21
- image.onload = () => {
22
- draw(ctx, canvas, image, 0);
23
- };
26
+ image.onload = () => {
27
+ if (!isMounted) return;
28
+ draw(ctx, canvas, image, 0);
29
+ const animate = () => {
30
+ const decayRate = 0.06;
31
+ if (currentLoudnessRef.current > targetLoudnessRef.current) {
32
+ currentLoudnessRef.current = Math.max(
33
+ targetLoudnessRef.current,
34
+ currentLoudnessRef.current - decayRate * currentLoudnessRef.current
35
+ );
36
+ } else {
37
+ currentLoudnessRef.current = targetLoudnessRef.current;
38
+ }
39
+ draw(ctx, canvas, image, currentLoudnessRef.current);
40
+ animationFrameRef.current = requestAnimationFrame(animate);
41
+ };
42
+ animationFrameRef.current = requestAnimationFrame(animate);
43
+ };
24
44
 
25
- const handleLoudness = (event: EventBusMessage) => {
26
- draw(ctx, canvas, image, event.data.loudness);
27
- };
45
+ const handleLoudness = ({ data }: EventBusMessage) => {
46
+ const newLoudness = data.loudness;
47
+ if (newLoudness > currentLoudnessRef.current) {
48
+ currentLoudnessRef.current = newLoudness;
49
+ }
50
+ targetLoudnessRef.current = newLoudness;
51
+ };
28
52
 
29
- // Subscribe to loudness changes
30
- const listenerId = EventBus.on('self.avatar.triggerLoudness', handleLoudness);
53
+ const listener = EventBus.on('self.avatar.triggerLoudness', handleLoudness);
31
54
 
32
- return () => {
33
- EventBus.off(listenerId);
34
- };
35
- }
36
- }
37
- }, [imageUrl]);
55
+ return () => {
56
+ isMounted = false;
57
+ listener.off();
58
+ if (animationFrameRef.current) {
59
+ cancelAnimationFrame(animationFrameRef.current);
60
+ }
61
+ };
62
+ }
63
+ }
64
+ }, [imageUrl]);
38
65
 
39
- // Function to draw on the canvas
40
- const draw = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, image: HTMLImageElement, loudness: number) => {
41
- if (canvas && ctx) {
42
- ctx.clearRect(0, 0, canvas.width, canvas.height);
66
+ const draw = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, image: HTMLImageElement, loudness: number) => {
67
+ if (canvas && ctx) {
68
+ ctx.clearRect(0, 0, canvas.width, canvas.height);
43
69
 
44
- // Draw pulsing circle
45
- const radius = Math.min(canvas.width, canvas.height) / 3;
46
- const centerX = canvas.width / 2;
47
- const centerY = canvas.height / 2;
48
- const pulseRadius = radius + loudness / 2.5; // Adjust the divisor for sensitivity
49
- ctx.beginPath();
50
- ctx.arc(centerX, centerY, pulseRadius, 0, Math.PI * 2, true);
51
- ctx.strokeStyle = 'rgba(0, 0, 0, 0.5)';
52
- ctx.lineWidth = 5;
53
- ctx.stroke();
70
+ const radius = Math.min(canvas.width, canvas.height) / 3;
71
+ const centerX = canvas.width / 2;
72
+ const centerY = canvas.height / 2;
73
+ const pulseRadius = radius + loudness / 2.5;
74
+ ctx.beginPath();
75
+ ctx.arc(centerX, centerY, pulseRadius, 0, Math.PI * 2, true);
76
+ ctx.strokeStyle = isDarkTheme ? 'rgba(255, 255, 255, 0.5)' : 'rgba(0, 0, 0, 0.5)';
77
+ ctx.lineWidth = 5;
78
+ ctx.stroke();
54
79
 
55
- // Draw image circle
56
- ctx.save();
57
- ctx.beginPath();
58
- ctx.arc(centerX, centerY, radius, 0, Math.PI * 2, true);
59
- ctx.closePath();
60
- ctx.clip();
61
- ctx.drawImage(image, centerX - radius, centerY - radius, radius * 2, radius * 2);
62
- ctx.restore();
80
+ ctx.save();
81
+ ctx.beginPath();
82
+ ctx.arc(centerX, centerY, radius, 0, Math.PI * 2, true);
83
+ ctx.closePath();
84
+ ctx.clip();
85
+ ctx.drawImage(image, centerX - radius, centerY - radius, radius * 2, radius * 2);
86
+ ctx.restore();
63
87
 
64
- // Draw circular frame around the image
65
- ctx.beginPath();
66
- ctx.arc(centerX, centerY, radius, 0, Math.PI * 2, true);
67
- ctx.strokeStyle = 'rgba(20,20, 20, 0.9)';
68
- ctx.lineWidth = 5; // Adjust the width of the frame as needed
69
- ctx.stroke();
70
- }
71
- };
88
+ ctx.beginPath();
89
+ ctx.arc(centerX, centerY, radius, 0, Math.PI * 2, true);
90
+ ctx.strokeStyle = isDarkTheme ? 'rgba(255, 255, 255, 0.9)' : 'rgba(0, 0, 0, 0.9)';
91
+ ctx.lineWidth = 5;
92
+ ctx.stroke();
93
+ }
94
+ };
72
95
 
73
- return <canvas ref={canvasRef} className={className} width={500} height={500} style={{ width }} />;
96
+ return <canvas ref={canvasRef} className={className} width={500} height={500} style={{ width }} />;
74
97
  };
75
98
 
@@ -88,4 +88,8 @@ export class MessageSender {
88
88
  callback(loudness);
89
89
  });
90
90
  }
91
+
92
+ public setOnEndOfSpeech(callback: () => void) {
93
+ this.player.setOnEndOfSpeech(callback);
94
+ }
91
95
  }
@@ -12,6 +12,7 @@ export class ChunkedAudioPlayer {
12
12
  private loudnessCallback: (value: number) => void = () => { };
13
13
  private currentIndex = 0;
14
14
  private startedPlaying = false;
15
+ private onEndOfSpeech: () => void = () => { };
15
16
 
16
17
  constructor() {
17
18
  this.init();
@@ -139,6 +140,7 @@ export class ChunkedAudioPlayer {
139
140
  // console.log('Loudness monitoring stopped.');
140
141
  cancelAnimationFrame(this.handle);
141
142
  this.loudnessCallback(0);
143
+ this.onEndOfSpeech();
142
144
  return;
143
145
  }
144
146
 
@@ -189,4 +191,8 @@ export class ChunkedAudioPlayer {
189
191
  this.isPlaying = false;
190
192
  this.init();
191
193
  }
194
+
195
+ public setOnEndOfSpeech(callback: () => void) {
196
+ this.onEndOfSpeech = callback;
197
+ }
192
198
  }
@@ -1,21 +1,32 @@
1
- import { useState, useRef, forwardRef, useImperativeHandle } from 'react';
2
- import { FaMicrophone } from 'react-icons/fa6';
1
+ import { useState, useRef, forwardRef, useImperativeHandle, useEffect } from 'react';
2
+ import { FaMicrophone, FaSpinner } from 'react-icons/fa6';
3
3
  import { usePlugin } from '../../../components';
4
4
 
5
5
  interface Props {
6
6
  iconSize?: string;
7
7
  className?: string;
8
+ disabled?: boolean;
9
+ loading?: boolean;
10
+ onRecordingStatusChange: (running: boolean) => void;
8
11
  onVoiceRecorded: (message: string) => void;
9
12
  }
10
13
 
11
- export const VoiceRecorder = forwardRef(({ onVoiceRecorded, iconSize, className }: Props, ref) => {
14
+ export const VoiceRecorder = forwardRef(({ onVoiceRecorded, iconSize, className, disabled, loading, onRecordingStatusChange }: Props, ref) => {
12
15
  const [isRecording, setIsRecording] = useState(false);
13
16
  const mediaRecorderRef = useRef<MediaRecorder | null>(null);
14
17
  const audioChunksRef = useRef<Blob[]>([]);
18
+ const mediaStreamRef = useRef<MediaStream | null>(null);
15
19
  const { llm } = usePlugin();
16
20
 
21
+ // Ref for latest onVoiceRecorded callback
22
+ const onVoiceRecordedRef = useRef(onVoiceRecorded);
23
+ useEffect(() => {
24
+ onVoiceRecordedRef.current = onVoiceRecorded;
25
+ }, [onVoiceRecorded]);
26
+
17
27
  const startRecording = async () => {
18
28
  const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
29
+ mediaStreamRef.current = stream;
19
30
  const mediaRecorder = new MediaRecorder(stream);
20
31
  mediaRecorderRef.current = mediaRecorder;
21
32
 
@@ -27,17 +38,23 @@ export const VoiceRecorder = forwardRef(({ onVoiceRecorded, iconSize, className
27
38
  const audioBlob = new Blob(audioChunksRef.current);
28
39
  audioChunksRef.current = [];
29
40
 
30
- onVoiceRecorded(await llm.getTextFromVoice(audioBlob));
41
+ onVoiceRecordedRef.current(await llm.getTextFromVoice(audioBlob));
31
42
  };
32
43
 
33
44
  mediaRecorder.start();
34
45
  setIsRecording(true);
46
+ onRecordingStatusChange(true);
35
47
  };
36
48
 
37
49
  const stopRecording = () => {
38
50
  if (mediaRecorderRef.current) {
39
51
  mediaRecorderRef.current.stop();
40
52
  setIsRecording(false);
53
+ onRecordingStatusChange(false);
54
+ }
55
+ if (mediaStreamRef.current) {
56
+ mediaStreamRef.current.getTracks().forEach(track => track.stop());
57
+ mediaStreamRef.current = null;
41
58
  }
42
59
  };
43
60
 
@@ -46,11 +63,37 @@ export const VoiceRecorder = forwardRef(({ onVoiceRecorded, iconSize, className
46
63
  stopRecording,
47
64
  }));
48
65
 
66
+ // push to talk feature
67
+ const spacePressedRef = useRef(false);
68
+
69
+ useEffect(() => {
70
+ const handleKeyDown = async (event: KeyboardEvent) => {
71
+ if (event.code === 'Space' && !spacePressedRef.current) {
72
+ spacePressedRef.current = true;
73
+ await startRecording();
74
+ }
75
+ };
76
+ const handleKeyUp = (event: KeyboardEvent) => {
77
+ if (event.code === 'Space' && spacePressedRef.current) {
78
+ spacePressedRef.current = false;
79
+ stopRecording();
80
+ }
81
+ };
82
+ window.addEventListener('keydown', handleKeyDown);
83
+ window.addEventListener('keyup', handleKeyUp);
84
+ return () => {
85
+ window.removeEventListener('keydown', handleKeyDown);
86
+ window.removeEventListener('keyup', handleKeyUp);
87
+ };
88
+ }, []);
89
+
49
90
  return (
50
- <div className={className}>
51
- <button onClick={isRecording ? stopRecording : startRecording}>
91
+ <button className={"w-16 h-16 flex text-4xl shadow-lg flex-row justify-center items-center rounded-full mx-auto bg-gray-400 dark:bg-gray-800 pl-[6px] disabled:opacity-50 " + className}
92
+ onClick={isRecording ? stopRecording : startRecording}
93
+ disabled={disabled || loading}>
94
+ {loading ? <FaSpinner className="animate-spin mr-[6px]" /> :
52
95
  <FaMicrophone size={iconSize} className={"h-7 w-7 mr-2 " + (isRecording ? "text-red-600" : "")} />
53
- </button>
54
- </div>
96
+ }
97
+ </button>
55
98
  );
56
99
  });
@@ -1,6 +1,6 @@
1
1
  export interface FirstMessages {
2
2
  instructions?: string;
3
- userMessage: string;
3
+ userMessage?: string;
4
4
  assistantMessage?: string;
5
5
  }
6
6
 
package/src/components.ts CHANGED
@@ -7,4 +7,5 @@ export * from "./hooks/UseChatHook";
7
7
  export * from "./plugin/ThemeSetter";
8
8
  export * from "./providers/PluginProvider";
9
9
  export * from "./components/ai/Avatar";
10
- export * from "./components/ai/Assistant";
10
+ export * from "./components/ai/Assistant";
11
+ export * from "./types/Actions";
@@ -69,7 +69,8 @@ export async function streamChatGPT(supabaseUrl: string, messages: Message[], to
69
69
  // console.log("AI response:", content);
70
70
 
71
71
  //content \n\n should be real line break when message is displayed
72
- onResponse(messageId, content.replace(/\\n/g, '\n'), false);
72
+ //content \"\" should be real double quote when message is displayed
73
+ onResponse(messageId, content.replace(/\\n/g, '\n').replace(/\\+"|"\\+/g, '"'), false);
73
74
  } else if (command === 'd') {
74
75
  // console.log("AI usage:", JSON.parse(line.substring(2)));
75
76
  done = true;
@@ -1,102 +1,101 @@
1
1
  import { SupabaseClient } from "@supabase/supabase-js";
2
2
  import { LanguageLevel } from "../utils/difficultyConverter";
3
+ import { Language } from "../utils/Language";
3
4
 
4
5
  export interface UserInfo {
5
- motherTongue: string;
6
- xp: number;
7
- listening_level: LanguageLevel;
8
- reading_level: LanguageLevel;
9
- speaking_level: LanguageLevel;
10
- writing_level: LanguageLevel;
11
- understanding_level: LanguageLevel;
12
- grammar_level: LanguageLevel;
13
- longterm_goal: string;
14
- motivation_type: string;
15
- study_buddy: string;
16
- preferred_genre: string;
17
- milestone: string;
18
- settings: {
19
- contextMenuOnSelect: boolean;
20
- }
6
+ skill_level_reading: LanguageLevel;
7
+ skill_level_writing: LanguageLevel;
8
+ skill_level_grammar: LanguageLevel;
9
+ skill_level_speaking: LanguageLevel;
10
+ skill_level_listening: LanguageLevel;
11
+ skill_level_understanding: LanguageLevel;
12
+ goal_longterm: string;
13
+ goal_weekly: string;
14
+ study_buddy: string;
15
+ story_genre: string;
16
+ study_duration: number;
17
+ mother_tongue: Language;
18
+ motivation_type: string;
19
+ onboarding_completed: boolean;
20
+ context_menu_on_select: boolean;
21
21
  }
22
22
 
23
23
  export class SettingsController {
24
- private pluginId: string;
25
- private supabase: SupabaseClient;
26
-
27
- constructor(supabase: SupabaseClient, pluginId: string) {
28
- this.supabase = supabase;
29
- this.pluginId = pluginId;
30
- }
24
+ private pluginId: string;
25
+ private supabase: SupabaseClient;
31
26
 
32
- private async fetchSettings(): Promise<any | null> {
33
- const { data } = await this.supabase.from("plugin_settings").select("*").eq("plugin_id", this.pluginId)
27
+ constructor(supabase: SupabaseClient, pluginId: string) {
28
+ this.supabase = supabase;
29
+ this.pluginId = pluginId;
30
+ }
34
31
 
35
- if (!data || data.length === 0) {
36
- return null;
37
- }
32
+ private async fetchSettings(): Promise<any | null> {
33
+ const { data } = await this.supabase.from("plugin_settings").select("*").eq("plugin_id", this.pluginId)
38
34
 
39
- return data[0].settings;
35
+ if (!data || data.length === 0) {
36
+ return null;
40
37
  }
41
38
 
42
- public async setSettings(settings: any): Promise<void> {
43
- await this.supabase.from("plugin_settings").upsert({ plugin_id: this.pluginId, settings });
39
+ return data[0].settings;
40
+ }
41
+
42
+ public async setSettings(settings: any): Promise<void> {
43
+ await this.supabase.from("plugin_settings").upsert({ plugin_id: this.pluginId, settings });
44
+ }
45
+
46
+ public async getUserInfo(): Promise<UserInfo> {
47
+ const { data } = await this.supabase.from("profiles").select("*");
48
+
49
+ if (!data || data.length === 0) {
50
+ return {
51
+ mother_tongue: "en",
52
+ skill_level_listening: "Pre-A1",
53
+ skill_level_reading: "Pre-A1",
54
+ skill_level_speaking: "Pre-A1",
55
+ skill_level_writing: "Pre-A1",
56
+ skill_level_understanding: "Pre-A1",
57
+ skill_level_grammar: "Pre-A1",
58
+ goal_longterm: "",
59
+ goal_weekly: "",
60
+ study_buddy: "clarence",
61
+ story_genre: "adventure",
62
+ study_duration: 30,
63
+ motivation_type: "self-motivated",
64
+ onboarding_completed: false,
65
+ context_menu_on_select: false,
66
+ }
44
67
  }
45
68
 
46
- public async getUserInfo(): Promise<UserInfo> {
47
- const { data } = await this.supabase.from("profiles").select("*");
48
-
49
- if (!data || data.length === 0) {
50
- return {
51
- motherTongue: "en",
52
- xp: 0,
53
- listening_level: "Pre-A1",
54
- reading_level: "Pre-A1",
55
- speaking_level: "Pre-A1",
56
- writing_level: "Pre-A1",
57
- understanding_level: "Pre-A1",
58
- grammar_level: "Pre-A1",
59
- longterm_goal: "",
60
- motivation_type: "self-motivated",
61
- study_buddy: "clarence",
62
- preferred_genre: "adventure",
63
- milestone: "",
64
- settings: {
65
- contextMenuOnSelect: false,
66
- }
67
- }
68
- }
69
-
70
- return data[0];
69
+ return data[0].settings;
70
+ }
71
+
72
+ /**
73
+ * Get the settings for the plugin. T can be any type of settings, UserSettings or SystemSettings.
74
+ * @param defaultSettings The default settings to use if no settings are found.
75
+ * @returns The settings for the plugin.
76
+ */
77
+ public async getSettings<T extends object>(defaultSettings: T): Promise<T> {
78
+ const storedSettings = await this.fetchSettings() as T | null;
79
+
80
+ if (!storedSettings) {
81
+ await this.setSettings(defaultSettings);
82
+ return defaultSettings;
71
83
  }
72
84
 
73
- /**
74
- * Get the settings for the plugin. T can be any type of settings, UserSettings or SystemSettings.
75
- * @param defaultSettings The default settings to use if no settings are found.
76
- * @returns The settings for the plugin.
77
- */
78
- public async getSettings<T extends object>(defaultSettings: T): Promise<T> {
79
- const storedSettings = await this.fetchSettings() as T | null;
80
-
81
- if (!storedSettings) {
82
- await this.setSettings(defaultSettings);
83
- return defaultSettings;
84
- }
85
-
86
- // Handle settings migration
87
- const storedKeys = Object.keys(storedSettings);
88
- const defaultKeys = Object.keys(defaultSettings);
89
-
90
- if (storedKeys.length !== defaultKeys.length) {
91
- const validStoredSettings = Object.fromEntries(
92
- Object.entries(storedSettings).filter(([key]) => defaultKeys.includes(key))
93
- );
94
- const mergedSettings = { ...defaultSettings, ...validStoredSettings } as T;
95
-
96
- await this.setSettings(mergedSettings);
97
- return mergedSettings;
98
- }
99
-
100
- return storedSettings;
85
+ // Handle settings migration
86
+ const storedKeys = Object.keys(storedSettings);
87
+ const defaultKeys = Object.keys(defaultSettings);
88
+
89
+ if (storedKeys.length !== defaultKeys.length) {
90
+ const validStoredSettings = Object.fromEntries(
91
+ Object.entries(storedSettings).filter(([key]) => defaultKeys.includes(key))
92
+ );
93
+ const mergedSettings = { ...defaultSettings, ...validStoredSettings } as T;
94
+
95
+ await this.setSettings(mergedSettings);
96
+ return mergedSettings;
101
97
  }
98
+
99
+ return storedSettings;
100
+ }
102
101
  }