@rimori/client 1.0.4 → 1.1.0

package/src/components/ai/Avatar.tsx

@@ -1,5 +1,5 @@
 import { Tool } from '../../core';
-import { useEffect, useMemo } from 'react';
+import { useEffect, useMemo, useState } from 'react';
 import { VoiceRecorder } from './EmbeddedAssistent/VoiceRecoder';
 import { MessageSender } from './EmbeddedAssistent/TTS/MessageSender';
 import { CircleAudioAvatar } from './EmbeddedAssistent/CircleAudioAvatar';
@@ -9,53 +9,70 @@ import { getFirstMessages } from './utils';
 import { FirstMessages } from './utils';
 
 interface Props {
-    title?: string;
-    voiceId: any;
-    avatarImageUrl: string;
-    agentTools: Tool[];
-    autoStartConversation?: FirstMessages;
+  voiceId: any;
+  agentTools: Tool[];
+  avatarImageUrl: string;
+  circleSize?: string;
+  isDarkTheme?: boolean;
+  children?: React.ReactNode;
+  autoStartConversation?: FirstMessages;
 }
 
-export function Avatar({ avatarImageUrl, voiceId, title, agentTools, autoStartConversation }: Props) {
-    const { llm, event } = usePlugin();
-    const sender = useMemo(() => new MessageSender(llm.getVoice, voiceId), []);
-    const { messages, append, isLoading, lastMessage, setMessages } = useChat(agentTools);
-
-    useEffect(() => {
-        console.log("messages", messages);
-    }, [messages]);
-
-    useEffect(() => {
-        sender.setOnLoudnessChange((value: number) => event.emit('self.avatar.triggerLoudness', value));
-
-        if (!autoStartConversation) return;
-
-        setMessages(getFirstMessages(autoStartConversation));
-        // append([{ role: 'user', content: autoStartConversation.userMessage }]);
-
-        if (autoStartConversation.assistantMessage) {
-            // console.log("autostartmessages", { autoStartConversation, isLoading });
-            sender.handleNewText(autoStartConversation.assistantMessage, isLoading);
-        } else if (autoStartConversation.userMessage) {
-            append([{ role: 'user', content: autoStartConversation.userMessage, id: messages.length.toString() }]);
-        }
-    }, []);
-
-    useEffect(() => {
-        if (lastMessage?.role === 'assistant') {
-            sender.handleNewText(lastMessage.content, isLoading);
-        }
-    }, [lastMessage, isLoading]);
-
-    return (
-        <div className='pb-8'>
-            {title && <p className="text-center mt-5 w-3/4 mx-auto rounded-lg dark:text-gray-100">{title}</p>}
-            <CircleAudioAvatar imageUrl={avatarImageUrl} width={"250px"} className='mx-auto' />
-            <div className='w-16 h-16 flex text-4xl shadow-lg flex-row justify-center items-center rounded-full mx-auto bg-gray-400 dark:bg-gray-800'>
-                <VoiceRecorder className='w-7' iconSize='300' onVoiceRecorded={(message) => {
-                    append([{ role: 'user', content: "Message(" + Math.floor((messages.length + 1) / 2) + "): " + message, id: messages.length.toString() }]);
-                }} />
-            </div>
-        </div>
-    );
+export function Avatar({ avatarImageUrl, voiceId, agentTools, autoStartConversation, children, isDarkTheme = false, circleSize = "300px" }: Props) {
+  const { llm, event } = usePlugin();
+  const [agentReplying, setAgentReplying] = useState(false);
+  const [isProcessingMessage, setIsProcessingMessage] = useState(false);
+  const sender = useMemo(() => new MessageSender(llm.getVoice, voiceId), []);
+  const { messages, append, isLoading, lastMessage, setMessages } = useChat(agentTools);
+
+  useEffect(() => {
+    console.log("messages", messages);
+  }, [messages]);
+
+  useEffect(() => {
+    if (!isLoading) setIsProcessingMessage(false);
+  }, [isLoading]);
+
+  useEffect(() => {
+    sender.setOnLoudnessChange((value) => event.emit('self.avatar.triggerLoudness', { loudness: value }));
+    sender.setOnEndOfSpeech(() => setAgentReplying(false));
+
+    if (!autoStartConversation) return;
+
+    setMessages(getFirstMessages(autoStartConversation));
+    // append([{ role: 'user', content: autoStartConversation.userMessage }]);
+
+    if (autoStartConversation.assistantMessage) {
+      // console.log("autostartmessages", { autoStartConversation, isLoading });
+      sender.handleNewText(autoStartConversation.assistantMessage, isLoading);
+    } else if (autoStartConversation.userMessage) {
+      append([{ role: 'user', content: autoStartConversation.userMessage, id: messages.length.toString() }]);
+    }
+  }, []);
+
+  useEffect(() => {
+    if (lastMessage?.role === 'assistant') {
+      sender.handleNewText(lastMessage.content, isLoading);
+    }
+  }, [lastMessage, isLoading]);
+
+  return (
+    <div className='pb-8'>
+      <CircleAudioAvatar
+        width={circleSize}
+        className='mx-auto'
+        imageUrl={avatarImageUrl}
+        isDarkTheme={isDarkTheme} />
+      {children}
+      <VoiceRecorder
+        iconSize='300'
+        disabled={agentReplying}
+        loading={isProcessingMessage}
+        onVoiceRecorded={(message) => {
+          setAgentReplying(true);
+          append([{ role: 'user', content: "Message(" + Math.floor((messages.length + 1) / 2) + "): " + message, id: messages.length.toString() }]);
+        }}
+        onRecordingStatusChange={(running) => !running && setIsProcessingMessage(true)} />
+    </div>
+  );
 };

package/src/components/ai/EmbeddedAssistent/CircleAudioAvatar.tsx

@@ -1,75 +1,98 @@
-import React, { useEffect, useRef } from 'react';
+import { useEffect, useRef } from 'react';
 import { EventBus, EventBusMessage } from '../../../core';
 
 interface CircleAudioAvatarProps {
-    width?: string;
-    imageUrl: string;
-    className?: string;
+  width?: string;
+  imageUrl: string;
+  className?: string;
+  isDarkTheme?: boolean;
 }
 
+export function CircleAudioAvatar({ imageUrl, className, isDarkTheme = false, width = "150px" }: CircleAudioAvatarProps) {
+  const canvasRef = useRef<HTMLCanvasElement>(null);
+  const currentLoudnessRef = useRef(0);
+  const targetLoudnessRef = useRef(0);
+  const animationFrameRef = useRef<number | null>(null);
 
-export function CircleAudioAvatar({ imageUrl, className, width = "150px" }: CircleAudioAvatarProps) {
-    const canvasRef = useRef<HTMLCanvasElement>(null);
+  useEffect(() => {
+    const canvas = canvasRef.current;
+    if (canvas) {
+      const ctx = canvas.getContext('2d');
+      if (ctx) {
+        const image = new Image();
+        image.src = imageUrl;
+        let isMounted = true;
 
-    useEffect(() => {
-        const canvas = canvasRef.current;
-        if (canvas) {
-            const ctx = canvas.getContext('2d');
-            if (ctx) {
-                const image = new Image();
-                image.src = imageUrl;
-                image.onload = () => {
-                    draw(ctx, canvas, image, 0);
-                };
+        image.onload = () => {
+          if (!isMounted) return;
+          draw(ctx, canvas, image, 0);
+          const animate = () => {
+            const decayRate = 0.06;
+            if (currentLoudnessRef.current > targetLoudnessRef.current) {
+              currentLoudnessRef.current = Math.max(
+                targetLoudnessRef.current,
+                currentLoudnessRef.current - decayRate * currentLoudnessRef.current
+              );
+            } else {
+              currentLoudnessRef.current = targetLoudnessRef.current;
+            }
+            draw(ctx, canvas, image, currentLoudnessRef.current);
+            animationFrameRef.current = requestAnimationFrame(animate);
+          };
+          animationFrameRef.current = requestAnimationFrame(animate);
+        };
 
-                const handleLoudness = (event: EventBusMessage) => {
-                    draw(ctx, canvas, image, event.data.loudness);
-                };
+        const handleLoudness = ({ data }: EventBusMessage) => {
+          const newLoudness = data.loudness;
+          if (newLoudness > currentLoudnessRef.current) {
+            currentLoudnessRef.current = newLoudness;
+          }
+          targetLoudnessRef.current = newLoudness;
+        };
 
-                // Subscribe to loudness changes
-                const listenerId = EventBus.on('self.avatar.triggerLoudness', handleLoudness);
+        const listener = EventBus.on('self.avatar.triggerLoudness', handleLoudness);
 
-                return () => {
-                    EventBus.off(listenerId);
-                };
-            }
-        }
-    }, [imageUrl]);
+        return () => {
+          isMounted = false;
+          listener.off();
+          if (animationFrameRef.current) {
+            cancelAnimationFrame(animationFrameRef.current);
+          }
+        };
+      }
+    }
+  }, [imageUrl]);
 
-    // Function to draw on the canvas
-    const draw = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, image: HTMLImageElement, loudness: number) => {
-        if (canvas && ctx) {
-            ctx.clearRect(0, 0, canvas.width, canvas.height);
+  const draw = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, image: HTMLImageElement, loudness: number) => {
+    if (canvas && ctx) {
+      ctx.clearRect(0, 0, canvas.width, canvas.height);
 
-            // Draw pulsing circle
-            const radius = Math.min(canvas.width, canvas.height) / 3;
-            const centerX = canvas.width / 2;
-            const centerY = canvas.height / 2;
-            const pulseRadius = radius + loudness / 2.5; // Adjust the divisor for sensitivity
-            ctx.beginPath();
-            ctx.arc(centerX, centerY, pulseRadius, 0, Math.PI * 2, true);
-            ctx.strokeStyle = 'rgba(0, 0, 0, 0.5)';
-            ctx.lineWidth = 5;
-            ctx.stroke();
+      const radius = Math.min(canvas.width, canvas.height) / 3;
+      const centerX = canvas.width / 2;
+      const centerY = canvas.height / 2;
+      const pulseRadius = radius + loudness / 2.5;
+      ctx.beginPath();
+      ctx.arc(centerX, centerY, pulseRadius, 0, Math.PI * 2, true);
+      ctx.strokeStyle = isDarkTheme ? 'rgba(255, 255, 255, 0.5)' : 'rgba(0, 0, 0, 0.5)';
+      ctx.lineWidth = 5;
+      ctx.stroke();
 
-            // Draw image circle
-            ctx.save();
-            ctx.beginPath();
-            ctx.arc(centerX, centerY, radius, 0, Math.PI * 2, true);
-            ctx.closePath();
-            ctx.clip();
-            ctx.drawImage(image, centerX - radius, centerY - radius, radius * 2, radius * 2);
-            ctx.restore();
+      ctx.save();
+      ctx.beginPath();
+      ctx.arc(centerX, centerY, radius, 0, Math.PI * 2, true);
+      ctx.closePath();
+      ctx.clip();
+      ctx.drawImage(image, centerX - radius, centerY - radius, radius * 2, radius * 2);
+      ctx.restore();
 
-            // Draw circular frame around the image
-            ctx.beginPath();
-            ctx.arc(centerX, centerY, radius, 0, Math.PI * 2, true);
-            ctx.strokeStyle = 'rgba(20,20, 20, 0.9)';
-            ctx.lineWidth = 5; // Adjust the width of the frame as needed
-            ctx.stroke();
-        }
-    };
+      ctx.beginPath();
+      ctx.arc(centerX, centerY, radius, 0, Math.PI * 2, true);
+      ctx.strokeStyle = isDarkTheme ? 'rgba(255, 255, 255, 0.9)' : 'rgba(0, 0, 0, 0.9)';
+      ctx.lineWidth = 5;
+      ctx.stroke();
+    }
+  };
 
-    return <canvas ref={canvasRef} className={className} width={500} height={500} style={{ width }} />;
+  return <canvas ref={canvasRef} className={className} width={500} height={500} style={{ width }} />;
 };
 

package/src/components/ai/EmbeddedAssistent/TTS/MessageSender.ts

@@ -88,4 +88,8 @@ export class MessageSender {
             callback(loudness);
         });
     }
+
+    public setOnEndOfSpeech(callback: () => void) {
+        this.player.setOnEndOfSpeech(callback);
+    }
 }

package/src/components/ai/EmbeddedAssistent/TTS/Player.ts

@@ -12,6 +12,7 @@ export class ChunkedAudioPlayer {
     private loudnessCallback: (value: number) => void = () => { };
     private currentIndex = 0;
     private startedPlaying = false;
+    private onEndOfSpeech: () => void = () => { };
 
     constructor() {
         this.init();
@@ -139,6 +140,7 @@ export class ChunkedAudioPlayer {
             // console.log('Loudness monitoring stopped.');
             cancelAnimationFrame(this.handle);
             this.loudnessCallback(0);
+            this.onEndOfSpeech();
             return;
         }
 
@@ -189,4 +191,8 @@ export class ChunkedAudioPlayer {
         this.isPlaying = false;
         this.init();
     }
+
+    public setOnEndOfSpeech(callback: () => void) {
+        this.onEndOfSpeech = callback;
+    }
 }

package/src/components/ai/EmbeddedAssistent/VoiceRecoder.tsx

@@ -1,21 +1,32 @@
-import { useState, useRef, forwardRef, useImperativeHandle } from 'react';
-import { FaMicrophone } from 'react-icons/fa6';
+import { useState, useRef, forwardRef, useImperativeHandle, useEffect } from 'react';
+import { FaMicrophone, FaSpinner } from 'react-icons/fa6';
 import { usePlugin } from '../../../components';
 
 interface Props {
   iconSize?: string;
   className?: string;
+  disabled?: boolean;
+  loading?: boolean;
+  onRecordingStatusChange: (running: boolean) => void;
   onVoiceRecorded: (message: string) => void;
 }
 
-export const VoiceRecorder = forwardRef(({ onVoiceRecorded, iconSize, className }: Props, ref) => {
+export const VoiceRecorder = forwardRef(({ onVoiceRecorded, iconSize, className, disabled, loading, onRecordingStatusChange }: Props, ref) => {
   const [isRecording, setIsRecording] = useState(false);
   const mediaRecorderRef = useRef<MediaRecorder | null>(null);
   const audioChunksRef = useRef<Blob[]>([]);
+  const mediaStreamRef = useRef<MediaStream | null>(null);
   const { llm } = usePlugin();
 
+  // Ref for latest onVoiceRecorded callback
+  const onVoiceRecordedRef = useRef(onVoiceRecorded);
+  useEffect(() => {
+    onVoiceRecordedRef.current = onVoiceRecorded;
+  }, [onVoiceRecorded]);
+
   const startRecording = async () => {
     const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+    mediaStreamRef.current = stream;
     const mediaRecorder = new MediaRecorder(stream);
     mediaRecorderRef.current = mediaRecorder;
 
@@ -27,17 +38,23 @@ export const VoiceRecorder = forwardRef(({ onVoiceRecorded, iconSize, className
       const audioBlob = new Blob(audioChunksRef.current);
       audioChunksRef.current = [];
 
-      onVoiceRecorded(await llm.getTextFromVoice(audioBlob));
+      onVoiceRecordedRef.current(await llm.getTextFromVoice(audioBlob));
     };
 
     mediaRecorder.start();
     setIsRecording(true);
+    onRecordingStatusChange(true);
   };
 
   const stopRecording = () => {
     if (mediaRecorderRef.current) {
       mediaRecorderRef.current.stop();
       setIsRecording(false);
+      onRecordingStatusChange(false);
+    }
+    if (mediaStreamRef.current) {
+      mediaStreamRef.current.getTracks().forEach(track => track.stop());
+      mediaStreamRef.current = null;
     }
   };
 
@@ -46,11 +63,37 @@ export const VoiceRecorder = forwardRef(({ onVoiceRecorded, iconSize, className
     stopRecording,
   }));
 
+  // push to talk feature
+  const spacePressedRef = useRef(false);
+
+  useEffect(() => {
+    const handleKeyDown = async (event: KeyboardEvent) => {
+      if (event.code === 'Space' && !spacePressedRef.current) {
+        spacePressedRef.current = true;
+        await startRecording();
+      }
+    };
+    const handleKeyUp = (event: KeyboardEvent) => {
+      if (event.code === 'Space' && spacePressedRef.current) {
+        spacePressedRef.current = false;
+        stopRecording();
+      }
+    };
+    window.addEventListener('keydown', handleKeyDown);
+    window.addEventListener('keyup', handleKeyUp);
+    return () => {
+      window.removeEventListener('keydown', handleKeyDown);
+      window.removeEventListener('keyup', handleKeyUp);
+    };
+  }, []);
+
   return (
-    <div className={className}>
-      <button onClick={isRecording ? stopRecording : startRecording}>
+    <button className={"w-16 h-16 flex text-4xl shadow-lg flex-row justify-center items-center rounded-full mx-auto bg-gray-400 dark:bg-gray-800 pl-[6px] disabled:opacity-50 " + className}
+      onClick={isRecording ? stopRecording : startRecording}
+      disabled={disabled || loading}>
+      {loading ? <FaSpinner className="animate-spin mr-[6px]" /> :
         <FaMicrophone size={iconSize} className={"h-7 w-7 mr-2 " + (isRecording ? "text-red-600" : "")} />
-      </button>
-    </div>
+      }
+    </button>
   );
 });

package/src/components/ai/utils.ts

@@ -1,6 +1,6 @@
 export interface FirstMessages {
   instructions?: string;
-  userMessage: string;
+  userMessage?: string;
   assistantMessage?: string;
 }
 

package/src/components.ts

@@ -7,4 +7,5 @@ export * from "./hooks/UseChatHook";
 export * from "./plugin/ThemeSetter";
 export * from "./providers/PluginProvider"; 
 export * from "./components/ai/Avatar";
-export * from "./components/ai/Assistant";
+export * from "./components/ai/Assistant";
+export * from "./types/Actions";

package/src/controller/AIController.ts

@@ -69,7 +69,8 @@ export async function streamChatGPT(supabaseUrl: string, messages: Message[], to
           // console.log("AI response:", content);
 
           //content \n\n should be real line break when message is displayed
-          onResponse(messageId, content.replace(/\\n/g, '\n'), false);
+          //content \"\" should be real double quote when message is displayed
+          onResponse(messageId, content.replace(/\\n/g, '\n').replace(/\\+"|"\\+/g, '"'), false);
         } else if (command === 'd') {
           // console.log("AI usage:", JSON.parse(line.substring(2)));
           done = true;

package/src/controller/SettingsController.ts

@@ -1,96 +1,101 @@
 import { SupabaseClient } from "@supabase/supabase-js";
 import { LanguageLevel } from "../utils/difficultyConverter";
-
-type SettingsType = "user" | "system" | "plugin";
+import { Language } from "../utils/Language";
 
 export interface UserInfo {
-    motherTongue: string;
-    languageLevel: LanguageLevel;
-    contextMenuOnSelect: boolean;
-}
-
-export interface SystemSettings {
-    // TODO: add system settings
+  skill_level_reading: LanguageLevel;
+  skill_level_writing: LanguageLevel;
+  skill_level_grammar: LanguageLevel;
+  skill_level_speaking: LanguageLevel;
+  skill_level_listening: LanguageLevel;
+  skill_level_understanding: LanguageLevel;
+  goal_longterm: string;
+  goal_weekly: string;
+  study_buddy: string;
+  story_genre: string;
+  study_duration: number;
+  mother_tongue: Language;
+  motivation_type: string;
+  onboarding_completed: boolean;
+  context_menu_on_select: boolean;
 }
 
 export class SettingsController {
-    private pluginId: string;
-    private supabase: SupabaseClient;
+  private pluginId: string;
+  private supabase: SupabaseClient;
+
+  constructor(supabase: SupabaseClient, pluginId: string) {
+    this.supabase = supabase;
+    this.pluginId = pluginId;
+  }
+
+  private async fetchSettings(): Promise<any | null> {
+    const { data } = await this.supabase.from("plugin_settings").select("*").eq("plugin_id", this.pluginId)
 
-    constructor(supabase: SupabaseClient, pluginId: string) {
-        this.supabase = supabase;
-        this.pluginId = pluginId;
+    if (!data || data.length === 0) {
+      return null;
     }
 
-    private getSettingsType(genericSettings?: "user" | "system"): SettingsType {
-        return genericSettings || "plugin";
+    return data[0].settings;
+  }
+
+  public async setSettings(settings: any): Promise<void> {
+    await this.supabase.from("plugin_settings").upsert({ plugin_id: this.pluginId, settings });
+  }
+
+  public async getUserInfo(): Promise<UserInfo> {
+    const { data } = await this.supabase.from("profiles").select("*");
+
+    if (!data || data.length === 0) {
+      return {
+        mother_tongue: "en",
+        skill_level_listening: "Pre-A1",
+        skill_level_reading: "Pre-A1",
+        skill_level_speaking: "Pre-A1",
+        skill_level_writing: "Pre-A1",
+        skill_level_understanding: "Pre-A1",
+        skill_level_grammar: "Pre-A1",
+        goal_longterm: "",
+        goal_weekly: "",
+        study_buddy: "clarence",
+        story_genre: "adventure",
+        study_duration: 30,
+        motivation_type: "self-motivated",
+        onboarding_completed: false,
+        context_menu_on_select: false,
+      }
     }
 
-    private async fetchSettings(type: SettingsType): Promise<any | null> {
-        const pluginId = type === "plugin" ? this.pluginId : type;
-        const { data } = await this.supabase.from("plugin_settings").select("*").eq("plugin_id", pluginId)
+    return data[0].settings;
+  }
 
-        if (!data || data.length === 0) {
-            return null;
-        }
+  /**
+   * Get the settings for the plugin. T can be any type of settings, UserSettings or SystemSettings.
+   * @param defaultSettings The default settings to use if no settings are found.
+   * @returns The settings for the plugin. 
+   */
+  public async getSettings<T extends object>(defaultSettings: T): Promise<T> {
+    const storedSettings = await this.fetchSettings() as T | null;
 
-        return data[0].settings;
+    if (!storedSettings) {
+      await this.setSettings(defaultSettings);
+      return defaultSettings;
     }
 
-    private async saveSettings(settings: any, type: SettingsType): Promise<void> {
-        if (type !== "plugin") {
-            throw new Error(`Cannot modify ${type} settings`);
-        }
+    // Handle settings migration
+    const storedKeys = Object.keys(storedSettings);
+    const defaultKeys = Object.keys(defaultSettings);
 
-        await this.supabase.from("plugin_settings").upsert({ plugin_id: this.pluginId, settings });
-    }
-
-    public async getUserInfo(): Promise<UserInfo> {
-        return this.getSettings<UserInfo>({
-            motherTongue: "sv",
-            languageLevel: "A1",
-            contextMenuOnSelect: true,
-        }, "user");
-    }
+    if (storedKeys.length !== defaultKeys.length) {
+      const validStoredSettings = Object.fromEntries(
+        Object.entries(storedSettings).filter(([key]) => defaultKeys.includes(key))
+      );
+      const mergedSettings = { ...defaultSettings, ...validStoredSettings } as T;
 
-    /**
-     * Get the settings for the plugin. T can be any type of settings, UserSettings or SystemSettings.
-     * @param defaultSettings The default settings to use if no settings are found.
-     * @param genericSettings The type of settings to get.
-     * @returns The settings for the plugin. 
-     */
-    public async getSettings<T extends object>(defaultSettings: T, genericSettings?: "user" | "system"): Promise<T> {
-        const type = this.getSettingsType(genericSettings);
-        const storedSettings = await this.fetchSettings(type) as T | null;
-
-        if (!storedSettings) {
-            if (type === "plugin") {
-                await this.saveSettings(defaultSettings, type);
-            }
-            return defaultSettings;
-        }
-
-        // Handle settings migration
-        const storedKeys = Object.keys(storedSettings);
-        const defaultKeys = Object.keys(defaultSettings);
-
-        if (storedKeys.length !== defaultKeys.length) {
-            const validStoredSettings = Object.fromEntries(
-                Object.entries(storedSettings).filter(([key]) => defaultKeys.includes(key))
-            );
-            const mergedSettings = { ...defaultSettings, ...validStoredSettings } as T;
-
-            if (type === "plugin") {
-                await this.saveSettings(mergedSettings, type);
-            }
-            return mergedSettings;
-        }
-
-        return storedSettings;
+      await this.setSettings(mergedSettings);
+      return mergedSettings;
     }
 
-    public async setSettings(settings: any, genericSettings?: "user" | "system"): Promise<void> {
-        const type = this.getSettingsType(genericSettings);
-        await this.saveSettings(settings, type);
-    }
+    return storedSettings;
+  }
 }