@rimori/client 1.2.0 → 1.3.0

package/src/components/ai/EmbeddedAssistent/VoiceRecoder.tsx

@@ -1,6 +1,7 @@
-import { useState, useRef, forwardRef, useImperativeHandle, useEffect } from 'react';
+import { useRimori } from '../../../components';
 import { FaMicrophone, FaSpinner } from 'react-icons/fa6';
-import { usePlugin } from '../../../components';
+import { AudioController } from '../../../plugin/AudioController';
+import { useState, useRef, forwardRef, useImperativeHandle, useEffect } from 'react';
 
 interface Props {
   iconSize?: string;
@@ -15,10 +16,8 @@ interface Props {
 export const VoiceRecorder = forwardRef(({ onVoiceRecorded, iconSize, className, disabled, loading, onRecordingStatusChange, enablePushToTalk = false }: Props, ref) => {
   const [isRecording, setIsRecording] = useState(false);
   const [internalIsProcessing, setInternalIsProcessing] = useState(false);
-  const mediaRecorderRef = useRef<MediaRecorder | null>(null);
-  const audioChunksRef = useRef<Blob[]>([]);
-  const mediaStreamRef = useRef<MediaStream | null>(null);
-  const { ai: llm } = usePlugin();
+  const audioControllerRef = useRef<AudioController | null>(null);
+  const { ai, plugin } = useRimori();
 
   // Ref for latest onVoiceRecorded callback
   const onVoiceRecordedRef = useRef(onVoiceRecorded);
@@ -27,41 +26,49 @@ export const VoiceRecorder = forwardRef(({ onVoiceRecorded, iconSize, className,
   }, [onVoiceRecorded]);
 
   const startRecording = async () => {
-    const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
-    mediaStreamRef.current = stream;
-    const mediaRecorder = new MediaRecorder(stream);
-    mediaRecorderRef.current = mediaRecorder;
+    try {
+      if (!audioControllerRef.current) {
+        audioControllerRef.current = new AudioController(plugin.pluginId);
+      }
 
-    mediaRecorder.ondataavailable = (event) => {
-      audioChunksRef.current.push(event.data);
-    };
+      await audioControllerRef.current.startRecording();
+      setIsRecording(true);
+      onRecordingStatusChange(true);
+    } catch (error) {
+      console.error('Failed to start recording:', error);
+      // Handle permission denied or other errors
+    }
+  };
 
-    mediaRecorder.onstop = async () => {
-      const audioBlob = new Blob(audioChunksRef.current);
-      audioChunksRef.current = [];
 
 
-      setInternalIsProcessing(true);
-      const text = await llm.getTextFromVoice(audioBlob);
-      setInternalIsProcessing(false);
-      onVoiceRecordedRef.current(text);
-    };
+  const stopRecording = async () => {
+    try {
+      if (audioControllerRef.current && isRecording) {
+        const audioResult = await audioControllerRef.current.stopRecording();
+        // console.log("audioResult: ", audioResult);
 
-    mediaRecorder.start();
-    setIsRecording(true);
-    onRecordingStatusChange(true);
-  };
+        setInternalIsProcessing(true);
+
+        // Play the recorded audio from the Blob
+        // const blobUrl = URL.createObjectURL(audioResult.recording);
+        // const audioRef = new Audio(blobUrl);
+        // audioRef.onended = () => URL.revokeObjectURL(blobUrl);
+        // audioRef.play().catch((e) => console.error('Playback error:', e));
 
-  const stopRecording = () => {
-    if (mediaRecorderRef.current) {
-      mediaRecorderRef.current.stop();
+        // console.log("audioBlob: ", audioResult.recording);
+        const text = await ai.getTextFromVoice(audioResult.recording);
+        // console.log("stt result", text);
+        // throw new Error("test");
+        setInternalIsProcessing(false);
+        onVoiceRecordedRef.current(text);
+      }
+    } catch (error) {
+      console.error('Failed to stop recording:', error);
+    } finally {
       setIsRecording(false);
       onRecordingStatusChange(false);
     }
-    if (mediaStreamRef.current) {
-      mediaStreamRef.current.getTracks().forEach(track => track.stop());
-      mediaStreamRef.current = null;
-    }
   };
 
   useImperativeHandle(ref, () => ({

package/src/components/audio/Playbutton.tsx

@@ -1,6 +1,6 @@
 import React, { useState, useEffect } from 'react';
 import { FaPlayCircle, FaStopCircle } from "react-icons/fa";
-import { usePlugin } from "../../providers/PluginProvider";
+import { useRimori } from "../../providers/PluginProvider";
 import { Spinner } from '../Spinner';
 import { EventBus } from '../../fromRimori/EventBus';
 
@@ -34,7 +34,7 @@ export const AudioPlayer: React.FC<AudioPlayerProps> = ({
     const [speed, setSpeed] = useState(initialSpeed);
     const [isPlaying, setIsPlaying] = useState(false);
     const [isLoading, setIsLoading] = useState(false);
-    const { ai } = usePlugin();
+    const { ai } = useRimori();
 
     useEffect(() => {
         if (!playListenerEvent) return;

package/src/components/components/ContextMenu.tsx

@@ -14,28 +14,64 @@ const ContextMenu = ({ client }: { client: RimoriClient }) => {
   const [actions, setActions] = useState<MenuEntry[]>([]);
   const [position, setPosition] = useState<Position>({ x: 0, y: 0 });
   const [openOnTextSelect, setOpenOnTextSelect] = useState(false);
+  const [menuWidth, setMenuWidth] = useState<number>(0);
   const menuRef = useRef<HTMLDivElement>(null);
+  const isMobile = window.innerWidth < 768;
+
+  /**
+   * Calculates position for mobile context menu based on selected text bounds.
+   * Centers the menu horizontally over the selected text and positions it 30px below the text's end.
+   * @param selectedText - The currently selected text
+   * @param menuWidth - The width of the menu to center properly
+   * @returns Position object with x and y coordinates
+   */
+  const calculateMobilePosition = (selectedText: string, menuWidth: number = 0): Position => {
+    const selection = window.getSelection();
+    if (!selection || !selectedText) {
+      return { x: 0, y: 0, text: selectedText };
+    }
+
+    const range = selection.getRangeAt(0);
+    const rect = range.getBoundingClientRect();
+
+    // Center horizontally over the selected text, accounting for menu width
+    const centerX = rect.left + (rect.width / 2) - (menuWidth / 2);
+
+    // Position 12px below where the text ends vertically
+    const textEndY = rect.bottom + 12;
+
+    return { x: centerX, y: textEndY, text: selectedText };
+  };
 
   useEffect(() => {
-    client.plugin.getInstalled().then(plugins => {
-      setActions(plugins.flatMap(p => p.context_menu_actions).filter(Boolean));
-    });
+    const actions = client.plugin.getPluginInfo().installedPlugins.flatMap(p => p.context_menu_actions).filter(Boolean);
+    setActions(actions);
+    setOpenOnTextSelect(client.plugin.getUserInfo().context_menu_on_select);
 
-    client.plugin.getUserInfo().then((userInfo) => {
-      setOpenOnTextSelect(userInfo.context_menu_on_select);
-    })
 
     EventBus.on<{ actions: MenuEntry[] }>("global.contextMenu.createActions", ({ data }) => {
       setActions([...data.actions, ...actions]);
     });
   }, []);
 
+  // Update menu width when menu is rendered
+  useEffect(() => {
+    if (isOpen && menuRef.current) {
+      setMenuWidth(menuRef.current.offsetWidth);
+    }
+  }, [isOpen, actions]);
+
   useEffect(() => {
     // Track mouse position globally
     const handleMouseMove = (e: MouseEvent) => {
       const selectedText = window.getSelection()?.toString().trim();
       if (isOpen && selectedText === position.text) return;
-      setPosition({ x: e.clientX, y: e.clientY, text: selectedText });
+
+      if (isMobile && selectedText) {
+        setPosition(calculateMobilePosition(selectedText, menuWidth));
+      } else {
+        setPosition({ x: e.clientX, y: e.clientY, text: selectedText });
+      }
     };
 
     const handleMouseUp = (e: MouseEvent) => {
@@ -64,18 +100,26 @@ const ContextMenu = ({ client }: { client: RimoriClient }) => {
         if (e.button === 2) {
           e.preventDefault();
         }
-        setPosition({ x: e.clientX, y: e.clientY, text: selectedText });
+
+        if (isMobile) {
+          setPosition(calculateMobilePosition(selectedText, menuWidth));
+        } else {
+          setPosition({ x: e.clientX, y: e.clientY, text: selectedText });
+        }
         setIsOpen(true);
       } else {
         setIsOpen(false);
       }
     };
 
-    // Add selectionchange listener to close menu if selection is cleared
+    // Add selectionchange listener to close menu if selection is cleared and update position for mobile
     const handleSelectionChange = () => {
       const selectedText = window.getSelection()?.toString().trim();
       if (!selectedText && isOpen) {
         setIsOpen(false);
+      } else if (selectedText && isOpen && isMobile) {
+        // Update position in real-time as text selection changes on mobile
+        setPosition(calculateMobilePosition(selectedText, menuWidth));
       }
     };
 

package/src/core/controller/AIController.ts

@@ -29,94 +29,255 @@ export async function streamChatGPT(backendUrl: string, messages: Message[], too
   const messageId = Math.random().toString(36).substring(3);
   let currentMessages: Message[] = [...messages];
 
+  console.log('Starting streamChatGPT with:', {
+    messageId,
+    messageCount: messages.length,
+    toolCount: tools.length,
+    backendUrl
+  });
+
   while (true) {
     const messagesForApi = currentMessages.map(({ id, ...rest }) => rest);
 
-    const response = await fetch(`${backendUrl}/ai/llm`, {
-      method: 'POST',
-      body: JSON.stringify({ messages: messagesForApi, tools, stream: true }),
-      headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json' }
-    });
+    try {
+      const response = await fetch(`${backendUrl}/ai/llm`, {
+        method: 'POST',
+        body: JSON.stringify({ messages: messagesForApi, tools, stream: true }),
+        headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json' }
+      });
 
-    if (!response.body) {
-      console.error('No response body.');
-      return;
-    }
+      if (!response.ok) {
+        throw new Error(`HTTP error! status: ${response.status}`);
+      }
+
+      if (!response.body) {
+        console.error('No response body.');
+        return;
+      }
 
-    const reader = response.body.getReader();
-    const decoder = new TextDecoder('utf-8');
-
-    let content = "";
-    let done = false;
-    let toolInvocations: { toolCallId: string, toolName: string, args: any }[] = [];
-    let finishReason = "";
-
-    while (!done) {
-      const { value, done: readerDone } = await reader.read();
-
-      if (value) {
-        const chunk = decoder.decode(value, { stream: true });
-        const lines = chunk.split('\n').filter(line => line.trim() !== '');
-
-        for (const line of lines) {
-          const command = line.substring(0, 1);
-
-          if (command === '0') {
-            const data = line.substring(3, line.length - 1);
-            content += data;
-            onResponse(messageId, content.replace(/\\n/g, '\n').replace(/\\+"/g, '"'), false);
-          } else if (command === 'd' || command === 'e') {
-            const eventData = JSON.parse(line.substring(2));
-            finishReason = eventData.finishReason;
-            done = true;
-            break;
-          } else if (command === '9') {
-            const toolInvocation = JSON.parse(line.substring(2));
-            toolInvocations.push(toolInvocation);
+      const reader = response.body.getReader();
+      const decoder = new TextDecoder('utf-8');
+
+      let content = "";
+      let done = false;
+      let toolInvocations: { toolCallId: string, toolName: string, args: any }[] = [];
+      let currentTextId = "";
+      let isToolCallMode = false;
+      let buffer = ""; // Buffer for incomplete chunks
+
+      while (!done) {
+        const { value, done: readerDone } = await reader.read();
+
+        if (value) {
+          const chunk = decoder.decode(value, { stream: true });
+          buffer += chunk;
+          
+          // Split by lines, but handle incomplete lines
+          const lines = buffer.split('\n');
+          
+          // Keep the last line in buffer if it's incomplete
+          if (lines.length > 1) {
+            buffer = lines.pop() || "";
+          }
+
+          for (const line of lines) {
+            if (line.trim() === '') continue;
+            
+            // Handle the new streaming format
+            if (line.startsWith('data: ')) {
+              const dataStr = line.substring(6); // Remove 'data: ' prefix
+              
+              // Handle [DONE] marker
+              if (dataStr === '[DONE]') {
+                done = true;
+                break;
+              }
+
+              try {
+                const data = JSON.parse(dataStr);
+                
+                // Log the first message to understand the format
+                if (!content && !isToolCallMode) {
+                  console.log('First stream message received:', data);
+                }
+                
+                switch (data.type) {
+                  case 'start':
+                    // Stream started, no action needed
+                    console.log('Stream started');
+                    break;
+                    
+                  case 'start-step':
+                    // Step started, no action needed
+                    console.log('Step started');
+                    break;
+                    
+                  case 'reasoning-start':
+                    // Reasoning started, no action needed
+                    console.log('Reasoning started:', data.id);
+                    break;
+                    
+                  case 'reasoning-end':
+                    // Reasoning ended, no action needed
+                    console.log('Reasoning ended:', data.id);
+                    break;
+                    
+                  case 'text-start':
+                    // Text generation started, store the ID
+                    currentTextId = data.id;
+                    console.log('Text generation started:', data.id);
+                    break;
+                    
+                  case 'text-delta':
+                    // Text delta received, append to content
+                    if (data.delta) {
+                      content += data.delta;
+                      onResponse(messageId, content, false);
+                    }
+                    break;
+                    
+                  case 'text-end':
+                    // Text generation ended
+                    console.log('Text generation ended:', data.id);
+                    break;
+                    
+                  case 'finish-step':
+                    // Step finished, no action needed
+                    console.log('Step finished');
+                    break;
+                    
+                  case 'finish':
+                    // Stream finished
+                    console.log('Stream finished');
+                    done = true;
+                    break;
+                    
+                  // Additional message types that might be present in the AI library
+                  case 'tool-call':
+                    // Tool call initiated
+                    console.log('Tool call initiated:', data);
+                    isToolCallMode = true;
+                    if (data.toolCallId && data.toolName && data.args) {
+                      toolInvocations.push({
+                        toolCallId: data.toolCallId,
+                        toolName: data.toolName,
+                        args: data.args
+                      });
+                    }
+                    break;
+                    
+                  case 'tool-call-delta':
+                    // Tool call delta (for streaming tool calls)
+                    console.log('Tool call delta:', data);
+                    break;
+                    
+                  case 'tool-call-end':
+                    // Tool call completed
+                    console.log('Tool call completed:', data);
+                    break;
+                    
+                  case 'tool-result':
+                    // Tool execution result
+                    console.log('Tool result:', data);
+                    break;
+                    
+                  case 'error':
+                    // Error occurred
+                    console.error('Stream error:', data);
+                    break;
+                    
+                  case 'usage':
+                    // Usage information
+                    console.log('Usage info:', data);
+                    break;
+                    
+                  case 'model':
+                    // Model information
+                    console.log('Model info:', data);
+                    break;
+                    
+                  case 'stop':
+                    // Stop signal
+                    console.log('Stop signal received');
+                    done = true;
+                    break;
+                    
+                  default:
+                    // Unknown type, log for debugging
+                    console.log('Unknown stream type:', data.type, data);
+                    break;
+                }
+              } catch (error) {
+                console.error('Error parsing stream data:', error, dataStr);
+              }
+            }
           }
         }
-      }
 
-      if (readerDone) {
-        done = true;
+        if (readerDone) {
+          done = true;
+        }
       }
-    }
-
-    if (content || toolInvocations.length > 0) {
-      currentMessages.push({
-        id: messageId,
-        role: "assistant",
-        content: content,
-        toolCalls: toolInvocations.length > 0 ? toolInvocations: undefined,
-      });
-    }
 
-    if (finishReason !== 'tool-calls') {
-      onResponse(messageId, content.replace(/\\n/g, '\n'), true, toolInvocations);
-      return;
-    }
+      // Check if we have content or if this was a tool call response
+      if (content || toolInvocations.length > 0) {
+        currentMessages.push({
+          id: messageId,
+          role: "assistant",
+          content: content,
+          toolCalls: toolInvocations.length > 0 ? toolInvocations: undefined,
+        });
+      }
 
-    const toolResults: Message[] = [];
-    for (const toolInvocation of toolInvocations) {
-      const tool = tools.find(t => t.name === toolInvocation.toolName);
-      if (tool && tool.execute) {
-        try {
-          const result = await tool.execute(toolInvocation.args);
-          toolResults.push({
-            id: Math.random().toString(36).substring(3),
-            role: "user",
-            content: `Tool '${toolInvocation.toolName}' returned: ${JSON.stringify(result)}`,
-          });
-        } catch (error) {
-          console.error(`Error executing tool ${toolInvocation.toolName}:`, error);
-          toolResults.push({
-            id: Math.random().toString(36).substring(3),
-            role: "user",
-            content: `Tool '${toolInvocation.toolName}' failed with error: ${error}`,
-          });
+      // Handle tool call scenario if tools were provided
+      if (tools.length > 0 && toolInvocations.length > 0) {
+        console.log('Tool calls detected, executing tools...');
+        
+        const toolResults: Message[] = [];
+        for (const toolInvocation of toolInvocations) {
+          const tool = tools.find(t => t.name === toolInvocation.toolName);
+          if (tool && tool.execute) {
+            try {
+              const result = await tool.execute(toolInvocation.args);
+              toolResults.push({
+                id: Math.random().toString(36).substring(3),
+                role: "user",
+                content: `Tool '${toolInvocation.toolName}' returned: ${JSON.stringify(result)}`,
+              });
+            } catch (error) {
+              console.error(`Error executing tool ${toolInvocation.toolName}:`, error);
+              toolResults.push({
+                id: Math.random().toString(36).substring(3),
+                role: "user",
+                content: `Tool '${toolInvocation.toolName}' failed with error: ${error}`,
+              });
+            }
+          }
+        }
+        
+        if (toolResults.length > 0) {
+          currentMessages.push(...toolResults);
+          // Continue the loop to handle the next response
+          continue;
         }
       }
+
+      // Since the new format doesn't seem to support tool calls in the same way,
+      // we'll assume the stream is complete when we reach the end
+      // If tools are provided and no content was generated, this might indicate a tool call
+      if (tools.length > 0 && !content && !isToolCallMode) {
+        // This might be a tool call scenario, but we need more information
+        // For now, we'll just finish the stream
+        console.log('No content generated, but tools provided - might be tool call scenario');
+      }
+
+      onResponse(messageId, content, true, toolInvocations);
+      return;
+      
+    } catch (error) {
+      console.error('Error in streamChatGPT:', error);
+      onResponse(messageId, `Error: ${error instanceof Error ? error.message : String(error)}`, true, []);
+      return;
     }
-    currentMessages.push(...toolResults);
   }
 }

package/src/core/controller/ObjectController.ts

@@ -41,8 +41,8 @@ export interface ObjectRequest {
   instructions: string;
 }
 
-export async function generateObject(supabaseUrl: string, request: ObjectRequest, token: string) {
-  return await fetch(`${supabaseUrl}/functions/v1/llm-object`, {
+export async function generateObject(backendUrl: string, request: ObjectRequest, token: string) {
+  return await fetch(`${backendUrl}/ai/llm-object`, {
     method: 'POST',
     body: JSON.stringify({
       stream: false,
@@ -50,16 +50,16 @@ export async function generateObject(supabaseUrl: string, request: ObjectRequest
       behaviour: request.behaviour,
       instructions: request.instructions,
     }),
-    headers: { 'Authorization': `Bearer ${token}` }
+    headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json' }
   }).then(response => response.json());
 }
 
 // TODO adjust stream to work with object
 export type OnLLMResponse = (id: string, response: string, finished: boolean, toolInvocations?: any[]) => void;
 
-export async function streamObject(supabaseUrl: string, request: ObjectRequest, onResponse: OnLLMResponse, token: string) {
+export async function streamObject(backendUrl: string, request: ObjectRequest, onResponse: OnLLMResponse, token: string) {
   const messageId = Math.random().toString(36).substring(3);
-  const response = await fetch(`${supabaseUrl}/functions/v1/llm-object`, {
+  const response = await fetch(`${backendUrl}/ai/llm-object`, {
     method: 'POST',
     body: JSON.stringify({
       stream: true,
@@ -67,7 +67,7 @@ export async function streamObject(supabaseUrl: string, request: ObjectRequest,
       systemInstructions: request.behaviour,
       secondaryInstructions: request.instructions,
     }),
-    headers: { 'Authorization': `Bearer ${token}` }
+    headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json' }
   });
 
   if (!response.body) {
@@ -98,7 +98,7 @@ export async function streamObject(supabaseUrl: string, request: ObjectRequest,
           // console.log("AI response:", content);
 
           //content \n\n should be real line break when message is displayed
-          onResponse(messageId, content.replace(/\\n/g, '\n'), false);
+          onResponse(messageId, content.replace(/\\n/g, '\n').replace(/\\+"/g, '"'), false);
         } else if (command === 'd') {
           // console.log("AI usage:", JSON.parse(line.substring(2)));
           done = true;
@@ -111,5 +111,5 @@ export async function streamObject(supabaseUrl: string, request: ObjectRequest,
       }
     }
   }
-  onResponse(messageId, content.replace(/\\n/g, '\n'), true, toolInvocations);
+  onResponse(messageId, content.replace(/\\n/g, '\n').replace(/\\+"/g, '"'), true, toolInvocations);
 }

package/src/core/controller/SettingsController.ts

@@ -23,11 +23,27 @@ export interface UserInfo {
   study_buddy: Buddy;
   story_genre: string;
   study_duration: number;
+  /**
+   * The 2 letter language code of the language the user speaks natively.
+   * With the function getLanguageName, the language name can be retrieved.
+   */
   mother_tongue: Language;
+  /**
+   * The language the user targets to learn.
+   */
+  target_language: Language;
   motivation_type: string;
   onboarding_completed: boolean;
   context_menu_on_select: boolean;
   user_name?: string;
+  /**
+   * ISO 3166-1 alpha-2 country code of user's location (exposed to plugins)
+   */
+  location_country: string;
+  /**
+   * Optional: nearest big city (>100,000) near user's location
+   */
+  location_city?: string;
 }
 
 export class SettingsController {