@rimori/client 1.1.10 → 1.3.0

package/src/{core → components}/components/ContextMenu.tsx

@@ -1,7 +1,7 @@
 import React, { useState, useEffect, useRef } from "react";
-import { EventBus } from "../../plugin/fromRimori/EventBus";
+import { EventBus } from "../../fromRimori/EventBus";
 import { RimoriClient } from "../../plugin/RimoriClient";
-import { MenuEntry } from "../../plugin/fromRimori/PluginTypes";
+import { MenuEntry } from "../../fromRimori/PluginTypes";
 
 export interface Position {
   x: number,
@@ -14,28 +14,64 @@ const ContextMenu = ({ client }: { client: RimoriClient }) => {
   const [actions, setActions] = useState<MenuEntry[]>([]);
   const [position, setPosition] = useState<Position>({ x: 0, y: 0 });
   const [openOnTextSelect, setOpenOnTextSelect] = useState(false);
+  const [menuWidth, setMenuWidth] = useState<number>(0);
   const menuRef = useRef<HTMLDivElement>(null);
+  const isMobile = window.innerWidth < 768;
+
+  /**
+   * Calculates position for mobile context menu based on selected text bounds.
+   * Centers the menu horizontally over the selected text and positions it 30px below the text's end.
+   * @param selectedText - The currently selected text
+   * @param menuWidth - The width of the menu to center properly
+   * @returns Position object with x and y coordinates
+   */
+  const calculateMobilePosition = (selectedText: string, menuWidth: number = 0): Position => {
+    const selection = window.getSelection();
+    if (!selection || !selectedText) {
+      return { x: 0, y: 0, text: selectedText };
+    }
+
+    const range = selection.getRangeAt(0);
+    const rect = range.getBoundingClientRect();
+
+    // Center horizontally over the selected text, accounting for menu width
+    const centerX = rect.left + (rect.width / 2) - (menuWidth / 2);
+
+    // Position 12px below where the text ends vertically
+    const textEndY = rect.bottom + 12;
+
+    return { x: centerX, y: textEndY, text: selectedText };
+  };
 
   useEffect(() => {
-    client.plugin.getInstalled().then(plugins => {
-      setActions(plugins.flatMap(p => p.context_menu_actions).filter(Boolean));
-    });
+    const actions = client.plugin.getPluginInfo().installedPlugins.flatMap(p => p.context_menu_actions).filter(Boolean);
+    setActions(actions);
+    setOpenOnTextSelect(client.plugin.getUserInfo().context_menu_on_select);
 
-    client.plugin.getUserInfo().then((userInfo) => {
-      setOpenOnTextSelect(userInfo.context_menu_on_select);
-    })
 
     EventBus.on<{ actions: MenuEntry[] }>("global.contextMenu.createActions", ({ data }) => {
       setActions([...data.actions, ...actions]);
     });
   }, []);
 
+  // Update menu width when menu is rendered
+  useEffect(() => {
+    if (isOpen && menuRef.current) {
+      setMenuWidth(menuRef.current.offsetWidth);
+    }
+  }, [isOpen, actions]);
+
   useEffect(() => {
     // Track mouse position globally
     const handleMouseMove = (e: MouseEvent) => {
       const selectedText = window.getSelection()?.toString().trim();
       if (isOpen && selectedText === position.text) return;
-      setPosition({ x: e.clientX, y: e.clientY, text: selectedText });
+
+      if (isMobile && selectedText) {
+        setPosition(calculateMobilePosition(selectedText, menuWidth));
+      } else {
+        setPosition({ x: e.clientX, y: e.clientY, text: selectedText });
+      }
     };
 
     const handleMouseUp = (e: MouseEvent) => {
@@ -64,18 +100,26 @@ const ContextMenu = ({ client }: { client: RimoriClient }) => {
         if (e.button === 2) {
           e.preventDefault();
         }
-        setPosition({ x: e.clientX, y: e.clientY, text: selectedText });
+
+        if (isMobile) {
+          setPosition(calculateMobilePosition(selectedText, menuWidth));
+        } else {
+          setPosition({ x: e.clientX, y: e.clientY, text: selectedText });
+        }
         setIsOpen(true);
       } else {
         setIsOpen(false);
       }
     };
 
-    // Add selectionchange listener to close menu if selection is cleared
+    // Add selectionchange listener to close menu if selection is cleared and update position for mobile
     const handleSelectionChange = () => {
       const selectedText = window.getSelection()?.toString().trim();
       if (!selectedText && isOpen) {
         setIsOpen(false);
+      } else if (selectedText && isOpen && isMobile) {
+        // Update position in real-time as text selection changes on mobile
+        setPosition(calculateMobilePosition(selectedText, menuWidth));
       }
     };
 
@@ -105,7 +149,7 @@ const ContextMenu = ({ client }: { client: RimoriClient }) => {
         <MenuEntryItem key={index} icon={action.icon} text={action.text} onClick={() => {
           setIsOpen(false);
           window.getSelection()?.removeAllRanges();
-          client.event.emitSidebarAction(action.pluginId, action.actionKey, position.text);
+          client.event.emitSidebarAction(action.plugin_id, action.action_key, position.text);
         }} />
       ))}
     </div>

package/src/components.ts

@@ -1,11 +1,11 @@
 // React components and hooks exports
-export * from "./components/MarkdownEditor";
+export * from "./components/ai/Assistant";
+export * from "./components/ai/Avatar";
+export * from "./components/ai/EmbeddedAssistent/VoiceRecoder";
+export * from "./components/audio/Playbutton";
 export * from "./components/CRUDModal";
+export * from "./components/MarkdownEditor";
 export * from "./components/Spinner";
-export * from "./components/audio/Playbutton";
 export * from "./hooks/UseChatHook";
 export * from "./plugin/ThemeSetter";
-export * from "./providers/PluginProvider"; 
-export * from "./components/ai/Avatar";
-export * from "./components/ai/Assistant";
-export * from "./types/Actions";
+export * from "./providers/PluginProvider";

package/src/core/controller/AIController.ts

@@ -0,0 +1,283 @@
+import { Tool } from "../../fromRimori/PluginTypes";
+
+export interface ToolInvocation {
+  toolCallId: string;
+  toolName: string;
+  args: Record<string, string>;
+}
+
+export interface Message {
+  id?: string;
+  role: "user" | "assistant" | "system"
+  content: string;
+  toolCalls?: ToolInvocation[];
+}
+
+export async function generateText(backendUrl: string, messages: Message[], tools: Tool[], token: string) {
+  const response = await fetch(`${backendUrl}/ai/llm`, {
+    method: 'POST',
+    body: JSON.stringify({ messages, tools }),
+    headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json' }
+  });
+
+  return await response.json();
+}
+
+export type OnLLMResponse = (id: string, response: string, finished: boolean, toolInvocations?: ToolInvocation[]) => void;
+
+export async function streamChatGPT(backendUrl: string, messages: Message[], tools: Tool[], onResponse: OnLLMResponse, token: string) {
+  const messageId = Math.random().toString(36).substring(3);
+  let currentMessages: Message[] = [...messages];
+
+  console.log('Starting streamChatGPT with:', {
+    messageId,
+    messageCount: messages.length,
+    toolCount: tools.length,
+    backendUrl
+  });
+
+  while (true) {
+    const messagesForApi = currentMessages.map(({ id, ...rest }) => rest);
+
+    try {
+      const response = await fetch(`${backendUrl}/ai/llm`, {
+        method: 'POST',
+        body: JSON.stringify({ messages: messagesForApi, tools, stream: true }),
+        headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json' }
+      });
+
+      if (!response.ok) {
+        throw new Error(`HTTP error! status: ${response.status}`);
+      }
+
+      if (!response.body) {
+        console.error('No response body.');
+        return;
+      }
+
+      const reader = response.body.getReader();
+      const decoder = new TextDecoder('utf-8');
+
+      let content = "";
+      let done = false;
+      let toolInvocations: { toolCallId: string, toolName: string, args: any }[] = [];
+      let currentTextId = "";
+      let isToolCallMode = false;
+      let buffer = ""; // Buffer for incomplete chunks
+
+      while (!done) {
+        const { value, done: readerDone } = await reader.read();
+
+        if (value) {
+          const chunk = decoder.decode(value, { stream: true });
+          buffer += chunk;
+          
+          // Split by lines, but handle incomplete lines
+          const lines = buffer.split('\n');
+          
+          // Keep the last line in buffer if it's incomplete
+          if (lines.length > 1) {
+            buffer = lines.pop() || "";
+          }
+
+          for (const line of lines) {
+            if (line.trim() === '') continue;
+            
+            // Handle the new streaming format
+            if (line.startsWith('data: ')) {
+              const dataStr = line.substring(6); // Remove 'data: ' prefix
+              
+              // Handle [DONE] marker
+              if (dataStr === '[DONE]') {
+                done = true;
+                break;
+              }
+
+              try {
+                const data = JSON.parse(dataStr);
+                
+                // Log the first message to understand the format
+                if (!content && !isToolCallMode) {
+                  console.log('First stream message received:', data);
+                }
+                
+                switch (data.type) {
+                  case 'start':
+                    // Stream started, no action needed
+                    console.log('Stream started');
+                    break;
+                    
+                  case 'start-step':
+                    // Step started, no action needed
+                    console.log('Step started');
+                    break;
+                    
+                  case 'reasoning-start':
+                    // Reasoning started, no action needed
+                    console.log('Reasoning started:', data.id);
+                    break;
+                    
+                  case 'reasoning-end':
+                    // Reasoning ended, no action needed
+                    console.log('Reasoning ended:', data.id);
+                    break;
+                    
+                  case 'text-start':
+                    // Text generation started, store the ID
+                    currentTextId = data.id;
+                    console.log('Text generation started:', data.id);
+                    break;
+                    
+                  case 'text-delta':
+                    // Text delta received, append to content
+                    if (data.delta) {
+                      content += data.delta;
+                      onResponse(messageId, content, false);
+                    }
+                    break;
+                    
+                  case 'text-end':
+                    // Text generation ended
+                    console.log('Text generation ended:', data.id);
+                    break;
+                    
+                  case 'finish-step':
+                    // Step finished, no action needed
+                    console.log('Step finished');
+                    break;
+                    
+                  case 'finish':
+                    // Stream finished
+                    console.log('Stream finished');
+                    done = true;
+                    break;
+                    
+                  // Additional message types that might be present in the AI library
+                  case 'tool-call':
+                    // Tool call initiated
+                    console.log('Tool call initiated:', data);
+                    isToolCallMode = true;
+                    if (data.toolCallId && data.toolName && data.args) {
+                      toolInvocations.push({
+                        toolCallId: data.toolCallId,
+                        toolName: data.toolName,
+                        args: data.args
+                      });
+                    }
+                    break;
+                    
+                  case 'tool-call-delta':
+                    // Tool call delta (for streaming tool calls)
+                    console.log('Tool call delta:', data);
+                    break;
+                    
+                  case 'tool-call-end':
+                    // Tool call completed
+                    console.log('Tool call completed:', data);
+                    break;
+                    
+                  case 'tool-result':
+                    // Tool execution result
+                    console.log('Tool result:', data);
+                    break;
+                    
+                  case 'error':
+                    // Error occurred
+                    console.error('Stream error:', data);
+                    break;
+                    
+                  case 'usage':
+                    // Usage information
+                    console.log('Usage info:', data);
+                    break;
+                    
+                  case 'model':
+                    // Model information
+                    console.log('Model info:', data);
+                    break;
+                    
+                  case 'stop':
+                    // Stop signal
+                    console.log('Stop signal received');
+                    done = true;
+                    break;
+                    
+                  default:
+                    // Unknown type, log for debugging
+                    console.log('Unknown stream type:', data.type, data);
+                    break;
+                }
+              } catch (error) {
+                console.error('Error parsing stream data:', error, dataStr);
+              }
+            }
+          }
+        }
+
+        if (readerDone) {
+          done = true;
+        }
+      }
+
+      // Check if we have content or if this was a tool call response
+      if (content || toolInvocations.length > 0) {
+        currentMessages.push({
+          id: messageId,
+          role: "assistant",
+          content: content,
+          toolCalls: toolInvocations.length > 0 ? toolInvocations: undefined,
+        });
+      }
+
+      // Handle tool call scenario if tools were provided
+      if (tools.length > 0 && toolInvocations.length > 0) {
+        console.log('Tool calls detected, executing tools...');
+        
+        const toolResults: Message[] = [];
+        for (const toolInvocation of toolInvocations) {
+          const tool = tools.find(t => t.name === toolInvocation.toolName);
+          if (tool && tool.execute) {
+            try {
+              const result = await tool.execute(toolInvocation.args);
+              toolResults.push({
+                id: Math.random().toString(36).substring(3),
+                role: "user",
+                content: `Tool '${toolInvocation.toolName}' returned: ${JSON.stringify(result)}`,
+              });
+            } catch (error) {
+              console.error(`Error executing tool ${toolInvocation.toolName}:`, error);
+              toolResults.push({
+                id: Math.random().toString(36).substring(3),
+                role: "user",
+                content: `Tool '${toolInvocation.toolName}' failed with error: ${error}`,
+              });
+            }
+          }
+        }
+        
+        if (toolResults.length > 0) {
+          currentMessages.push(...toolResults);
+          // Continue the loop to handle the next response
+          continue;
+        }
+      }
+
+      // Since the new format doesn't seem to support tool calls in the same way,
+      // we'll assume the stream is complete when we reach the end
+      // If tools are provided and no content was generated, this might indicate a tool call
+      if (tools.length > 0 && !content && !isToolCallMode) {
+        // This might be a tool call scenario, but we need more information
+        // For now, we'll just finish the stream
+        console.log('No content generated, but tools provided - might be tool call scenario');
+      }
+
+      onResponse(messageId, content, true, toolInvocations);
+      return;
+      
+    } catch (error) {
+      console.error('Error in streamChatGPT:', error);
+      onResponse(messageId, `Error: ${error instanceof Error ? error.message : String(error)}`, true, []);
+      return;
+    }
+  }
+}

package/src/core/controller/ObjectController.ts

@@ -0,0 +1,115 @@
+type PrimitiveType = 'string' | 'number' | 'boolean';
+
+// This is the type that can appear in the `type` property
+type ObjectToolParameterType =
+  | PrimitiveType
+  | { [key: string]: ObjectToolParameter }  // for nested objects
+  | [{ [key: string]: ObjectToolParameter }];  // for arrays of objects (notice the tuple type)
+
+interface ObjectToolParameter {
+  type: ObjectToolParameterType;
+  description?: string;
+  enum?: string[];
+  optional?: boolean;
+}
+
+/**
+ * The tools that the AI can use.
+ * 
+ * The key is the name of the tool.
+ * The value is the parameter of the tool.
+ * 
+ */
+export type ObjectTool = {
+  [key: string]: ObjectToolParameter;
+};
+
+export interface ObjectRequest {
+  /**
+   * The tools that the AI can use.
+   */
+  tool: ObjectTool;
+  /**
+   * High level instructions for the AI to follow. Behaviour, tone, restrictions, etc.
+   * Example: "Act like a recipe writer."
+   */
+  behaviour?: string;
+  /**
+   * The specific instruction for the AI to follow.
+   * Example: "Generate a recipe using chicken, rice and vegetables."
+   */
+  instructions: string;
+}
+
+export async function generateObject(backendUrl: string, request: ObjectRequest, token: string) {
+  return await fetch(`${backendUrl}/ai/llm-object`, {
+    method: 'POST',
+    body: JSON.stringify({
+      stream: false,
+      tool: request.tool,
+      behaviour: request.behaviour,
+      instructions: request.instructions,
+    }),
+    headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json' }
+  }).then(response => response.json());
+}
+
+// TODO adjust stream to work with object
+export type OnLLMResponse = (id: string, response: string, finished: boolean, toolInvocations?: any[]) => void;
+
+export async function streamObject(backendUrl: string, request: ObjectRequest, onResponse: OnLLMResponse, token: string) {
+  const messageId = Math.random().toString(36).substring(3);
+  const response = await fetch(`${backendUrl}/ai/llm-object`, {
+    method: 'POST',
+    body: JSON.stringify({
+      stream: true,
+      tools: request.tool,
+      systemInstructions: request.behaviour,
+      secondaryInstructions: request.instructions,
+    }),
+    headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json' }
+  });
+
+  if (!response.body) {
+    console.error('No response body.');
+    return;
+  }
+
+  const reader = response.body.getReader();
+  const decoder = new TextDecoder('utf-8');
+
+  let content = "";
+  let done = false;
+  let toolInvocations: any[] = [];
+  while (!done) {
+    const { value } = await reader.read();
+
+    if (value) {
+      const chunk = decoder.decode(value, { stream: true });
+      const lines = chunk.split('\n').filter(line => line.trim() !== '');
+
+      for (const line of lines) {
+        const data = line.substring(3, line.length - 1);
+        const command = line.substring(0, 1);
+        // console.log("data: ", { line, data, command });
+
+        if (command === '0') {
+          content += data;
+          // console.log("AI response:", content);
+
+          //content \n\n should be real line break when message is displayed
+          onResponse(messageId, content.replace(/\\n/g, '\n').replace(/\\+"/g, '"'), false);
+        } else if (command === 'd') {
+          // console.log("AI usage:", JSON.parse(line.substring(2)));
+          done = true;
+          break;
+        } else if (command === '9') {
+          // console.log("tool call:", JSON.parse(line.substring(2)));
+          // console.log("tools", tools);
+          toolInvocations.push(JSON.parse(line.substring(2)));
+        }
+      }
+    }
+  }
+  onResponse(messageId, content.replace(/\\n/g, '\n').replace(/\\+"/g, '"'), true, toolInvocations);
+}

package/src/{controller → core/controller}/SettingsController.ts

@@ -1,6 +1,15 @@
 import { SupabaseClient } from "@supabase/supabase-js";
-import { LanguageLevel } from "../utils/difficultyConverter";
-import { Language } from "../utils/Language";
+import { LanguageLevel } from "../../utils/difficultyConverter";
+import { Language } from "../../utils/Language";
+
+export interface Buddy {
+  id: string;
+  name: string;
+  description: string;
+  avatarUrl: string;
+  voiceId: string;
+  aiPersonality: string;
+}
 
 export interface UserInfo {
   skill_level_reading: LanguageLevel;
@@ -11,13 +20,30 @@ export interface UserInfo {
   skill_level_understanding: LanguageLevel;
   goal_longterm: string;
   goal_weekly: string;
-  study_buddy: string;
+  study_buddy: Buddy;
   story_genre: string;
   study_duration: number;
+  /**
+   * The 2 letter language code of the language the user speaks natively.
+   * With the function getLanguageName, the language name can be retrieved.
+   */
   mother_tongue: Language;
+  /**
+   * The language the user targets to learn.
+   */
+  target_language: Language;
   motivation_type: string;
   onboarding_completed: boolean;
   context_menu_on_select: boolean;
+  user_name?: string;
+  /**
+   * ISO 3166-1 alpha-2 country code of user's location (exposed to plugins)
+   */
+  location_country: string;
+  /**
+   * Optional: nearest big city (>100,000) near user's location
+   */
+  location_city?: string;
 }
 
 export class SettingsController {
@@ -43,32 +69,6 @@ export class SettingsController {
     await this.supabase.from("plugin_settings").upsert({ plugin_id: this.pluginId, settings });
   }
 
-  public async getUserInfo(): Promise<UserInfo> {
-    const { data } = await this.supabase.from("profiles").select("*");
-
-    if (!data || data.length === 0) {
-      return {
-        mother_tongue: "en",
-        skill_level_listening: "Pre-A1",
-        skill_level_reading: "Pre-A1",
-        skill_level_speaking: "Pre-A1",
-        skill_level_writing: "Pre-A1",
-        skill_level_understanding: "Pre-A1",
-        skill_level_grammar: "Pre-A1",
-        goal_longterm: "",
-        goal_weekly: "",
-        study_buddy: "clarence",
-        story_genre: "adventure",
-        study_duration: 30,
-        motivation_type: "self-motivated",
-        onboarding_completed: false,
-        context_menu_on_select: false,
-      }
-    }
-
-    return data[0].settings;
-  }
-
   /**
    * Get the settings for the plugin. T can be any type of settings, UserSettings or SystemSettings.
    * @param defaultSettings The default settings to use if no settings are found.