@rimori/client 1.0.2 → 1.0.4

package/src/components/ai/EmbeddedAssistent/TTS/Player.ts

@@ -0,0 +1,192 @@
+export class ChunkedAudioPlayer {
+
+    private audioContext!: AudioContext;
+    private chunkQueue: ArrayBuffer[] = [];
+    private isPlaying = false;
+    private analyser!: AnalyserNode;
+    private dataArray!: Uint8Array;
+    private shouldMonitorLoudness = true;
+    private isMonitoring = false;
+    private handle = 0;
+    private volume = 1.0;
+    private loudnessCallback: (value: number) => void = () => { };
+    private currentIndex = 0;
+    private startedPlaying = false;
+
+    constructor() {
+        this.init();
+    }
+
+    private init(): void {
+        this.audioContext = new AudioContext();
+        this.analyser = this.audioContext.createAnalyser();
+        this.analyser.fftSize = 256; // Set the FFT size (smaller values provide faster updates, larger ones give better resolution)
+        const bufferLength = this.analyser.frequencyBinCount;
+        this.dataArray = new Uint8Array(bufferLength); // Array to hold frequency data
+    }
+
+    public setOnLoudnessChange(callback: (value: number) => void) {
+        this.loudnessCallback = callback;
+    }
+
+    public setVolume(volume: number) {
+        this.volume = volume;
+    }
+
+    public async addChunk(chunk: ArrayBuffer, position: number): Promise<void> {
+        console.log('Adding chunk', position, chunk);
+        this.chunkQueue[position] = chunk;
+        // console.log("received chunk", {
+        //     chunkQueue: this.chunkQueue.length,
+        //     isPlaying: this.isPlaying,
+        // })
+
+        if (position === 0 && !this.startedPlaying) {
+            this.startedPlaying = true;
+            this.playChunks();
+        }
+    }
+
+    private playChunks(): void {
+        // console.log({ isPlaying: this.isPlaying });
+        if (this.isPlaying) return;
+        if (!this.chunkQueue[this.currentIndex]) {
+            // wait until the correct chunk arrives
+            setTimeout(() => this.playChunks(), 10);
+        }
+        this.isPlaying = true;
+
+        this.playChunk(this.chunkQueue[this.currentIndex]).then(() => {
+            this.isPlaying = false;
+            this.currentIndex++;
+            if (this.chunkQueue[this.currentIndex]) {
+                this.shouldMonitorLoudness = true;
+                this.playChunks();
+            } else {
+                // console.log('Playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
+                setTimeout(() => {
+                    // console.log('Check again if really playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
+                    if (this.chunkQueue.length > this.currentIndex) {
+                        this.playChunks();
+                    } else {
+                        this.startedPlaying = false;
+                        this.shouldMonitorLoudness = false;
+                    }
+                }, 1000);
+            }
+        });
+    }
+
+    public stopPlayback(): void {
+        // console.log('Stopping playback');
+        // Implement logic to stop the current playback
+        this.isPlaying = false;
+        this.chunkQueue = [];
+        this.startedPlaying = false;
+        this.shouldMonitorLoudness = false;
+        cancelAnimationFrame(this.handle);
+    }
+
+    private playChunk(chunk: ArrayBuffer): Promise<void> {
+        console.log({queue: this.chunkQueue})
+        if (!chunk) {
+            return Promise.resolve();
+        }
+
+        // console.log('Playing chunk', chunk);
+        return new Promise((resolve) => {
+            const source = this.audioContext.createBufferSource();
+            this.audioContext.decodeAudioData(chunk.slice(0)).then((audioBuffer) => {
+                source.buffer = audioBuffer;
+
+                // Create a GainNode for volume control
+                const gainNode = this.audioContext.createGain();
+                gainNode.gain.value = this.volume;
+
+                // Connect the source to the GainNode, then to the analyser node, then to the destination (speakers)
+                source.connect(gainNode);
+                gainNode.connect(this.analyser);
+                this.analyser.connect(this.audioContext.destination);
+
+                source.start(0);
+                // console.log('Playing chunk', this.currentIndex);
+                gainNode.gain.value = this.volume;
+                source.onended = () => {
+                    // console.log('Chunk ended');
+                    resolve();
+                };
+
+                // Start monitoring loudness only once
+                if (!this.isMonitoring) {
+                    this.isMonitoring = true;
+                    this.shouldMonitorLoudness = true;
+                    this.monitorLoudness();
+                }
+            });
+        });
+    }
+
+    async playAgain(): Promise<void> {
+        console.log('Playing again');
+        if (this.chunkQueue.length > 0 && !this.isPlaying) {
+            this.playChunks();
+        }
+    }
+
+    private monitorLoudness(): void {
+        // Stop monitoring when the flag is false
+        if (!this.shouldMonitorLoudness) {
+            // console.log('Loudness monitoring stopped.');
+            cancelAnimationFrame(this.handle);
+            this.loudnessCallback(0);
+            return;
+        }
+
+        // Get the time domain data from the analyser (this is a snapshot of the waveform)
+        this.analyser.getByteTimeDomainData(this.dataArray);
+
+        // Calculate the RMS (root mean square) of the waveform values to get the perceived loudness
+        let sum = 0;
+        for (let i = 0; i < this.dataArray.length; i++) {
+            const value = this.dataArray[i] / 128.0 - 1.0; // Normalize between -1 and 1
+            sum += value * value;
+        }
+
+        const rms = Math.sqrt(sum / this.dataArray.length);
+
+        // Handle the case where RMS is 0 to avoid log10(0)
+        if (rms === 0) {
+            // console.log('Current loudness: Silent');
+        } else {
+            let loudnessInDb = 20 * Math.log10(rms); // Convert to dB
+            // console.log('Current loudness:' + loudnessInDb);
+            const minDb = -57;
+            const maxDb = -15;
+
+            if (loudnessInDb < minDb) {
+                loudnessInDb = minDb;
+            }
+            if (loudnessInDb > maxDb) {
+                loudnessInDb = maxDb;
+            }
+
+            const loudnessScale = ((loudnessInDb - minDb) / (maxDb - minDb)) * 100;
+            // console.log("root:corrent loudness", loudnessScale);
+
+            this.loudnessCallback(loudnessScale);
+        }
+
+        // Call this method again at regular intervals if you want continuous loudness monitoring
+        this.handle = requestAnimationFrame(() => this.monitorLoudness());
+    }
+    public reset() {
+        // console.log('Resetting player');
+        this.stopPlayback();
+        this.currentIndex = 0;
+        this.shouldMonitorLoudness = true;
+        //reset to the beginning when the class gets initialized
+        this.isMonitoring = false;
+        this.isPlaying = false;
+        this.init();
+    }
+}

package/src/components/ai/EmbeddedAssistent/VoiceRecoder.tsx

@@ -0,0 +1,56 @@
+import { useState, useRef, forwardRef, useImperativeHandle } from 'react';
+import { FaMicrophone } from 'react-icons/fa6';
+import { usePlugin } from '../../../components';
+
+interface Props {
+  iconSize?: string;
+  className?: string;
+  onVoiceRecorded: (message: string) => void;
+}
+
+export const VoiceRecorder = forwardRef(({ onVoiceRecorded, iconSize, className }: Props, ref) => {
+  const [isRecording, setIsRecording] = useState(false);
+  const mediaRecorderRef = useRef<MediaRecorder | null>(null);
+  const audioChunksRef = useRef<Blob[]>([]);
+  const { llm } = usePlugin();
+
+  const startRecording = async () => {
+    const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+    const mediaRecorder = new MediaRecorder(stream);
+    mediaRecorderRef.current = mediaRecorder;
+
+    mediaRecorder.ondataavailable = (event) => {
+      audioChunksRef.current.push(event.data);
+    };
+
+    mediaRecorder.onstop = async () => {
+      const audioBlob = new Blob(audioChunksRef.current);
+      audioChunksRef.current = [];
+
+      onVoiceRecorded(await llm.getTextFromVoice(audioBlob));
+    };
+
+    mediaRecorder.start();
+    setIsRecording(true);
+  };
+
+  const stopRecording = () => {
+    if (mediaRecorderRef.current) {
+      mediaRecorderRef.current.stop();
+      setIsRecording(false);
+    }
+  };
+
+  useImperativeHandle(ref, () => ({
+    startRecording,
+    stopRecording,
+  }));
+
+  return (
+    <div className={className}>
+      <button onClick={isRecording ? stopRecording : startRecording}>
+        <FaMicrophone size={iconSize} className={"h-7 w-7 mr-2 " + (isRecording ? "text-red-600" : "")} />
+      </button>
+    </div>
+  );
+});

package/src/components/ai/utils.ts

@@ -0,0 +1,23 @@
+export interface FirstMessages {
+  instructions?: string;
+  userMessage: string;
+  assistantMessage?: string;
+}
+
+export function getFirstMessages(instructions: FirstMessages): any[] {
+  const messages = [];
+
+  if (instructions.instructions) {
+      messages.push({ id: '1', role: 'system', content: instructions.instructions });
+  }
+  if (instructions.userMessage) {
+      messages.push({ id: '2', role: 'user', content: instructions.userMessage });
+  }
+  if (instructions.assistantMessage) {
+      messages.push({ id: '3', role: 'assistant', content: instructions.assistantMessage });
+  }
+
+  console.log("getFirstMessages", messages);
+
+  return messages;
+}

package/src/components/audio/Playbutton.tsx

@@ -2,7 +2,7 @@ import React, { useState, useEffect } from 'react';
 import { FaPlayCircle, FaStopCircle } from "react-icons/fa";
 import { usePlugin } from "../../providers/PluginProvider";
 import { Spinner } from '../Spinner';
-import { EmitterSingleton } from '../../providers/EventEmitter';
+import { EventBus } from '../../plugin/fromRimori/EventBus';
 
 type AudioPlayerProps = {
     text: string;
@@ -34,12 +34,11 @@ export const AudioPlayer: React.FC<AudioPlayerProps> = ({
     const [speed, setSpeed] = useState(initialSpeed);
     const [isPlaying, setIsPlaying] = useState(false);
     const [isLoading, setIsLoading] = useState(false);
-    const { getVoiceResponse, } = usePlugin();
-    const emitter = EmitterSingleton;
+    const { llm } = usePlugin();
 
     useEffect(() => {
         if (!playListenerEvent) return;
-        emitter.on(playListenerEvent, () => togglePlayback());
+        EventBus.on(playListenerEvent, () => togglePlayback());
     }, [playListenerEvent]);
 
     useEffect(() => {
@@ -53,7 +52,7 @@ export const AudioPlayer: React.FC<AudioPlayerProps> = ({
     const generateAudio = async () => {
         setIsLoading(true);
 
-        const blob = await getVoiceResponse(text, voice || (language ? "aws_default" : "openai_alloy"), 1, language);
+        const blob = await llm.getVoice(text, voice || (language ? "aws_default" : "openai_alloy"), 1, language);
         setAudioUrl(URL.createObjectURL(blob));
         setIsLoading(false);
     };

package/src/components.ts

@@ -0,0 +1,10 @@
+// React components and hooks exports
+export * from "./components/MarkdownEditor";
+export * from "./components/CRUDModal";
+export * from "./components/Spinner";
+export * from "./components/audio/Playbutton";
+export * from "./hooks/UseChatHook";
+export * from "./plugin/ThemeSetter";
+export * from "./providers/PluginProvider"; 
+export * from "./components/ai/Avatar";
+export * from "./components/ai/Assistant";

package/src/controller/AIController.ts

@@ -1,87 +1,111 @@
-import { env } from "../utils/constants";
-
 export interface ToolInvocation {
-    toolName: string;
-    args: Record<string, string>;
+  toolName: string;
+  args: Record<string, string>;
 }
 
 export interface Tool {
+  name: string;
+  description: string;
+  parameters: {
     name: string;
     description: string;
-    parameters: {
-        name: string;
-        type: "string" | "number" | "boolean";
-        description: string;
-    }[];
+    type: "string" | "number" | "boolean";
+  }[];
+  execute?: <T = Record<string, string | boolean | number>>(args: T) => Promise<any> | void;
 }
 
 export interface Message {
-    id: string;
-    role: string;
-    content: string;
-    toolInvocations?: ToolInvocation[];
+  id: string;
+  role: string;
+  content: string;
+  toolInvocations?: ToolInvocation[];
 }
 
-export async function generateText(messages: Message[], tools: Tool[], token: string) {
-    const response = await fetch(`${env.SUPABASE_URL}/functions/v1/llm`, {
-        method: 'POST',
-        body: JSON.stringify({ messages, tools }),
-        headers: { 'Authorization': `Bearer ${token}` }
-    });
+export async function generateText(supabaseUrl: string, messages: Message[], tools: Tool[], token: string) {
+  const response = await fetch(`${supabaseUrl}/functions/v1/llm`, {
+    method: 'POST',
+    body: JSON.stringify({ messages, tools }),
+    headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json' }
+  });
 
-    return await response.json();
+  return await response.json();
 }
 
 export type OnLLMResponse = (id: string, response: string, finished: boolean, toolInvocations?: ToolInvocation[]) => void;
 
-export async function streamChatGPT(messages: Message[], tools: Tool[], onResponse: OnLLMResponse, token: string) {
-    const messageId = Math.random().toString(36).substring(3);
-    const response = await fetch(`${env.SUPABASE_URL}/functions/v1/llm`, {
-        method: 'POST',
-        body: JSON.stringify({ messages, tools, stream: true }),
-        headers: { 'Authorization': `Bearer ${token}` }
-    });
+export async function streamChatGPT(supabaseUrl: string, messages: Message[], tools: Tool[], onResponse: OnLLMResponse, token: string) {
+  const messageId = Math.random().toString(36).substring(3);
+  const response = await fetch(`${supabaseUrl}/functions/v1/llm`, {
+    method: 'POST',
+    body: JSON.stringify({ messages, tools, stream: true }),
+    headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json' }
+  });
 
-    if (!response.body) {
-        console.error('No response body.');
-        return;
-    }
+  if (!response.body) {
+    console.error('No response body.');
+    return;
+  }
+
+  const reader = response.body.getReader();
+  const decoder = new TextDecoder('utf-8');
 
-    const reader = response.body.getReader();
-    const decoder = new TextDecoder('utf-8');
+  let content = "";
+  let done = false;
+  let toolInvocations: ToolInvocation[] = [];
+  while (!done) {
+    const { value } = await reader.read();
 
-    let content = "";
-    let done = false;
-    let toolInvocations: ToolInvocation[] = [];
-    while (!done) {
-        const { value } = await reader.read();
+    if (value) {
+      const chunk = decoder.decode(value, { stream: true });
+      const lines = chunk.split('\n').filter(line => line.trim() !== '');
 
-        if (value) {
-            const chunk = decoder.decode(value, { stream: true });
-            const lines = chunk.split('\n').filter(line => line.trim() !== '');
+      for (const line of lines) {
+        const data = line.substring(3, line.length - 1);
+        const command = line.substring(0, 1);
+        // console.log("data: ", { line, data, command });
 
-            for (const line of lines) {
-                const data = line.substring(3, line.length - 1);
-                const command = line.substring(0, 1);
-                // console.log("data: ", { line, data, command });
+        if (command === '0') {
+          content += data;
+          // console.log("AI response:", content);
 
-                if (command === '0') {
-                    content += data;
-                    // console.log("AI response:", content);
+          //content \n\n should be real line break when message is displayed
+          onResponse(messageId, content.replace(/\\n/g, '\n'), false);
+        } else if (command === 'd') {
+          // console.log("AI usage:", JSON.parse(line.substring(2)));
+          done = true;
+          break;
+        } else if (command === '9') {
+          // console.log("tool call:", JSON.parse(line.substring(2)));
+          // console.log("tools", tools);
+          const toolInvocation = JSON.parse(line.substring(2));
+          toolInvocations.push(toolInvocation);
 
-                    //content \n\n should be real line break when message is displayed
-                    onResponse(messageId, content.replace(/\\n/g, '\n'), false);
-                } else if (command === 'd') {
-                    // console.log("AI usage:", JSON.parse(line.substring(2)));
-                    done = true;
-                    break;
-                } else if (command === '9') {
-                    // console.log("tool call:", JSON.parse(line.substring(2)));
-                    // console.log("tools", tools);
-                    toolInvocations.push(JSON.parse(line.substring(2)));
-                }
+          // Find the tool and execute it if it has an execute function
+          const tool = tools.find(t => t.name === toolInvocation.toolName);
+          // console.log("tool", tool);
+          if (tool && tool.execute) {
+            try {
+              const result = await tool.execute(toolInvocation.args);
+              const toolResult = { success: true, result: result }
+              // Add the tool result as a message
+              messages.push({
+                id: Math.random().toString(36).substring(3),
+                role: "function",
+                content: JSON.stringify(toolResult),
+                toolInvocations: [{
+                  args: toolInvocation.args,
+                  toolName: toolInvocation.toolName,
+                }]
+              });
+            } catch (error) {
+              console.error(`Error executing tool ${toolInvocation.toolName}:`, error);
             }
+          }
         }
+      }
     }
-    onResponse(messageId, content.replace(/\\n/g, '\n'), true, toolInvocations);
+    // Wait for 5ms to avoid blocking the main thread
+    await new Promise(resolve => setTimeout(resolve, 5));
+  }
+  onResponse(messageId, content.replace(/\\n/g, '\n'), true, toolInvocations);
 }

package/src/controller/ObjectController.ts

@@ -1,5 +1,3 @@
-import { env } from "../utils/constants";
-
 type PrimitiveType = 'string' | 'number' | 'boolean';
 
 // This is the type that can appear in the `type` property
@@ -35,8 +33,8 @@ export interface ObjectRequest {
     instructions: string;
 }
 
-export async function generateObject(request: ObjectRequest, token: string) {
-    return await fetch(`${env.SUPABASE_URL}/functions/v1/llm-object`, {
+export async function generateObject(supabaseUrl: string, request: ObjectRequest, token: string) {
+    return await fetch(`${supabaseUrl}/functions/v1/llm-object`, {
         method: 'POST',
         body: JSON.stringify({
             stream: false,
@@ -51,9 +49,9 @@ export async function generateObject(request: ObjectRequest, token: string) {
 // TODO adjust stream to work with object
 export type OnLLMResponse = (id: string, response: string, finished: boolean, toolInvocations?: any[]) => void;
 
-export async function streamObject(request: ObjectRequest, onResponse: OnLLMResponse, token: string) {
+export async function streamObject(supabaseUrl: string, request: ObjectRequest, onResponse: OnLLMResponse, token: string) {
     const messageId = Math.random().toString(36).substring(3);
-    const response = await fetch(`${env.SUPABASE_URL}/functions/v1/llm-object`, {
+    const response = await fetch(`${supabaseUrl}/functions/v1/llm-object`, {
         method: 'POST',
         body: JSON.stringify({
             stream: true,

package/src/controller/SettingsController.ts

@@ -3,9 +3,10 @@ import { LanguageLevel } from "../utils/difficultyConverter";
 
 type SettingsType = "user" | "system" | "plugin";
 
-export interface UserSettings {
+export interface UserInfo {
     motherTongue: string;
     languageLevel: LanguageLevel;
+    contextMenuOnSelect: boolean;
 }
 
 export interface SystemSettings {
@@ -44,6 +45,14 @@ export class SettingsController {
         await this.supabase.from("plugin_settings").upsert({ plugin_id: this.pluginId, settings });
     }
 
+    public async getUserInfo(): Promise<UserInfo> {
+        return this.getSettings<UserInfo>({
+            motherTongue: "sv",
+            languageLevel: "A1",
+            contextMenuOnSelect: true,
+        }, "user");
+    }
+
     /**
      * Get the settings for the plugin. T can be any type of settings, UserSettings or SystemSettings.
      * @param defaultSettings The default settings to use if no settings are found.

package/src/controller/SharedContentController.ts

@@ -24,14 +24,14 @@ export class SharedContentController {
         filter?: { column: string, value: string | number | boolean },
     ): Promise<R[]> {
         const queryParameter = { filter_column: filter?.column || null, filter_value: filter?.value || null, unread: true }
-        const { data: newAssignments } = await this.rimoriClient.rpc(type + "_entries", queryParameter)
+        const { data: newAssignments } = await this.rimoriClient.db.rpc(type + "_entries", queryParameter)
         console.log('newAssignments:', newAssignments);
 
         if ((newAssignments as any[]).length > 0) {
             return newAssignments as R[];
         }
         // generate new assignments
-        const { data: oldAssignments } = await this.rimoriClient.rpc(type + "_entries", { ...queryParameter, unread: false })
+        const { data: oldAssignments } = await this.rimoriClient.db.rpc(type + "_entries", { ...queryParameter, unread: false })
         console.log('oldAssignments:', oldAssignments);
         const reservedTopics = this.getReservedTopics(oldAssignments as BasicAssignment[]);
 
@@ -39,7 +39,7 @@ export class SharedContentController {
         if (!request.tool.keywords || !request.tool.topic) {
             throw new Error("topic or keywords not found in the request schema");
         }
-        const instructions = await this.rimoriClient.generateObject(request);
+        const instructions = await this.rimoriClient.llm.getObject(request);
         console.log('instructions:', instructions);
 
         const preparedData = {
@@ -47,7 +47,7 @@ export class SharedContentController {
             ...instructions,
             keywords: this.purifyStringArray(instructions.keywords),
         };
-        return await this.rimoriClient.from(type).insert(preparedData).then(() => [preparedData] as R[]);
+        return await this.rimoriClient.db.from(type).insert(preparedData).then(() => [preparedData] as R[]);
     }
 
     private getReservedTopics(oldAssignments: BasicAssignment[]) {
@@ -62,10 +62,10 @@ export class SharedContentController {
     }
 
     public async getSharedContent<T extends BasicAssignment>(type: string, id: string): Promise<T> {
-        return await this.rimoriClient.from(type).select().eq('id', id).single() as unknown as T;
+        return await this.rimoriClient.db.from(type).select().eq('id', id).single() as unknown as T;
     }
 
     public async completeSharedContent(type: string, assignmentId: string) {
-        await this.rimoriClient.from(type + "_result").insert({ assignment_id: assignmentId });
+        await this.rimoriClient.db.from(type + "_result").insert({ assignment_id: assignmentId });
     }
 }

package/src/controller/SidePluginController.ts

@@ -0,0 +1,36 @@
+import { SupabaseClient } from '@supabase/supabase-js';
+
+export interface Plugin {
+    id: string;
+    title: string;
+    icon_url: string;
+    website: string;
+    context_menu_actions: string;
+    plugin_pages: string;
+    sidebar_pages: string;
+    settings_page: string;
+    version: string;
+    external_hosted_url: string;
+}
+
+export async function getPlugins(supabase: SupabaseClient): Promise<Plugin[]> {
+    let { data, error } = await supabase.from('plugins').select('*');
+    
+    if (error) {
+        console.error(error);
+        return [];
+    }
+
+    return (data || []).map((plugin: any) => ({
+        id: plugin.id,
+        title: plugin.title,
+        icon_url: plugin.icon_url,
+        website: plugin.website,
+        context_menu_actions: plugin.context_menu_actions,
+        plugin_pages: plugin.plugin_pages,
+        sidebar_pages: plugin.sidebar_pages,
+        settings_page: plugin.settings_page,
+        version: plugin.version,
+        external_hosted_url: plugin.external_hosted_url,
+    }));
+}

package/src/controller/VoiceController.ts

@@ -4,7 +4,7 @@ export async function getSTTResponse(supabase: SupabaseClient, audio: Blob) {
     const formData = new FormData();
     formData.append('file', audio);
 
-    return await supabase.functions.invoke('speech', { method: 'POST', body: formData }).then((r: any) => r.text);
+    return await supabase.functions.invoke('speech', { method: 'POST', body: formData }).then(({ data }) => data.text);
 }
 
 export async function getTTSResponse(supabaseUrl: string, request: TTSRequest, token: string) {

package/src/core.ts

@@ -0,0 +1,10 @@
+// Core functionality exports
+export * from "./controller/AIController";
+export * from "./controller/SharedContentController";
+export * from "./controller/SettingsController";
+export * from "./plugin/RimoriClient";
+export * from "./plugin/PluginController";
+export * from "./utils/difficultyConverter";
+export * from "./utils/PluginUtils";
+export * from "./worker/WorkerSetup";
+export * from "./plugin/fromRimori/EventBus";