@rimori/client 1.0.3 → 1.0.4

package/src/components/ai/EmbeddedAssistent/CircleAudioAvatar.tsx

@@ -0,0 +1,75 @@
+import React, { useEffect, useRef } from 'react';
+import { EventBus, EventBusMessage } from '../../../core';
+
+interface CircleAudioAvatarProps {
+    width?: string;
+    imageUrl: string;
+    className?: string;
+}
+
+
+export function CircleAudioAvatar({ imageUrl, className, width = "150px" }: CircleAudioAvatarProps) {
+    const canvasRef = useRef<HTMLCanvasElement>(null);
+
+    useEffect(() => {
+        const canvas = canvasRef.current;
+        if (canvas) {
+            const ctx = canvas.getContext('2d');
+            if (ctx) {
+                const image = new Image();
+                image.src = imageUrl;
+                image.onload = () => {
+                    draw(ctx, canvas, image, 0);
+                };
+
+                const handleLoudness = (event: EventBusMessage) => {
+                    draw(ctx, canvas, image, event.data.loudness);
+                };
+
+                // Subscribe to loudness changes
+                const listenerId = EventBus.on('self.avatar.triggerLoudness', handleLoudness);
+
+                return () => {
+                    EventBus.off(listenerId);
+                };
+            }
+        }
+    }, [imageUrl]);
+
+    // Function to draw on the canvas
+    const draw = (ctx: CanvasRenderingContext2D, canvas: HTMLCanvasElement, image: HTMLImageElement, loudness: number) => {
+        if (canvas && ctx) {
+            ctx.clearRect(0, 0, canvas.width, canvas.height);
+
+            // Draw pulsing circle
+            const radius = Math.min(canvas.width, canvas.height) / 3;
+            const centerX = canvas.width / 2;
+            const centerY = canvas.height / 2;
+            const pulseRadius = radius + loudness / 2.5; // Adjust the divisor for sensitivity
+            ctx.beginPath();
+            ctx.arc(centerX, centerY, pulseRadius, 0, Math.PI * 2, true);
+            ctx.strokeStyle = 'rgba(0, 0, 0, 0.5)';
+            ctx.lineWidth = 5;
+            ctx.stroke();
+
+            // Draw image circle
+            ctx.save();
+            ctx.beginPath();
+            ctx.arc(centerX, centerY, radius, 0, Math.PI * 2, true);
+            ctx.closePath();
+            ctx.clip();
+            ctx.drawImage(image, centerX - radius, centerY - radius, radius * 2, radius * 2);
+            ctx.restore();
+
+            // Draw circular frame around the image
+            ctx.beginPath();
+            ctx.arc(centerX, centerY, radius, 0, Math.PI * 2, true);
+            ctx.strokeStyle = 'rgba(20,20, 20, 0.9)';
+            ctx.lineWidth = 5; // Adjust the width of the frame as needed
+            ctx.stroke();
+        }
+    };
+
+    return <canvas ref={canvasRef} className={className} width={500} height={500} style={{ width }} />;
+};
+

package/src/components/ai/EmbeddedAssistent/TTS/MessageSender.ts

@@ -0,0 +1,91 @@
+import { ChunkedAudioPlayer } from './Player';
+
+type VoiceBackend = (text: string, voice?: string, speed?: number) => Promise<Blob>;
+
+export class MessageSender {
+    private player = new ChunkedAudioPlayer();
+    private fetchedSentences = new Set<string>();
+    private lastLoading = false;
+    private voice: string;
+    private model: string;
+    private voiceBackend: VoiceBackend;
+
+    constructor(voiceBackend: VoiceBackend, voice: string = 'alloy', model = 'openai') {
+        this.voiceBackend = voiceBackend;
+        this.voice = voice;
+        this.model = model;
+    }
+
+    private getCompletedSentences(currentText: string, isLoading: boolean): string[] {
+        // Split the text based on the following characters: .,?!
+        // Only split on : when followed by a space
+        const pattern = /(.+?[,.?!]|.+?:\s+|.+?\n+)/g;
+        const result: string[] = [];
+        let match;
+        while ((match = pattern.exec(currentText)) !== null) {
+            const sentence = match[0].trim();
+            if (sentence.length > 0) {
+                result.push(sentence);
+            }
+        }
+        if (!isLoading) {
+            const lastFullSentence = result[result.length - 1];
+            const leftoverIndex = currentText.lastIndexOf(lastFullSentence) + lastFullSentence.length;
+            if (leftoverIndex < currentText.length) {
+                result.push(currentText.slice(leftoverIndex).trim());
+            }
+        }
+        return result;
+    }
+
+    public async handleNewText(currentText: string | undefined, isLoading: boolean) {
+        if (!this.lastLoading && isLoading) {
+            this.reset();
+        }
+        this.lastLoading = isLoading;
+
+        if (!currentText) {
+            return;
+        }
+
+        const sentences = this.getCompletedSentences(currentText, isLoading);
+
+        for (let i = 0; i < sentences.length; i++) {
+            const sentence = sentences[i];
+            if (!this.fetchedSentences.has(sentence)) {
+                this.fetchedSentences.add(sentence);
+                const audioData = await this.generateSpeech(sentence);
+                await this.player.addChunk(audioData, i);
+            }
+        }
+    }
+
+    private async generateSpeech(sentence: string): Promise<ArrayBuffer> {
+        const blob = await this.voiceBackend(sentence, this.voice, 1.0);
+        return await blob.arrayBuffer();
+    }
+
+    public play() {
+        this.player.playAgain();
+    }
+
+    public stop() {
+        this.player.stopPlayback();
+    }
+
+    private reset() {
+        this.stop();
+        this.fetchedSentences.clear();
+        this.player.reset();
+    }
+
+    public setVolume(volume: number) {
+        this.player.setVolume(volume);
+    }
+
+    public setOnLoudnessChange(callback: (value: number) => void) {
+        this.player.setOnLoudnessChange((loudness) => {
+            callback(loudness);
+        });
+    }
+}

package/src/components/ai/EmbeddedAssistent/TTS/Player.ts

@@ -0,0 +1,192 @@
+export class ChunkedAudioPlayer {
+
+    private audioContext!: AudioContext;
+    private chunkQueue: ArrayBuffer[] = [];
+    private isPlaying = false;
+    private analyser!: AnalyserNode;
+    private dataArray!: Uint8Array;
+    private shouldMonitorLoudness = true;
+    private isMonitoring = false;
+    private handle = 0;
+    private volume = 1.0;
+    private loudnessCallback: (value: number) => void = () => { };
+    private currentIndex = 0;
+    private startedPlaying = false;
+
+    constructor() {
+        this.init();
+    }
+
+    private init(): void {
+        this.audioContext = new AudioContext();
+        this.analyser = this.audioContext.createAnalyser();
+        this.analyser.fftSize = 256; // Set the FFT size (smaller values provide faster updates, larger ones give better resolution)
+        const bufferLength = this.analyser.frequencyBinCount;
+        this.dataArray = new Uint8Array(bufferLength); // Array to hold frequency data
+    }
+
+    public setOnLoudnessChange(callback: (value: number) => void) {
+        this.loudnessCallback = callback;
+    }
+
+    public setVolume(volume: number) {
+        this.volume = volume;
+    }
+
+    public async addChunk(chunk: ArrayBuffer, position: number): Promise<void> {
+        console.log('Adding chunk', position, chunk);
+        this.chunkQueue[position] = chunk;
+        // console.log("received chunk", {
+        //     chunkQueue: this.chunkQueue.length,
+        //     isPlaying: this.isPlaying,
+        // })
+
+        if (position === 0 && !this.startedPlaying) {
+            this.startedPlaying = true;
+            this.playChunks();
+        }
+    }
+
+    private playChunks(): void {
+        // console.log({ isPlaying: this.isPlaying });
+        if (this.isPlaying) return;
+        if (!this.chunkQueue[this.currentIndex]) {
+            // wait until the correct chunk arrives
+            setTimeout(() => this.playChunks(), 10);
+        }
+        this.isPlaying = true;
+
+        this.playChunk(this.chunkQueue[this.currentIndex]).then(() => {
+            this.isPlaying = false;
+            this.currentIndex++;
+            if (this.chunkQueue[this.currentIndex]) {
+                this.shouldMonitorLoudness = true;
+                this.playChunks();
+            } else {
+                // console.log('Playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
+                setTimeout(() => {
+                    // console.log('Check again if really playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
+                    if (this.chunkQueue.length > this.currentIndex) {
+                        this.playChunks();
+                    } else {
+                        this.startedPlaying = false;
+                        this.shouldMonitorLoudness = false;
+                    }
+                }, 1000);
+            }
+        });
+    }
+
+    public stopPlayback(): void {
+        // console.log('Stopping playback');
+        // Implement logic to stop the current playback
+        this.isPlaying = false;
+        this.chunkQueue = [];
+        this.startedPlaying = false;
+        this.shouldMonitorLoudness = false;
+        cancelAnimationFrame(this.handle);
+    }
+
+    private playChunk(chunk: ArrayBuffer): Promise<void> {
+        console.log({queue: this.chunkQueue})
+        if (!chunk) {
+            return Promise.resolve();
+        }
+
+        // console.log('Playing chunk', chunk);
+        return new Promise((resolve) => {
+            const source = this.audioContext.createBufferSource();
+            this.audioContext.decodeAudioData(chunk.slice(0)).then((audioBuffer) => {
+                source.buffer = audioBuffer;
+
+                // Create a GainNode for volume control
+                const gainNode = this.audioContext.createGain();
+                gainNode.gain.value = this.volume;
+
+                // Connect the source to the GainNode, then to the analyser node, then to the destination (speakers)
+                source.connect(gainNode);
+                gainNode.connect(this.analyser);
+                this.analyser.connect(this.audioContext.destination);
+
+                source.start(0);
+                // console.log('Playing chunk', this.currentIndex);
+                gainNode.gain.value = this.volume;
+                source.onended = () => {
+                    // console.log('Chunk ended');
+                    resolve();
+                };
+
+                // Start monitoring loudness only once
+                if (!this.isMonitoring) {
+                    this.isMonitoring = true;
+                    this.shouldMonitorLoudness = true;
+                    this.monitorLoudness();
+                }
+            });
+        });
+    }
+
+    async playAgain(): Promise<void> {
+        console.log('Playing again');
+        if (this.chunkQueue.length > 0 && !this.isPlaying) {
+            this.playChunks();
+        }
+    }
+
+    private monitorLoudness(): void {
+        // Stop monitoring when the flag is false
+        if (!this.shouldMonitorLoudness) {
+            // console.log('Loudness monitoring stopped.');
+            cancelAnimationFrame(this.handle);
+            this.loudnessCallback(0);
+            return;
+        }
+
+        // Get the time domain data from the analyser (this is a snapshot of the waveform)
+        this.analyser.getByteTimeDomainData(this.dataArray);
+
+        // Calculate the RMS (root mean square) of the waveform values to get the perceived loudness
+        let sum = 0;
+        for (let i = 0; i < this.dataArray.length; i++) {
+            const value = this.dataArray[i] / 128.0 - 1.0; // Normalize between -1 and 1
+            sum += value * value;
+        }
+
+        const rms = Math.sqrt(sum / this.dataArray.length);
+
+        // Handle the case where RMS is 0 to avoid log10(0)
+        if (rms === 0) {
+            // console.log('Current loudness: Silent');
+        } else {
+            let loudnessInDb = 20 * Math.log10(rms); // Convert to dB
+            // console.log('Current loudness:' + loudnessInDb);
+            const minDb = -57;
+            const maxDb = -15;
+
+            if (loudnessInDb < minDb) {
+                loudnessInDb = minDb;
+            }
+            if (loudnessInDb > maxDb) {
+                loudnessInDb = maxDb;
+            }
+
+            const loudnessScale = ((loudnessInDb - minDb) / (maxDb - minDb)) * 100;
+            // console.log("root:corrent loudness", loudnessScale);
+
+            this.loudnessCallback(loudnessScale);
+        }
+
+        // Call this method again at regular intervals if you want continuous loudness monitoring
+        this.handle = requestAnimationFrame(() => this.monitorLoudness());
+    }
+    public reset() {
+        // console.log('Resetting player');
+        this.stopPlayback();
+        this.currentIndex = 0;
+        this.shouldMonitorLoudness = true;
+        //reset to the beginning when the class gets initialized
+        this.isMonitoring = false;
+        this.isPlaying = false;
+        this.init();
+    }
+}

package/src/components/ai/EmbeddedAssistent/VoiceRecoder.tsx

@@ -0,0 +1,56 @@
+import { useState, useRef, forwardRef, useImperativeHandle } from 'react';
+import { FaMicrophone } from 'react-icons/fa6';
+import { usePlugin } from '../../../components';
+
+interface Props {
+  iconSize?: string;
+  className?: string;
+  onVoiceRecorded: (message: string) => void;
+}
+
+export const VoiceRecorder = forwardRef(({ onVoiceRecorded, iconSize, className }: Props, ref) => {
+  const [isRecording, setIsRecording] = useState(false);
+  const mediaRecorderRef = useRef<MediaRecorder | null>(null);
+  const audioChunksRef = useRef<Blob[]>([]);
+  const { llm } = usePlugin();
+
+  const startRecording = async () => {
+    const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+    const mediaRecorder = new MediaRecorder(stream);
+    mediaRecorderRef.current = mediaRecorder;
+
+    mediaRecorder.ondataavailable = (event) => {
+      audioChunksRef.current.push(event.data);
+    };
+
+    mediaRecorder.onstop = async () => {
+      const audioBlob = new Blob(audioChunksRef.current);
+      audioChunksRef.current = [];
+
+      onVoiceRecorded(await llm.getTextFromVoice(audioBlob));
+    };
+
+    mediaRecorder.start();
+    setIsRecording(true);
+  };
+
+  const stopRecording = () => {
+    if (mediaRecorderRef.current) {
+      mediaRecorderRef.current.stop();
+      setIsRecording(false);
+    }
+  };
+
+  useImperativeHandle(ref, () => ({
+    startRecording,
+    stopRecording,
+  }));
+
+  return (
+    <div className={className}>
+      <button onClick={isRecording ? stopRecording : startRecording}>
+        <FaMicrophone size={iconSize} className={"h-7 w-7 mr-2 " + (isRecording ? "text-red-600" : "")} />
+      </button>
+    </div>
+  );
+});

package/src/components/ai/utils.ts

@@ -0,0 +1,23 @@
+export interface FirstMessages {
+  instructions?: string;
+  userMessage: string;
+  assistantMessage?: string;
+}
+
+export function getFirstMessages(instructions: FirstMessages): any[] {
+  const messages = [];
+
+  if (instructions.instructions) {
+      messages.push({ id: '1', role: 'system', content: instructions.instructions });
+  }
+  if (instructions.userMessage) {
+      messages.push({ id: '2', role: 'user', content: instructions.userMessage });
+  }
+  if (instructions.assistantMessage) {
+      messages.push({ id: '3', role: 'assistant', content: instructions.assistantMessage });
+  }
+
+  console.log("getFirstMessages", messages);
+
+  return messages;
+}

package/src/components/audio/Playbutton.tsx

@@ -2,7 +2,7 @@ import React, { useState, useEffect } from 'react';
 import { FaPlayCircle, FaStopCircle } from "react-icons/fa";
 import { usePlugin } from "../../providers/PluginProvider";
 import { Spinner } from '../Spinner';
-import { EmitterSingleton } from '../../providers/EventEmitter';
+import { EventBus } from '../../plugin/fromRimori/EventBus';
 
 type AudioPlayerProps = {
     text: string;
@@ -34,12 +34,11 @@ export const AudioPlayer: React.FC<AudioPlayerProps> = ({
     const [speed, setSpeed] = useState(initialSpeed);
     const [isPlaying, setIsPlaying] = useState(false);
     const [isLoading, setIsLoading] = useState(false);
-    const { getVoiceResponse, } = usePlugin();
-    const emitter = EmitterSingleton;
+    const { llm } = usePlugin();
 
     useEffect(() => {
         if (!playListenerEvent) return;
-        emitter.on(playListenerEvent, () => togglePlayback());
+        EventBus.on(playListenerEvent, () => togglePlayback());
     }, [playListenerEvent]);
 
     useEffect(() => {
@@ -53,7 +52,7 @@ export const AudioPlayer: React.FC<AudioPlayerProps> = ({
     const generateAudio = async () => {
         setIsLoading(true);
 
-        const blob = await getVoiceResponse(text, voice || (language ? "aws_default" : "openai_alloy"), 1, language);
+        const blob = await llm.getVoice(text, voice || (language ? "aws_default" : "openai_alloy"), 1, language);
         setAudioUrl(URL.createObjectURL(blob));
         setIsLoading(false);
     };

package/src/components.ts

@@ -0,0 +1,10 @@
+// React components and hooks exports
+export * from "./components/MarkdownEditor";
+export * from "./components/CRUDModal";
+export * from "./components/Spinner";
+export * from "./components/audio/Playbutton";
+export * from "./hooks/UseChatHook";
+export * from "./plugin/ThemeSetter";
+export * from "./providers/PluginProvider"; 
+export * from "./components/ai/Avatar";
+export * from "./components/ai/Assistant";

package/src/controller/AIController.ts

@@ -1,87 +1,111 @@
-import { env } from "../utils/constants";
-
 export interface ToolInvocation {
-    toolName: string;
-    args: Record<string, string>;
+  toolName: string;
+  args: Record<string, string>;
 }
 
 export interface Tool {
+  name: string;
+  description: string;
+  parameters: {
     name: string;
     description: string;
-    parameters: {
-        name: string;
-        type: "string" | "number" | "boolean";
-        description: string;
-    }[];
+    type: "string" | "number" | "boolean";
+  }[];
+  execute?: <T = Record<string, string | boolean | number>>(args: T) => Promise<any> | void;
 }
 
 export interface Message {
-    id: string;
-    role: string;
-    content: string;
-    toolInvocations?: ToolInvocation[];
+  id: string;
+  role: string;
+  content: string;
+  toolInvocations?: ToolInvocation[];
 }
 
-export async function generateText(messages: Message[], tools: Tool[], token: string) {
-    const response = await fetch(`${env.SUPABASE_URL}/functions/v1/llm`, {
-        method: 'POST',
-        body: JSON.stringify({ messages, tools }),
-        headers: { 'Authorization': `Bearer ${token}` }
-    });
+export async function generateText(supabaseUrl: string, messages: Message[], tools: Tool[], token: string) {
+  const response = await fetch(`${supabaseUrl}/functions/v1/llm`, {
+    method: 'POST',
+    body: JSON.stringify({ messages, tools }),
+    headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json' }
+  });
 
-    return await response.json();
+  return await response.json();
 }
 
 export type OnLLMResponse = (id: string, response: string, finished: boolean, toolInvocations?: ToolInvocation[]) => void;
 
-export async function streamChatGPT(messages: Message[], tools: Tool[], onResponse: OnLLMResponse, token: string) {
-    const messageId = Math.random().toString(36).substring(3);
-    const response = await fetch(`${env.SUPABASE_URL}/functions/v1/llm`, {
-        method: 'POST',
-        body: JSON.stringify({ messages, tools, stream: true }),
-        headers: { 'Authorization': `Bearer ${token}` }
-    });
+export async function streamChatGPT(supabaseUrl: string, messages: Message[], tools: Tool[], onResponse: OnLLMResponse, token: string) {
+  const messageId = Math.random().toString(36).substring(3);
+  const response = await fetch(`${supabaseUrl}/functions/v1/llm`, {
+    method: 'POST',
+    body: JSON.stringify({ messages, tools, stream: true }),
+    headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json' }
+  });
 
-    if (!response.body) {
-        console.error('No response body.');
-        return;
-    }
+  if (!response.body) {
+    console.error('No response body.');
+    return;
+  }
+
+  const reader = response.body.getReader();
+  const decoder = new TextDecoder('utf-8');
 
-    const reader = response.body.getReader();
-    const decoder = new TextDecoder('utf-8');
+  let content = "";
+  let done = false;
+  let toolInvocations: ToolInvocation[] = [];
+  while (!done) {
+    const { value } = await reader.read();
 
-    let content = "";
-    let done = false;
-    let toolInvocations: ToolInvocation[] = [];
-    while (!done) {
-        const { value } = await reader.read();
+    if (value) {
+      const chunk = decoder.decode(value, { stream: true });
+      const lines = chunk.split('\n').filter(line => line.trim() !== '');
 
-        if (value) {
-            const chunk = decoder.decode(value, { stream: true });
-            const lines = chunk.split('\n').filter(line => line.trim() !== '');
+      for (const line of lines) {
+        const data = line.substring(3, line.length - 1);
+        const command = line.substring(0, 1);
+        // console.log("data: ", { line, data, command });
 
-            for (const line of lines) {
-                const data = line.substring(3, line.length - 1);
-                const command = line.substring(0, 1);
-                // console.log("data: ", { line, data, command });
+        if (command === '0') {
+          content += data;
+          // console.log("AI response:", content);
 
-                if (command === '0') {
-                    content += data;
-                    // console.log("AI response:", content);
+          //content \n\n should be real line break when message is displayed
+          onResponse(messageId, content.replace(/\\n/g, '\n'), false);
+        } else if (command === 'd') {
+          // console.log("AI usage:", JSON.parse(line.substring(2)));
+          done = true;
+          break;
+        } else if (command === '9') {
+          // console.log("tool call:", JSON.parse(line.substring(2)));
+          // console.log("tools", tools);
+          const toolInvocation = JSON.parse(line.substring(2));
+          toolInvocations.push(toolInvocation);
 
-                    //content \n\n should be real line break when message is displayed
-                    onResponse(messageId, content.replace(/\\n/g, '\n'), false);
-                } else if (command === 'd') {
-                    // console.log("AI usage:", JSON.parse(line.substring(2)));
-                    done = true;
-                    break;
-                } else if (command === '9') {
-                    // console.log("tool call:", JSON.parse(line.substring(2)));
-                    // console.log("tools", tools);
-                    toolInvocations.push(JSON.parse(line.substring(2)));
-                }
+          // Find the tool and execute it if it has an execute function
+          const tool = tools.find(t => t.name === toolInvocation.toolName);
+          // console.log("tool", tool);
+          if (tool && tool.execute) {
+            try {
+              const result = await tool.execute(toolInvocation.args);
+              const toolResult = { success: true, result: result }
+              // Add the tool result as a message
+              messages.push({
+                id: Math.random().toString(36).substring(3),
+                role: "function",
+                content: JSON.stringify(toolResult),
+                toolInvocations: [{
+                  args: toolInvocation.args,
+                  toolName: toolInvocation.toolName,
+                }]
+              });
+            } catch (error) {
+              console.error(`Error executing tool ${toolInvocation.toolName}:`, error);
             }
+          }
         }
+      }
     }
-    onResponse(messageId, content.replace(/\\n/g, '\n'), true, toolInvocations);
+    // Wait for 5ms to avoid blocking the main thread
+    await new Promise(resolve => setTimeout(resolve, 5));
+  }
+  onResponse(messageId, content.replace(/\\n/g, '\n'), true, toolInvocations);
 }