@rimori/client 1.0.2 → 1.0.4

package/README.md

@@ -0,0 +1,51 @@
+# Rimori Client Package
+
+This is the React connection package required by plugins to be able to
+communicate with the Rimori platform.
+
+## Usage
+
+In order to use the package first install the package with
+
+```bash
+npm i @rimori/client
+```
+
+Then wrap your app the following way to get started:
+
+```typescript
+import { lazy } from "react";
+import { PluginProvider } from "@rimori/client";
+import { HashRouter, Route, Routes } from "react-router-dom";
+
+// adding the theme setter
+
+const queryClient = new QueryClient();
+
+// load all pages lazy for fast loading speed
+const SettingsPage = lazy(() => import("./pages/settings/SettingsPage"));
+const DiscussionsPage = lazy(() => import("./pages/discussions/page"));
+
+const App = () => (
+    // this provides connectivity to Rimori
+    <PluginProvider  pluginId="my-rimori-plugin-id">
+        //allows using the routes set the plugin settings
+        <HashRouter future={{ v7_startTransition: true, v7_relativeSplatPath: true }}>
+            <Routes>
+                // the plugins pages
+                <Route path="/discussions" element={<DiscussionsPage />} />
+                // the settings page
+                <Route path="/settings" element={<SettingsPage />} />
+            </Routes>
+        </HashRouter>
+    </PluginProvider>
+);
+
+export default App;
+```
+
+Inside the pages simply use the `usePlugin` hook.
+
+```typescript
+const { getSettings, ... } = usePlugin();
+```

package/dist/components/CRUDModal.js

@@ -1,4 +1,3 @@
-"use client";
 import { jsx as _jsx, jsxs as _jsxs, Fragment as _Fragment } from "react/jsx-runtime";
 import { useEffect, useRef } from "react";
 export function CRUDModal({ actionbuttons, children, title, buttonText, className, closeAble = true, show = false, onClose }) {

package/dist/components/ai/Assistant.d.ts

@@ -0,0 +1,9 @@
+import { FirstMessages } from './utils';
+interface Props {
+    voiceId: any;
+    avatarImageUrl: string;
+    onComplete: (result: any) => void;
+    autoStartConversation?: FirstMessages;
+}
+export declare function AssistantChat({ avatarImageUrl, voiceId, onComplete, autoStartConversation }: Props): import("react/jsx-runtime").JSX.Element;
+export {};

package/dist/components/ai/Assistant.js

@@ -0,0 +1,59 @@
+import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
+import React, { useEffect, useMemo } from 'react';
+import { CircleAudioAvatar } from './EmbeddedAssistent/CircleAudioAvatar';
+import { AudioInputField } from './EmbeddedAssistent/AudioInputField';
+import { MessageSender } from './EmbeddedAssistent/TTS/MessageSender';
+import Markdown from 'react-markdown';
+import { useChat } from '../../hooks/UseChatHook';
+import { usePlugin } from '../../components';
+import { getFirstMessages } from './utils';
+export function AssistantChat({ avatarImageUrl, voiceId, onComplete, autoStartConversation }) {
+    var _a;
+    const [oralCommunication, setOralCommunication] = React.useState(true);
+    const { llm, event } = usePlugin();
+    const sender = useMemo(() => new MessageSender(llm.getVoice, voiceId), []);
+    const { messages, append, isLoading, setMessages } = useChat();
+    const lastAssistantMessage = (_a = [...messages].filter((m) => m.role === 'assistant').pop()) === null || _a === void 0 ? void 0 : _a.content;
+    useEffect(() => {
+        sender.setOnLoudnessChange((value) => event.emit('self.avatar.triggerLoudness', value));
+        if (!autoStartConversation) {
+            return;
+        }
+        setMessages(getFirstMessages(autoStartConversation));
+        // append([{ role: 'user', content: autoStartConversation.userMessage }]);
+        if (autoStartConversation.assistantMessage) {
+            // console.log("autostartmessages", { autoStartConversation, isLoading });
+            sender.handleNewText(autoStartConversation.assistantMessage, isLoading);
+        }
+    }, []);
+    useEffect(() => {
+        var _a;
+        let message = lastAssistantMessage;
+        if (message !== ((_a = messages[messages.length - 1]) === null || _a === void 0 ? void 0 : _a.content)) {
+            message = undefined;
+        }
+        sender.handleNewText(message, isLoading);
+    }, [messages, isLoading]);
+    const lastMessage = messages[messages.length - 1];
+    useEffect(() => {
+        console.log("lastMessage", lastMessage);
+        const toolInvocations = lastMessage === null || lastMessage === void 0 ? void 0 : lastMessage.toolInvocations;
+        if (toolInvocations && toolInvocations.length > 0) {
+            console.log("toolInvocations", toolInvocations);
+            onComplete(toolInvocations[0].args);
+        }
+    }, [lastMessage]);
+    if ((lastMessage === null || lastMessage === void 0 ? void 0 : lastMessage.toolInvocations) && lastMessage.toolInvocations.length > 0) {
+        console.log("lastMessage test2", lastMessage);
+        const args = lastMessage.toolInvocations[0].args;
+        const success = args.explanationUnderstood === "TRUE" || args.studentKnowsTopic === "TRUE";
+        return _jsxs("div", { className: "px-5 pt-5 overflow-y-auto text-center", style: { height: "478px" }, children: [_jsx("h1", { className: 'text-center mt-5 mb-5', children: success ? "Great job!" : "You failed" }), _jsx("p", { children: args.improvementHints })] });
+    }
+    return (_jsxs("div", { children: [oralCommunication && _jsx(CircleAudioAvatar, { imageUrl: avatarImageUrl, className: 'mx-auto my-10' }), _jsx("div", { className: "w-full", children: lastAssistantMessage && _jsx("div", { className: "px-5 pt-5 overflow-y-auto remirror-theme", style: { height: "4k78px" }, children: _jsx(Markdown, { children: lastAssistantMessage }) }) }), _jsx(AudioInputField, { blockSubmission: isLoading, onSubmit: message => {
+                    append([{ role: 'user', content: message, id: messages.length.toString() }]);
+                }, onAudioControl: voice => {
+                    setOralCommunication(voice);
+                    sender.setVolume(voice ? 1 : 0);
+                } })] }));
+}
+;

package/dist/components/ai/Avatar.d.ts

@@ -0,0 +1,11 @@
+import { Tool } from '../../core';
+import { FirstMessages } from './utils';
+interface Props {
+    title?: string;
+    voiceId: any;
+    avatarImageUrl: string;
+    agentTools: Tool[];
+    autoStartConversation?: FirstMessages;
+}
+export declare function Avatar({ avatarImageUrl, voiceId, title, agentTools, autoStartConversation }: Props): import("react/jsx-runtime").JSX.Element;
+export {};

package/dist/components/ai/Avatar.js

@@ -0,0 +1,39 @@
+import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
+import { useEffect, useMemo } from 'react';
+import { VoiceRecorder } from './EmbeddedAssistent/VoiceRecoder';
+import { MessageSender } from './EmbeddedAssistent/TTS/MessageSender';
+import { CircleAudioAvatar } from './EmbeddedAssistent/CircleAudioAvatar';
+import { useChat } from '../../hooks/UseChatHook';
+import { usePlugin } from '../../components';
+import { getFirstMessages } from './utils';
+export function Avatar({ avatarImageUrl, voiceId, title, agentTools, autoStartConversation }) {
+    const { llm, event } = usePlugin();
+    const sender = useMemo(() => new MessageSender(llm.getVoice, voiceId), []);
+    const { messages, append, isLoading, lastMessage, setMessages } = useChat(agentTools);
+    useEffect(() => {
+        console.log("messages", messages);
+    }, [messages]);
+    useEffect(() => {
+        sender.setOnLoudnessChange((value) => event.emit('self.avatar.triggerLoudness', value));
+        if (!autoStartConversation)
+            return;
+        setMessages(getFirstMessages(autoStartConversation));
+        // append([{ role: 'user', content: autoStartConversation.userMessage }]);
+        if (autoStartConversation.assistantMessage) {
+            // console.log("autostartmessages", { autoStartConversation, isLoading });
+            sender.handleNewText(autoStartConversation.assistantMessage, isLoading);
+        }
+        else if (autoStartConversation.userMessage) {
+            append([{ role: 'user', content: autoStartConversation.userMessage, id: messages.length.toString() }]);
+        }
+    }, []);
+    useEffect(() => {
+        if ((lastMessage === null || lastMessage === void 0 ? void 0 : lastMessage.role) === 'assistant') {
+            sender.handleNewText(lastMessage.content, isLoading);
+        }
+    }, [lastMessage, isLoading]);
+    return (_jsxs("div", { className: 'pb-8', children: [title && _jsx("p", { className: "text-center mt-5 w-3/4 mx-auto rounded-lg dark:text-gray-100", children: title }), _jsx(CircleAudioAvatar, { imageUrl: avatarImageUrl, width: "250px", className: 'mx-auto' }), _jsx("div", { className: 'w-16 h-16 flex text-4xl shadow-lg flex-row justify-center items-center rounded-full mx-auto bg-gray-400 dark:bg-gray-800', children: _jsx(VoiceRecorder, { className: 'w-7', iconSize: '300', onVoiceRecorded: (message) => {
+                        append([{ role: 'user', content: "Message(" + Math.floor((messages.length + 1) / 2) + "): " + message, id: messages.length.toString() }]);
+                    } }) })] }));
+}
+;

package/dist/components/ai/EmbeddedAssistent/AudioInputField.d.ts

@@ -0,0 +1,7 @@
+interface AudioInputFieldProps {
+    onSubmit: (text: string) => void;
+    onAudioControl?: (voice: boolean) => void;
+    blockSubmission?: boolean;
+}
+export declare function AudioInputField({ onSubmit, onAudioControl, blockSubmission }: AudioInputFieldProps): import("react/jsx-runtime").JSX.Element;
+export {};

package/dist/components/ai/EmbeddedAssistent/AudioInputField.js

@@ -0,0 +1,38 @@
+import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
+import { useState } from 'react';
+import { VoiceRecorder } from './VoiceRecoder';
+import { BiSolidRightArrow } from "react-icons/bi";
+import { HiMiniSpeakerXMark, HiMiniSpeakerWave } from "react-icons/hi2";
+export function AudioInputField({ onSubmit, onAudioControl, blockSubmission = false }) {
+    const [text, setText] = useState('');
+    const [audioEnabled, setAudioEnabled] = useState(true);
+    const handleSubmit = (manualText) => {
+        if (blockSubmission)
+            return;
+        const sendableText = manualText || text;
+        if (sendableText.trim()) {
+            onSubmit(sendableText);
+            setTimeout(() => {
+                setText('');
+            }, 100);
+        }
+    };
+    const handleKeyDown = (e) => {
+        if (blockSubmission)
+            return;
+        if (e.key === 'Enter' && e.ctrlKey) {
+            setText(text + '\n');
+        }
+        else if (e.key === 'Enter') {
+            handleSubmit();
+        }
+    };
+    return (_jsxs("div", { className: "flex items-center bg-gray-600 pt-2 pb-2 p-2", children: [onAudioControl && _jsx("button", { onClick: () => {
+                    onAudioControl(!audioEnabled);
+                    setAudioEnabled(!audioEnabled);
+                }, className: "cursor-default", children: audioEnabled ? _jsx(HiMiniSpeakerWave, { className: 'w-9 h-9 cursor-pointer' }) : _jsx(HiMiniSpeakerXMark, { className: 'w-9 h-9 cursor-pointer' }) }), _jsx(VoiceRecorder, { onVoiceRecorded: (m) => {
+                    console.log('onVoiceRecorded', m);
+                    handleSubmit(m);
+                } }), _jsx("textarea", { value: text, onChange: (e) => setText(e.target.value), onKeyDown: handleKeyDown, className: "flex-1 border-none rounded-lg p-2 text-gray-800 focus::outline-none", placeholder: 'Type a message...', disabled: blockSubmission }), _jsx("button", { onClick: () => handleSubmit(), className: "cursor-default", disabled: blockSubmission, children: _jsx(BiSolidRightArrow, { className: 'w-9 h-10 cursor-pointer' }) })] }));
+}
+;

package/dist/components/ai/EmbeddedAssistent/CircleAudioAvatar.d.ts

@@ -0,0 +1,7 @@
+interface CircleAudioAvatarProps {
+    width?: string;
+    imageUrl: string;
+    className?: string;
+}
+export declare function CircleAudioAvatar({ imageUrl, className, width }: CircleAudioAvatarProps): import("react/jsx-runtime").JSX.Element;
+export {};

package/dist/components/ai/EmbeddedAssistent/CircleAudioAvatar.js

@@ -0,0 +1,59 @@
+import { jsx as _jsx } from "react/jsx-runtime";
+import { useEffect, useRef } from 'react';
+import { EventBus } from '../../../core';
+export function CircleAudioAvatar({ imageUrl, className, width = "150px" }) {
+    const canvasRef = useRef(null);
+    useEffect(() => {
+        const canvas = canvasRef.current;
+        if (canvas) {
+            const ctx = canvas.getContext('2d');
+            if (ctx) {
+                const image = new Image();
+                image.src = imageUrl;
+                image.onload = () => {
+                    draw(ctx, canvas, image, 0);
+                };
+                const handleLoudness = (event) => {
+                    draw(ctx, canvas, image, event.data.loudness);
+                };
+                // Subscribe to loudness changes
+                const listenerId = EventBus.on('self.avatar.triggerLoudness', handleLoudness);
+                return () => {
+                    EventBus.off(listenerId);
+                };
+            }
+        }
+    }, [imageUrl]);
+    // Function to draw on the canvas
+    const draw = (ctx, canvas, image, loudness) => {
+        if (canvas && ctx) {
+            ctx.clearRect(0, 0, canvas.width, canvas.height);
+            // Draw pulsing circle
+            const radius = Math.min(canvas.width, canvas.height) / 3;
+            const centerX = canvas.width / 2;
+            const centerY = canvas.height / 2;
+            const pulseRadius = radius + loudness / 2.5; // Adjust the divisor for sensitivity
+            ctx.beginPath();
+            ctx.arc(centerX, centerY, pulseRadius, 0, Math.PI * 2, true);
+            ctx.strokeStyle = 'rgba(0, 0, 0, 0.5)';
+            ctx.lineWidth = 5;
+            ctx.stroke();
+            // Draw image circle
+            ctx.save();
+            ctx.beginPath();
+            ctx.arc(centerX, centerY, radius, 0, Math.PI * 2, true);
+            ctx.closePath();
+            ctx.clip();
+            ctx.drawImage(image, centerX - radius, centerY - radius, radius * 2, radius * 2);
+            ctx.restore();
+            // Draw circular frame around the image
+            ctx.beginPath();
+            ctx.arc(centerX, centerY, radius, 0, Math.PI * 2, true);
+            ctx.strokeStyle = 'rgba(20,20, 20, 0.9)';
+            ctx.lineWidth = 5; // Adjust the width of the frame as needed
+            ctx.stroke();
+        }
+    };
+    return _jsx("canvas", { ref: canvasRef, className: className, width: 500, height: 500, style: { width } });
+}
+;

package/dist/components/ai/EmbeddedAssistent/TTS/MessageSender.d.ts

@@ -0,0 +1,19 @@
+type VoiceBackend = (text: string, voice?: string, speed?: number) => Promise<Blob>;
+export declare class MessageSender {
+    private player;
+    private fetchedSentences;
+    private lastLoading;
+    private voice;
+    private model;
+    private voiceBackend;
+    constructor(voiceBackend: VoiceBackend, voice?: string, model?: string);
+    private getCompletedSentences;
+    handleNewText(currentText: string | undefined, isLoading: boolean): Promise<void>;
+    private generateSpeech;
+    play(): void;
+    stop(): void;
+    private reset;
+    setVolume(volume: number): void;
+    setOnLoudnessChange(callback: (value: number) => void): void;
+}
+export {};

package/dist/components/ai/EmbeddedAssistent/TTS/MessageSender.js

@@ -0,0 +1,86 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { ChunkedAudioPlayer } from './Player';
+export class MessageSender {
+    constructor(voiceBackend, voice = 'alloy', model = 'openai') {
+        this.player = new ChunkedAudioPlayer();
+        this.fetchedSentences = new Set();
+        this.lastLoading = false;
+        this.voiceBackend = voiceBackend;
+        this.voice = voice;
+        this.model = model;
+    }
+    getCompletedSentences(currentText, isLoading) {
+        // Split the text based on the following characters: .,?!
+        // Only split on : when followed by a space
+        const pattern = /(.+?[,.?!]|.+?:\s+|.+?\n+)/g;
+        const result = [];
+        let match;
+        while ((match = pattern.exec(currentText)) !== null) {
+            const sentence = match[0].trim();
+            if (sentence.length > 0) {
+                result.push(sentence);
+            }
+        }
+        if (!isLoading) {
+            const lastFullSentence = result[result.length - 1];
+            const leftoverIndex = currentText.lastIndexOf(lastFullSentence) + lastFullSentence.length;
+            if (leftoverIndex < currentText.length) {
+                result.push(currentText.slice(leftoverIndex).trim());
+            }
+        }
+        return result;
+    }
+    handleNewText(currentText, isLoading) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.lastLoading && isLoading) {
+                this.reset();
+            }
+            this.lastLoading = isLoading;
+            if (!currentText) {
+                return;
+            }
+            const sentences = this.getCompletedSentences(currentText, isLoading);
+            for (let i = 0; i < sentences.length; i++) {
+                const sentence = sentences[i];
+                if (!this.fetchedSentences.has(sentence)) {
+                    this.fetchedSentences.add(sentence);
+                    const audioData = yield this.generateSpeech(sentence);
+                    yield this.player.addChunk(audioData, i);
+                }
+            }
+        });
+    }
+    generateSpeech(sentence) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const blob = yield this.voiceBackend(sentence, this.voice, 1.0);
+            return yield blob.arrayBuffer();
+        });
+    }
+    play() {
+        this.player.playAgain();
+    }
+    stop() {
+        this.player.stopPlayback();
+    }
+    reset() {
+        this.stop();
+        this.fetchedSentences.clear();
+        this.player.reset();
+    }
+    setVolume(volume) {
+        this.player.setVolume(volume);
+    }
+    setOnLoudnessChange(callback) {
+        this.player.setOnLoudnessChange((loudness) => {
+            callback(loudness);
+        });
+    }
+}

package/dist/components/ai/EmbeddedAssistent/TTS/Player.d.ts

@@ -0,0 +1,25 @@
+export declare class ChunkedAudioPlayer {
+    private audioContext;
+    private chunkQueue;
+    private isPlaying;
+    private analyser;
+    private dataArray;
+    private shouldMonitorLoudness;
+    private isMonitoring;
+    private handle;
+    private volume;
+    private loudnessCallback;
+    private currentIndex;
+    private startedPlaying;
+    constructor();
+    private init;
+    setOnLoudnessChange(callback: (value: number) => void): void;
+    setVolume(volume: number): void;
+    addChunk(chunk: ArrayBuffer, position: number): Promise<void>;
+    private playChunks;
+    stopPlayback(): void;
+    private playChunk;
+    playAgain(): Promise<void>;
+    private monitorLoudness;
+    reset(): void;
+}

package/dist/components/ai/EmbeddedAssistent/TTS/Player.js

@@ -0,0 +1,180 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+export class ChunkedAudioPlayer {
+    constructor() {
+        this.chunkQueue = [];
+        this.isPlaying = false;
+        this.shouldMonitorLoudness = true;
+        this.isMonitoring = false;
+        this.handle = 0;
+        this.volume = 1.0;
+        this.loudnessCallback = () => { };
+        this.currentIndex = 0;
+        this.startedPlaying = false;
+        this.init();
+    }
+    init() {
+        this.audioContext = new AudioContext();
+        this.analyser = this.audioContext.createAnalyser();
+        this.analyser.fftSize = 256; // Set the FFT size (smaller values provide faster updates, larger ones give better resolution)
+        const bufferLength = this.analyser.frequencyBinCount;
+        this.dataArray = new Uint8Array(bufferLength); // Array to hold frequency data
+    }
+    setOnLoudnessChange(callback) {
+        this.loudnessCallback = callback;
+    }
+    setVolume(volume) {
+        this.volume = volume;
+    }
+    addChunk(chunk, position) {
+        return __awaiter(this, void 0, void 0, function* () {
+            console.log('Adding chunk', position, chunk);
+            this.chunkQueue[position] = chunk;
+            // console.log("received chunk", {
+            //     chunkQueue: this.chunkQueue.length,
+            //     isPlaying: this.isPlaying,
+            // })
+            if (position === 0 && !this.startedPlaying) {
+                this.startedPlaying = true;
+                this.playChunks();
+            }
+        });
+    }
+    playChunks() {
+        // console.log({ isPlaying: this.isPlaying });
+        if (this.isPlaying)
+            return;
+        if (!this.chunkQueue[this.currentIndex]) {
+            // wait until the correct chunk arrives
+            setTimeout(() => this.playChunks(), 10);
+        }
+        this.isPlaying = true;
+        this.playChunk(this.chunkQueue[this.currentIndex]).then(() => {
+            this.isPlaying = false;
+            this.currentIndex++;
+            if (this.chunkQueue[this.currentIndex]) {
+                this.shouldMonitorLoudness = true;
+                this.playChunks();
+            }
+            else {
+                // console.log('Playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
+                setTimeout(() => {
+                    // console.log('Check again if really playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
+                    if (this.chunkQueue.length > this.currentIndex) {
+                        this.playChunks();
+                    }
+                    else {
+                        this.startedPlaying = false;
+                        this.shouldMonitorLoudness = false;
+                    }
+                }, 1000);
+            }
+        });
+    }
+    stopPlayback() {
+        // console.log('Stopping playback');
+        // Implement logic to stop the current playback
+        this.isPlaying = false;
+        this.chunkQueue = [];
+        this.startedPlaying = false;
+        this.shouldMonitorLoudness = false;
+        cancelAnimationFrame(this.handle);
+    }
+    playChunk(chunk) {
+        console.log({ queue: this.chunkQueue });
+        if (!chunk) {
+            return Promise.resolve();
+        }
+        // console.log('Playing chunk', chunk);
+        return new Promise((resolve) => {
+            const source = this.audioContext.createBufferSource();
+            this.audioContext.decodeAudioData(chunk.slice(0)).then((audioBuffer) => {
+                source.buffer = audioBuffer;
+                // Create a GainNode for volume control
+                const gainNode = this.audioContext.createGain();
+                gainNode.gain.value = this.volume;
+                // Connect the source to the GainNode, then to the analyser node, then to the destination (speakers)
+                source.connect(gainNode);
+                gainNode.connect(this.analyser);
+                this.analyser.connect(this.audioContext.destination);
+                source.start(0);
+                // console.log('Playing chunk', this.currentIndex);
+                gainNode.gain.value = this.volume;
+                source.onended = () => {
+                    // console.log('Chunk ended');
+                    resolve();
+                };
+                // Start monitoring loudness only once
+                if (!this.isMonitoring) {
+                    this.isMonitoring = true;
+                    this.shouldMonitorLoudness = true;
+                    this.monitorLoudness();
+                }
+            });
+        });
+    }
+    playAgain() {
+        return __awaiter(this, void 0, void 0, function* () {
+            console.log('Playing again');
+            if (this.chunkQueue.length > 0 && !this.isPlaying) {
+                this.playChunks();
+            }
+        });
+    }
+    monitorLoudness() {
+        // Stop monitoring when the flag is false
+        if (!this.shouldMonitorLoudness) {
+            // console.log('Loudness monitoring stopped.');
+            cancelAnimationFrame(this.handle);
+            this.loudnessCallback(0);
+            return;
+        }
+        // Get the time domain data from the analyser (this is a snapshot of the waveform)
+        this.analyser.getByteTimeDomainData(this.dataArray);
+        // Calculate the RMS (root mean square) of the waveform values to get the perceived loudness
+        let sum = 0;
+        for (let i = 0; i < this.dataArray.length; i++) {
+            const value = this.dataArray[i] / 128.0 - 1.0; // Normalize between -1 and 1
+            sum += value * value;
+        }
+        const rms = Math.sqrt(sum / this.dataArray.length);
+        // Handle the case where RMS is 0 to avoid log10(0)
+        if (rms === 0) {
+            // console.log('Current loudness: Silent');
+        }
+        else {
+            let loudnessInDb = 20 * Math.log10(rms); // Convert to dB
+            // console.log('Current loudness:' + loudnessInDb);
+            const minDb = -57;
+            const maxDb = -15;
+            if (loudnessInDb < minDb) {
+                loudnessInDb = minDb;
+            }
+            if (loudnessInDb > maxDb) {
+                loudnessInDb = maxDb;
+            }
+            const loudnessScale = ((loudnessInDb - minDb) / (maxDb - minDb)) * 100;
+            // console.log("root:corrent loudness", loudnessScale);
+            this.loudnessCallback(loudnessScale);
+        }
+        // Call this method again at regular intervals if you want continuous loudness monitoring
+        this.handle = requestAnimationFrame(() => this.monitorLoudness());
+    }
+    reset() {
+        // console.log('Resetting player');
+        this.stopPlayback();
+        this.currentIndex = 0;
+        this.shouldMonitorLoudness = true;
+        //reset to the beginning when the class gets initialized
+        this.isMonitoring = false;
+        this.isPlaying = false;
+        this.init();
+    }
+}

package/dist/components/ai/EmbeddedAssistent/VoiceRecoder.d.ts

@@ -0,0 +1,7 @@
+interface Props {
+    iconSize?: string;
+    className?: string;
+    onVoiceRecorded: (message: string) => void;
+}
+export declare const VoiceRecorder: import("react").ForwardRefExoticComponent<Props & import("react").RefAttributes<unknown>>;
+export {};

package/dist/components/ai/EmbeddedAssistent/VoiceRecoder.js

@@ -0,0 +1,45 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { jsx as _jsx } from "react/jsx-runtime";
+import { useState, useRef, forwardRef, useImperativeHandle } from 'react';
+import { FaMicrophone } from 'react-icons/fa6';
+import { usePlugin } from '../../../components';
+export const VoiceRecorder = forwardRef(({ onVoiceRecorded, iconSize, className }, ref) => {
+    const [isRecording, setIsRecording] = useState(false);
+    const mediaRecorderRef = useRef(null);
+    const audioChunksRef = useRef([]);
+    const { llm } = usePlugin();
+    const startRecording = () => __awaiter(void 0, void 0, void 0, function* () {
+        const stream = yield navigator.mediaDevices.getUserMedia({ audio: true });
+        const mediaRecorder = new MediaRecorder(stream);
+        mediaRecorderRef.current = mediaRecorder;
+        mediaRecorder.ondataavailable = (event) => {
+            audioChunksRef.current.push(event.data);
+        };
+        mediaRecorder.onstop = () => __awaiter(void 0, void 0, void 0, function* () {
+            const audioBlob = new Blob(audioChunksRef.current);
+            audioChunksRef.current = [];
+            onVoiceRecorded(yield llm.getTextFromVoice(audioBlob));
+        });
+        mediaRecorder.start();
+        setIsRecording(true);
+    });
+    const stopRecording = () => {
+        if (mediaRecorderRef.current) {
+            mediaRecorderRef.current.stop();
+            setIsRecording(false);
+        }
+    };
+    useImperativeHandle(ref, () => ({
+        startRecording,
+        stopRecording,
+    }));
+    return (_jsx("div", { className: className, children: _jsx("button", { onClick: isRecording ? stopRecording : startRecording, children: _jsx(FaMicrophone, { size: iconSize, className: "h-7 w-7 mr-2 " + (isRecording ? "text-red-600" : "") }) }) }));
+});