@rimori/react-client 0.4.8 → 0.4.9-next.1

package/dist/components/ai/Assistant.js

@@ -2,7 +2,7 @@ import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
 import React, { useEffect, useMemo } from 'react';
 import { CircleAudioAvatar } from './EmbeddedAssistent/CircleAudioAvatar';
 import { AudioInputField } from './EmbeddedAssistent/AudioInputField';
-import { MessageSender } from './EmbeddedAssistent/TTS/MessageSender';
+import { MessageSender } from '@rimori/client';
 import Markdown from 'react-markdown';
 import { useChat } from '../../hooks/UseChatHook';
 import { useRimori } from '../../providers/PluginProvider';

package/dist/components/ai/Avatar.d.ts

@@ -9,6 +9,7 @@ interface Props {
     children?: React.ReactNode;
     autoStartConversation?: FirstMessages;
     className?: string;
+    knowledgeId?: string;
 }
-export declare function Avatar({ avatarImageUrl, voiceId, agentTools, autoStartConversation, children, circleSize, className, cache, }: Props): import("react/jsx-runtime").JSX.Element;
+export declare function Avatar({ avatarImageUrl, voiceId, agentTools, autoStartConversation, children, circleSize, className, cache, knowledgeId, }: Props): import("react/jsx-runtime").JSX.Element;
 export {};

package/dist/components/ai/Avatar.js

@@ -1,19 +1,19 @@
 import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
 import { useEffect, useMemo, useState } from 'react';
 import { VoiceRecorder } from './EmbeddedAssistent/VoiceRecorder';
-import { MessageSender } from './EmbeddedAssistent/TTS/MessageSender';
+import { MessageSender } from '@rimori/client';
 import { CircleAudioAvatar } from './EmbeddedAssistent/CircleAudioAvatar';
 import { useChat } from '../../hooks/UseChatHook';
 import { useRimori } from '../../providers/PluginProvider';
 import { getFirstMessages } from './utils';
 import { useTheme } from '../../hooks/ThemeSetter';
-export function Avatar({ avatarImageUrl, voiceId, agentTools, autoStartConversation, children, circleSize = '300px', className, cache = false, }) {
+export function Avatar({ avatarImageUrl, voiceId, agentTools, autoStartConversation, children, circleSize = '300px', className, cache = false, knowledgeId, }) {
     const { ai, event, plugin } = useRimori();
     const { isDark: isDarkThemeValue } = useTheme(plugin.theme);
     const [agentReplying, setAgentReplying] = useState(false);
     const [isProcessingMessage, setIsProcessingMessage] = useState(false);
     const sender = useMemo(() => new MessageSender((...args) => ai.getVoice(...args), voiceId, cache), [voiceId, ai, cache]);
-    const { messages, append, isLoading, lastMessage, setMessages } = useChat(agentTools);
+    const { messages, append, isLoading, lastMessage, setMessages } = useChat(agentTools, { knowledgeId });
     useEffect(() => {
         console.log('messages', messages);
     }, [messages]);

package/dist/components/audio/Playbutton.js

@@ -8,7 +8,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
-import { useState, useEffect, useRef } from 'react';
+import { useState, useEffect, useLayoutEffect, useRef } from 'react';
 import { FaPlayCircle, FaStopCircle } from 'react-icons/fa';
 import { useRimori } from '../../providers/PluginProvider';
 import { EventBus } from '@rimori/client';
@@ -105,10 +105,14 @@ export const AudioPlayer = ({ text, voice, language, hide, playListenerEvent, in
             setIsPlaying((prev) => !prev);
         }
     };
+    const togglePlaybackRef = useRef(togglePlayback);
+    useLayoutEffect(() => {
+        togglePlaybackRef.current = togglePlayback;
+    });
     useEffect(() => {
         if (!playListenerEvent)
             return;
-        const handler = () => togglePlayback();
+        const handler = () => togglePlaybackRef.current();
         const listener = EventBus.on(playListenerEvent, handler);
         eventBusListenerRef.current = listener;
         return () => {

package/dist/hooks/UseChatHook.d.ts

@@ -1,7 +1,9 @@
 import React from 'react';
 import { Tool } from '@rimori/client';
 import { Message } from '@rimori/client';
-export declare function useChat(tools?: Tool[]): {
+export declare function useChat(tools?: Tool[], options?: {
+    knowledgeId?: string | (() => string | undefined);
+}): {
     messages: Message[];
     append: (appendMessages: Message[]) => void;
     isLoading: boolean;

package/dist/hooks/UseChatHook.js

@@ -1,12 +1,13 @@
 import React from 'react';
 import { useRimori } from '../providers/PluginProvider';
-export function useChat(tools) {
+export function useChat(tools, options) {
     const [messages, setMessages] = React.useState([]);
     const [isLoading, setIsLoading] = React.useState(false);
     const { ai } = useRimori();
     const append = (appendMessages) => {
         const allMessages = [...messages, ...appendMessages];
         setMessages(allMessages);
+        const knowledgeId = typeof (options === null || options === void 0 ? void 0 : options.knowledgeId) === 'function' ? options.knowledgeId() : options === null || options === void 0 ? void 0 : options.knowledgeId;
         ai.getSteamedText(allMessages, (id, message, finished, toolInvocations) => {
             const lastMessage = messages[messages.length - 1];
             setIsLoading(!finished);
@@ -17,7 +18,7 @@ export function useChat(tools) {
             else {
                 setMessages([...allMessages, { id, role: 'assistant', content: message, toolCalls: toolInvocations }]);
             }
-        }, tools);
+        }, tools, false, undefined, knowledgeId);
     };
     return {
         messages,

package/dist/index.d.ts

@@ -1,7 +1,6 @@
 export * from './hooks/UseChatHook';
 export * from './providers/PluginProvider';
 export * from './components/audio/Playbutton';
-export * from './components/ai/EmbeddedAssistent/TTS/Player';
 export * from './components/ai/Avatar';
 export { FirstMessages } from './components/ai/utils';
 export { useTranslation } from './hooks/I18nHooks';

package/dist/index.js

@@ -2,7 +2,6 @@
 export * from './hooks/UseChatHook';
 export * from './providers/PluginProvider';
 export * from './components/audio/Playbutton';
-export * from './components/ai/EmbeddedAssistent/TTS/Player';
 export * from './components/ai/Avatar';
 export { useTranslation } from './hooks/I18nHooks';
 export { Avatar } from './components/ai/Avatar';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rimori/react-client",
-  "version": "0.4.8",
+  "version": "0.4.9-next.1",
   "license": "Apache-2.0",
   "repository": {
     "type": "git",
@@ -23,7 +23,7 @@
     "format": "prettier --write ."
   },
   "peerDependencies": {
-    "@rimori/client": "^2.5.13",
+    "@rimori/client": "2.5.17-next.0",
     "react": "^18.1.0",
     "react-dom": "^18.1.0"
   },
@@ -34,7 +34,7 @@
   },
   "devDependencies": {
     "@eslint/js": "^9.37.0",
-    "@rimori/client": "^2.5.13",
+    "@rimori/client": "2.5.17-next.0",
     "@types/react": "^18.3.21",
     "eslint-config-prettier": "^10.1.8",
     "eslint-plugin-prettier": "^5.5.4",

package/src/components/ai/Assistant.tsx

@@ -1,7 +1,7 @@
 import React, { useEffect, useMemo } from 'react';
 import { CircleAudioAvatar } from './EmbeddedAssistent/CircleAudioAvatar';
 import { AudioInputField } from './EmbeddedAssistent/AudioInputField';
-import { MessageSender } from './EmbeddedAssistent/TTS/MessageSender';
+import { MessageSender } from '@rimori/client';
 import Markdown from 'react-markdown';
 import { useChat } from '../../hooks/UseChatHook';
 import { useRimori } from '../../providers/PluginProvider';

package/src/components/ai/Avatar.tsx

@@ -1,6 +1,6 @@
 import { useEffect, useMemo, useState } from 'react';
 import { VoiceRecorder } from './EmbeddedAssistent/VoiceRecorder';
-import { MessageSender } from './EmbeddedAssistent/TTS/MessageSender';
+import { MessageSender } from '@rimori/client';
 import { CircleAudioAvatar } from './EmbeddedAssistent/CircleAudioAvatar';
 import { Tool } from '@rimori/client';
 import { useChat } from '../../hooks/UseChatHook';
@@ -18,6 +18,7 @@ interface Props {
   children?: React.ReactNode;
   autoStartConversation?: FirstMessages;
   className?: string;
+  knowledgeId?: string;
 }
 
 export function Avatar({
@@ -29,6 +30,7 @@ export function Avatar({
   circleSize = '300px',
   className,
   cache = false,
+  knowledgeId,
 }: Props) {
   const { ai, event, plugin } = useRimori();
   const { isDark: isDarkThemeValue } = useTheme(plugin.theme);
@@ -38,7 +40,7 @@ export function Avatar({
     () => new MessageSender((...args) => ai.getVoice(...args), voiceId, cache),
     [voiceId, ai, cache],
   );
-  const { messages, append, isLoading, lastMessage, setMessages } = useChat(agentTools);
+  const { messages, append, isLoading, lastMessage, setMessages } = useChat(agentTools, { knowledgeId });
 
   useEffect(() => {
     console.log('messages', messages);

package/src/components/audio/Playbutton.tsx

@@ -1,4 +1,4 @@
-import React, { useState, useEffect, useRef } from 'react';
+import React, { useState, useEffect, useLayoutEffect, useRef } from 'react';
 import { FaPlayCircle, FaStopCircle } from 'react-icons/fa';
 import { useRimori } from '../../providers/PluginProvider';
 import { EventBus } from '@rimori/client';
@@ -129,9 +129,14 @@ export const AudioPlayer: React.FC<AudioPlayerProps> = ({
     }
   };
 
+  const togglePlaybackRef = useRef(togglePlayback);
+  useLayoutEffect(() => {
+    togglePlaybackRef.current = togglePlayback;
+  });
+
   useEffect(() => {
     if (!playListenerEvent) return;
-    const handler = () => togglePlayback();
+    const handler = () => togglePlaybackRef.current();
     const listener = EventBus.on(playListenerEvent, handler);
     eventBusListenerRef.current = listener;
 

package/src/hooks/UseChatHook.ts

@@ -3,7 +3,7 @@ import { Tool } from '@rimori/client';
 import { useRimori } from '../providers/PluginProvider';
 import { Message, ToolInvocation } from '@rimori/client';
 
-export function useChat(tools?: Tool[]) {
+export function useChat(tools?: Tool[], options?: { knowledgeId?: string | (() => string | undefined) }) {
   const [messages, setMessages] = React.useState<Message[]>([]);
   const [isLoading, setIsLoading] = React.useState(false);
   const { ai } = useRimori();
@@ -11,6 +11,7 @@ export function useChat(tools?: Tool[]) {
   const append = (appendMessages: Message[]) => {
     const allMessages = [...messages, ...appendMessages];
     setMessages(allMessages);
+    const knowledgeId = typeof options?.knowledgeId === 'function' ? options.knowledgeId() : options?.knowledgeId;
     ai.getSteamedText(
       allMessages,
       (id, message, finished: boolean, toolInvocations?: ToolInvocation[]) => {
@@ -25,6 +26,9 @@ export function useChat(tools?: Tool[]) {
         }
       },
       tools,
+      false,
+      undefined,
+      knowledgeId,
     );
   };
 

package/src/index.ts

@@ -2,7 +2,6 @@
 export * from './hooks/UseChatHook';
 export * from './providers/PluginProvider';
 export * from './components/audio/Playbutton';
-export * from './components/ai/EmbeddedAssistent/TTS/Player';
 export * from './components/ai/Avatar';
 export { FirstMessages } from './components/ai/utils';
 export { useTranslation } from './hooks/I18nHooks';

package/dist/components/ai/EmbeddedAssistent/TTS/MessageSender.d.ts

@@ -1,21 +0,0 @@
-type VoiceBackend = (text: string, voice?: string, speed?: number, language?: string, cache?: boolean) => Promise<Blob>;
-export declare class MessageSender {
-    private player;
-    private fetchedSentences;
-    private lastLoading;
-    private voice;
-    private voiceBackend;
-    private cache;
-    constructor(voiceBackend: VoiceBackend, voice: string, cache?: boolean);
-    private getCompletedSentences;
-    handleNewText(currentText: string | undefined, isLoading: boolean): Promise<void>;
-    private generateSpeech;
-    play(): void;
-    stop(): void;
-    cleanup(): void;
-    private reset;
-    setVolume(volume: number): void;
-    setOnLoudnessChange(callback: (value: number) => void): void;
-    setOnEndOfSpeech(callback: () => void): void;
-}
-export {};

package/dist/components/ai/EmbeddedAssistent/TTS/MessageSender.js

@@ -1,95 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import { ChunkedAudioPlayer } from './Player';
-export class MessageSender {
-    constructor(voiceBackend, voice, cache = false) {
-        this.player = new ChunkedAudioPlayer();
-        this.fetchedSentences = new Set();
-        this.lastLoading = false;
-        if ((voice === null || voice === void 0 ? void 0 : voice.split('_').length) !== 2) {
-            throw new Error("Invalid voice id format '" + voice + "'. Voice id needs to look like <provider>_<voice_id>");
-        }
-        this.voiceBackend = voiceBackend;
-        this.voice = voice;
-        this.cache = cache;
-    }
-    getCompletedSentences(currentText, isLoading) {
-        // Split the text based on the following characters: .?!
-        // Only split on : when followed by a space
-        const pattern = /(.+?[.?!]|.+?:\s+|.+?\n+)/g;
-        const result = [];
-        let match;
-        while ((match = pattern.exec(currentText)) !== null) {
-            const sentence = match[0].trim();
-            if (sentence.length > 0) {
-                result.push(sentence);
-            }
-        }
-        if (!isLoading) {
-            const lastFullSentence = result[result.length - 1];
-            const leftoverIndex = currentText.lastIndexOf(lastFullSentence) + lastFullSentence.length;
-            if (leftoverIndex < currentText.length) {
-                result.push(currentText.slice(leftoverIndex).trim());
-            }
-        }
-        return result;
-    }
-    handleNewText(currentText, isLoading) {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (!this.lastLoading && isLoading) {
-                this.reset();
-            }
-            this.lastLoading = isLoading;
-            if (!currentText) {
-                return;
-            }
-            const sentences = this.getCompletedSentences(currentText, isLoading);
-            for (let i = 0; i < sentences.length; i++) {
-                const sentence = sentences[i];
-                if (!this.fetchedSentences.has(sentence)) {
-                    this.fetchedSentences.add(sentence);
-                    const audioData = yield this.generateSpeech(sentence);
-                    yield this.player.addChunk(audioData, i);
-                }
-            }
-        });
-    }
-    generateSpeech(sentence) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const blob = yield this.voiceBackend(sentence, this.voice, 1.0, undefined, this.cache);
-            return yield blob.arrayBuffer();
-        });
-    }
-    play() {
-        this.player.playAgain();
-    }
-    stop() {
-        this.player.stopPlayback();
-    }
-    cleanup() {
-        this.player.cleanup();
-    }
-    reset() {
-        this.stop();
-        this.fetchedSentences.clear();
-        this.player.reset();
-    }
-    setVolume(volume) {
-        this.player.setVolume(volume);
-    }
-    setOnLoudnessChange(callback) {
-        this.player.setOnLoudnessChange((loudness) => {
-            callback(loudness);
-        });
-    }
-    setOnEndOfSpeech(callback) {
-        this.player.setOnEndOfSpeech(callback);
-    }
-}

package/dist/components/ai/EmbeddedAssistent/TTS/Player.d.ts

@@ -1,29 +0,0 @@
-export declare class ChunkedAudioPlayer {
-    private audioContext;
-    private chunkQueue;
-    private isPlaying;
-    private analyser;
-    private dataArray;
-    private shouldMonitorLoudness;
-    private isMonitoring;
-    private handle;
-    private volume;
-    private loudnessCallback;
-    private currentIndex;
-    private startedPlaying;
-    private onEndOfSpeech;
-    private readonly backgroundNoiseLevel;
-    constructor();
-    private init;
-    setOnLoudnessChange(callback: (value: number) => void): void;
-    setVolume(volume: number): void;
-    addChunk(chunk: ArrayBuffer, position: number): Promise<void>;
-    private playChunks;
-    stopPlayback(): void;
-    cleanup(): void;
-    private playChunk;
-    playAgain(): Promise<void>;
-    private monitorLoudness;
-    reset(): void;
-    setOnEndOfSpeech(callback: () => void): void;
-}

package/dist/components/ai/EmbeddedAssistent/TTS/Player.js

@@ -1,205 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-export class ChunkedAudioPlayer {
-    constructor() {
-        this.chunkQueue = [];
-        this.isPlaying = false;
-        this.shouldMonitorLoudness = true;
-        this.isMonitoring = false;
-        this.handle = 0;
-        this.volume = 1.0;
-        this.loudnessCallback = () => { };
-        this.currentIndex = 0;
-        this.startedPlaying = false;
-        this.onEndOfSpeech = () => { };
-        this.backgroundNoiseLevel = 30; // Background noise level that should be treated as baseline (0)
-        this.init();
-    }
-    init() {
-        this.audioContext = new AudioContext();
-        this.analyser = this.audioContext.createAnalyser();
-        this.analyser.fftSize = 256; // Set the FFT size (smaller values provide faster updates, larger ones give better resolution)
-        const bufferLength = this.analyser.frequencyBinCount;
-        this.dataArray = new Uint8Array(bufferLength); // Array to hold frequency data
-    }
-    setOnLoudnessChange(callback) {
-        this.loudnessCallback = callback;
-    }
-    setVolume(volume) {
-        this.volume = volume;
-    }
-    addChunk(chunk, position) {
-        return __awaiter(this, void 0, void 0, function* () {
-            console.log('Adding chunk', position, chunk);
-            this.chunkQueue[position] = chunk;
-            // console.log("received chunk", {
-            //     chunkQueue: this.chunkQueue.length,
-            //     isPlaying: this.isPlaying,
-            // })
-            if (position === 0 && !this.startedPlaying) {
-                this.startedPlaying = true;
-                this.playChunks();
-            }
-        });
-    }
-    playChunks() {
-        // console.log({ isPlaying: this.isPlaying });
-        if (this.isPlaying)
-            return;
-        if (!this.chunkQueue[this.currentIndex]) {
-            // wait until the correct chunk arrives
-            setTimeout(() => this.playChunks(), 10);
-        }
-        this.isPlaying = true;
-        this.playChunk(this.chunkQueue[this.currentIndex]).then(() => {
-            this.isPlaying = false;
-            this.currentIndex++;
-            if (this.chunkQueue[this.currentIndex]) {
-                this.shouldMonitorLoudness = true;
-                this.playChunks();
-            }
-            else {
-                // console.log('Playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
-                setTimeout(() => {
-                    // console.log('Check again if really playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
-                    if (this.chunkQueue.length > this.currentIndex) {
-                        this.playChunks();
-                    }
-                    else {
-                        this.startedPlaying = false;
-                        this.shouldMonitorLoudness = false;
-                    }
-                }, 1000);
-            }
-        });
-    }
-    stopPlayback() {
-        // console.log('Stopping playback');
-        // Implement logic to stop the current playback
-        this.isPlaying = false;
-        this.chunkQueue = [];
-        this.startedPlaying = false;
-        this.shouldMonitorLoudness = false;
-        cancelAnimationFrame(this.handle);
-    }
-    cleanup() {
-        // Stop playback first
-        this.stopPlayback();
-        // Close AudioContext to free resources
-        if (this.audioContext && this.audioContext.state !== 'closed') {
-            this.audioContext.close().catch((e) => {
-                console.warn('Error closing AudioContext:', e);
-            });
-        }
-    }
-    playChunk(chunk) {
-        // console.log({queue: this.chunkQueue})
-        if (!chunk) {
-            return Promise.resolve();
-        }
-        // console.log('Playing chunk', chunk);
-        return new Promise((resolve) => {
-            const source = this.audioContext.createBufferSource();
-            this.audioContext.decodeAudioData(chunk.slice(0)).then((audioBuffer) => {
-                source.buffer = audioBuffer;
-                // Create a GainNode for volume control
-                const gainNode = this.audioContext.createGain();
-                gainNode.gain.value = this.volume;
-                // Connect the source to the GainNode, then to the analyser node, then to the destination (speakers)
-                source.connect(gainNode);
-                gainNode.connect(this.analyser);
-                this.analyser.connect(this.audioContext.destination);
-                source.start(0);
-                // console.log('Playing chunk', this.currentIndex);
-                gainNode.gain.value = this.volume;
-                source.onended = () => {
-                    // console.log('Chunk ended');
-                    resolve();
-                };
-                // Start monitoring loudness only once
-                if (!this.isMonitoring) {
-                    this.isMonitoring = true;
-                    this.shouldMonitorLoudness = true;
-                    this.monitorLoudness();
-                }
-            });
-        });
-    }
-    playAgain() {
-        return __awaiter(this, void 0, void 0, function* () {
-            console.log('Playing again');
-            if (this.chunkQueue.length > 0 && !this.isPlaying) {
-                this.playChunks();
-            }
-        });
-    }
-    monitorLoudness() {
-        // Stop monitoring when the flag is false
-        if (!this.shouldMonitorLoudness) {
-            // console.log('Loudness monitoring stopped.');
-            cancelAnimationFrame(this.handle);
-            this.loudnessCallback(0);
-            this.onEndOfSpeech();
-            return;
-        }
-        // Get the time domain data from the analyser (this is a snapshot of the waveform)
-        this.analyser.getByteTimeDomainData(this.dataArray);
-        // Calculate the RMS (root mean square) of the waveform values to get the perceived loudness
-        let sum = 0;
-        for (let i = 0; i < this.dataArray.length; i++) {
-            const value = this.dataArray[i] / 128.0 - 1.0; // Normalize between -1 and 1
-            sum += value * value;
-        }
-        const rms = Math.sqrt(sum / this.dataArray.length);
-        // Handle the case where RMS is 0 to avoid log10(0)
-        if (rms === 0) {
-            // console.log('Current loudness: Silent');
-        }
-        else {
-            let loudnessInDb = 20 * Math.log10(rms); // Convert to dB
-            // console.log('Current loudness:' + loudnessInDb);
-            const minDb = -57;
-            const maxDb = -15;
-            if (loudnessInDb < minDb) {
-                loudnessInDb = minDb;
-            }
-            if (loudnessInDb > maxDb) {
-                loudnessInDb = maxDb;
-            }
-            let loudnessScale = ((loudnessInDb - minDb) / (maxDb - minDb)) * 100;
-            // Adjust loudness: shift zero level up by background noise amount
-            // Values below background noise level are set to 0
-            // Values above are remapped to 0-100 scale
-            if (loudnessScale < this.backgroundNoiseLevel) {
-                loudnessScale = 0;
-            }
-            else {
-                // Remap from [backgroundNoiseLevel, 100] to [0, 100]
-                loudnessScale = ((loudnessScale - this.backgroundNoiseLevel) / (100 - this.backgroundNoiseLevel)) * 100;
-            }
-            this.loudnessCallback(Math.round(loudnessScale));
-        }
-        // Call this method again at regular intervals if you want continuous loudness monitoring
-        this.handle = requestAnimationFrame(() => this.monitorLoudness());
-    }
-    reset() {
-        // console.log('Resetting player');
-        this.stopPlayback();
-        this.currentIndex = 0;
-        this.shouldMonitorLoudness = true;
-        //reset to the beginning when the class gets initialized
-        this.isMonitoring = false;
-        this.isPlaying = false;
-        this.init();
-    }
-    setOnEndOfSpeech(callback) {
-        this.onEndOfSpeech = callback;
-    }
-}

package/src/components/ai/EmbeddedAssistent/TTS/MessageSender.ts

@@ -1,102 +0,0 @@
-import { ChunkedAudioPlayer } from './Player';
-
-type VoiceBackend = (text: string, voice?: string, speed?: number, language?: string, cache?: boolean) => Promise<Blob>;
-
-export class MessageSender {
-  private player = new ChunkedAudioPlayer();
-  private fetchedSentences = new Set<string>();
-  private lastLoading = false;
-  private voice: string;
-  private voiceBackend: VoiceBackend;
-  private cache: boolean;
-
-  constructor(voiceBackend: VoiceBackend, voice: string, cache = false) {
-    if (voice?.split('_').length !== 2) {
-      throw new Error("Invalid voice id format '" + voice + "'. Voice id needs to look like <provider>_<voice_id>");
-    }
-    this.voiceBackend = voiceBackend;
-    this.voice = voice;
-    this.cache = cache;
-  }
-
-  private getCompletedSentences(currentText: string, isLoading: boolean): string[] {
-    // Split the text based on the following characters: .?!
-    // Only split on : when followed by a space
-    const pattern = /(.+?[.?!]|.+?:\s+|.+?\n+)/g;
-    const result: string[] = [];
-    let match;
-    while ((match = pattern.exec(currentText)) !== null) {
-      const sentence = match[0].trim();
-      if (sentence.length > 0) {
-        result.push(sentence);
-      }
-    }
-    if (!isLoading) {
-      const lastFullSentence = result[result.length - 1];
-      const leftoverIndex = currentText.lastIndexOf(lastFullSentence) + lastFullSentence.length;
-      if (leftoverIndex < currentText.length) {
-        result.push(currentText.slice(leftoverIndex).trim());
-      }
-    }
-    return result;
-  }
-
-  public async handleNewText(currentText: string | undefined, isLoading: boolean) {
-    if (!this.lastLoading && isLoading) {
-      this.reset();
-    }
-    this.lastLoading = isLoading;
-
-    if (!currentText) {
-      return;
-    }
-
-    const sentences = this.getCompletedSentences(currentText, isLoading);
-
-    for (let i = 0; i < sentences.length; i++) {
-      const sentence = sentences[i];
-      if (!this.fetchedSentences.has(sentence)) {
-        this.fetchedSentences.add(sentence);
-        const audioData = await this.generateSpeech(sentence);
-        await this.player.addChunk(audioData, i);
-      }
-    }
-  }
-
-  private async generateSpeech(sentence: string): Promise<ArrayBuffer> {
-    const blob = await this.voiceBackend(sentence, this.voice, 1.0, undefined, this.cache);
-    return await blob.arrayBuffer();
-  }
-
-  public play() {
-    this.player.playAgain();
-  }
-
-  public stop() {
-    this.player.stopPlayback();
-  }
-
-  public cleanup() {
-    this.player.cleanup();
-  }
-
-  private reset() {
-    this.stop();
-    this.fetchedSentences.clear();
-    this.player.reset();
-  }
-
-  public setVolume(volume: number) {
-    this.player.setVolume(volume);
-  }
-
-  public setOnLoudnessChange(callback: (value: number) => void) {
-    this.player.setOnLoudnessChange((loudness) => {
-      callback(loudness);
-    });
-  }
-
-  public setOnEndOfSpeech(callback: () => void) {
-    this.player.setOnEndOfSpeech(callback);
-  }
-}

package/src/components/ai/EmbeddedAssistent/TTS/Player.ts

@@ -1,218 +0,0 @@
-export class ChunkedAudioPlayer {
-  private audioContext!: AudioContext;
-  private chunkQueue: ArrayBuffer[] = [];
-  private isPlaying = false;
-  private analyser!: AnalyserNode;
-  private dataArray!: Uint8Array<ArrayBuffer>;
-  private shouldMonitorLoudness = true;
-  private isMonitoring = false;
-  private handle = 0;
-  private volume = 1.0;
-  private loudnessCallback: (value: number) => void = () => {};
-  private currentIndex = 0;
-  private startedPlaying = false;
-  private onEndOfSpeech: () => void = () => {};
-  private readonly backgroundNoiseLevel = 30; // Background noise level that should be treated as baseline (0)
-
-  constructor() {
-    this.init();
-  }
-
-  private init(): void {
-    this.audioContext = new AudioContext();
-    this.analyser = this.audioContext.createAnalyser();
-    this.analyser.fftSize = 256; // Set the FFT size (smaller values provide faster updates, larger ones give better resolution)
-    const bufferLength = this.analyser.frequencyBinCount;
-    this.dataArray = new Uint8Array(bufferLength); // Array to hold frequency data
-  }
-
-  public setOnLoudnessChange(callback: (value: number) => void) {
-    this.loudnessCallback = callback;
-  }
-
-  public setVolume(volume: number) {
-    this.volume = volume;
-  }
-
-  public async addChunk(chunk: ArrayBuffer, position: number): Promise<void> {
-    console.log('Adding chunk', position, chunk);
-    this.chunkQueue[position] = chunk;
-    // console.log("received chunk", {
-    //     chunkQueue: this.chunkQueue.length,
-    //     isPlaying: this.isPlaying,
-    // })
-
-    if (position === 0 && !this.startedPlaying) {
-      this.startedPlaying = true;
-      this.playChunks();
-    }
-  }
-
-  private playChunks(): void {
-    // console.log({ isPlaying: this.isPlaying });
-    if (this.isPlaying) return;
-    if (!this.chunkQueue[this.currentIndex]) {
-      // wait until the correct chunk arrives
-      setTimeout(() => this.playChunks(), 10);
-    }
-    this.isPlaying = true;
-
-    this.playChunk(this.chunkQueue[this.currentIndex]).then(() => {
-      this.isPlaying = false;
-      this.currentIndex++;
-      if (this.chunkQueue[this.currentIndex]) {
-        this.shouldMonitorLoudness = true;
-        this.playChunks();
-      } else {
-        // console.log('Playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
-        setTimeout(() => {
-          // console.log('Check again if really playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
-          if (this.chunkQueue.length > this.currentIndex) {
-            this.playChunks();
-          } else {
-            this.startedPlaying = false;
-            this.shouldMonitorLoudness = false;
-          }
-        }, 1000);
-      }
-    });
-  }
-
-  public stopPlayback(): void {
-    // console.log('Stopping playback');
-    // Implement logic to stop the current playback
-    this.isPlaying = false;
-    this.chunkQueue = [];
-    this.startedPlaying = false;
-    this.shouldMonitorLoudness = false;
-    cancelAnimationFrame(this.handle);
-  }
-
-  public cleanup(): void {
-    // Stop playback first
-    this.stopPlayback();
-    // Close AudioContext to free resources
-    if (this.audioContext && this.audioContext.state !== 'closed') {
-      this.audioContext.close().catch((e) => {
-        console.warn('Error closing AudioContext:', e);
-      });
-    }
-  }
-
-  private playChunk(chunk: ArrayBuffer): Promise<void> {
-    // console.log({queue: this.chunkQueue})
-    if (!chunk) {
-      return Promise.resolve();
-    }
-
-    // console.log('Playing chunk', chunk);
-    return new Promise((resolve) => {
-      const source = this.audioContext.createBufferSource();
-      this.audioContext.decodeAudioData(chunk.slice(0)).then((audioBuffer) => {
-        source.buffer = audioBuffer;
-
-        // Create a GainNode for volume control
-        const gainNode = this.audioContext.createGain();
-        gainNode.gain.value = this.volume;
-
-        // Connect the source to the GainNode, then to the analyser node, then to the destination (speakers)
-        source.connect(gainNode);
-        gainNode.connect(this.analyser);
-        this.analyser.connect(this.audioContext.destination);
-
-        source.start(0);
-        // console.log('Playing chunk', this.currentIndex);
-        gainNode.gain.value = this.volume;
-        source.onended = () => {
-          // console.log('Chunk ended');
-          resolve();
-        };
-
-        // Start monitoring loudness only once
-        if (!this.isMonitoring) {
-          this.isMonitoring = true;
-          this.shouldMonitorLoudness = true;
-          this.monitorLoudness();
-        }
-      });
-    });
-  }
-
-  async playAgain(): Promise<void> {
-    console.log('Playing again');
-    if (this.chunkQueue.length > 0 && !this.isPlaying) {
-      this.playChunks();
-    }
-  }
-
-  private monitorLoudness(): void {
-    // Stop monitoring when the flag is false
-    if (!this.shouldMonitorLoudness) {
-      // console.log('Loudness monitoring stopped.');
-      cancelAnimationFrame(this.handle);
-      this.loudnessCallback(0);
-      this.onEndOfSpeech();
-      return;
-    }
-
-    // Get the time domain data from the analyser (this is a snapshot of the waveform)
-    this.analyser.getByteTimeDomainData(this.dataArray);
-
-    // Calculate the RMS (root mean square) of the waveform values to get the perceived loudness
-    let sum = 0;
-    for (let i = 0; i < this.dataArray.length; i++) {
-      const value = this.dataArray[i] / 128.0 - 1.0; // Normalize between -1 and 1
-      sum += value * value;
-    }
-
-    const rms = Math.sqrt(sum / this.dataArray.length);
-
-    // Handle the case where RMS is 0 to avoid log10(0)
-    if (rms === 0) {
-      // console.log('Current loudness: Silent');
-    } else {
-      let loudnessInDb = 20 * Math.log10(rms); // Convert to dB
-      // console.log('Current loudness:' + loudnessInDb);
-      const minDb = -57;
-      const maxDb = -15;
-
-      if (loudnessInDb < minDb) {
-        loudnessInDb = minDb;
-      }
-      if (loudnessInDb > maxDb) {
-        loudnessInDb = maxDb;
-      }
-
-      let loudnessScale = ((loudnessInDb - minDb) / (maxDb - minDb)) * 100;
-
-      // Adjust loudness: shift zero level up by background noise amount
-      // Values below background noise level are set to 0
-      // Values above are remapped to 0-100 scale
-      if (loudnessScale < this.backgroundNoiseLevel) {
-        loudnessScale = 0;
-      } else {
-        // Remap from [backgroundNoiseLevel, 100] to [0, 100]
-        loudnessScale = ((loudnessScale - this.backgroundNoiseLevel) / (100 - this.backgroundNoiseLevel)) * 100;
-      }
-
-      this.loudnessCallback(Math.round(loudnessScale));
-    }
-
-    // Call this method again at regular intervals if you want continuous loudness monitoring
-    this.handle = requestAnimationFrame(() => this.monitorLoudness());
-  }
-  public reset() {
-    // console.log('Resetting player');
-    this.stopPlayback();
-    this.currentIndex = 0;
-    this.shouldMonitorLoudness = true;
-    //reset to the beginning when the class gets initialized
-    this.isMonitoring = false;
-    this.isPlaying = false;
-    this.init();
-  }
-
-  public setOnEndOfSpeech(callback: () => void) {
-    this.onEndOfSpeech = callback;
-  }
-}