@rimori/client 1.3.1 → 1.4.3

package/src/components/ai/EmbeddedAssistent/TTS/MessageSender.ts

@@ -3,93 +3,94 @@ import { ChunkedAudioPlayer } from './Player';
 type VoiceBackend = (text: string, voice?: string, speed?: number) => Promise<Blob>;
 
 export class MessageSender {
-    private player = new ChunkedAudioPlayer();
-    private fetchedSentences = new Set<string>();
-    private lastLoading = false;
-    private voice: string;
-    private model: string;
-    private voiceBackend: VoiceBackend;
+  private player = new ChunkedAudioPlayer();
+  private fetchedSentences = new Set<string>();
+  private lastLoading = false;
+  private voice: string;
+  private voiceBackend: VoiceBackend;
 
-    constructor(voiceBackend: VoiceBackend, voice: string = 'alloy', model = 'openai') {
-        this.voiceBackend = voiceBackend;
-        this.voice = voice;
-        this.model = model;
+  constructor(voiceBackend: VoiceBackend, voice: string) {
+    if (voice?.split('_').length !== 2) {
+      throw new Error("Invalid voice id format '" + voice + "'. Voice id needs to look like <provider>_<voice_id>");
     }
+    this.voiceBackend = voiceBackend;
+    this.voice = voice;
+  }
 
-    private getCompletedSentences(currentText: string, isLoading: boolean): string[] {
-        // Split the text based on the following characters: .,?!
-        // Only split on : when followed by a space
-        const pattern = /(.+?[,.?!]|.+?:\s+|.+?\n+)/g;
-        const result: string[] = [];
-        let match;
-        while ((match = pattern.exec(currentText)) !== null) {
-            const sentence = match[0].trim();
-            if (sentence.length > 0) {
-                result.push(sentence);
-            }
-        }
-        if (!isLoading) {
-            const lastFullSentence = result[result.length - 1];
-            const leftoverIndex = currentText.lastIndexOf(lastFullSentence) + lastFullSentence.length;
-            if (leftoverIndex < currentText.length) {
-                result.push(currentText.slice(leftoverIndex).trim());
-            }
-        }
-        return result;
+  private getCompletedSentences(currentText: string, isLoading: boolean): string[] {
+    // Split the text based on the following characters: .,?!
+    // Only split on : when followed by a space
+    const pattern = /(.+?[,.?!]|.+?:\s+|.+?\n+)/g;
+    const result: string[] = [];
+    let match;
+    while ((match = pattern.exec(currentText)) !== null) {
+      const sentence = match[0].trim();
+      if (sentence.length > 0) {
+        result.push(sentence);
+      }
     }
+    if (!isLoading) {
+      const lastFullSentence = result[result.length - 1];
+      const leftoverIndex = currentText.lastIndexOf(lastFullSentence) + lastFullSentence.length;
+      if (leftoverIndex < currentText.length) {
+        result.push(currentText.slice(leftoverIndex).trim());
+      }
+    }
+    return result;
+  }
 
-    public async handleNewText(currentText: string | undefined, isLoading: boolean) {
-        if (!this.lastLoading && isLoading) {
-            this.reset();
-        }
-        this.lastLoading = isLoading;
+  public async handleNewText(currentText: string | undefined, isLoading: boolean) {
+    if (!this.lastLoading && isLoading) {
+      this.reset();
+    }
+    this.lastLoading = isLoading;
 
-        if (!currentText) {
-            return;
-        }
+    if (!currentText) {
+      return;
+    }
 
-        const sentences = this.getCompletedSentences(currentText, isLoading);
+    const sentences = this.getCompletedSentences(currentText, isLoading);
 
-        for (let i = 0; i < sentences.length; i++) {
-            const sentence = sentences[i];
-            if (!this.fetchedSentences.has(sentence)) {
-                this.fetchedSentences.add(sentence);
-                const audioData = await this.generateSpeech(sentence);
-                await this.player.addChunk(audioData, i);
-            }
-        }
+    for (let i = 0; i < sentences.length; i++) {
+      const sentence = sentences[i];
+      if (!this.fetchedSentences.has(sentence)) {
+        this.fetchedSentences.add(sentence);
+        const audioData = await this.generateSpeech(sentence);
+        await this.player.addChunk(audioData, i);
+      }
     }
+  }
 
-    private async generateSpeech(sentence: string): Promise<ArrayBuffer> {
-        const blob = await this.voiceBackend(sentence, this.voice, 1.0);
-        return await blob.arrayBuffer();
-    }
+  private async generateSpeech(sentence: string): Promise<ArrayBuffer> {
+    const blob = await this.voiceBackend(sentence, this.voice, 1.0);
+    return await blob.arrayBuffer();
+  }
 
-    public play() {
-        this.player.playAgain();
-    }
+  public play() {
+    this.player.playAgain();
+  }
 
-    public stop() {
-        this.player.stopPlayback();
-    }
+  public stop() {
+    this.player.stopPlayback();
+  }
 
-    private reset() {
-        this.stop();
-        this.fetchedSentences.clear();
-        this.player.reset();
-    }
+  private reset() {
+    this.stop();
+    this.fetchedSentences.clear();
+    this.player.reset();
+  }
 
-    public setVolume(volume: number) {
-        this.player.setVolume(volume);
-    }
+  public setVolume(volume: number) {
+    this.player.setVolume(volume);
+  }
 
-    public setOnLoudnessChange(callback: (value: number) => void) {
-        this.player.setOnLoudnessChange((loudness) => {
-            callback(loudness);
-        });
-    }
+  public setOnLoudnessChange(callback: (value: number) => void) {
+    this.player.setOnLoudnessChange((loudness) => {
+      callback(loudness);
+    });
+  }
 
-    public setOnEndOfSpeech(callback: () => void) {
-        this.player.setOnEndOfSpeech(callback);
-    }
-}
+  public setOnEndOfSpeech(callback: () => void) {
+    this.player.setOnEndOfSpeech(callback);
+  }
+}

package/src/components/ai/EmbeddedAssistent/TTS/Player.ts

@@ -1,198 +1,197 @@
 export class ChunkedAudioPlayer {
-
-    private audioContext!: AudioContext;
-    private chunkQueue: ArrayBuffer[] = [];
-    private isPlaying = false;
-    private analyser!: AnalyserNode;
-    private dataArray!: Uint8Array;
-    private shouldMonitorLoudness = true;
-    private isMonitoring = false;
-    private handle = 0;
-    private volume = 1.0;
-    private loudnessCallback: (value: number) => void = () => { };
-    private currentIndex = 0;
-    private startedPlaying = false;
-    private onEndOfSpeech: () => void = () => { };
-
-    constructor() {
-        this.init();
-    }
-
-    private init(): void {
-        this.audioContext = new AudioContext();
-        this.analyser = this.audioContext.createAnalyser();
-        this.analyser.fftSize = 256; // Set the FFT size (smaller values provide faster updates, larger ones give better resolution)
-        const bufferLength = this.analyser.frequencyBinCount;
-        this.dataArray = new Uint8Array(bufferLength); // Array to hold frequency data
+  private audioContext!: AudioContext;
+  private chunkQueue: ArrayBuffer[] = [];
+  private isPlaying = false;
+  private analyser!: AnalyserNode;
+  private dataArray!: Uint8Array<ArrayBuffer>;
+  private shouldMonitorLoudness = true;
+  private isMonitoring = false;
+  private handle = 0;
+  private volume = 1.0;
+  private loudnessCallback: (value: number) => void = () => {};
+  private currentIndex = 0;
+  private startedPlaying = false;
+  private onEndOfSpeech: () => void = () => {};
+
+  constructor() {
+    this.init();
+  }
+
+  private init(): void {
+    this.audioContext = new AudioContext();
+    this.analyser = this.audioContext.createAnalyser();
+    this.analyser.fftSize = 256; // Set the FFT size (smaller values provide faster updates, larger ones give better resolution)
+    const bufferLength = this.analyser.frequencyBinCount;
+    this.dataArray = new Uint8Array(bufferLength); // Array to hold frequency data
+  }
+
+  public setOnLoudnessChange(callback: (value: number) => void) {
+    this.loudnessCallback = callback;
+  }
+
+  public setVolume(volume: number) {
+    this.volume = volume;
+  }
+
+  public async addChunk(chunk: ArrayBuffer, position: number): Promise<void> {
+    console.log('Adding chunk', position, chunk);
+    this.chunkQueue[position] = chunk;
+    // console.log("received chunk", {
+    //     chunkQueue: this.chunkQueue.length,
+    //     isPlaying: this.isPlaying,
+    // })
+
+    if (position === 0 && !this.startedPlaying) {
+      this.startedPlaying = true;
+      this.playChunks();
     }
-
-    public setOnLoudnessChange(callback: (value: number) => void) {
-        this.loudnessCallback = callback;
+  }
+
+  private playChunks(): void {
+    // console.log({ isPlaying: this.isPlaying });
+    if (this.isPlaying) return;
+    if (!this.chunkQueue[this.currentIndex]) {
+      // wait until the correct chunk arrives
+      setTimeout(() => this.playChunks(), 10);
     }
+    this.isPlaying = true;
 
-    public setVolume(volume: number) {
-        this.volume = volume;
-    }
-
-    public async addChunk(chunk: ArrayBuffer, position: number): Promise<void> {
-        console.log('Adding chunk', position, chunk);
-        this.chunkQueue[position] = chunk;
-        // console.log("received chunk", {
-        //     chunkQueue: this.chunkQueue.length,
-        //     isPlaying: this.isPlaying,
-        // })
-
-        if (position === 0 && !this.startedPlaying) {
-            this.startedPlaying = true;
+    this.playChunk(this.chunkQueue[this.currentIndex]).then(() => {
+      this.isPlaying = false;
+      this.currentIndex++;
+      if (this.chunkQueue[this.currentIndex]) {
+        this.shouldMonitorLoudness = true;
+        this.playChunks();
+      } else {
+        // console.log('Playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
+        setTimeout(() => {
+          // console.log('Check again if really playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
+          if (this.chunkQueue.length > this.currentIndex) {
             this.playChunks();
-        }
+          } else {
+            this.startedPlaying = false;
+            this.shouldMonitorLoudness = false;
+          }
+        }, 1000);
+      }
+    });
+  }
+
+  public stopPlayback(): void {
+    // console.log('Stopping playback');
+    // Implement logic to stop the current playback
+    this.isPlaying = false;
+    this.chunkQueue = [];
+    this.startedPlaying = false;
+    this.shouldMonitorLoudness = false;
+    cancelAnimationFrame(this.handle);
+  }
+
+  private playChunk(chunk: ArrayBuffer): Promise<void> {
+    // console.log({queue: this.chunkQueue})
+    if (!chunk) {
+      return Promise.resolve();
     }
 
-    private playChunks(): void {
-        // console.log({ isPlaying: this.isPlaying });
-        if (this.isPlaying) return;
-        if (!this.chunkQueue[this.currentIndex]) {
-            // wait until the correct chunk arrives
-            setTimeout(() => this.playChunks(), 10);
+    // console.log('Playing chunk', chunk);
+    return new Promise((resolve) => {
+      const source = this.audioContext.createBufferSource();
+      this.audioContext.decodeAudioData(chunk.slice(0)).then((audioBuffer) => {
+        source.buffer = audioBuffer;
+
+        // Create a GainNode for volume control
+        const gainNode = this.audioContext.createGain();
+        gainNode.gain.value = this.volume;
+
+        // Connect the source to the GainNode, then to the analyser node, then to the destination (speakers)
+        source.connect(gainNode);
+        gainNode.connect(this.analyser);
+        this.analyser.connect(this.audioContext.destination);
+
+        source.start(0);
+        // console.log('Playing chunk', this.currentIndex);
+        gainNode.gain.value = this.volume;
+        source.onended = () => {
+          // console.log('Chunk ended');
+          resolve();
+        };
+
+        // Start monitoring loudness only once
+        if (!this.isMonitoring) {
+          this.isMonitoring = true;
+          this.shouldMonitorLoudness = true;
+          this.monitorLoudness();
         }
-        this.isPlaying = true;
-
-        this.playChunk(this.chunkQueue[this.currentIndex]).then(() => {
-            this.isPlaying = false;
-            this.currentIndex++;
-            if (this.chunkQueue[this.currentIndex]) {
-                this.shouldMonitorLoudness = true;
-                this.playChunks();
-            } else {
-                // console.log('Playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
-                setTimeout(() => {
-                    // console.log('Check again if really playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
-                    if (this.chunkQueue.length > this.currentIndex) {
-                        this.playChunks();
-                    } else {
-                        this.startedPlaying = false;
-                        this.shouldMonitorLoudness = false;
-                    }
-                }, 1000);
-            }
-        });
+      });
+    });
+  }
+
+  async playAgain(): Promise<void> {
+    console.log('Playing again');
+    if (this.chunkQueue.length > 0 && !this.isPlaying) {
+      this.playChunks();
     }
-
-    public stopPlayback(): void {
-        // console.log('Stopping playback');
-        // Implement logic to stop the current playback
-        this.isPlaying = false;
-        this.chunkQueue = [];
-        this.startedPlaying = false;
-        this.shouldMonitorLoudness = false;
-        cancelAnimationFrame(this.handle);
+  }
+
+  private monitorLoudness(): void {
+    // Stop monitoring when the flag is false
+    if (!this.shouldMonitorLoudness) {
+      // console.log('Loudness monitoring stopped.');
+      cancelAnimationFrame(this.handle);
+      this.loudnessCallback(0);
+      this.onEndOfSpeech();
+      return;
     }
 
-    private playChunk(chunk: ArrayBuffer): Promise<void> {
-        console.log({queue: this.chunkQueue})
-        if (!chunk) {
-            return Promise.resolve();
-        }
+    // Get the time domain data from the analyser (this is a snapshot of the waveform)
+    this.analyser.getByteTimeDomainData(this.dataArray);
 
-        // console.log('Playing chunk', chunk);
-        return new Promise((resolve) => {
-            const source = this.audioContext.createBufferSource();
-            this.audioContext.decodeAudioData(chunk.slice(0)).then((audioBuffer) => {
-                source.buffer = audioBuffer;
-
-                // Create a GainNode for volume control
-                const gainNode = this.audioContext.createGain();
-                gainNode.gain.value = this.volume;
-
-                // Connect the source to the GainNode, then to the analyser node, then to the destination (speakers)
-                source.connect(gainNode);
-                gainNode.connect(this.analyser);
-                this.analyser.connect(this.audioContext.destination);
-
-                source.start(0);
-                // console.log('Playing chunk', this.currentIndex);
-                gainNode.gain.value = this.volume;
-                source.onended = () => {
-                    // console.log('Chunk ended');
-                    resolve();
-                };
-
-                // Start monitoring loudness only once
-                if (!this.isMonitoring) {
-                    this.isMonitoring = true;
-                    this.shouldMonitorLoudness = true;
-                    this.monitorLoudness();
-                }
-            });
-        });
+    // Calculate the RMS (root mean square) of the waveform values to get the perceived loudness
+    let sum = 0;
+    for (let i = 0; i < this.dataArray.length; i++) {
+      const value = this.dataArray[i] / 128.0 - 1.0; // Normalize between -1 and 1
+      sum += value * value;
     }
 
-    async playAgain(): Promise<void> {
-        console.log('Playing again');
-        if (this.chunkQueue.length > 0 && !this.isPlaying) {
-            this.playChunks();
-        }
-    }
-
-    private monitorLoudness(): void {
-        // Stop monitoring when the flag is false
-        if (!this.shouldMonitorLoudness) {
-            // console.log('Loudness monitoring stopped.');
-            cancelAnimationFrame(this.handle);
-            this.loudnessCallback(0);
-            this.onEndOfSpeech();
-            return;
-        }
-
-        // Get the time domain data from the analyser (this is a snapshot of the waveform)
-        this.analyser.getByteTimeDomainData(this.dataArray);
-
-        // Calculate the RMS (root mean square) of the waveform values to get the perceived loudness
-        let sum = 0;
-        for (let i = 0; i < this.dataArray.length; i++) {
-            const value = this.dataArray[i] / 128.0 - 1.0; // Normalize between -1 and 1
-            sum += value * value;
-        }
+    const rms = Math.sqrt(sum / this.dataArray.length);
 
-        const rms = Math.sqrt(sum / this.dataArray.length);
+    // Handle the case where RMS is 0 to avoid log10(0)
+    if (rms === 0) {
+      // console.log('Current loudness: Silent');
+    } else {
+      let loudnessInDb = 20 * Math.log10(rms); // Convert to dB
+      // console.log('Current loudness:' + loudnessInDb);
+      const minDb = -57;
+      const maxDb = -15;
 
-        // Handle the case where RMS is 0 to avoid log10(0)
-        if (rms === 0) {
-            // console.log('Current loudness: Silent');
-        } else {
-            let loudnessInDb = 20 * Math.log10(rms); // Convert to dB
-            // console.log('Current loudness:' + loudnessInDb);
-            const minDb = -57;
-            const maxDb = -15;
+      if (loudnessInDb < minDb) {
+        loudnessInDb = minDb;
+      }
+      if (loudnessInDb > maxDb) {
+        loudnessInDb = maxDb;
+      }
 
-            if (loudnessInDb < minDb) {
-                loudnessInDb = minDb;
-            }
-            if (loudnessInDb > maxDb) {
-                loudnessInDb = maxDb;
-            }
+      const loudnessScale = ((loudnessInDb - minDb) / (maxDb - minDb)) * 100;
+      // console.log("root:corrent loudness", loudnessScale);
 
-            const loudnessScale = ((loudnessInDb - minDb) / (maxDb - minDb)) * 100;
-            // console.log("root:corrent loudness", loudnessScale);
-
-            this.loudnessCallback(loudnessScale);
-        }
-
-        // Call this method again at regular intervals if you want continuous loudness monitoring
-        this.handle = requestAnimationFrame(() => this.monitorLoudness());
-    }
-    public reset() {
-        // console.log('Resetting player');
-        this.stopPlayback();
-        this.currentIndex = 0;
-        this.shouldMonitorLoudness = true;
-        //reset to the beginning when the class gets initialized
-        this.isMonitoring = false;
-        this.isPlaying = false;
-        this.init();
+      this.loudnessCallback(loudnessScale);
     }
 
-    public setOnEndOfSpeech(callback: () => void) {
-        this.onEndOfSpeech = callback;
-    }
-}
+    // Call this method again at regular intervals if you want continuous loudness monitoring
+    this.handle = requestAnimationFrame(() => this.monitorLoudness());
+  }
+  public reset() {
+    // console.log('Resetting player');
+    this.stopPlayback();
+    this.currentIndex = 0;
+    this.shouldMonitorLoudness = true;
+    //reset to the beginning when the class gets initialized
+    this.isMonitoring = false;
+    this.isPlaying = false;
+    this.init();
+  }
+
+  public setOnEndOfSpeech(callback: () => void) {
+    this.onEndOfSpeech = callback;
+  }
+}