@rimori/client 2.5.14-next.2 → 2.5.14-next.3

package/dist/index.d.ts

@@ -17,3 +17,4 @@ export type { TriggerAction } from './plugin/module/ExerciseModule';
 export type { MacroAccomplishmentPayload, MicroAccomplishmentPayload } from './controller/AccomplishmentController';
 export type { EventBusMessage } from './fromRimori/EventBus';
 export type { Theme, ApplicationMode } from './plugin/module/PluginModule';
+export * from './plugin/TTS/MessageSender';

package/dist/index.js

@@ -9,3 +9,4 @@ export * from './plugin/StandaloneClient';
 export { setupWorker } from './worker/WorkerSetup';
 export { AudioController } from './controller/AudioController';
 export { Translator } from './controller/TranslationController';
+export * from './plugin/TTS/MessageSender';

package/dist/modules.d.ts

@@ -2,3 +2,4 @@ export * from './plugin/module/AIModule';
 export * from './fromRimori/PluginTypes';
 export * from './fromRimori/EventBus';
 export * from './cli/types/DatabaseTypes';
+export * from './plugin/TTS/MessageSender';

package/dist/modules.js

@@ -3,3 +3,4 @@ export * from './plugin/module/AIModule';
 export * from './fromRimori/PluginTypes';
 export * from './fromRimori/EventBus';
 export * from './cli/types/DatabaseTypes';
+export * from './plugin/TTS/MessageSender';

package/dist/plugin/TTS/ChunkedAudioPlayer.d.ts

@@ -0,0 +1,29 @@
+export declare class ChunkedAudioPlayer {
+    private audioContext;
+    private chunkQueue;
+    private isPlaying;
+    private analyser;
+    private dataArray;
+    private shouldMonitorLoudness;
+    private isMonitoring;
+    private handle;
+    private volume;
+    private loudnessCallback;
+    private currentIndex;
+    private startedPlaying;
+    private onEndOfSpeech;
+    private readonly backgroundNoiseLevel;
+    constructor();
+    private init;
+    setOnLoudnessChange(callback: (value: number) => void): void;
+    setVolume(volume: number): void;
+    addChunk(chunk: ArrayBuffer, position: number): Promise<void>;
+    private playChunks;
+    stopPlayback(): void;
+    cleanup(): void;
+    private playChunk;
+    playAgain(): Promise<void>;
+    private monitorLoudness;
+    reset(): void;
+    setOnEndOfSpeech(callback: () => void): void;
+}

package/dist/plugin/TTS/ChunkedAudioPlayer.js

@@ -0,0 +1,205 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+export class ChunkedAudioPlayer {
+    constructor() {
+        this.chunkQueue = [];
+        this.isPlaying = false;
+        this.shouldMonitorLoudness = true;
+        this.isMonitoring = false;
+        this.handle = 0;
+        this.volume = 1.0;
+        this.loudnessCallback = () => { };
+        this.currentIndex = 0;
+        this.startedPlaying = false;
+        this.onEndOfSpeech = () => { };
+        this.backgroundNoiseLevel = 30; // Background noise level that should be treated as baseline (0)
+        this.init();
+    }
+    init() {
+        this.audioContext = new AudioContext();
+        this.analyser = this.audioContext.createAnalyser();
+        this.analyser.fftSize = 256; // Set the FFT size (smaller values provide faster updates, larger ones give better resolution)
+        const bufferLength = this.analyser.frequencyBinCount;
+        this.dataArray = new Uint8Array(bufferLength); // Array to hold frequency data
+    }
+    setOnLoudnessChange(callback) {
+        this.loudnessCallback = callback;
+    }
+    setVolume(volume) {
+        this.volume = volume;
+    }
+    addChunk(chunk, position) {
+        return __awaiter(this, void 0, void 0, function* () {
+            console.log('Adding chunk', position, chunk);
+            this.chunkQueue[position] = chunk;
+            // console.log("received chunk", {
+            //     chunkQueue: this.chunkQueue.length,
+            //     isPlaying: this.isPlaying,
+            // })
+            if (position === 0 && !this.startedPlaying) {
+                this.startedPlaying = true;
+                this.playChunks();
+            }
+        });
+    }
+    playChunks() {
+        // console.log({ isPlaying: this.isPlaying });
+        if (this.isPlaying)
+            return;
+        if (!this.chunkQueue[this.currentIndex]) {
+            // wait until the correct chunk arrives
+            setTimeout(() => this.playChunks(), 10);
+        }
+        this.isPlaying = true;
+        this.playChunk(this.chunkQueue[this.currentIndex]).then(() => {
+            this.isPlaying = false;
+            this.currentIndex++;
+            if (this.chunkQueue[this.currentIndex]) {
+                this.shouldMonitorLoudness = true;
+                this.playChunks();
+            }
+            else {
+                // console.log('Playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
+                setTimeout(() => {
+                    // console.log('Check again if really playback finished', { currentIndex: this.currentIndex, chunkQueue: this.chunkQueue });
+                    if (this.chunkQueue.length > this.currentIndex) {
+                        this.playChunks();
+                    }
+                    else {
+                        this.startedPlaying = false;
+                        this.shouldMonitorLoudness = false;
+                    }
+                }, 1000);
+            }
+        });
+    }
+    stopPlayback() {
+        // console.log('Stopping playback');
+        // Implement logic to stop the current playback
+        this.isPlaying = false;
+        this.chunkQueue = [];
+        this.startedPlaying = false;
+        this.shouldMonitorLoudness = false;
+        cancelAnimationFrame(this.handle);
+    }
+    cleanup() {
+        // Stop playback first
+        this.stopPlayback();
+        // Close AudioContext to free resources
+        if (this.audioContext && this.audioContext.state !== 'closed') {
+            this.audioContext.close().catch((e) => {
+                console.warn('Error closing AudioContext:', e);
+            });
+        }
+    }
+    playChunk(chunk) {
+        // console.log({queue: this.chunkQueue})
+        if (!chunk) {
+            return Promise.resolve();
+        }
+        // console.log('Playing chunk', chunk);
+        return new Promise((resolve) => {
+            const source = this.audioContext.createBufferSource();
+            this.audioContext.decodeAudioData(chunk.slice(0)).then((audioBuffer) => {
+                source.buffer = audioBuffer;
+                // Create a GainNode for volume control
+                const gainNode = this.audioContext.createGain();
+                gainNode.gain.value = this.volume;
+                // Connect the source to the GainNode, then to the analyser node, then to the destination (speakers)
+                source.connect(gainNode);
+                gainNode.connect(this.analyser);
+                this.analyser.connect(this.audioContext.destination);
+                source.start(0);
+                // console.log('Playing chunk', this.currentIndex);
+                gainNode.gain.value = this.volume;
+                source.onended = () => {
+                    // console.log('Chunk ended');
+                    resolve();
+                };
+                // Start monitoring loudness only once
+                if (!this.isMonitoring) {
+                    this.isMonitoring = true;
+                    this.shouldMonitorLoudness = true;
+                    this.monitorLoudness();
+                }
+            });
+        });
+    }
+    playAgain() {
+        return __awaiter(this, void 0, void 0, function* () {
+            console.log('Playing again');
+            if (this.chunkQueue.length > 0 && !this.isPlaying) {
+                this.playChunks();
+            }
+        });
+    }
+    monitorLoudness() {
+        // Stop monitoring when the flag is false
+        if (!this.shouldMonitorLoudness) {
+            // console.log('Loudness monitoring stopped.');
+            cancelAnimationFrame(this.handle);
+            this.loudnessCallback(0);
+            this.onEndOfSpeech();
+            return;
+        }
+        // Get the time domain data from the analyser (this is a snapshot of the waveform)
+        this.analyser.getByteTimeDomainData(this.dataArray);
+        // Calculate the RMS (root mean square) of the waveform values to get the perceived loudness
+        let sum = 0;
+        for (let i = 0; i < this.dataArray.length; i++) {
+            const value = this.dataArray[i] / 128.0 - 1.0; // Normalize between -1 and 1
+            sum += value * value;
+        }
+        const rms = Math.sqrt(sum / this.dataArray.length);
+        // Handle the case where RMS is 0 to avoid log10(0)
+        if (rms === 0) {
+            // console.log('Current loudness: Silent');
+        }
+        else {
+            let loudnessInDb = 20 * Math.log10(rms); // Convert to dB
+            // console.log('Current loudness:' + loudnessInDb);
+            const minDb = -57;
+            const maxDb = -15;
+            if (loudnessInDb < minDb) {
+                loudnessInDb = minDb;
+            }
+            if (loudnessInDb > maxDb) {
+                loudnessInDb = maxDb;
+            }
+            let loudnessScale = ((loudnessInDb - minDb) / (maxDb - minDb)) * 100;
+            // Adjust loudness: shift zero level up by background noise amount
+            // Values below background noise level are set to 0
+            // Values above are remapped to 0-100 scale
+            if (loudnessScale < this.backgroundNoiseLevel) {
+                loudnessScale = 0;
+            }
+            else {
+                // Remap from [backgroundNoiseLevel, 100] to [0, 100]
+                loudnessScale = ((loudnessScale - this.backgroundNoiseLevel) / (100 - this.backgroundNoiseLevel)) * 100;
+            }
+            this.loudnessCallback(Math.round(loudnessScale));
+        }
+        // Call this method again at regular intervals if you want continuous loudness monitoring
+        this.handle = requestAnimationFrame(() => this.monitorLoudness());
+    }
+    reset() {
+        // console.log('Resetting player');
+        this.stopPlayback();
+        this.currentIndex = 0;
+        this.shouldMonitorLoudness = true;
+        //reset to the beginning when the class gets initialized
+        this.isMonitoring = false;
+        this.isPlaying = false;
+        this.init();
+    }
+    setOnEndOfSpeech(callback) {
+        this.onEndOfSpeech = callback;
+    }
+}

package/dist/plugin/TTS/MessageSender.d.ts

@@ -0,0 +1,21 @@
+type VoiceBackend = (text: string, voice?: string, speed?: number, language?: string, cache?: boolean) => Promise<Blob>;
+export declare class MessageSender {
+    private player;
+    private fetchedSentences;
+    private lastLoading;
+    private voice;
+    private voiceBackend;
+    private cache;
+    constructor(voiceBackend: VoiceBackend, voice: string, cache?: boolean);
+    private getCompletedSentences;
+    handleNewText(currentText: string | undefined, isLoading: boolean): Promise<void>;
+    private generateSpeech;
+    play(): void;
+    stop(): void;
+    cleanup(): void;
+    private reset;
+    setVolume(volume: number): void;
+    setOnLoudnessChange(callback: (value: number) => void): void;
+    setOnEndOfSpeech(callback: () => void): void;
+}
+export {};

package/dist/plugin/TTS/MessageSender.js

@@ -0,0 +1,95 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { ChunkedAudioPlayer } from './ChunkedAudioPlayer';
+export class MessageSender {
+    constructor(voiceBackend, voice, cache = false) {
+        this.player = new ChunkedAudioPlayer();
+        this.fetchedSentences = new Set();
+        this.lastLoading = false;
+        if ((voice === null || voice === void 0 ? void 0 : voice.split('_').length) !== 2) {
+            throw new Error("Invalid voice id format '" + voice + "'. Voice id needs to look like <provider>_<voice_id>");
+        }
+        this.voiceBackend = voiceBackend;
+        this.voice = voice;
+        this.cache = cache;
+    }
+    getCompletedSentences(currentText, isLoading) {
+        // Split the text based on the following characters: .?!
+        // Only split on : when followed by a space
+        const pattern = /(.+?[.?!]|.+?:\s+|.+?\n+)/g;
+        const result = [];
+        let match;
+        while ((match = pattern.exec(currentText)) !== null) {
+            const sentence = match[0].trim();
+            if (sentence.length > 0) {
+                result.push(sentence);
+            }
+        }
+        if (!isLoading) {
+            const lastFullSentence = result[result.length - 1];
+            const leftoverIndex = currentText.lastIndexOf(lastFullSentence) + lastFullSentence.length;
+            if (leftoverIndex < currentText.length) {
+                result.push(currentText.slice(leftoverIndex).trim());
+            }
+        }
+        return result;
+    }
+    handleNewText(currentText, isLoading) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.lastLoading && isLoading) {
+                this.reset();
+            }
+            this.lastLoading = isLoading;
+            if (!currentText) {
+                return;
+            }
+            const sentences = this.getCompletedSentences(currentText, isLoading);
+            for (let i = 0; i < sentences.length; i++) {
+                const sentence = sentences[i];
+                if (!this.fetchedSentences.has(sentence)) {
+                    this.fetchedSentences.add(sentence);
+                    const audioData = yield this.generateSpeech(sentence);
+                    yield this.player.addChunk(audioData, i);
+                }
+            }
+        });
+    }
+    generateSpeech(sentence) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const blob = yield this.voiceBackend(sentence, this.voice, 1.0, undefined, this.cache);
+            return yield blob.arrayBuffer();
+        });
+    }
+    play() {
+        this.player.playAgain();
+    }
+    stop() {
+        this.player.stopPlayback();
+    }
+    cleanup() {
+        this.player.cleanup();
+    }
+    reset() {
+        this.stop();
+        this.fetchedSentences.clear();
+        this.player.reset();
+    }
+    setVolume(volume) {
+        this.player.setVolume(volume);
+    }
+    setOnLoudnessChange(callback) {
+        this.player.setOnLoudnessChange((loudness) => {
+            callback(loudness);
+        });
+    }
+    setOnEndOfSpeech(callback) {
+        this.player.setOnEndOfSpeech(callback);
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rimori/client",
-  "version": "2.5.14-next.2",
+  "version": "2.5.14-next.3",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "repository": {