@creativeorange/azure-text-to-speech 2.1.1 → 2.2.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@creativeorange/azure-text-to-speech",
-  "version": "2.1.1",
+  "version": "2.2.1",
   "main": "dist/co-azure-tts.umd.js",
   "browser": "dist/co-azure-tts.es.js",
   "scripts": {

package/src/TextToSpeech.ts

@@ -36,6 +36,10 @@ export class TextToSpeech {
     wordBoundaryOffset: number = 0;
     prevTextOffset: number = 0;
     url: string = '';
+    prefetchedAudio: Map<string, any> = new Map();
+    prefetchPromises: Map<string, Promise<any>> = new Map();
+    activePrefetchedAudioUrl: string = '';
+    playbackSegments: any[] = [];
 
     constructor(key: string, region: string, voice: string, rate: number = 0, pitch: number = 0, url: string = '') {
         this.key = key;
@@ -52,18 +56,21 @@ export class TextToSpeech {
 
     setVoice(voice: string) {
         this.voice = voice;
+        this.clearPrefetchedAudio();
 
         return this;
     }
 
     setRate(rate: number) {
         this.rate = rate;
+        this.clearPrefetchedAudio();
 
         return this;
     }
 
     setPitch(pitch: number) {
         this.pitch = pitch;
+        this.clearPrefetchedAudio();
 
         return this;
     }
@@ -234,9 +241,15 @@ export class TextToSpeech {
         this.originalHighlightDivInnerHTML = '';
         this.wordBoundryList = [];
         this.wordEncounters = [];
+        this.resetPlaybackSegments();
+        this.playbackSegments = [];
         if (this.player !== undefined) {
             this.player.pause();
         }
+        if (this.activePrefetchedAudioUrl !== '') {
+            URL.revokeObjectURL(this.activePrefetchedAudioUrl);
+            this.activePrefetchedAudioUrl = '';
+        }
         this.player = undefined;
         this.highlightDiv = undefined;
         this.prevTextOffset = 0;
@@ -257,11 +270,32 @@ export class TextToSpeech {
             this.wordBoundryList.push(e);
         };
 
+        const playbackChain = this.collectPlaybackChain(this.clickedNode);
+        const isChainedPlayback = playbackChain.length > 1;
+
+        if (isChainedPlayback) {
+            this.preparePlaybackChain(playbackChain);
+        } else {
+            this.playbackSegments = [];
+        }
+
         this.player.onAudioEnd = async () => {
+            const wasChainedPlayback = this.playbackSegments.length > 0;
             this.stopPlayer();
 
+            if (wasChainedPlayback) {
+                document.dispatchEvent(new CustomEvent('COAzureTTSFinishedPlaying', {}));
+
+                return;
+            }
+
             if (this.clickedNode.hasAttribute('co-tts.next')) {
                 const nextNode = document.getElementById(this.clickedNode.getAttribute('co-tts.next'));
+
+                if (nextNode && await this.playPrefetchedNode(nextNode)) {
+                    return;
+                }
+
                 if (nextNode && nextNode.attributes.getNamedItem('co-tts.text')) {
                     this.handleWithoutClick(nextNode, nextNode.attributes.getNamedItem('co-tts.text'));
                 } else if (nextNode) {
@@ -276,6 +310,10 @@ export class TextToSpeech {
             document.dispatchEvent(new CustomEvent('COAzureTTSStartedPlaying', {}));
         };
 
+        if (!isChainedPlayback) {
+            this.prefetchNextNode(this.clickedNode);
+        }
+
         this.synthesizer.speakSsmlAsync(this.buildSSML(this.textToRead),
             () => {
                 this.synthesizer.close();
@@ -287,6 +325,295 @@ export class TextToSpeech {
             });
     }
 
+    collectPlaybackChain(node: any) {
+        const chain = [];
+        const seenNodeIds = new Set<string>();
+        let currentNode = node;
+        let offset = 0;
+
+        while (currentNode && !seenNodeIds.has(currentNode.id)) {
+            seenNodeIds.add(currentNode.id);
+
+            const attr = currentNode.attributes.getNamedItem('co-tts.text') ?? currentNode.attributes.getNamedItem('co-tts');
+            const text = this.getNodeText(currentNode, attr);
+
+            if (text === '') {
+                break;
+            }
+
+            const highlightDiv = this.getHighlightDivForNode(currentNode);
+
+            chain.push({
+                node: currentNode,
+                text,
+                start: offset,
+                end: offset + text.length,
+                highlightDiv,
+                originalHighlightDivInnerHTML: highlightDiv?.innerHTML ?? '',
+            });
+
+            offset += text.length + 2;
+
+            if (!currentNode.hasAttribute('co-tts.next')) {
+                break;
+            }
+
+            currentNode = document.getElementById(currentNode.getAttribute('co-tts.next'));
+        }
+
+        return chain;
+    }
+
+    preparePlaybackChain(chain: any[]) {
+        this.playbackSegments = chain;
+        this.textToRead = chain.map((segment) => segment.text).join('. ');
+        this.highlightDiv = undefined;
+        this.originalHighlightDivInnerHTML = '';
+        this.wordEncounters = [];
+        this.previousWordBoundary = undefined;
+        this.prevTextOffset = 0;
+        this.currentWord = '';
+        this.currentOffset = 0;
+        this.wordBoundaryOffset = 0;
+    }
+
+    getHighlightDivForNode(node: any) {
+        if (!node.hasAttribute('co-tts.highlight')) {
+            return undefined;
+        }
+
+        if (node.attributes.getNamedItem('co-tts.highlight')?.value !== '') {
+            return document.getElementById(node.attributes.getNamedItem('co-tts.highlight').value);
+        }
+
+        return node;
+    }
+
+    resetPlaybackSegments() {
+        this.playbackSegments.forEach((segment) => {
+            if (segment.highlightDiv) {
+                segment.highlightDiv.innerHTML = segment.originalHighlightDivInnerHTML;
+            }
+        });
+    }
+
+    updateChainedHighlight(wordBoundary: any) {
+        if (~['.', ',', '!', '?', '*', '(', ')', '&', '\\', '/', '^', '[', ']', '<', '>', ':'].indexOf(wordBoundary.text)) {
+            wordBoundary = this.previousWordBoundary ?? undefined;
+        }
+
+        if (!wordBoundary) {
+            this.resetPlaybackSegments();
+
+            return;
+        }
+
+        const segment = this.playbackSegments.find((candidate) => (
+            wordBoundary.textOffset >= candidate.start && wordBoundary.textOffset < candidate.end
+        ));
+
+        this.resetPlaybackSegments();
+
+        if (!segment?.highlightDiv) {
+            this.previousWordBoundary = wordBoundary;
+
+            return;
+        }
+
+        const relativeTextOffset = wordBoundary.textOffset - segment.start;
+        const currentOffset = this.getPosition(segment.originalHighlightDivInnerHTML, wordBoundary.text, relativeTextOffset);
+
+        if (currentOffset === Number.MAX_SAFE_INTEGER) {
+            this.previousWordBoundary = wordBoundary;
+
+            return;
+        }
+
+        const startOfString = segment.originalHighlightDivInnerHTML.substring(0, currentOffset);
+        const endOffset = currentOffset + wordBoundary.wordLength;
+        const endOfString = segment.originalHighlightDivInnerHTML.substring(endOffset);
+
+        segment.highlightDiv.innerHTML = `
+            ${startOfString}<mark class='co-tts-highlight'>${wordBoundary.text}</mark>${endOfString}
+        `;
+        this.previousWordBoundary = wordBoundary;
+    }
+
+    getNodeText(node: any, attr?: Attr | null) {
+        if (attr && attr.value !== '') {
+            return attr.value;
+        }
+
+        if (node.hasAttribute('co-tts.text') && node.getAttribute('co-tts.text') !== '') {
+            return node.getAttribute('co-tts.text') ?? '';
+        }
+
+        return node.innerText;
+    }
+
+    getPrefetchKey(node: any, text: string) {
+        return [
+            node.id,
+            text,
+            this.voice,
+            this.rate,
+            this.pitch,
+            this.url,
+        ].join('|');
+    }
+
+    clearPrefetchedAudio() {
+        this.prefetchedAudio.forEach((prefetch) => {
+            if (prefetch.url) {
+                URL.revokeObjectURL(prefetch.url);
+            }
+        });
+
+        this.prefetchedAudio.clear();
+        this.prefetchPromises.clear();
+    }
+
+    async prefetchNextNode(node: any) {
+        if (!node || !node.hasAttribute('co-tts.next')) {
+            return;
+        }
+
+        const nextNode = document.getElementById(node.getAttribute('co-tts.next'));
+
+        if (!nextNode) {
+            return;
+        }
+
+        const attr = nextNode.attributes.getNamedItem('co-tts.text') ?? nextNode.attributes.getNamedItem('co-tts');
+        const text = this.getNodeText(nextNode, attr);
+
+        if (text === '') {
+            return;
+        }
+
+        const key = this.getPrefetchKey(nextNode, text);
+
+        if (this.prefetchedAudio.has(key) || this.prefetchPromises.has(key)) {
+            return;
+        }
+
+        const speechConfig = SpeechConfig.fromSubscription(this.key, this.region);
+        speechConfig.speechSynthesisVoiceName = `Microsoft Server Speech Text to Speech Voice (${this.voice})`;
+        speechConfig.speechSynthesisOutputFormat = SpeechSynthesisOutputFormat.Audio24Khz160KBitRateMonoMp3;
+
+        const synthesizer = new SpeechSynthesizer(speechConfig, null);
+        const wordBoundryList: any[] = [];
+
+        synthesizer.wordBoundary = (s: any, e: any) => {
+            wordBoundryList.push(e);
+        };
+
+        const prefetchPromise = new Promise((resolve) => {
+            synthesizer.speakSsmlAsync(this.buildSSML(text),
+                (result: any) => {
+                    synthesizer.close();
+
+                    if (!result?.audioData) {
+                        resolve(null);
+
+                        return;
+                    }
+
+                    const blob = new Blob([result.audioData], {type: 'audio/mpeg'});
+                    const url = URL.createObjectURL(blob);
+                    const prefetch = {
+                        key,
+                        nodeId: nextNode.id,
+                        text,
+                        url,
+                        wordBoundryList,
+                    };
+
+                    this.prefetchedAudio.set(key, prefetch);
+                    resolve(prefetch);
+                },
+                () => {
+                    synthesizer.close();
+                    resolve(null);
+                });
+        }).finally(() => {
+            this.prefetchPromises.delete(key);
+        });
+
+        this.prefetchPromises.set(key, prefetchPromise);
+    }
+
+    async playPrefetchedNode(node: any) {
+        const attr = node.attributes.getNamedItem('co-tts.text') ?? node.attributes.getNamedItem('co-tts');
+        const text = this.getNodeText(node, attr);
+        const key = this.getPrefetchKey(node, text);
+        const prefetch = this.prefetchedAudio.get(key) ?? await this.prefetchPromises.get(key);
+
+        if (!prefetch) {
+            return false;
+        }
+
+        this.prefetchedAudio.delete(key);
+        this.clickedNode = node;
+        this.textToRead = prefetch.text;
+        this.wordBoundryList = prefetch.wordBoundryList;
+        this.wordEncounters = [];
+        this.previousWordBoundary = undefined;
+        this.prevTextOffset = 0;
+        this.currentWord = '';
+        this.currentOffset = 0;
+        this.wordBoundaryOffset = 0;
+
+        if (node.hasAttribute('co-tts.highlight')) {
+            if (node.attributes.getNamedItem('co-tts.highlight')?.value !== '') {
+                const newReferenceDiv = document.getElementById(node.attributes.getNamedItem('co-tts.highlight').value);
+
+                this.highlightDiv = newReferenceDiv;
+                if (newReferenceDiv !== null) {
+                    this.originalHighlightDivInnerHTML = newReferenceDiv.innerHTML;
+                }
+            } else {
+                this.highlightDiv = node;
+                this.originalHighlightDivInnerHTML = node.innerHTML;
+            }
+        }
+
+        await this.createInterval();
+
+        const audio = new Audio(prefetch.url);
+        this.activePrefetchedAudioUrl = prefetch.url;
+        this.player = audio;
+
+        audio.addEventListener('play', () => {
+            document.dispatchEvent(new CustomEvent('COAzureTTSStartedPlaying', {}));
+        }, {once: true});
+
+        audio.addEventListener('ended', async () => {
+            this.stopPlayer();
+
+            if (this.clickedNode.hasAttribute('co-tts.next')) {
+                const nextNode = document.getElementById(this.clickedNode.getAttribute('co-tts.next'));
+
+                if (nextNode && await this.playPrefetchedNode(nextNode)) {
+                    return;
+                }
+
+                if (nextNode && nextNode.attributes.getNamedItem('co-tts.text')) {
+                    this.handleWithoutClick(nextNode, nextNode.attributes.getNamedItem('co-tts.text'));
+                } else if (nextNode) {
+                    nextNode.dispatchEvent(new Event('click'));
+                }
+            } else {
+                document.dispatchEvent(new CustomEvent('COAzureTTSFinishedPlaying', {}));
+            }
+        }, {once: true});
+
+        this.prefetchNextNode(node);
+        await audio.play();
+
+        return true;
+    }
+
     async clearInterval() {
         clearInterval(this.interval);
     }
@@ -305,6 +632,12 @@ export class TextToSpeech {
                 }
 
                 if (wordBoundary !== undefined) {
+                    if (this.playbackSegments.length > 0) {
+                        this.updateChainedHighlight(wordBoundary);
+
+                        return;
+                    }
+
                     if (~['.', ',', '!', '?', '*', '(', ')', '&', '\\', '/', '^', '[', ']', '<', '>', ':']
                         .indexOf(wordBoundary.text)) {
                         wordBoundary = this.previousWordBoundary ?? undefined;