@creativeorange/azure-text-to-speech 2.2.2 → 3.0.0

package/src/TextToSpeech.ts

@@ -5,9 +5,21 @@ import {
     SpeechSynthesizer,
     SpeechSynthesisOutputFormat,
 } from 'microsoft-cognitiveservices-speech-sdk';
+import {
+    SpeechAuthenticationOptions,
+    SpeechAuthorizationManager,
+    toSafeErrorDetail,
+} from './authentication';
+
+export type TextToSpeechOptions = SpeechAuthenticationOptions & {
+    region?: string;
+    voice: string;
+    rate?: number;
+    pitch?: number;
+    url?: string;
+};
 
 export class TextToSpeech {
-    key: string;
     region: string;
     voice: string;
     rate: number;
@@ -34,6 +46,7 @@ export class TextToSpeech {
     currentWord: string = '';
     currentOffset: number = 0;
     wordBoundaryOffset: number = 0;
+    playbackTextOffsetBase: number | undefined;
     prevTextOffset: number = 0;
     url: string = '';
     prefetchedAudio: Map<string, any> = new Map();
@@ -41,13 +54,19 @@ export class TextToSpeech {
     activePrefetchedAudioUrl: string = '';
     playbackSegments: any[] = [];
 
-    constructor(key: string, region: string, voice: string, rate: number = 0, pitch: number = 0, url: string = '') {
-        this.key = key;
-        this.region = region;
-        this.voice = voice;
-        this.rate = rate;
-        this.pitch = pitch;
-        this.url = url;
+    private readonly authorizationManager: SpeechAuthorizationManager;
+
+    constructor(options: TextToSpeechOptions) {
+        if (!options || typeof options.voice !== 'string' || options.voice.trim() === '') {
+            throw new Error('A voice is required.');
+        }
+
+        this.authorizationManager = new SpeechAuthorizationManager(options);
+        this.region = options.region?.trim() ?? '';
+        this.voice = options.voice;
+        this.rate = options.rate ?? 0;
+        this.pitch = options.pitch ?? 0;
+        this.url = options.url ?? '';
     }
 
     async start() {
@@ -120,7 +139,7 @@ export class TextToSpeech {
             if (referenceDiv.hasAttribute('co-tts.text') && referenceDiv.getAttribute('co-tts.text') !== '') {
                 this.textToRead = referenceDiv.getAttribute('co-tts.text') ?? '';
             } else {
-                this.textToRead = referenceDiv.innerText;
+                this.textToRead = referenceDiv.innerText ?? referenceDiv.textContent ?? '';
             }
 
             if (referenceDiv.hasAttribute('co-tts.highlight')) {
@@ -136,7 +155,7 @@ export class TextToSpeech {
                 }
             }
 
-            this.startSynthesizer(node, attr);
+            await this.startSynthesizer(node, attr);
         });
     }
 
@@ -151,7 +170,7 @@ export class TextToSpeech {
 
             this.textToRead = await response.text();
 
-            this.startSynthesizer(node, attr);
+            await this.startSynthesizer(node, attr);
         });
     }
 
@@ -172,12 +191,12 @@ export class TextToSpeech {
                 }
             }
             if (attr.value === '') {
-                this.textToRead = node.innerText;
+                this.textToRead = node.innerText ?? node.textContent ?? '';
             } else {
                 this.textToRead = attr.value;
             }
 
-            this.startSynthesizer(node, attr);
+            await this.startSynthesizer(node, attr);
         });
     }
 
@@ -199,12 +218,12 @@ export class TextToSpeech {
             }
         }
         if (attr.value === '') {
-            this.textToRead = node.innerText;
+            this.textToRead = node.innerText ?? node.textContent ?? '';
         } else {
             this.textToRead = attr.value;
         }
 
-        this.startSynthesizer(node, attr);
+        await this.startSynthesizer(node, attr);
     }
 
     async handleStopModifier(node: any, attr: Attr) {
@@ -250,79 +269,105 @@ export class TextToSpeech {
             URL.revokeObjectURL(this.activePrefetchedAudioUrl);
             this.activePrefetchedAudioUrl = '';
         }
+        this.closeSynthesizer();
         this.player = undefined;
+        this.audioConfig = undefined;
+        this.speechConfig = undefined;
         this.highlightDiv = undefined;
         this.prevTextOffset = 0;
+        this.playbackTextOffsetBase = undefined;
     }
 
-    async startSynthesizer(node: any, attr: Attr) {
-        this.speechConfig = SpeechConfig.fromSubscription(this.key, this.region);
+    private async createSpeechConfig(): Promise<SpeechConfig> {
+        const authorization =
+            await this.authorizationManager.getAuthorization();
 
-        this.speechConfig.speechSynthesisVoiceName = `Microsoft Server Speech Text to Speech Voice (${this.voice})`;
-        this.speechConfig.speechSynthesisOutputFormat = SpeechSynthesisOutputFormat.Audio24Khz160KBitRateMonoMp3;
+        const speechConfig = SpeechConfig.fromAuthorizationToken(
+            authorization.token,
+            authorization.region,
+        );
 
-        this.player = new SpeakerAudioDestination();
+        speechConfig.speechSynthesisVoiceName =
+            `Microsoft Server Speech Text to Speech Voice (${this.voice})`;
 
-        this.audioConfig = AudioConfig.fromSpeakerOutput(this.player);
-        this.synthesizer = new SpeechSynthesizer(this.speechConfig, this.audioConfig);
+        speechConfig.speechSynthesisOutputFormat =
+            SpeechSynthesisOutputFormat.Audio24Khz160KBitRateMonoMp3;
 
-        this.synthesizer.wordBoundary = (s: any, e: any) => {
-            this.wordBoundryList.push(e);
-        };
+        this.region = authorization.region;
 
-        const playbackChain = this.collectPlaybackChain(this.clickedNode);
-        const isChainedPlayback = playbackChain.length > 1;
+        return speechConfig;
+    }
 
-        if (isChainedPlayback) {
-            this.preparePlaybackChain(playbackChain);
-        } else {
-            this.playbackSegments = [];
-        }
+    async startSynthesizer(node: any, attr: Attr) {
+        try {
+            this.speechConfig = await this.createSpeechConfig();
 
-        this.player.onAudioEnd = async () => {
-            const wasChainedPlayback = this.playbackSegments.length > 0;
-            this.stopPlayer();
+            this.player = new SpeakerAudioDestination();
 
-            if (wasChainedPlayback) {
-                document.dispatchEvent(new CustomEvent('COAzureTTSFinishedPlaying', {}));
+            this.audioConfig = AudioConfig.fromSpeakerOutput(this.player);
+            this.synthesizer = new SpeechSynthesizer(this.speechConfig, this.audioConfig);
 
-                return;
+            this.synthesizer.wordBoundary = (s: any, e: any) => {
+                this.wordBoundryList.push(e);
+            };
+
+            const playbackChain = this.collectPlaybackChain(this.clickedNode);
+            const isChainedPlayback = playbackChain.length > 1;
+
+            if (isChainedPlayback) {
+                this.preparePlaybackChain(playbackChain);
+            } else {
+                this.playbackSegments = [];
             }
 
-            if (this.clickedNode.hasAttribute('co-tts.next')) {
-                const nextNode = document.getElementById(this.clickedNode.getAttribute('co-tts.next'));
+            this.player.onAudioEnd = async () => {
+                const wasChainedPlayback = this.playbackSegments.length > 0;
+                this.stopPlayer();
+
+                if (wasChainedPlayback) {
+                    document.dispatchEvent(new CustomEvent('COAzureTTSFinishedPlaying', {}));
 
-                if (nextNode && await this.playPrefetchedNode(nextNode)) {
                     return;
                 }
 
-                if (nextNode && nextNode.attributes.getNamedItem('co-tts.text')) {
-                    this.handleWithoutClick(nextNode, nextNode.attributes.getNamedItem('co-tts.text'));
-                } else if (nextNode) {
-                    nextNode.dispatchEvent(new Event('click'));
+                if (this.clickedNode.hasAttribute('co-tts.next')) {
+                    const nextNode = document.getElementById(this.clickedNode.getAttribute('co-tts.next'));
+
+                    if (nextNode && await this.playPrefetchedNode(nextNode)) {
+                        return;
+                    }
+
+                    if (nextNode && nextNode.attributes.getNamedItem('co-tts.text')) {
+                        this.handleWithoutClick(nextNode, nextNode.attributes.getNamedItem('co-tts.text'));
+                    } else if (nextNode) {
+                        nextNode.dispatchEvent(new Event('click'));
+                    }
+                } else {
+                    document.dispatchEvent(new CustomEvent('COAzureTTSFinishedPlaying', {}));
                 }
-            } else {
-                document.dispatchEvent(new CustomEvent('COAzureTTSFinishedPlaying', {}));
-            }
-        };
+            };
 
-        this.player.onAudioStart = async () => {
-            document.dispatchEvent(new CustomEvent('COAzureTTSStartedPlaying', {}));
-        };
+            this.player.onAudioStart = async () => {
+                document.dispatchEvent(new CustomEvent('COAzureTTSStartedPlaying', {}));
+            };
 
-        if (!isChainedPlayback) {
-            this.prefetchNextNode(this.clickedNode);
-        }
+            if (!isChainedPlayback) {
+                this.prefetchNextNode(this.clickedNode);
+            }
 
-        this.synthesizer.speakSsmlAsync(this.buildSSML(this.textToRead),
-            () => {
-                this.synthesizer.close();
-                this.synthesizer = undefined;
-            },
-            () => {
-                this.synthesizer.close();
-                this.synthesizer = undefined;
-            });
+            this.synthesizer.speakSsmlAsync(this.buildSSML(this.textToRead),
+                () => {
+                    this.closeSynthesizer();
+                },
+                (error: unknown) => {
+                    this.dispatchError(error, 'SYNTHESIS_ERROR');
+                    this.closeSynthesizer();
+                    this.stopPlayer();
+                });
+        } catch (error) {
+            this.dispatchError(error);
+            await this.stopPlayer();
+        }
     }
 
     collectPlaybackChain(node: any) {
@@ -372,6 +417,7 @@ export class TextToSpeech {
         this.wordEncounters = [];
         this.previousWordBoundary = undefined;
         this.prevTextOffset = 0;
+        this.playbackTextOffsetBase = undefined;
         this.currentWord = '';
         this.currentOffset = 0;
         this.wordBoundaryOffset = 0;
@@ -408,8 +454,13 @@ export class TextToSpeech {
             return;
         }
 
+        if (this.playbackTextOffsetBase === undefined) {
+            this.playbackTextOffsetBase = wordBoundary.textOffset;
+        }
+
+        const normalizedTextOffset = Math.max(0, wordBoundary.textOffset - this.playbackTextOffsetBase);
         const segment = this.playbackSegments.find((candidate) => (
-            wordBoundary.textOffset >= candidate.start && wordBoundary.textOffset < candidate.end
+            normalizedTextOffset >= candidate.start && normalizedTextOffset < candidate.end
         ));
 
         this.resetPlaybackSegments();
@@ -420,7 +471,7 @@ export class TextToSpeech {
             return;
         }
 
-        const relativeTextOffset = wordBoundary.textOffset - segment.start;
+        const relativeTextOffset = normalizedTextOffset - segment.start;
         const currentOffset = this.getPosition(segment.originalHighlightDivInnerHTML, wordBoundary.text, relativeTextOffset);
 
         if (currentOffset === Number.MAX_SAFE_INTEGER) {
@@ -448,7 +499,7 @@ export class TextToSpeech {
             return node.getAttribute('co-tts.text') ?? '';
         }
 
-        return node.innerText;
+        return node.innerText ?? node.textContent ?? '';
     }
 
     getPrefetchKey(node: any, text: string) {
@@ -497,46 +548,57 @@ export class TextToSpeech {
             return;
         }
 
-        const speechConfig = SpeechConfig.fromSubscription(this.key, this.region);
-        speechConfig.speechSynthesisVoiceName = `Microsoft Server Speech Text to Speech Voice (${this.voice})`;
-        speechConfig.speechSynthesisOutputFormat = SpeechSynthesisOutputFormat.Audio24Khz160KBitRateMonoMp3;
-
-        const synthesizer = new SpeechSynthesizer(speechConfig, null);
-        const wordBoundryList: any[] = [];
-
-        synthesizer.wordBoundary = (s: any, e: any) => {
-            wordBoundryList.push(e);
-        };
-
-        const prefetchPromise = new Promise((resolve) => {
-            synthesizer.speakSsmlAsync(this.buildSSML(text),
-                (result: any) => {
-                    synthesizer.close();
-
-                    if (!result?.audioData) {
-                        resolve(null);
-
-                        return;
-                    }
-
-                    const blob = new Blob([result.audioData], {type: 'audio/mpeg'});
-                    const url = URL.createObjectURL(blob);
-                    const prefetch = {
-                        key,
-                        nodeId: nextNode.id,
-                        text,
-                        url,
-                        wordBoundryList,
-                    };
-
-                    this.prefetchedAudio.set(key, prefetch);
-                    resolve(prefetch);
-                },
-                () => {
-                    synthesizer.close();
-                    resolve(null);
+        let synthesizer: SpeechSynthesizer | undefined;
+
+        const prefetchPromise = (async () => {
+            try {
+                const speechConfig = await this.createSpeechConfig();
+                synthesizer = new SpeechSynthesizer(speechConfig, null);
+                const wordBoundryList: any[] = [];
+
+                synthesizer.wordBoundary = (s: any, e: any) => {
+                    wordBoundryList.push(e);
+                };
+
+                return await new Promise((resolve) => {
+                    synthesizer?.speakSsmlAsync(this.buildSSML(text),
+                        (result: any) => {
+                            this.closeResource(synthesizer);
+                            synthesizer = undefined;
+
+                            if (!result?.audioData) {
+                                resolve(null);
+
+                                return;
+                            }
+
+                            const blob = new Blob([result.audioData], {type: 'audio/mpeg'});
+                            const url = URL.createObjectURL(blob);
+                            const prefetch = {
+                                key,
+                                nodeId: nextNode.id,
+                                text,
+                                url,
+                                wordBoundryList,
+                            };
+
+                            this.prefetchedAudio.set(key, prefetch);
+                            resolve(prefetch);
+                        },
+                        (error: unknown) => {
+                            this.closeResource(synthesizer);
+                            synthesizer = undefined;
+                            this.dispatchError(error, 'PREFETCH_ERROR');
+                            resolve(null);
+                        });
                 });
-        }).finally(() => {
+            } catch (error) {
+                this.closeResource(synthesizer);
+                synthesizer = undefined;
+                this.dispatchError(error);
+                return null;
+            }
+        })().finally(() => {
             this.prefetchPromises.delete(key);
         });
 
@@ -580,38 +642,53 @@ export class TextToSpeech {
 
         await this.createInterval();
 
-        const audio = new Audio(prefetch.url);
-        this.activePrefetchedAudioUrl = prefetch.url;
-        this.player = audio;
+        try {
+            const audio = new Audio(prefetch.url);
+            this.activePrefetchedAudioUrl = prefetch.url;
+            this.player = audio;
 
-        audio.addEventListener('play', () => {
-            document.dispatchEvent(new CustomEvent('COAzureTTSStartedPlaying', {}));
-        }, {once: true});
+            audio.addEventListener('play', () => {
+                document.dispatchEvent(new CustomEvent('COAzureTTSStartedPlaying', {}));
+            }, {once: true});
 
-        audio.addEventListener('ended', async () => {
-            this.stopPlayer();
+            audio.addEventListener('ended', async () => {
+                this.stopPlayer();
 
-            if (this.clickedNode.hasAttribute('co-tts.next')) {
-                const nextNode = document.getElementById(this.clickedNode.getAttribute('co-tts.next'));
+                if (this.clickedNode.hasAttribute('co-tts.next')) {
+                    const nextNode = document.getElementById(this.clickedNode.getAttribute('co-tts.next'));
 
-                if (nextNode && await this.playPrefetchedNode(nextNode)) {
-                    return;
-                }
+                    if (nextNode && await this.playPrefetchedNode(nextNode)) {
+                        return;
+                    }
 
-                if (nextNode && nextNode.attributes.getNamedItem('co-tts.text')) {
-                    this.handleWithoutClick(nextNode, nextNode.attributes.getNamedItem('co-tts.text'));
-                } else if (nextNode) {
-                    nextNode.dispatchEvent(new Event('click'));
+                    if (nextNode && nextNode.attributes.getNamedItem('co-tts.text')) {
+                        this.handleWithoutClick(nextNode, nextNode.attributes.getNamedItem('co-tts.text'));
+                    } else if (nextNode) {
+                        nextNode.dispatchEvent(new Event('click'));
+                    }
+                } else {
+                    document.dispatchEvent(new CustomEvent('COAzureTTSFinishedPlaying', {}));
                 }
-            } else {
-                document.dispatchEvent(new CustomEvent('COAzureTTSFinishedPlaying', {}));
-            }
-        }, {once: true});
-
-        this.prefetchNextNode(node);
-        await audio.play();
+            }, {once: true});
+
+            audio.addEventListener('error', () => {
+                this.dispatchError(
+                    new Error('Prefetched audio resource closed unexpectedly.'),
+                    'AUDIO_RESOURCE_ERROR',
+                );
+                this.stopPlayer();
+            }, {once: true});
+
+            this.prefetchNextNode(node);
+            await audio.play();
+
+            return true;
+        } catch (error) {
+            this.dispatchError(error, 'AUDIO_RESOURCE_ERROR');
+            await this.stopPlayer();
 
-        return true;
+            return false;
+        }
     }
 
     async clearInterval() {
@@ -620,7 +697,7 @@ export class TextToSpeech {
 
     async createInterval() {
         this.interval = setInterval(() => {
-            if (this.player !== undefined && this.highlightDiv) {
+            if (this.player !== undefined && (this.highlightDiv || this.playbackSegments.length > 0)) {
                 const currentTime = this.player.currentTime;
                 let wordBoundary;
                 for (const e of this.wordBoundryList) {
@@ -674,6 +751,8 @@ export class TextToSpeech {
                             `;
                         }
                     }
+                } else if (this.playbackSegments.length > 0) {
+                    this.resetPlaybackSegments();
                 } else {
                     this.highlightDiv.innerHTML = this.originalHighlightDivInnerHTML;
                 }
@@ -732,4 +811,36 @@ export class TextToSpeech {
 
         return p.innerHTML;
     }
+
+    private closeSynthesizer() {
+        this.closeResource(this.synthesizer);
+        this.synthesizer = undefined;
+    }
+
+    private closeResource(resource: {close?: () => void} | undefined) {
+        if (!resource || typeof resource.close !== 'function') {
+            return;
+        }
+
+        try {
+            resource.close();
+        } catch {
+            // Resource may already be closed.
+        }
+    }
+
+    private dispatchError(error: unknown, code?: string) {
+        const detail = toSafeErrorDetail(error);
+        if (code && !detail.code) {
+            detail.code = code;
+        }
+
+        document.dispatchEvent(
+            new CustomEvent('COAzureTTSError', {
+                detail: {
+                    error: detail,
+                },
+            }),
+        );
+    }
 }