@pie-players/pie-assessment-toolkit 0.3.64 → 0.3.66

package/dist/services/TTSService.js

@@ -97,6 +97,13 @@ export class TTSService {
     sentenceHighlightSegments = [];
     currentSeekSegmentIndex = 0;
     activeSentenceStartOffset = null;
+    playbackStartDeferredRunId = null;
+    pendingPlaybackStartHighlights = [];
+    playbackStartBarrierCleanup = null;
+    playbackRateWriteQueue = Promise.resolve();
+    playbackRateRequestId = 0;
+    pendingPlaybackRateRequest = null;
+    activePlaybackRate = null;
     activeHighlightMode = "word";
     lastRenderedRegionTarget = null;
     telemetryReporter = null;
@@ -130,6 +137,61 @@ export class TTSService {
             console.debug(message, detail);
         }
     }
+    clearPlaybackStartBarrier(cleanup = this.playbackStartBarrierCleanup) {
+        cleanup?.();
+        if (this.playbackStartBarrierCleanup === cleanup) {
+            this.playbackStartBarrierCleanup = null;
+        }
+    }
+    notifyPlaybackStarted(runId) {
+        if (this.playbackStartDeferredRunId !== runId ||
+            runId !== this.speakRunId) {
+            return;
+        }
+        this.setState(PlaybackState.PLAYING);
+        const pendingHighlights = this.pendingPlaybackStartHighlights.splice(0);
+        for (const applyHighlight of pendingHighlights) {
+            applyHighlight();
+        }
+    }
+    installPlaybackStartBarrier(runId, hasMediaStartSignal = false) {
+        this.clearPlaybackStartBarrier();
+        const provider = this.provider;
+        const startAwareProvider = provider && "onPlaybackStart" in provider ? provider : null;
+        if (!startAwareProvider && !hasMediaStartSignal)
+            return null;
+        const previousOnPlaybackStart = startAwareProvider?.onPlaybackStart;
+        const onPlaybackStart = () => {
+            try {
+                this.notifyPlaybackStarted(runId);
+            }
+            finally {
+                previousOnPlaybackStart?.();
+            }
+        };
+        if (startAwareProvider) {
+            startAwareProvider.onPlaybackStart = onPlaybackStart;
+        }
+        this.playbackStartDeferredRunId = runId;
+        const cleanup = () => {
+            if (startAwareProvider?.onPlaybackStart === onPlaybackStart) {
+                startAwareProvider.onPlaybackStart = previousOnPlaybackStart;
+            }
+            if (this.playbackStartDeferredRunId === runId) {
+                this.playbackStartDeferredRunId = null;
+                this.pendingPlaybackStartHighlights = [];
+            }
+        };
+        this.playbackStartBarrierCleanup = cleanup;
+        return cleanup;
+    }
+    runWhenPlaybackStarts(runId, callback) {
+        if (this.playbackStartDeferredRunId === runId) {
+            this.pendingPlaybackStartHighlights.push(callback);
+            return;
+        }
+        callback();
+    }
     async emitTelemetry(eventName, payload) {
         try {
             await this.telemetryReporter?.(eventName, payload);
@@ -810,7 +872,9 @@ export class TTSService {
             }
             this.currentBoundaryOffset = segment.startOffset;
             if (shouldTrackSentenceProgress) {
-                this.highlightSentenceSegment(segment.startOffset, segment.text);
+                this.runWhenPlaybackStarts(runId, () => {
+                    this.highlightSentenceSegment(segment.startOffset, segment.text);
+                });
             }
             await this.provider.speak(segment.text);
             const pauseMs = segment.pauseMsAfter ?? 0;
@@ -923,6 +987,8 @@ export class TTSService {
             throw new Error("TTS service not initialized");
         }
         const runId = ++this.speakRunId;
+        this.clearPlaybackStartBarrier();
+        let playbackStartBarrier = null;
         try {
             await this.applyLanguageSettings(options);
             if (runId !== this.speakRunId)
@@ -962,11 +1028,13 @@ export class TTSService {
                     playbackSegments: this.seekSegments,
                 });
             this.setState(PlaybackState.LOADING);
+            playbackStartBarrier = this.installPlaybackStartBarrier(runId, !!this.playbackChunks[0]?.audio);
             this.prepareHighlightsForSpeak({
                 contentToSpeak: highlightText,
                 options,
                 highlightMode,
                 shouldUsePlan,
+                runId,
             });
             if (speechMatchesVisibleText) {
                 this.configureWordBoundaryHighlighting({
@@ -977,7 +1045,10 @@ export class TTSService {
             else {
                 this.clearWordBoundaryHighlighting();
             }
-            this.setState(PlaybackState.PLAYING);
+            if (!playbackStartBarrier) {
+                this.setState(PlaybackState.PLAYING);
+            }
+            this.activePlaybackRate = this.normalizePlaybackRate(Number(this.ttsConfig.rate ?? 1));
             await this.executeSpeakPlayback({
                 shouldUsePlan,
                 runId,
@@ -1001,13 +1072,30 @@ export class TTSService {
             this.clearHighlightsAndTracking();
             throw finalError;
         }
+        finally {
+            this.clearPlaybackStartBarrier(playbackStartBarrier);
+        }
     }
     async switchProviderToBrowser(reason, context) {
         if (!this.currentProvider)
             return false;
         const fallbackProvider = new BrowserTTSProvider();
         const previousProviderId = this.currentProvider.providerId;
-        const browserConfig = { ...this.ttsConfig };
+        // Voice identifiers and provider extensions belong to the provider that
+        // rejected initialization. Carry only settings whose meaning is portable
+        // when crossing the provider boundary; otherwise a Polly/Google/SC voice
+        // such as "Joanna" can incorrectly become an explicit Browser voice.
+        const browserConfig = {};
+        if (this.ttsConfig.rate !== undefined) {
+            browserConfig.rate = this.ttsConfig.rate;
+        }
+        if (this.ttsConfig.pitch !== undefined) {
+            browserConfig.pitch = this.ttsConfig.pitch;
+        }
+        if (this.ttsConfig.mathTokenHighlighting !== undefined) {
+            browserConfig.mathTokenHighlighting =
+                this.ttsConfig.mathTokenHighlighting;
+        }
         const operation = context?.operation || "tts-initialize";
         const offendingText = context?.contentToSpeak ?? "";
         const offendingPreview = offendingText
@@ -1431,20 +1519,22 @@ export class TTSService {
             return;
         }
         const initialSegment = this.sentenceHighlightSegments[0] || this.seekSegments[0];
-        if (initialSegment) {
-            this.highlightSentenceSegment(initialSegment.startOffset, initialSegment.text);
-            this.debugLog("[TTSService] Applied initial sentence highlighting");
-        }
-        else {
-            try {
-                const range = document.createRange();
-                range.selectNodeContents(this.currentContentElement);
-                this.paintTTSSentenceRanges([range]);
+        this.runWhenPlaybackStarts(args.runId, () => {
+            if (initialSegment) {
+                this.highlightSentenceSegment(initialSegment.startOffset, initialSegment.text);
+                this.debugLog("[TTSService] Applied initial sentence highlighting");
             }
-            catch {
-                // No-op fallback when DOM range creation fails.
+            else {
+                try {
+                    const range = document.createRange();
+                    range.selectNodeContents(this.currentContentElement);
+                    this.paintTTSSentenceRanges([range]);
+                }
+                catch {
+                    // No-op fallback when DOM range creation fails.
+                }
             }
-        }
+        });
     }
     configureWordBoundaryHighlighting(args) {
         if (!this.provider ||
@@ -1605,6 +1695,16 @@ export class TTSService {
             // fallback, so this cannot recurse.
             if (chunk.plainFallback && runId === this.speakRunId) {
                 this.debugLog("[TTSService] SSML chunk speak failed; retrying plain text", { message: error instanceof Error ? error.message : String(error) });
+                // A recorded-first run can own the start barrier even when its fallback
+                // provider has no formal start signal. Drop the failed recording's queued
+                // highlight and resume that non-start-aware provider's immediate-start lifecycle.
+                if (this.playbackStartDeferredRunId === runId &&
+                    this.provider &&
+                    !("onPlaybackStart" in this.provider)) {
+                    this.pendingPlaybackStartHighlights = [];
+                    this.clearPlaybackStartBarrier();
+                    this.setState(PlaybackState.PLAYING);
+                }
                 await this.speakCatalogChunkOnce(chunk.plainFallback, runId);
                 return;
             }
@@ -1621,7 +1721,7 @@ export class TTSService {
      * would highlight the wrong words confidently rather than the right region
      * vaguely.
      */
-    async playRecordedAudio(media) {
+    async playRecordedAudio(media, onPlaybackStart) {
         if (typeof document === "undefined") {
             throw new Error("[tts] no document available to play recorded audio");
         }
@@ -1632,9 +1732,18 @@ export class TTSService {
         element.playbackRate = this.normalizePlaybackRate(Number(this.ttsConfig.rate || 1));
         await new Promise((resolve, reject) => {
             let endGuard;
+            let didStart = false;
+            let settled = false;
+            const markStarted = () => {
+                if (settled || didStart)
+                    return;
+                didStart = true;
+                onPlaybackStart?.();
+            };
             const cleanup = () => {
                 element.removeEventListener("ended", onEnded);
                 element.removeEventListener("error", onError);
+                element.removeEventListener("playing", markStarted);
                 if (endGuard !== undefined)
                     clearInterval(endGuard);
                 if (this.activeRecordedAudio?.element === element) {
@@ -1642,10 +1751,16 @@ export class TTSService {
                 }
             };
             const onEnded = () => {
+                if (settled)
+                    return;
+                settled = true;
                 cleanup();
                 resolve();
             };
             const onError = () => {
+                if (settled)
+                    return;
+                settled = true;
                 cleanup();
                 reject(new Error(`[tts] recorded audio failed to play: ${source.src}`));
             };
@@ -1655,6 +1770,9 @@ export class TTSService {
             this.activeRecordedAudio = {
                 element,
                 cancel: () => {
+                    if (settled)
+                        return;
+                    settled = true;
                     cleanup();
                     element.pause();
                     resolve();
@@ -1662,6 +1780,7 @@ export class TTSService {
             };
             element.addEventListener("ended", onEnded);
             element.addEventListener("error", onError);
+            element.addEventListener("playing", markStarted);
             // Browsers honour a Media Fragments start offset but are inconsistent
             // about the end bound, so the end is enforced here — the same reason the
             // signing region enforces its own.
@@ -1672,7 +1791,7 @@ export class TTSService {
                         onEnded();
                 }, 100);
             }
-            Promise.resolve(element.play()).catch(onError);
+            Promise.resolve(element.play()).then(markStarted).catch(onError);
         }).finally(() => {
             if (this.activeRecordedAudio?.element === element) {
                 this.activeRecordedAudio = null;
@@ -1693,8 +1812,12 @@ export class TTSService {
             return;
         this.lastRenderedRegionTarget = null;
         if (chunk.audio) {
-            this.highlightCatalogRegion(chunk);
-            await this.playRecordedAudio(chunk.audio);
+            this.runWhenPlaybackStarts(runId, () => {
+                this.highlightCatalogRegion(chunk);
+            });
+            await this.playRecordedAudio(chunk.audio, () => {
+                this.notifyPlaybackStarted(runId);
+            });
             return;
         }
         const contentRoot = chunk.regionElement || chunk.sourceElement || this.currentContentElement;
@@ -1713,12 +1836,14 @@ export class TTSService {
                 mathTokenHighlighting,
             })
             : null;
-        if (highlightPlan && pipelineChunk && chunk.mathAlignment) {
-            this.renderHighlightDecision(highlightPlan.resolveInitial(pipelineChunk.id));
-        }
-        else {
-            this.highlightCatalogRegion(chunk);
-        }
+        this.runWhenPlaybackStarts(runId, () => {
+            if (highlightPlan && pipelineChunk && chunk.mathAlignment) {
+                this.renderHighlightDecision(highlightPlan.resolveInitial(pipelineChunk.id));
+            }
+            else {
+                this.highlightCatalogRegion(chunk);
+            }
+        });
         const canUseChunkBoundaries = chunk.sourceElement &&
             ((chunk.mathAlignment && chunk.mathAlignment.speech.tokens.length > 0) ||
                 (chunk.mathAlignments && chunk.mathAlignments.length > 0) ||
@@ -1805,6 +1930,8 @@ export class TTSService {
         }
     }
     clearHighlightsAndTracking() {
+        this.pendingPlaybackStartHighlights = [];
+        this.activePlaybackRate = null;
         if (this.highlightCoordinator) {
             this.highlightCoordinator.clearTTS();
         }
@@ -1923,13 +2050,21 @@ export class TTSService {
             ? this.playbackChunks.slice(safeTargetIndex)
             : null;
         this.speakRunId += 1;
+        this.clearPlaybackStartBarrier();
         this.cancelRecordedAudio();
         this.provider.onWordBoundary = undefined;
         this.provider.stop();
         this.currentSeekSegmentIndex = safeTargetIndex;
         const runId = ++this.speakRunId;
         const restartSegments = this.seekSegments.slice(safeTargetIndex);
-        this.setState(PlaybackState.PLAYING);
+        this.highlightCoordinator?.clearTTS();
+        this.activeSentenceStartOffset = null;
+        this.setState(PlaybackState.LOADING);
+        const playbackStartBarrier = this.installPlaybackStartBarrier(runId, !!restartChunks?.[0]?.audio);
+        if (!playbackStartBarrier) {
+            this.setState(PlaybackState.PLAYING);
+        }
+        this.activePlaybackRate = this.normalizePlaybackRate(Number(this.ttsConfig.rate ?? 1));
         try {
             this.configureWordBoundaryHighlighting({
                 highlightMode: this.activeHighlightMode,
@@ -1968,17 +2103,68 @@ export class TTSService {
             this.highlightTargetResolverProvider = null;
             throw error;
         }
+        finally {
+            this.clearPlaybackStartBarrier(playbackStartBarrier);
+        }
     }
     async setPlaybackRate(rate) {
+        if (!this.provider) {
+            throw new Error("TTSService not initialized. Call initialize() first.");
+        }
         const nextRate = this.normalizePlaybackRate(rate);
-        await this.updateSettings({ rate: nextRate });
-        if (this.state !== PlaybackState.PLAYING ||
-            !this.currentText ||
-            this.seekSegments.length === 0 ||
-            this.hasExplicitBreakSemantics(this.currentText)) {
+        const pendingRequest = this.pendingPlaybackRateRequest;
+        if (pendingRequest?.rate === nextRate) {
+            await pendingRequest.promise;
             return;
         }
-        await this.restartFromSeekIndex(this.getCurrentSeekSegmentIndex());
+        const requestId = ++this.playbackRateRequestId;
+        const settingsWrite = this.playbackRateWriteQueue.then(async () => {
+            const playbackRunId = this.speakRunId;
+            // A newer distinct target supersedes queued work before it mutates either
+            // provider settings or playback. An already-running write cannot be
+            // cancelled, so its post-write generation check still suppresses restart.
+            if (requestId !== this.playbackRateRequestId) {
+                return { didUpdate: false, playbackRunId };
+            }
+            // An omitted rate is the provider-default 1× rate. Re-check inside the
+            // serialized operation so overlapping callers observe the preceding write.
+            const currentRate = this.normalizePlaybackRate(Number(this.ttsConfig.rate ?? 1));
+            if (nextRate === currentRate) {
+                return { didUpdate: false, playbackRunId };
+            }
+            await this.updateSettings({ rate: nextRate });
+            return { didUpdate: true, playbackRunId };
+        });
+        this.playbackRateWriteQueue = settingsWrite.then(() => undefined, () => undefined);
+        const operation = settingsWrite.then(async (result) => {
+            if (requestId !== this.playbackRateRequestId ||
+                result.playbackRunId !== this.speakRunId) {
+                return;
+            }
+            const activePlaybackNeedsRate = this.activePlaybackRate !== null &&
+                this.activePlaybackRate !== nextRate;
+            if (!result.didUpdate && !activePlaybackNeedsRate)
+                return;
+            if ((this.state !== PlaybackState.PLAYING &&
+                this.state !== PlaybackState.LOADING) ||
+                !this.currentText ||
+                this.seekSegments.length === 0 ||
+                this.hasExplicitBreakSemantics(this.currentText)) {
+                return;
+            }
+            await this.restartFromSeekIndex(this.getCurrentSeekSegmentIndex());
+        });
+        let trackedOperation;
+        trackedOperation = operation.finally(() => {
+            if (this.pendingPlaybackRateRequest?.promise === trackedOperation) {
+                this.pendingPlaybackRateRequest = null;
+            }
+        });
+        this.pendingPlaybackRateRequest = {
+            rate: nextRate,
+            promise: trackedOperation,
+        };
+        await trackedOperation;
     }
     async seekBy(units) {
         if (!this.provider || !this.currentText)
@@ -2015,6 +2201,7 @@ export class TTSService {
         if (!this.provider)
             return;
         this.speakRunId += 1;
+        this.clearPlaybackStartBarrier();
         this.cancelRecordedAudio();
         this.provider.onWordBoundary = undefined;
         this.provider.stop();
@@ -2036,6 +2223,7 @@ export class TTSService {
         this.sentenceHighlightSegments = [];
         this.currentSeekSegmentIndex = 0;
         this.activeSentenceStartOffset = null;
+        this.activePlaybackRate = null;
     }
     /**
      * Request UI-level TTS controls to hand off/deactivate.