@pie-players/pie-assessment-toolkit 0.3.63 → 0.3.65

package/dist/services/TTSService.js

@@ -15,9 +15,11 @@
  *
  * Part of PIE Assessment Toolkit.
  */
+import { applyMediaFragment } from "./catalog-media.js";
 import { HighlightColor, HighlightType } from "./HighlightCoordinator.js";
+import { resolveSpokenAudioMedia, } from "./spoken-audio-cards.js";
 import { BrowserTTSProvider } from "./tts/browser-provider.js";
-import { collectVisibleTextAndMap, isElementHiddenForTTS, isNodeHiddenForTTS, normalizeTextForSpeech, } from "./tts/text-processing.js";
+import { collectRangeTextForSpeech, collectVisibleTextAndMap, isElementHiddenForTTS, isNodeExcludedFromSpeech, isNodeSuppressedForTTS, normalizeTextForSpeech, } from "./tts/text-processing.js";
 import { createCatalogSpanAlignment, } from "./tts/catalog-span-alignment.js";
 import { createMathAwareAlignment, } from "./tts/math-alignment/index.js";
 import { collectMathAwareTextAndMap } from "./tts/math-aware-text-processing.js";
@@ -88,6 +90,10 @@ export class TTSService {
     activeWordBoundaryOffset = 0;
     seekSegments = [];
     playbackChunks = [];
+    // The recording currently playing, if any. `cancel` settles the pending play
+    // promise as well as stopping the element, so stop/seek cannot wedge the
+    // chunk loop on a run that has already been superseded.
+    activeRecordedAudio = null;
     sentenceHighlightSegments = [];
     currentSeekSegmentIndex = 0;
     activeSentenceStartOffset = null;
@@ -234,6 +240,31 @@ export class TTSService {
     setCatalogResolver(resolver) {
         this.catalogResolver = resolver;
     }
+    /**
+     * Whether a catalog holds spoken content this service could actually speak.
+     *
+     * Exists because `data-catalog-idref` is one attribute with several readers:
+     * a node can be docked to a catalog that carries only a signing card, and the
+     * nearest docked ancestor of a selection is therefore not necessarily the one
+     * holding its authored SSML. A caller resolving a selection climbs ancestors
+     * and asks this before settling on an id — see the TTS tool's
+     * `findSpokenCatalogId`. Without it, a signing card docked on an inner node
+     * silently shadows authored speech on an outer one.
+     */
+    hasSpokenAlternate(catalogId, language = "en-US") {
+        if (!catalogId || !this.catalogResolver)
+            return false;
+        const resolved = this.catalogResolver.getAlternative(catalogId, {
+            type: "spoken",
+            language,
+            useFallback: true,
+            // The string form specifically. A node may also carry a recording of the
+            // same script; that one is not what this reports on, and card order must
+            // not decide the answer.
+            form: "content",
+        });
+        return resolved?.content !== undefined;
+    }
     getHighlightResolverRuntime() {
         let provided = null;
         try {
@@ -584,7 +615,9 @@ export class TTSService {
         while (currentNode) {
             const textNode = currentNode;
             const parent = textNode.parentElement;
-            if (parent && !this.isElementHidden(parent)) {
+            if (parent &&
+                !this.isElementHidden(parent) &&
+                !isNodeSuppressedForTTS(textNode)) {
                 const boundary = this.getBoundaryAnchor(textNode, contentElement);
                 const boundaryPoint = nodeStartOffsets.get(textNode);
                 if (boundary &&
@@ -1083,16 +1116,20 @@ export class TTSService {
                 language: options.language || "en-US",
                 useFallback: true,
                 context: options.catalogContext,
+                form: "content",
             });
-            if (catalogContent) {
+            // A card with no string form has nothing to speak — a signing card, for
+            // instance. Fall through to generated TTS rather than speaking "".
+            const spokenText = catalogContent?.content;
+            if (catalogContent && spokenText !== undefined) {
                 const visibleText = options?.contentElement
                     ? collectMathAwareTextAndMap(options.contentElement, this.getTextProcessingOptions(options.language)).visibleText || normalizedInputText
                     : normalizedInputText;
-                const normalizedCatalogText = normalizeTextForSpeech(catalogContent.content);
+                const normalizedCatalogText = normalizeTextForSpeech(spokenText);
                 this.debugLog(`[TTSService] Using catalog content for "${options.catalogId}" (${catalogContent.language})`);
                 return {
-                    contentToSpeak: catalogContent.content,
-                    speechText: catalogContent.content,
+                    contentToSpeak: spokenText,
+                    speechText: spokenText,
                     visibleText,
                     highlightText: visibleText,
                     usedCatalogSpoken: true,
@@ -1166,12 +1203,32 @@ export class TTSService {
             const catalogIdRef = element.getAttribute("data-catalog-idref");
             if (!catalogIdRef)
                 return null;
-            return this.catalogResolver.getAlternative(catalogIdRef, {
+            const lookup = {
                 type: "spoken",
                 language: options.language || "en-US",
                 useFallback: true,
                 context: options.catalogContext,
+            };
+            // Two lookups rather than one, because `form` is a preference: asking for
+            // the payload form happily returns a script card when no recording
+            // exists, so the answer has to be checked rather than assumed.
+            const audioCard = this.catalogResolver.getAlternative(catalogIdRef, {
+                ...lookup,
+                form: "payload",
             });
+            const audio = audioCard?.payload
+                ? (resolveSpokenAudioMedia({ payload: audioCard.payload }) ?? undefined)
+                : undefined;
+            const scriptCard = this.catalogResolver.getAlternative(catalogIdRef, {
+                ...lookup,
+                form: "content",
+            });
+            // Only a card with a string form can contribute synthesized speech. The
+            // same docking node may also carry a signing card; that one is not ours.
+            const script = scriptCard?.content;
+            if (audio === undefined && script === undefined)
+                return null;
+            return { script, audio };
         };
         const getSingleMathElementForAlignment = (element, visibleText) => {
             const mathElements = getMathElementsForAlignment(element);
@@ -1192,7 +1249,10 @@ export class TTSService {
             return mathElements;
         };
         const visit = (node) => {
-            if (isNodeHiddenForTTS(node, root))
+            // Before `resolveCatalog`, so suppression beats an authored `spoken`
+            // card on the same node: the card says how to speak this content, the
+            // suppression says it must not be spoken at all.
+            if (isNodeExcludedFromSpeech(node, root))
                 return;
             if (node.nodeType === Node.TEXT_NODE) {
                 textBuffer += ` ${node.textContent || ""}`;
@@ -1207,39 +1267,71 @@ export class TTSService {
                 const collectedVisible = collectMathAwareTextAndMap(element, this.getTextProcessingOptions(options.language));
                 const visibleText = collectedVisible.visibleText ||
                     normalizeTextForSpeech(element.textContent || "");
-                const alignment = createCatalogSpanAlignment({
-                    speechText: catalog.content,
-                    visibleText,
-                });
-                const mathElement = getSingleMathElementForAlignment(element, visibleText);
-                const mathAlignment = mathElement
-                    ? createMathAwareAlignment({
-                        mathElement,
-                        speechText: catalog.content,
-                    })
-                    : undefined;
-                const mathAlignments = mathAlignment
-                    ? undefined
-                    : getMathElementsForAlignment(element).map((candidate) => ({
-                        element: candidate,
-                        alignment: createMathAwareAlignment({
-                            mathElement: candidate,
-                            speechText: catalog.content,
-                        }),
-                    }));
-                const speechText = normalizeTextForSpeech(catalog.content);
-                chunks.push({
-                    speechText: catalog.content,
-                    visibleText,
-                    sourceElement: element,
-                    regionElement: resolveReadableRegion(element, root),
-                    speechMatchesVisibleText: speechText === visibleText,
-                    playbackMode: alignment.playbackMode,
-                    alignment,
-                    mathAlignment,
-                    mathAlignments,
-                    visibleMap: collectedVisible.map,
-                });
+                const regionElement = resolveReadableRegion(element, root);
+                // The script chunk, when there is a script. Built exactly as before,
+                // and it doubles as the recording's fallback: word-level alignment is
+                // only meaningful for the synthesized path, which is the one that runs
+                // if the audio cannot play.
+                const scriptChunk = catalog.script !== undefined
+                    ? (() => {
+                        const script = catalog.script;
+                        const alignment = createCatalogSpanAlignment({
+                            speechText: script,
+                            visibleText,
+                        });
+                        const mathElement = getSingleMathElementForAlignment(element, visibleText);
+                        const mathAlignment = mathElement
+                            ? createMathAwareAlignment({
+                                mathElement,
+                                speechText: script,
+                            })
+                            : undefined;
+                        const mathAlignments = mathAlignment
+                            ? undefined
+                            : getMathElementsForAlignment(element).map((candidate) => ({
+                                element: candidate,
+                                alignment: createMathAwareAlignment({
+                                    mathElement: candidate,
+                                    speechText: script,
+                                }),
+                            }));
+                        return {
+                            speechText: script,
+                            visibleText,
+                            sourceElement: element,
+                            regionElement,
+                            speechMatchesVisibleText: normalizeTextForSpeech(script) === visibleText,
+                            playbackMode: alignment.playbackMode,
+                            alignment,
+                            mathAlignment,
+                            mathAlignments,
+                            visibleMap: collectedVisible.map,
+                        };
+                    })()
+                    : null;
+                if (catalog.audio) {
+                    chunks.push(scriptChunk
+                        ? {
+                            ...scriptChunk,
+                            audio: catalog.audio,
+                            plainFallback: scriptChunk,
+                        }
+                        : {
+                            // No script to fall back to, and nothing synthesizes the
+                            // recording's words, so `speechText` exists only to keep seek
+                            // and offset bookkeeping consistent with the visible text.
+                            speechText: visibleText,
+                            visibleText,
+                            sourceElement: element,
+                            regionElement,
+                            speechMatchesVisibleText: false,
+                            visibleMap: collectedVisible.map,
+                            audio: catalog.audio,
+                        });
+                    return;
+                }
+                if (scriptChunk)
+                    chunks.push(scriptChunk);
                 return;
             }
             for (const child of Array.from(element.childNodes)) {
@@ -1519,10 +1611,92 @@ export class TTSService {
             throw error;
         }
     }
+    /**
+     * Play a recorded spoken alternate, resolving when it finishes.
+     *
+     * Rejects if the clip cannot play, which is what routes playback to the
+     * chunk's `plainFallback` — the reading script — in `speakCatalogChunk`.
+     * Highlighting is the docked node as a block for the clip's duration: a
+     * recording emits no word boundaries, and inventing them from its duration
+     * would highlight the wrong words confidently rather than the right region
+     * vaguely.
+     */
+    async playRecordedAudio(media) {
+        if (typeof document === "undefined") {
+            throw new Error("[tts] no document available to play recorded audio");
+        }
+        const source = media.sources[0];
+        const element = document.createElement("audio");
+        element.preload = "auto";
+        element.src = applyMediaFragment(source.src, media.fragment);
+        element.playbackRate = this.normalizePlaybackRate(Number(this.ttsConfig.rate || 1));
+        await new Promise((resolve, reject) => {
+            let endGuard;
+            const cleanup = () => {
+                element.removeEventListener("ended", onEnded);
+                element.removeEventListener("error", onError);
+                if (endGuard !== undefined)
+                    clearInterval(endGuard);
+                if (this.activeRecordedAudio?.element === element) {
+                    this.activeRecordedAudio = null;
+                }
+            };
+            const onEnded = () => {
+                cleanup();
+                resolve();
+            };
+            const onError = () => {
+                cleanup();
+                reject(new Error(`[tts] recorded audio failed to play: ${source.src}`));
+            };
+            // Cancellation has to settle this promise, not just stop the element:
+            // `stop()` bumps the run id, and a pending play that never resolves would
+            // wedge the chunk loop on a run nobody is listening to any more.
+            this.activeRecordedAudio = {
+                element,
+                cancel: () => {
+                    cleanup();
+                    element.pause();
+                    resolve();
+                },
+            };
+            element.addEventListener("ended", onEnded);
+            element.addEventListener("error", onError);
+            // Browsers honour a Media Fragments start offset but are inconsistent
+            // about the end bound, so the end is enforced here — the same reason the
+            // signing region enforces its own.
+            const endSeconds = media.fragment?.endSeconds;
+            if (endSeconds !== undefined) {
+                endGuard = setInterval(() => {
+                    if (element.currentTime >= endSeconds)
+                        onEnded();
+                }, 100);
+            }
+            Promise.resolve(element.play()).catch(onError);
+        }).finally(() => {
+            if (this.activeRecordedAudio?.element === element) {
+                this.activeRecordedAudio = null;
+            }
+            element.pause();
+            element.removeAttribute("src");
+        });
+    }
+    cancelRecordedAudio() {
+        const active = this.activeRecordedAudio;
+        if (!active)
+            return;
+        this.activeRecordedAudio = null;
+        active.cancel();
+    }
     async speakCatalogChunkOnce(chunk, runId) {
         if (!this.provider)
             return;
         this.lastRenderedRegionTarget = null;
+        if (chunk.audio) {
+            this.highlightCatalogRegion(chunk);
+            await this.playRecordedAudio(chunk.audio);
+            return;
+        }
         const contentRoot = chunk.regionElement || chunk.sourceElement || this.currentContentElement;
         const pipelineChunk = contentRoot
             ? normalizeSpeechChunks({
@@ -1657,9 +1831,13 @@ export class TTSService {
         if (!this.provider) {
             throw new Error("TTS service not initialized");
         }
-        const text = range.toString().trim();
-        if (!text)
+        // Enforced from the live ancestors rather than from `root`, because the
+        // element carrying the suppression may sit above whatever root the caller
+        // passed or the range happens to resolve to.
+        if (isNodeSuppressedForTTS(range.commonAncestorContainer)) {
+            console.warn("[tts] selection lies inside content marked not-to-be-spoken; nothing was spoken. Selecting a node and pressing read-aloud must not become a way around suppression.");
             return;
+        }
         // Use explicit content root when provided; otherwise keep highlighting scoped
         // to the selected range's nearest element ancestor.
         const fromOptions = options?.contentRoot || null;
@@ -1673,11 +1851,23 @@ export class TTSService {
         }
         if (!root)
             return;
-        // Calculate the offset of the range start within the root element
+        const selected = collectRangeTextForSpeech(range, root);
+        const text = selected.text.trim();
+        if (!text) {
+            if (selected.filtered) {
+                console.warn("[tts] every part of the selection is either hidden or marked not-to-be-spoken; nothing was spoken.");
+            }
+            return;
+        }
+        // Calculate the offset of the range start within the root element. Filtered
+        // the same way as the speech itself: the offset indexes into the highlight
+        // text, which comes from the exclusion-aware collectors, so counting
+        // characters here that never reach that text would shift every highlight
+        // after the excluded node.
         const beforeRange = document.createRange();
         beforeRange.selectNodeContents(root);
         beforeRange.setEnd(range.startContainer, range.startOffset);
-        const textBeforeRange = beforeRange.toString();
+        const textBeforeRange = collectRangeTextForSpeech(beforeRange, root).text;
         const normalizedTextBeforeRange = normalizeTextForSpeech(textBeforeRange);
         const offset = normalizedTextBeforeRange.length +
             (/\s$/.test(textBeforeRange) && normalizedTextBeforeRange ? 1 : 0);
@@ -1701,6 +1891,7 @@ export class TTSService {
         if (!this.provider)
             return;
         if (this.state === PlaybackState.PLAYING) {
+            this.activeRecordedAudio?.element.pause();
             this.provider.pause();
             this.setState(PlaybackState.PAUSED);
         }
@@ -1712,6 +1903,9 @@ export class TTSService {
         if (!this.provider)
             return;
         if (this.state === PlaybackState.PAUSED) {
+            const recorded = this.activeRecordedAudio?.element;
+            if (recorded)
+                void Promise.resolve(recorded.play()).catch(() => { });
             this.provider.resume();
             this.setState(PlaybackState.PLAYING);
         }
@@ -1729,6 +1923,7 @@ export class TTSService {
             ? this.playbackChunks.slice(safeTargetIndex)
             : null;
         this.speakRunId += 1;
+        this.cancelRecordedAudio();
         this.provider.onWordBoundary = undefined;
         this.provider.stop();
         this.currentSeekSegmentIndex = safeTargetIndex;
@@ -1820,6 +2015,7 @@ export class TTSService {
         if (!this.provider)
             return;
         this.speakRunId += 1;
+        this.cancelRecordedAudio();
         this.provider.onWordBoundary = undefined;
         this.provider.stop();
         this.highlightTargetResolverProvider = null;