@pie-players/pie-assessment-toolkit 0.2.7 → 0.2.10

package/dist/services/TTSService.js

@@ -15,6 +15,7 @@
  *
  * Part of PIE Assessment Toolkit.
  */
+import { collectVisibleTextAndMap, extractVisibleText as extractVisibleTextFromDOM, isElementHiddenForTTS, normalizeTextForSpeech, } from "./tts/text-processing.js";
 /**
  * Playback state
  */
@@ -38,11 +39,14 @@ export class TTSService {
     highlightCoordinator = null;
     catalogResolver = null;
     state = PlaybackState.IDLE;
+    ttsConfig = {};
     currentText = null;
     currentContentElement = null;
     normalizedToDOM = new Map();
     listeners = new Map();
     lastError = null;
+    speakRunId = 0;
+    currentBoundaryOffset = 0;
     constructor() { }
     /**
      * Initialize TTS service with a provider
@@ -52,6 +56,7 @@ export class TTSService {
      */
     async initialize(provider, config = {}) {
         this.currentProvider = provider;
+        this.ttsConfig = { ...config };
         // Initialize provider and get implementation
         this.provider = await provider.initialize(config);
     }
@@ -71,6 +76,7 @@ export class TTSService {
         if ("updateSettings" in this.provider &&
             typeof this.provider.updateSettings === "function") {
             await this.provider.updateSettings(settings);
+            this.ttsConfig = { ...this.ttsConfig, ...settings };
         }
         else {
             // Fallback: Reinitialize with merged config
@@ -106,15 +112,23 @@ export class TTSService {
      * Build a character-by-character map from normalized text positions to DOM positions
      * This handles the complexity of whitespace normalization
      */
-    buildPositionMap(element, spokenText) {
+    getTextProcessingOptions(language) {
+        const providerOptions = (this.ttsConfig.providerOptions || {});
+        const textNormalization = (providerOptions.textNormalization || {});
+        const mode = textNormalization.boundarySpacingMode;
+        const boundarySpacingMode = mode === "none" || mode === "alnum" || mode === "segmenterPreferred"
+            ? mode
+            : undefined;
+        const locale = language ||
+            (typeof textNormalization.locale === "string"
+                ? textNormalization.locale
+                : undefined);
+        return { locale, boundarySpacingMode };
+    }
+    buildPositionMap(element, spokenText, language) {
         this.normalizedToDOM.clear();
-        // Get the actual DOM text as it will be rendered
-        const range = document.createRange();
-        range.selectNodeContents(element);
-        const domText = range.toString();
-        // Normalize the DOM text the same way the spoken text is normalized
-        // (trim + collapse whitespace)
-        const normalizedDomText = domText.trim().replace(/\s+/g, " ");
+        const { text: normalizedDomText, map } = collectVisibleTextAndMap(element, this.getTextProcessingOptions(language));
+        this.normalizedToDOM = map;
         console.log("[TTSService] Text comparison:", {
             spokenLength: spokenText.length,
             normalizedDomLength: normalizedDomText.length,
@@ -138,64 +152,6 @@ export class TTSService {
                     return null;
                 })(),
         });
-        // Build map by walking through nodes and tracking normalized position
-        let normalizedPos = 0;
-        let inLeadingWhitespace = true;
-        let lastCharWasWhitespace = false;
-        const walk = (node) => {
-            if (node.nodeType === Node.TEXT_NODE) {
-                const textNode = node;
-                const text = textNode.textContent || "";
-                for (let i = 0; i < text.length; i++) {
-                    const char = text[i];
-                    const isWhitespace = /\s/.test(char);
-                    if (inLeadingWhitespace) {
-                        // Skip leading whitespace entirely - don't map it
-                        if (!isWhitespace) {
-                            inLeadingWhitespace = false;
-                            this.normalizedToDOM.set(normalizedPos, {
-                                node: textNode,
-                                offset: i,
-                            });
-                            normalizedPos++;
-                            lastCharWasWhitespace = false;
-                        }
-                    }
-                    else {
-                        // Past leading whitespace
-                        if (isWhitespace) {
-                            // Only map the first whitespace in a sequence (collapse multiple)
-                            if (!lastCharWasWhitespace) {
-                                this.normalizedToDOM.set(normalizedPos, {
-                                    node: textNode,
-                                    offset: i,
-                                });
-                                normalizedPos++;
-                            }
-                            lastCharWasWhitespace = true;
-                        }
-                        else {
-                            // Regular character - always map it
-                            this.normalizedToDOM.set(normalizedPos, {
-                                node: textNode,
-                                offset: i,
-                            });
-                            normalizedPos++;
-                            lastCharWasWhitespace = false;
-                        }
-                    }
-                }
-            }
-            else if (node.nodeType === Node.ELEMENT_NODE) {
-                const element = node;
-                if (element.tagName !== "SCRIPT" && element.tagName !== "STYLE") {
-                    for (const child of Array.from(node.childNodes)) {
-                        walk(child);
-                    }
-                }
-            }
-        };
-        walk(element);
         console.log("[TTSService] Position map built:", {
             entries: this.normalizedToDOM.size,
             spokenTextLength: spokenText.length,
@@ -211,6 +167,231 @@ export class TTSService {
             })),
         });
     }
+    getStructuralPauseProfile() {
+        const providerOptions = (this.ttsConfig.providerOptions || {});
+        const custom = (providerOptions.structuralPauses || {});
+        const customUnits = (custom.units || {});
+        const toNumber = (value, fallback) => {
+            if (typeof value === "number" && Number.isFinite(value))
+                return value;
+            if (typeof value === "string" && value.trim()) {
+                const parsed = Number(value);
+                return Number.isFinite(parsed) ? parsed : fallback;
+            }
+            return fallback;
+        };
+        return {
+            baseMs: toNumber(custom.baseMs, 280),
+            units: {
+                minor: toNumber(customUnits.minor, 0.65),
+                section: toNumber(customUnits.section, 1.0),
+                major: toNumber(customUnits.major, 1.35),
+            },
+            minMs: toNumber(custom.minMs, 120),
+            maxMs: toNumber(custom.maxMs, 900),
+        };
+    }
+    resolvePauseMsFromUnits(units) {
+        const profile = this.getStructuralPauseProfile();
+        const rate = Math.max(0.25, Math.min(4, Number(this.ttsConfig.rate || 1)));
+        const raw = (profile.baseMs * Math.max(0, units)) / rate;
+        return Math.max(profile.minMs, Math.min(profile.maxMs, Math.round(raw)));
+    }
+    resolveHighlightMode() {
+        const providerOptions = (this.ttsConfig.providerOptions || {});
+        const configuredMode = providerOptions.highlightMode;
+        if (configuredMode === "word" || configuredMode === "sentence") {
+            return configuredMode;
+        }
+        const providerId = (this.currentProvider?.providerId || "").toLowerCase();
+        if (providerId === "browser") {
+            return "sentence";
+        }
+        const supportsWordBoundary = this.currentProvider?.getCapabilities().supportsWordBoundary ?? false;
+        return supportsWordBoundary ? "word" : "sentence";
+    }
+    hasExplicitBreakSemantics(text) {
+        if (!text.includes("<"))
+            return false;
+        return /<\s*break\b|<\s*speak\b|<\s*prosody\b|<\s*p\b|<\s*s\b/i.test(text);
+    }
+    isElementHidden(element) {
+        return isElementHiddenForTTS(element);
+    }
+    extractVisibleText(element, language) {
+        return extractVisibleTextFromDOM(element, this.getTextProcessingOptions(language));
+    }
+    getBoundaryStrength(element) {
+        const explicitBreakMs = Number(element.getAttribute("data-tts-break-ms"));
+        if (Number.isFinite(explicitBreakMs) && explicitBreakMs > 0)
+            return "major";
+        const tagName = element.tagName.toUpperCase();
+        if (tagName.match(/^H[1-6]$/))
+            return "major";
+        const role = (element.getAttribute("role") || "").toLowerCase();
+        if (role === "heading")
+            return "major";
+        if (role === "listitem" ||
+            role === "row" ||
+            role === "option" ||
+            role === "radio" ||
+            tagName === "LI") {
+            return "section";
+        }
+        if (typeof window !== "undefined") {
+            const display = window.getComputedStyle(element).display;
+            if (display === "block" ||
+                display === "list-item" ||
+                display.startsWith("table") ||
+                display === "flex" ||
+                display === "grid") {
+                return "minor";
+            }
+        }
+        return "minor";
+    }
+    getBoundaryAnchor(textNode, root) {
+        let current = textNode.parentElement;
+        let best = null;
+        while (current && current !== root) {
+            if (this.isElementHidden(current))
+                return null;
+            const explicitBreakMs = Number(current.getAttribute("data-tts-break-ms"));
+            if (Number.isFinite(explicitBreakMs) && explicitBreakMs > 0) {
+                return {
+                    anchor: current,
+                    units: explicitBreakMs / this.getStructuralPauseProfile().baseMs,
+                };
+            }
+            const role = (current.getAttribute("role") || "").toLowerCase();
+            const tagName = current.tagName.toUpperCase();
+            if (tagName.match(/^H[1-6]$/) || role === "heading" || role === "listitem" || tagName === "LI") {
+                best = current;
+                break;
+            }
+            if (!best) {
+                if (typeof window !== "undefined") {
+                    const display = window.getComputedStyle(current).display;
+                    if (display === "block" ||
+                        display === "list-item" ||
+                        display.startsWith("table") ||
+                        display === "flex" ||
+                        display === "grid") {
+                        best = current;
+                    }
+                }
+            }
+            current = current.parentElement;
+        }
+        if (!best)
+            return null;
+        const strength = this.getBoundaryStrength(best);
+        return {
+            anchor: best,
+            units: this.getStructuralPauseProfile().units[strength],
+        };
+    }
+    createSpeechPlan(contentElement, normalizedText) {
+        const boundaries = new Map();
+        let previousBoundaryAnchor = null;
+        const { map } = collectVisibleTextAndMap(contentElement, this.getTextProcessingOptions());
+        const nodeStartOffsets = new Map();
+        for (const [normalizedIndex, mapping] of map.entries()) {
+            if (!nodeStartOffsets.has(mapping.node)) {
+                nodeStartOffsets.set(mapping.node, normalizedIndex);
+            }
+        }
+        const walker = document.createTreeWalker(contentElement, NodeFilter.SHOW_TEXT);
+        let currentNode = walker.nextNode();
+        while (currentNode) {
+            const textNode = currentNode;
+            const parent = textNode.parentElement;
+            if (parent && !this.isElementHidden(parent)) {
+                const boundary = this.getBoundaryAnchor(textNode, contentElement);
+                const boundaryPoint = nodeStartOffsets.get(textNode);
+                if (boundary &&
+                    boundaryPoint !== undefined &&
+                    boundary.anchor !== previousBoundaryAnchor &&
+                    boundaryPoint > 0) {
+                    boundaries.set(boundaryPoint, Math.max(boundaries.get(boundaryPoint) || 0, boundary.units));
+                    previousBoundaryAnchor = boundary.anchor;
+                }
+            }
+            currentNode = walker.nextNode();
+        }
+        const points = Array.from(boundaries.keys())
+            .filter((point) => point > 0 && point < normalizedText.length)
+            .sort((a, b) => a - b);
+        const segments = [];
+        let start = 0;
+        for (const point of points) {
+            const raw = normalizedText.substring(start, point);
+            const leading = raw.match(/^\s*/)?.[0].length || 0;
+            const trailing = raw.match(/\s*$/)?.[0].length || 0;
+            const segmentStart = start + leading;
+            const segmentEnd = point - trailing;
+            if (segmentEnd > segmentStart) {
+                const text = normalizedText.substring(segmentStart, segmentEnd);
+                segments.push({
+                    text,
+                    startOffset: segmentStart,
+                    pauseMsAfter: this.resolvePauseMsFromUnits(boundaries.get(point) || 0),
+                });
+            }
+            start = point;
+        }
+        const tail = normalizedText.substring(start);
+        const tailLeading = tail.match(/^\s*/)?.[0].length || 0;
+        const tailStart = start + tailLeading;
+        if (tailStart < normalizedText.length) {
+            segments.push({
+                text: normalizedText.substring(tailStart).trimEnd(),
+                startOffset: tailStart,
+                pauseMsAfter: 0,
+            });
+        }
+        return segments.filter((segment) => segment.text.trim().length > 0);
+    }
+    async speakWithPlan(segments, runId, options) {
+        if (!this.provider || segments.length === 0)
+            return;
+        const shouldTrackSentenceProgress = options?.highlightMode === "sentence";
+        const providerWithPlan = this.provider;
+        if (!shouldTrackSentenceProgress &&
+            typeof providerWithPlan.speakSegments === "function") {
+            this.currentBoundaryOffset = 0;
+            await providerWithPlan.speakSegments(segments);
+            return;
+        }
+        for (const segment of segments) {
+            if (runId !== this.speakRunId)
+                return;
+            this.currentBoundaryOffset = segment.startOffset;
+            if (shouldTrackSentenceProgress) {
+                this.highlightSentenceSegment(segment.startOffset, segment.text);
+            }
+            await this.provider.speak(segment.text);
+            const pauseMs = segment.pauseMsAfter ?? 0;
+            if (pauseMs > 0) {
+                await new Promise((resolve) => setTimeout(resolve, pauseMs));
+            }
+        }
+    }
+    highlightSentenceSegment(startOffset, text) {
+        if (!this.highlightCoordinator || typeof document === "undefined")
+            return;
+        const length = text.trimEnd().length;
+        if (length <= 0)
+            return;
+        const start = this.normalizedToDOM.get(startOffset);
+        const end = this.normalizedToDOM.get(startOffset + length - 1);
+        if (!start || !end)
+            return;
+        const range = document.createRange();
+        range.setStart(start.node, start.offset);
+        range.setEnd(end.node, end.offset + 1);
+        this.highlightCoordinator.highlightTTSSentence([range]);
+    }
     /**
      * Find text node and offsets for highlighting a word
      *
@@ -270,11 +451,46 @@ export class TTSService {
         if (!this.provider) {
             throw new Error("TTS service not initialized");
         }
-        // Normalize the input text to ensure consistency
-        // This handles cases where the caller didn't normalize
-        const normalizedText = text.trim().replace(/\s+/g, " ");
+        const runId = ++this.speakRunId;
+        if (options?.language) {
+            const providerOptions = (this.ttsConfig.providerOptions ||
+                {});
+            const textNormalization = (providerOptions.textNormalization ||
+                {});
+            const segmenter = (providerOptions.segmenter || {});
+            const mergedProviderOptions = {
+                ...providerOptions,
+                locale: options.language,
+                textNormalization: {
+                    ...textNormalization,
+                    locale: options.language,
+                },
+                segmenter: {
+                    ...segmenter,
+                    locale: options.language,
+                },
+            };
+            this.ttsConfig = {
+                ...this.ttsConfig,
+                providerOptions: mergedProviderOptions,
+            };
+            if ("updateSettings" in this.provider &&
+                typeof this.provider.updateSettings ===
+                    "function") {
+                await this.provider.updateSettings({ providerOptions: mergedProviderOptions });
+            }
+        }
+        // Normalize input text and prefer visible DOM-derived text when available.
+        // This keeps generated speech aligned to what learners can actually see.
+        const normalizedInputText = normalizeTextForSpeech(text);
+        const normalizedText = options?.contentElement
+            ? this.extractVisibleText(options.contentElement, options.language) ||
+                normalizedInputText
+            : normalizedInputText;
         // Try to resolve from accessibility catalog if catalogId provided
         let contentToSpeak = normalizedText;
+        let usedCatalogSpoken = false;
+        let speechSource = "dom-or-input";
         if (options?.catalogId && this.catalogResolver) {
             const catalogContent = this.catalogResolver.getAlternative(options.catalogId, {
                 type: "spoken",
@@ -284,43 +500,78 @@ export class TTSService {
             if (catalogContent) {
                 // Use pre-authored spoken content from catalog
                 contentToSpeak = catalogContent.content;
+                usedCatalogSpoken = true;
+                speechSource = "catalog-spoken";
                 console.debug(`[TTSService] Using catalog content for "${options.catalogId}" (${catalogContent.language})`);
             }
             else {
                 console.debug(`[TTSService] No catalog found for "${options.catalogId}", falling back to generated TTS`);
             }
         }
+        const preview = contentToSpeak.replace(/\s+/g, " ").trim().slice(0, 200);
+        console.debug("[TTSService] Speak resolved content", {
+            source: speechSource,
+            catalogId: options?.catalogId || null,
+            length: contentToSpeak.length,
+            preview,
+        });
         this.currentText = contentToSpeak;
         this.currentContentElement = options?.contentElement || null;
         this.lastError = null; // Clear previous error
+        this.currentBoundaryOffset = 0;
+        const highlightMode = options?.highlightModeOverride || this.resolveHighlightMode();
+        const shouldUsePlan = !!this.currentContentElement &&
+            !usedCatalogSpoken &&
+            !this.hasExplicitBreakSemantics(contentToSpeak);
         this.setState(PlaybackState.LOADING);
         // Build position map for highlighting if we have a content element
         if (this.currentContentElement && this.highlightCoordinator) {
-            this.buildPositionMap(this.currentContentElement, contentToSpeak);
-            // Apply sentence-level highlighting as base layer
-            const range = document.createRange();
-            range.selectNodeContents(this.currentContentElement);
-            this.highlightCoordinator.highlightTTSSentence([range]);
-            console.log("[TTSService] Applied sentence-level highlighting");
+            this.buildPositionMap(this.currentContentElement, contentToSpeak, options?.language);
+            // Fallback sentence-level highlight for non-planned flows.
+            // Planned sentence mode is updated progressively in speakWithPlan().
+            if (!(highlightMode === "sentence" && shouldUsePlan)) {
+                const range = document.createRange();
+                range.selectNodeContents(this.currentContentElement);
+                this.highlightCoordinator.highlightTTSSentence([range]);
+                console.log("[TTSService] Applied sentence-level highlighting");
+            }
         }
         try {
             // Setup word boundary highlighting
-            if (this.highlightCoordinator && this.currentContentElement) {
+            if (highlightMode === "word" &&
+                this.highlightCoordinator &&
+                this.currentContentElement) {
                 this.provider.onWordBoundary = (word, charIndex, length) => {
                     const wordLength = length || word.length;
-                    const highlightRange = this.findHighlightRange(charIndex, wordLength);
+                    const globalIndex = charIndex +
+                        this.currentBoundaryOffset +
+                        (options?.wordBoundaryOffset || 0);
+                    const highlightRange = this.findHighlightRange(globalIndex, wordLength);
                     if (highlightRange && this.highlightCoordinator) {
                         const highlightText = highlightRange.node.textContent?.substring(highlightRange.start, highlightRange.end) || "";
-                        console.log(`[TTSService] Highlighting "${highlightText}" (word: "${word}") at position ${charIndex}`);
+                        console.log(`[TTSService] Highlighting "${highlightText}" (word: "${word}") at position ${globalIndex}`);
                         this.highlightCoordinator.highlightTTSWord(highlightRange.node, highlightRange.start, highlightRange.end);
                     }
                     else {
-                        console.warn(`[TTSService] Could not find highlight range for position ${charIndex}, length ${wordLength}`);
+                        console.warn(`[TTSService] Could not find highlight range for position ${globalIndex}, length ${wordLength}`);
                     }
                 };
             }
             this.setState(PlaybackState.PLAYING);
-            await this.provider.speak(contentToSpeak);
+            if (shouldUsePlan && this.currentContentElement) {
+                const segments = this.createSpeechPlan(this.currentContentElement, normalizedText);
+                if (segments.length > 0) {
+                    await this.speakWithPlan(segments, runId, { highlightMode });
+                }
+                else {
+                    await this.provider.speak(contentToSpeak);
+                }
+            }
+            else {
+                await this.provider.speak(contentToSpeak);
+            }
+            if (runId !== this.speakRunId)
+                return;
             this.setState(PlaybackState.IDLE);
             // Clear highlights when done
             if (this.highlightCoordinator) {
@@ -329,10 +580,13 @@ export class TTSService {
             // Clear tracking
             this.currentContentElement = null;
             this.normalizedToDOM.clear();
+            this.currentBoundaryOffset = 0;
         }
         catch (error) {
             console.error("TTS error:", error);
             this.lastError = error instanceof Error ? error.message : String(error);
+            if (runId !== this.speakRunId)
+                return;
             this.setState(PlaybackState.ERROR);
             // Clear highlights on error
             if (this.highlightCoordinator) {
@@ -341,6 +595,7 @@ export class TTSService {
             // Clear tracking
             this.currentContentElement = null;
             this.normalizedToDOM.clear();
+            this.currentBoundaryOffset = 0;
             throw error;
         }
     }
@@ -352,61 +607,43 @@ export class TTSService {
      *
      * @param range DOM Range to speak
      */
-    async speakRange(range) {
+    async speakRange(range, options) {
         if (!this.provider) {
             throw new Error("TTS service not initialized");
         }
         const text = range.toString().trim();
         if (!text)
             return;
-        // Find the root content element (closest element with data-pie-content or body)
-        let root = null;
-        let node = range.commonAncestorContainer;
-        while (node && node.nodeType !== Node.ELEMENT_NODE) {
-            node = node.parentNode;
-        }
-        if (node) {
-            root = node.closest("[data-pie-content]") || document.body;
-        }
-        else {
-            root = document.body;
+        // Use explicit content root when provided; otherwise keep highlighting scoped
+        // to the selected range's nearest element ancestor.
+        const fromOptions = options?.contentRoot || null;
+        let root = fromOptions;
+        if (!root) {
+            const ancestor = range.commonAncestorContainer;
+            root =
+                ancestor.nodeType === Node.ELEMENT_NODE
+                    ? ancestor
+                    : ancestor.parentElement;
         }
+        if (!root)
+            return;
         // Calculate the offset of the range start within the root element
         const beforeRange = document.createRange();
         beforeRange.selectNodeContents(root);
         beforeRange.setEnd(range.startContainer, range.startOffset);
         const textBeforeRange = beforeRange.toString();
-        const offset = textBeforeRange.trim().replace(/\s+/g, " ").length;
+        const offset = normalizeTextForSpeech(textBeforeRange).length;
         console.log("[TTSService] speakRange offset calculation:", {
             selectedText: text,
             textBeforeRange: textBeforeRange.substring(0, 100),
             offset,
             rootTag: root.tagName,
         });
-        // Store the offset for word boundary calculations
-        const originalOnWordBoundary = this.provider.onWordBoundary;
-        // Wrap the onWordBoundary handler to adjust character indices
-        this.provider.onWordBoundary = (word, charIndex, length) => {
-            // Adjust the character index by the offset
-            const adjustedIndex = charIndex + offset;
-            // Call the handler from speak() method which will handle the highlighting
-            // Pass the adjusted index so highlighting aligns with the actual selection
-            if (this.highlightCoordinator && this.currentContentElement) {
-                const wordLength = length || word.length;
-                const highlightRange = this.findHighlightRange(adjustedIndex, wordLength);
-                if (highlightRange) {
-                    this.highlightCoordinator.highlightTTSWord(highlightRange.node, highlightRange.start, highlightRange.end);
-                }
-            }
-        };
-        try {
-            // Speak the text with the root element as context
-            await this.speak(text, { contentElement: root });
-        }
-        finally {
-            // Restore original handler
-            this.provider.onWordBoundary = originalOnWordBoundary;
-        }
+        // Speak the text with the root element as context
+        await this.speak(text, {
+            contentElement: root,
+            wordBoundaryOffset: offset,
+        });
     }
     /**
      * Pause playback
@@ -436,6 +673,7 @@ export class TTSService {
     stop() {
         if (!this.provider)
             return;
+        this.speakRunId += 1;
         this.provider.stop();
         this.setState(PlaybackState.IDLE);
         this.currentText = null;
@@ -446,6 +684,7 @@ export class TTSService {
         // Clear tracking
         this.currentContentElement = null;
         this.normalizedToDOM.clear();
+        this.currentBoundaryOffset = 0;
     }
     /**
      * Check if currently playing