@pie-players/pie-assessment-toolkit 0.3.73 → 0.3.75

package/dist/services/tts/browser-provider.js

@@ -6,9 +6,81 @@
  *
  * Part of PIE Assessment Toolkit.
  */
+import { createPieLogger, isTtsDebugEnabled, } from "@pie-players/pie-players-shared/pie";
+import { extractSpokenText, isSsmlDocument } from "./ssml/spoken-text.js";
 import { segmentSentences as segmentTextToSentences } from "./text-segmentation.js";
+const logger = createPieLogger("browser-tts-provider", isTtsDebugEnabled);
 const NATIVE_START_TIMEOUT_MS = 5_000;
 const VOICE_INVENTORY_TIMEOUT_MS = 2_000;
+// The provider options the toolkit and this provider read. Every other option,
+// and `voice`, belongs to the provider it was configured for: a Polly voice such
+// as "Joanna" resolves to no browser voice, and an unresolved voice fails speak.
+const PORTABLE_PROVIDER_OPTIONS = [
+    "__pieTelemetry",
+    "contentLanguage",
+    "highlightMode",
+    "locale",
+    "mathSpeech",
+    "segmenter",
+    "structuralPauses",
+    "textNormalization",
+];
+/**
+ * The part of another provider's configuration that keeps its meaning on
+ * browser speech. Every path that replaces a provider with browser speech
+ * starts it with this.
+ */
+export const browserFallbackConfig = (config) => {
+    const fallback = {};
+    if (config.rate !== undefined)
+        fallback.rate = config.rate;
+    if (config.pitch !== undefined)
+        fallback.pitch = config.pitch;
+    if (config.mathTokenHighlighting !== undefined) {
+        fallback.mathTokenHighlighting = config.mathTokenHighlighting;
+    }
+    const options = config.providerOptions;
+    if (options && typeof options === "object") {
+        const portable = {};
+        for (const key of PORTABLE_PROVIDER_OPTIONS) {
+            if (options[key] !== undefined)
+                portable[key] = options[key];
+        }
+        if (Object.keys(portable).length > 0)
+            fallback.providerOptions = portable;
+    }
+    return fallback;
+};
+/**
+ * The Web Speech API reads SSML tags aloud, so a `<speak>` document is voiced
+ * as its spoken text. Boundary offsets still index the text as sent: that is
+ * the space the highlight pipeline resolves a raw-SSML chunk in.
+ */
+const toUtteranceText = (text) => {
+    if (!isSsmlDocument(text)) {
+        return { text, sourceOffset: (offset) => offset };
+    }
+    const { spokenText, rawToSpokenOffsetMap } = extractSpokenText(text);
+    const spokenToRaw = [];
+    for (const [raw, spoken] of rawToSpokenOffsetMap) {
+        const known = spokenToRaw[spoken];
+        if (known === undefined || raw < known)
+            spokenToRaw[spoken] = raw;
+    }
+    return {
+        text: spokenText,
+        // Alias text (`<sub alias>`) has no raw offset; it takes the nearest
+        // preceding one.
+        sourceOffset: (offset) => {
+            for (let index = offset; index >= 0; index--) {
+                const raw = spokenToRaw[index];
+                if (raw !== undefined)
+                    return raw;
+            }
+            return 0;
+        },
+    };
+};
 const normalizeLanguageCode = (value) => String(value || "")
     .trim()
     .toLowerCase();
@@ -18,19 +90,28 @@ const browserLanguage = () => {
         : "";
     return normalizeLanguageCode(navigatorLanguage || "en-US");
 };
-const findBrowserVoice = (voices, preferredVoice) => {
+/**
+ * Whether a voice speaks `contentLanguage`, or the browser's own language when
+ * the content names none: the same language, or the same primary subtag.
+ */
+export const browserVoiceMatchesLanguage = (voice, contentLanguage) => {
+    const language = normalizeLanguageCode(contentLanguage) || browserLanguage();
+    const languagePrefix = language.split("-")[0] || "en";
+    const voiceLanguage = normalizeLanguageCode(voice.lang);
+    return (voiceLanguage === language || voiceLanguage.startsWith(`${languagePrefix}-`));
+};
+/**
+ * The voice to read with: the configured one, else one for `contentLanguage`
+ * (the language of the content being read), else one for the browser's own
+ * language.
+ */
+export const findBrowserVoice = (voices, preferredVoice, contentLanguage) => {
     if (preferredVoice) {
         return (voices.find((voice) => voice.voiceURI === preferredVoice) ||
             voices.find((voice) => voice.name === preferredVoice) ||
             null);
     }
-    const language = browserLanguage();
-    const languagePrefix = language.split("-")[0] || "en";
-    const matchesLanguage = (voice) => {
-        const voiceLanguage = normalizeLanguageCode(voice.lang);
-        return (voiceLanguage === language ||
-            voiceLanguage.startsWith(`${languagePrefix}-`));
-    };
+    const matchesLanguage = (voice) => browserVoiceMatchesLanguage(voice, contentLanguage);
     const ranked = [
         (voice) => voice.localService && matchesLanguage(voice),
         (voice) => voice.default && matchesLanguage(voice),
@@ -45,6 +126,64 @@ const findBrowserVoice = (voices, preferredVoice) => {
     }
     return voices[0] || null;
 };
+/**
+ * Waits for `synth` to publish its voice inventory, which browsers fill
+ * asynchronously after page load. Resolves `true` once `getVoices()` is
+ * non-empty, at once when it already is; `false` after `timeoutMs`, once
+ * `isCurrent` reports the wait superseded, or when `synth` offers no
+ * `voiceschanged` channel to wait on. A previous `onvoiceschanged` handler
+ * keeps running and is restored afterwards.
+ */
+export const waitForBrowserVoices = (synth, timeoutMs, isCurrent = () => true) => {
+    if (synth.getVoices().length > 0)
+        return Promise.resolve(true);
+    const canUseEventTarget = typeof synth.addEventListener === "function" &&
+        typeof synth.removeEventListener === "function";
+    if (!canUseEventTarget && !("onvoiceschanged" in synth)) {
+        return Promise.resolve(false);
+    }
+    return new Promise((resolve) => {
+        const previousHandler = canUseEventTarget ? null : synth.onvoiceschanged;
+        let settled = false;
+        const finish = (voicesAvailable) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timeout);
+            if (canUseEventTarget) {
+                synth.removeEventListener("voiceschanged", onVoicesChanged);
+            }
+            else if (synth.onvoiceschanged === propertyHandler) {
+                synth.onvoiceschanged = previousHandler;
+            }
+            resolve(voicesAvailable);
+        };
+        const inspectVoiceInventory = () => {
+            if (!isCurrent())
+                finish(false);
+            else if (synth.getVoices().length > 0)
+                finish(true);
+        };
+        const onVoicesChanged = () => inspectVoiceInventory();
+        const propertyHandler = (event) => {
+            try {
+                previousHandler?.call(synth, event);
+            }
+            finally {
+                inspectVoiceInventory();
+            }
+        };
+        const timeout = setTimeout(() => finish(false), timeoutMs);
+        if (canUseEventTarget) {
+            synth.addEventListener("voiceschanged", onVoicesChanged);
+        }
+        else {
+            synth.onvoiceschanged = propertyHandler;
+        }
+        // Close the getVoices()/listener-registration race without polling.
+        inspectVoiceInventory();
+    });
+};
 const shouldAssignBrowserVoice = (voice) => !voice.default;
 /**
  * Browser TTS Provider
@@ -64,17 +203,20 @@ export class BrowserTTSProvider {
         if (!("speechSynthesis" in window)) {
             throw new Error("Browser does not support Speech Synthesis API");
         }
+        // Only a configured voice needs the inventory; waiting otherwise would
+        // hold readiness on a browser that publishes no voices.
+        if (typeof config.voice === "string" && config.voice.trim()) {
+            await waitForBrowserVoices(window.speechSynthesis, VOICE_INVENTORY_TIMEOUT_MS);
+        }
         return new BrowserTTSProviderImpl(config);
     }
-    supportsFeature(feature) {
-        // Browser supports all TTS features
-        return true;
-    }
     getCapabilities() {
         return {
             supportsPause: true,
             supportsResume: true,
             supportsWordBoundary: true,
+            // Boundary events depend on the voice: several network voices send none.
+            defaultHighlightMode: "sentence",
             supportsVoiceSelection: true,
             supportsRateControl: true,
             supportsPitchControl: true,
@@ -95,99 +237,45 @@ class BrowserTTSProviderImpl {
     utterance = null;
     config = null;
     _isPlaying = false;
+    // The run's pause, which outlasts the utterance it paused: a pause between
+    // utterances holds the next one until resume.
     _isPaused = false;
     speakRunId = 0;
-    settlePendingVoiceWait = null;
+    // The run speaking now, between its utterances as well as during them.
+    activeRunId = null;
+    releasePauseHold = null;
     settlePendingChunk = null;
+    settlePendingGap = null;
     onPlaybackStart = undefined;
     constructor(config) {
         this.config = config;
     }
-    waitForBrowserVoices(runId, configuredVoice) {
-        return new Promise((resolve, reject) => {
-            const synth = speechSynthesis;
-            const canUseEventTarget = typeof synth.addEventListener === "function" &&
-                typeof synth.removeEventListener === "function";
-            const previousHandler = canUseEventTarget ? null : synth.onvoiceschanged;
-            let settled = false;
-            let timeout = null;
-            let pendingSettlement = null;
-            const cleanup = () => {
-                if (timeout !== null) {
-                    clearTimeout(timeout);
-                    timeout = null;
-                }
-                if (canUseEventTarget) {
-                    synth.removeEventListener("voiceschanged", onVoicesChanged);
-                }
-                else if (synth.onvoiceschanged === propertyHandler) {
-                    synth.onvoiceschanged = previousHandler;
-                }
-                if (pendingSettlement &&
-                    this.settlePendingVoiceWait === pendingSettlement) {
-                    this.settlePendingVoiceWait = null;
-                }
-            };
-            const finish = (voicesAvailable, error) => {
-                if (settled)
-                    return;
-                settled = true;
-                cleanup();
-                if (error) {
-                    reject(error);
-                }
-                else {
-                    resolve(voicesAvailable);
-                }
-            };
-            const inspectVoiceInventory = () => {
-                if (runId !== this.speakRunId) {
-                    finish(false);
-                    return;
-                }
-                if (synth.getVoices().length > 0)
-                    finish(true);
-            };
-            const onVoicesChanged = () => inspectVoiceInventory();
-            const propertyHandler = (event) => {
-                try {
-                    previousHandler?.call(synth, event);
-                }
-                finally {
-                    inspectVoiceInventory();
-                }
-            };
-            pendingSettlement = () => finish(false);
-            this.settlePendingVoiceWait = pendingSettlement;
-            timeout = setTimeout(() => finish(false, new Error(`Configured browser voice "${configuredVoice}" could not be resolved because the browser did not publish its voice inventory within ${VOICE_INVENTORY_TIMEOUT_MS / 1_000} seconds.`)), VOICE_INVENTORY_TIMEOUT_MS);
-            if (canUseEventTarget) {
-                synth.addEventListener("voiceschanged", onVoicesChanged);
-            }
-            else {
-                synth.onvoiceschanged = propertyHandler;
-            }
-            // Close the getVoices()/listener-registration race without polling.
-            inspectVoiceInventory();
-        });
+    /** Whether run `runId` is still current once any pause between utterances ends. */
+    async waitWhilePaused(runId) {
+        while (this._isPaused && runId === this.speakRunId) {
+            await new Promise((resolve) => {
+                this.releasePauseHold = resolve;
+            });
+        }
+        return runId === this.speakRunId;
+    }
+    releaseHold() {
+        const release = this.releasePauseHold;
+        this.releasePauseHold = null;
+        release?.();
     }
-    async resolveBrowserVoice(runId) {
+    /** The voice to read with; `initialize()` already waited for the inventory. */
+    resolveBrowserVoice() {
         const configuredVoice = typeof this.config?.voice === "string" ? this.config.voice.trim() : "";
-        let voices = speechSynthesis.getVoices();
+        const voices = speechSynthesis.getVoices();
         if (configuredVoice && voices.length === 0) {
-            const voicesAvailable = await this.waitForBrowserVoices(runId, configuredVoice);
-            if (!voicesAvailable || runId !== this.speakRunId) {
-                return { shouldContinue: false, voice: null };
-            }
-            voices = speechSynthesis.getVoices();
-        }
-        if (runId !== this.speakRunId) {
-            return { shouldContinue: false, voice: null };
+            throw new Error(`Configured browser voice "${configuredVoice}" could not be resolved because the browser did not publish its voice inventory within ${VOICE_INVENTORY_TIMEOUT_MS / 1_000} seconds.`);
         }
-        const voice = findBrowserVoice(voices, configuredVoice || undefined);
+        const voice = findBrowserVoice(voices, configuredVoice || undefined, this.getContentLocale());
         if (configuredVoice && !voice) {
             throw new Error(`Configured browser voice "${configuredVoice}" is unavailable. Select a voice exposed by this browser using its voiceURI or name.`);
         }
-        return { shouldContinue: true, voice };
+        return voice;
     }
     async speak(text) {
         if (!this.config) {
@@ -196,19 +284,24 @@ class BrowserTTSProviderImpl {
         // Invalidate any in-flight run and cancel current utterance.
         this.stop();
         const runId = this.speakRunId;
-        const voiceResolution = await this.resolveBrowserVoice(runId);
-        if (!voiceResolution.shouldContinue)
-            return;
-        const chunks = this.splitIntoChunks(text);
-        for (const chunk of chunks) {
-            if (runId !== this.speakRunId) {
-                break;
-            }
-            const shouldContinue = await this.speakChunk(chunk.text, chunk.offset, runId, voiceResolution.voice);
-            if (!shouldContinue) {
-                break;
+        this.activeRunId = runId;
+        try {
+            const voice = this.resolveBrowserVoice();
+            const utterance = toUtteranceText(text);
+            const chunks = this.splitIntoChunks(utterance.text);
+            for (const chunk of chunks) {
+                if (this._isPaused && !(await this.waitWhilePaused(runId)))
+                    break;
+                const shouldContinue = await this.speakChunk(chunk.text, chunk.offset, runId, voice, utterance.sourceOffset);
+                if (!shouldContinue) {
+                    break;
+                }
             }
         }
+        finally {
+            if (this.activeRunId === runId)
+                this.activeRunId = null;
+        }
     }
     async speakSegments(segments) {
         if (!this.config) {
@@ -216,30 +309,45 @@ class BrowserTTSProviderImpl {
         }
         this.stop();
         const runId = this.speakRunId;
-        const voiceResolution = await this.resolveBrowserVoice(runId);
-        if (!voiceResolution.shouldContinue)
-            return;
-        for (const segment of segments) {
-            if (runId !== this.speakRunId)
-                break;
-            const chunks = this.splitIntoChunks(segment.text);
-            for (const chunk of chunks) {
+        this.activeRunId = runId;
+        try {
+            const voice = this.resolveBrowserVoice();
+            for (const segment of segments) {
                 if (runId !== this.speakRunId)
                     break;
-                const shouldContinue = await this.speakChunk(chunk.text, segment.startOffset + chunk.offset, runId, voiceResolution.voice);
-                if (!shouldContinue)
-                    break;
-            }
-            const pauseMsAfter = Math.max(0, Number(segment.pauseMsAfter || 0));
-            if (pauseMsAfter > 0 && runId === this.speakRunId) {
-                const shouldContinue = await this.waitForPause(pauseMsAfter, runId);
-                if (!shouldContinue)
-                    break;
+                const utterance = toUtteranceText(segment.text);
+                const chunks = this.splitIntoChunks(utterance.text);
+                for (const chunk of chunks) {
+                    if (this._isPaused && !(await this.waitWhilePaused(runId)))
+                        break;
+                    const shouldContinue = await this.speakChunk(chunk.text, chunk.offset, runId, voice, (offset) => segment.startOffset + utterance.sourceOffset(offset));
+                    if (!shouldContinue)
+                        break;
+                }
+                const pauseMsAfter = Math.max(0, Number(segment.pauseMsAfter || 0));
+                if (pauseMsAfter > 0 && runId === this.speakRunId) {
+                    const shouldContinue = await this.waitForPause(pauseMsAfter, runId);
+                    if (!shouldContinue)
+                        break;
+                }
             }
         }
+        finally {
+            if (this.activeRunId === runId)
+                this.activeRunId = null;
+        }
     }
     async waitForPause(pauseMs, runId) {
-        await new Promise((resolve) => setTimeout(resolve, pauseMs));
+        await new Promise((resolve) => {
+            const settle = () => {
+                clearTimeout(timer);
+                if (this.settlePendingGap === settle)
+                    this.settlePendingGap = null;
+                resolve();
+            };
+            const timer = setTimeout(settle, pauseMs);
+            this.settlePendingGap = settle;
+        });
         return runId === this.speakRunId;
     }
     splitIntoChunks(text) {
@@ -277,13 +385,19 @@ class BrowserTTSProviderImpl {
         }
         return chunks.length ? chunks : [{ text, offset: 0 }];
     }
-    getHighlightMode() {
-        const providerOptions = (this.config?.providerOptions || {});
-        return providerOptions.highlightMode === "word" ? "word" : "sentence";
+    /**
+     * The language of the content being read, which the toolkit sets per speak as
+     * `providerOptions.contentLanguage` when the content or the host names one.
+     */
+    getContentLocale() {
+        const language = this.config?.providerOptions?.contentLanguage;
+        return typeof language === "string" && language.trim()
+            ? language.trim()
+            : undefined;
     }
     getSegmentationPolicy() {
-        const providerOptions = (this.config?.providerOptions || {});
-        const segmenter = (providerOptions.segmenter || {});
+        const providerOptions = this.config?.providerOptions || {};
+        const segmenter = providerOptions.segmenter || {};
         const mode = segmenter.mode;
         const useSegmenter = mode !== "regexOnly";
         const locale = typeof segmenter.locale === "string" && segmenter.locale.trim().length > 0
@@ -311,7 +425,11 @@ class BrowserTTSProviderImpl {
         const match = slice.match(/^\s*([^\s]+)/);
         return match?.[1]?.length || 1;
     }
-    async speakChunk(chunkText, chunkOffset, runId, voice) {
+    /**
+     * `chunkOffset` places the chunk in the utterance text; `sourceOffset` maps an
+     * utterance offset to the boundary position reported for it.
+     */
+    async speakChunk(chunkText, chunkOffset, runId, voice, sourceOffset) {
         return new Promise((resolve, reject) => {
             if (runId !== this.speakRunId) {
                 resolve(false);
@@ -319,7 +437,11 @@ class BrowserTTSProviderImpl {
             }
             const utterance = new SpeechSynthesisUtterance(chunkText);
             this.utterance = utterance;
-            // Apply config
+            // Apply config. The language tells the engine how to read the text, and
+            // lets it choose a voice when the default one is left in place.
+            const contentLocale = this.getContentLocale();
+            if (contentLocale)
+                utterance.lang = contentLocale;
             if (voice && shouldAssignBrowserVoice(voice))
                 utterance.voice = voice;
             if (this.config?.rate)
@@ -362,7 +484,6 @@ class BrowserTTSProviderImpl {
                 }
                 if (runId === this.speakRunId) {
                     this._isPlaying = false;
-                    this._isPaused = false;
                 }
                 if (error) {
                     reject(error);
@@ -384,7 +505,9 @@ class BrowserTTSProviderImpl {
                 didStart = true;
                 clearStartTimeout();
                 this._isPlaying = true;
-                this._isPaused = false;
+                // A pause that landed while the engine was starting holds this utterance.
+                if (this._isPaused)
+                    speechSynthesis.pause();
                 try {
                     this.onPlaybackStart?.();
                 }
@@ -427,12 +550,9 @@ class BrowserTTSProviderImpl {
             utterance.onboundary = (event) => {
                 if (runId !== this.speakRunId)
                     return;
-                console.log("[BrowserProvider] Boundary event:", event.name, "charIndex:", event.charIndex, "charLength:", event.charLength);
+                logger.debug(`boundary event: ${event.name}, charIndex ${event.charIndex}, charLength ${event.charLength}`);
                 if (event.name !== "word" || !this.onWordBoundary)
                     return;
-                if (this.getHighlightMode() === "sentence") {
-                    return;
-                }
                 const charIndex = Math.max(0, Math.min(event.charIndex, Math.max(0, chunkText.length - 1)));
                 const reportedLength = Number(event.charLength || 0);
                 const inferredLength = this.inferWordLength(chunkText, charIndex);
@@ -445,9 +565,13 @@ class BrowserTTSProviderImpl {
                 const word = chunkText
                     .substring(charIndex, Math.min(chunkText.length, charIndex + wordLength))
                     .trim();
-                const absoluteBoundaryStart = chunkOffset + charIndex;
-                console.log("[BrowserProvider] Calling onWordBoundary with word:", word, "at position:", absoluteBoundaryStart);
-                this.onWordBoundary(word, absoluteBoundaryStart, wordLength);
+                const utteranceStart = chunkOffset + charIndex;
+                const absoluteBoundaryStart = sourceOffset(utteranceStart);
+                const boundaryLength = sourceOffset(utteranceStart + wordLength - 1) +
+                    1 -
+                    absoluteBoundaryStart;
+                logger.debug(`word boundary "${word}" at ${absoluteBoundaryStart}`);
+                this.onWordBoundary(word, absoluteBoundaryStart, boundaryLength);
             };
             startTimeout = setTimeout(() => {
                 if (settled || didStart || runId !== this.speakRunId)
@@ -466,27 +590,39 @@ class BrowserTTSProviderImpl {
         });
     }
     pause() {
-        if (this._isPlaying && !this._isPaused) {
+        if (this._isPaused)
+            return;
+        if (this._isPlaying) {
             speechSynthesis.pause();
+            this._isPaused = true;
+        }
+        else if (this.activeRunId === this.speakRunId) {
+            // Between utterances, or in a structural gap: the next one holds.
+            this._isPaused = true;
         }
     }
     resume() {
-        if (this._isPlaying && this._isPaused) {
+        if (!this._isPaused)
+            return;
+        this._isPaused = false;
+        if (this._isPlaying)
             speechSynthesis.resume();
-        }
+        this.releaseHold();
     }
     stop() {
         const utterance = this.utterance;
         const shouldCancel = this._isPlaying || utterance !== null;
         this.speakRunId += 1;
-        this.settlePendingVoiceWait?.();
+        this.activeRunId = null;
         this.settlePendingChunk?.();
+        this.settlePendingGap?.();
         if (shouldCancel) {
             speechSynthesis.cancel();
         }
         this.utterance = null;
         this._isPlaying = false;
         this._isPaused = false;
+        this.releaseHold();
     }
     isPlaying() {
         return this._isPlaying && !this._isPaused;

package/dist/services/tts/catalog-span-alignment.d.ts

@@ -1,11 +1,5 @@
+import { type SpeechSourceTokenization } from "./math-alignment/speech-tokenizer.js";
 export type CatalogChunkPlaybackMode = "exact-word" | "anchor-span" | "region-fallback";
-export type BoundaryOffsetMode = "plain-spoken-text" | "raw-ssml" | "unsupported";
-export interface CatalogTextToken {
-    text: string;
-    normalized: string;
-    start: number;
-    end: number;
-}
 export interface CatalogSpanAnchor {
     spokenStart: number;
     spokenEnd: number;
@@ -14,25 +8,16 @@ export interface CatalogSpanAnchor {
     score: number;
 }
 export interface CatalogSpanAlignment {
-    speechText: string;
-    spokenText: string;
+    speech: SpeechSourceTokenization;
     visibleText: string;
     playbackMode: CatalogChunkPlaybackMode;
-    boundaryOffsetMode: BoundaryOffsetMode;
     anchors: CatalogSpanAnchor[];
     confidence: number;
-    spokenTokens: CatalogTextToken[];
-    visibleTokens: CatalogTextToken[];
-    rawToSpokenOffsetMap: Map<number, number>;
 }
 export declare const createCatalogSpanAlignment: (args: {
     speechText: string;
     visibleText: string;
 }) => CatalogSpanAlignment;
-export declare const resolveSpokenBoundaryOffset: (alignment: CatalogSpanAlignment, position: number, length?: number, boundaryWord?: string) => {
-    start: number;
-    length: number;
-} | null;
 export declare const resolveVisibleSpanForBoundary: (alignment: CatalogSpanAlignment, spokenOffset: number) => {
     start: number;
     end: number;