bloom-player 2.20.0-alpha.2 → 2.20.0-alpha.4

package/package.json

@@ -3,7 +3,7 @@
     "description": "A library for displaying Bloom books in iframes or WebViews",
     "author": "SIL Global",
     "license": "MIT",
-    "version": "2.20.0-alpha.2",
+    "version": "2.20.0-alpha.4",
     "packageManager": "pnpm@11.1.2",
     "private": false,
     "// sideeffects might need to be ['*.css'] to avoid 'shaking' our CSS, if we ever get tree shaking working": "",

package/src/shared/dragActivityRuntime.ts

@@ -235,6 +235,9 @@ export function prepareActivity(
 
     prepareOrderSentenceActivity(page);
 
+    // Start preloading correct/wrong sounds now so they are buffered by the time the user answers.
+    preloadSoundsForActivity(page);
+
     // Slider:     // for drag-word-chooser-slider
     //     setupWordChooserSlider(page);
     //     setSlideablesVisibility(page, false);
@@ -346,6 +349,9 @@ export function undoPrepareActivity(page: HTMLElement) {
     ).forEach((elt: HTMLElement) => {
         elt.parentElement?.removeChild(elt);
     });
+
+    cleanupActivitySounds(page);
+
     const inPlayer = page.closest(".swiper-slide") !== null;
     doShowAnswersInTargets(!inPlayer, page);
     //Slider: setSlideablesVisibility(page, true);
@@ -848,14 +854,95 @@ export function setDefaultSoundUrls(
     defaultWrongSoundUrl = wrongSoundUrl;
 }
 
-function showCorrectOrWrongItems(page: HTMLElement, correct: boolean) {
-    classSetter(page, "drag-activity-correct", correct);
-    classSetter(page, "drag-activity-wrong", !correct);
+// Attribute used to mark audio elements preloaded for correct/wrong sounds.
+const kPreloadSoundAttr = "data-preload-sound";
+
+// Remove any audio elements previously added by preloadSoundsForActivity.
+// Called from undoPrepareActivity and ActivityContext.stop().
+export function cleanupActivitySounds(page: HTMLElement): void {
+    page.querySelectorAll<HTMLAudioElement>(
+        `audio[${kPreloadSoundAttr}]`,
+    ).forEach((audio) => {
+        audio.src = "";
+        audio.remove();
+    });
+}
+
+// Preload both correct and wrong sounds so they are buffered before the user answers.
+// Called from prepareActivity so loading starts as soon as the page is shown.
+// Also called from ActivityContext so the sounds are ready for non-drag activities.
+export function preloadSoundsForActivity(page: HTMLElement): void {
+    const preloadIfNeeded = (
+        soundFile: string | null,
+        defaultUrl: string | undefined,
+    ): void => {
+        const addPrefix = soundFile !== null;
+        if (soundFile === null) {
+            soundFile = defaultUrl ?? null;
+        } else if (soundFile === "none") {
+            return;
+        }
+        if (!soundFile) return;
+
+        const url = (addPrefix ? urlPrefix() + "/audio/" : "") + soundFile;
+        // Avoid duplicate preloads (e.g. if prepareActivity is called more than once)
+        const already = Array.from(
+            page.querySelectorAll<HTMLAudioElement>(`audio[${kPreloadSoundAttr}]`),
+        ).find((a) => a.dataset.preloadSound === url);
+        if (already) return;
+
+        const audio = document.createElement("audio");
+        audio.dataset.preloadSound = url;
+        audio.style.visibility = "hidden";
+        if (IsRunningOnBloomDesktop(page)) {
+            audio.classList.add("bloom-ui");
+        }
+        audio.src = url;
+        audio.load();
+        page.append(audio);
+    };
 
-    // play sound
+    preloadIfNeeded(
+        page.getAttribute("data-correct-sound"),
+        defaultCorrectSoundUrl,
+    );
+    preloadIfNeeded(
+        page.getAttribute("data-wrong-sound"),
+        defaultWrongSoundUrl,
+    );
+}
+
+// Play the correct or wrong sound for the given page, then call `then` when done (or immediately
+// if there is no sound). Handles the data-correct-sound / data-wrong-sound attribute and the
+// registered default URLs. Exported so ActivityContext can share this logic.
+export function playCorrectOrWrongSound(
+    page: HTMLElement,
+    correct: boolean,
+    then?: () => void,
+): void {
     let soundFile = page.getAttribute(
         correct ? "data-correct-sound" : "data-wrong-sound",
     );
+    // if the attribute is not there at all, use the default sound, if one has been set.
+    // The default is not relative to the audio folder in the book, so we won't add a prefix.
+    const addPrefix = soundFile !== null;
+    if (soundFile === null) {
+        soundFile = correct ? defaultCorrectSoundUrl : defaultWrongSoundUrl;
+    } else if (soundFile === "none") {
+        // explicitly no sound
+        soundFile = undefined;
+    }
+    if (soundFile) {
+        playSound(page, soundFile, addPrefix, then);
+    } else {
+        then?.();
+    }
+}
+
+function showCorrectOrWrongItems(page: HTMLElement, correct: boolean) {
+    classSetter(page, "drag-activity-correct", correct);
+    classSetter(page, "drag-activity-wrong", !correct);
+
     const playOtherStuff = () => {
         const elementsMadeVisible = Array.from(
             page.getElementsByClassName(
@@ -871,21 +958,7 @@ function showCorrectOrWrongItems(page: HTMLElement, correct: boolean) {
         const playables = getAudioSentences(possibleNarrationElements);
         playAllVideo(videoElements, () => playAllAudio(playables, page));
     };
-    // if the attribute is not there at all, use the default sound, if one has been set.
-    // This is not in relative to the audio folder in the book, so we won't add a prefix
-    // in that case.
-    const addPrefix = soundFile !== null;
-    if (soundFile === null) {
-        soundFile = correct ? defaultCorrectSoundUrl : defaultWrongSoundUrl;
-    } else if (soundFile === "none") {
-        // explicity no sound, go straight to other stuff if any
-        soundFile = undefined;
-    }
-    if (soundFile) {
-        playSound(page, soundFile, addPrefix, playOtherStuff);
-    } else {
-        playOtherStuff();
-    }
+    playCorrectOrWrongSound(page, correct, playOtherStuff);
 }
 
 function playSound(
@@ -894,9 +967,31 @@ function playSound(
     addPrefix = true,
     then?: () => void,
 ) {
-    const audio = new Audio(
-        (addPrefix ? urlPrefix() + "/audio/" : "") + soundFile,
-    );
+    const url = (addPrefix ? urlPrefix() + "/audio/" : "") + soundFile;
+
+    // Reuse a preloaded element if one was prepared by preloadSoundsForActivity.
+    // This avoids re-downloading the file and removes the delay before playback starts.
+    const preloaded = Array.from(
+        someElt.querySelectorAll<HTMLAudioElement>(`audio[${kPreloadSoundAttr}]`),
+    ).find((a) => a.dataset.preloadSound === url);
+
+    let audio: HTMLAudioElement;
+    if (preloaded) {
+        delete preloaded.dataset.preloadSound; // mark as consumed
+        preloaded.currentTime = 0;
+        audio = preloaded;
+    } else {
+        audio = new Audio(url);
+        if (IsRunningOnBloomDesktop(someElt)) {
+            audio.classList.add("bloom-ui"); // in case remove code fails, should make sure it doesn't get saved.
+        }
+        audio.style.visibility = "hidden";
+        // To my surprise, in BP storybook it works without adding the audio to any document.
+        // But in Bloom proper, it does not. I think it is because this code is part of the toolbox,
+        // so the audio element doesn't have the right context to interpret the relative URL.
+        someElt.append(audio);
+    }
+
     let finished = false;
     const finish = () => {
         if (finished) {
@@ -905,14 +1000,6 @@ function playSound(
         finished = true;
         then?.();
     };
-    if (IsRunningOnBloomDesktop(someElt)) {
-        audio.classList.add("bloom-ui"); // in case remove code fails, should make sure it doesn't get saved.
-    }
-    audio.style.visibility = "hidden";
-    // To my surprise, in BP storybook it works without adding the audio to any document.
-    // But in Bloom proper, it does not. I think it is because this code is part of the toolbox,
-    // so the audio element doesn't have the right context to interpret the relative URL.
-    someElt.append(audio);
     const playPromise = audio.play();
     playPromise?.catch(() => {
         finish();

package/src/shared/narration.ts

@@ -312,6 +312,15 @@ export function playAllAudio(elements: HTMLElement[], page: HTMLElement): void {
     }
 
     const firstElementToPlay = elementsToPlayConsecutivelyStack[stackSize - 1]; // Remember to pop it when you're done playing it. (i.e., in playEnded)
+
+    // Discard any preloaded audio from the previous page, then start preloading the second
+    // segment now so the browser has the full duration of the first segment to download it.
+    // (The first segment is loaded normally by the setSoundAndHighlight call below.)
+    clearPreloadedAudio();
+    if (stackSize >= 2) {
+        preloadAudioForElement(elementsToPlayConsecutivelyStack[stackSize - 2]);
+    }
+
     // At one point it seemed to help something to delete the media player and make a new one each time.
     // I didn't comment this at the time, but my recollection is that this could help with some cases
     // where the old one was in a bad state, such as in the middle of pausing.
@@ -558,6 +567,18 @@ function playCurrentInternal() {
             ++currentAudioSessionNum;
             audioPlayCurrentStartTime = new Date().getTime();
             highlightNextSubElement(currentAudioSessionNum);
+
+            // While this segment plays, preload the next one so its audio is buffered by the
+            // time we need it.  The current segment is at stack[length-1] (not yet popped), so
+            // the next is at stack[length-2].
+            const nextPreloadIndex =
+                elementsToPlayConsecutivelyStack.length - 2;
+            if (nextPreloadIndex >= 0) {
+                preloadAudioForElement(
+                    elementsToPlayConsecutivelyStack[nextPreloadIndex],
+                );
+            }
+
             handlePlayPromise(promise);
         }
     }
@@ -796,7 +817,12 @@ function setHighlightTo({
     // narration and drag activity text.  See BL-14797
     const hasError = mediaPlayer.error !== null;
     if (!hasError && disableHighlightIfNoAudio) {
-        const isAlreadyPlaying = mediaPlayer.currentTime > 0;
+        // Use paused/ended to detect whether audio is actively playing.
+        // currentTime > 0 was the old check, but it also returns true right after a clip ends
+        // (the player is paused but currentTime is non-zero), which incorrectly skipped suppression
+        // for the next segment. The Soft Split case (one audio file, multiple highlighted sentences)
+        // still works because the player is neither paused nor ended while playing.
+        const isAlreadyPlaying = !mediaPlayer.paused && !mediaPlayer.ended;
         // If it's already playing, no need to disable (Especially in the Soft Split case, where only one file is playing but multiple sentences need to be highlighted).
         if (!isAlreadyPlaying) {
             // Start off in a highlight-disabled state so we don't display any momentary highlight for cases where there is no audio for this element.
@@ -805,13 +831,36 @@ function setHighlightTo({
             newElement.classList.add(kSuppressHighlightClass);
             // When it starts playing, we know we really have such an audio file, so we can stop
             // suppressing the highlight.
-            mediaPlayer.addEventListener("playing", () => {
+            // Remove any previous suppress listeners before adding new ones so they don't
+            // accumulate across segments (we keep explicit references instead of { once: true }
+            // because { once: true } on the error listener broke auto-advance).
+            if (currentSuppressPlayingListener) {
+                mediaPlayer.removeEventListener(
+                    "playing",
+                    currentSuppressPlayingListener,
+                );
+            }
+            if (currentSuppressErrorListener) {
+                mediaPlayer.removeEventListener(
+                    "error",
+                    currentSuppressErrorListener,
+                );
+            }
+            currentSuppressPlayingListener = () => {
                 newElement.classList.remove(kSuppressHighlightClass);
-            });
-            mediaPlayer.addEventListener("error", () => {
+            };
+            currentSuppressErrorListener = () => {
                 newElement.classList.remove("ui-audioCurrent");
                 newElement.classList.remove(kSuppressHighlightClass);
-            });
+            };
+            mediaPlayer.addEventListener(
+                "playing",
+                currentSuppressPlayingListener,
+            );
+            mediaPlayer.addEventListener(
+                "error",
+                currentSuppressErrorListener,
+            );
         }
     }
 
@@ -957,6 +1006,55 @@ function setCurrentAudioId(id: string) {
     }
 }
 
+// When we know which audio element comes next, we preload it on a hidden audio element so the
+// browser starts downloading before we actually need to play it. We generate the URL once (with
+// a fixed timestamp) and store it; updatePlayerStatus reuses that same URL so the browser can
+// serve the response from its cache rather than fetching again.
+let preloadedAudioId: string = "";
+let preloadedAudioSrc: string = "";
+
+// Suppress-highlight listeners attached to the shared media player.  We keep explicit references
+// so we can remove them before attaching new ones for the next segment — ensuring at most one of
+// each exists at any time without relying on { once: true }, which caused auto-advance to break.
+let currentSuppressPlayingListener: EventListener | null = null;
+let currentSuppressErrorListener: EventListener | null = null;
+
+function getPreloadPlayer(): HTMLAudioElement {
+    let el = document.getElementById(
+        "bloom-audio-preload",
+    ) as HTMLAudioElement | null;
+    if (!el) {
+        el = document.createElement("audio");
+        el.id = "bloom-audio-preload";
+        document.body.appendChild(el);
+    }
+    return el;
+}
+
+function preloadAudioForElement(element: Element): void {
+    const firstAudioSentence = getFirstAudioSentenceWithinElement(element);
+    const id = firstAudioSentence ? firstAudioSentence.id : element.id;
+    if (preloadedAudioId === id) return; // already preloading this one
+
+    const url =
+        currentAudioUrl(id) +
+        "?nocache=" +
+        new Date().getTime() +
+        "&optional=true";
+    preloadedAudioId = id;
+    preloadedAudioSrc = url;
+
+    const preloadPlayer = getPreloadPlayer();
+    preloadPlayer.src = url;
+    // load() tells the browser to actively fetch the resource rather than waiting.
+    preloadPlayer.load();
+}
+
+function clearPreloadedAudio(): void {
+    preloadedAudioId = "";
+    preloadedAudioSrc = "";
+}
+
 function updatePlayerStatus() {
     const player = getPlayer();
     if (!player) {
@@ -970,15 +1068,21 @@ function updatePlayerStatus() {
     }
     const url = currentAudioUrl(currentAudioId);
     logNarration(url);
-    // because this code is meant to work in both Bloom and BloomPlayer, we can't call a Bloom API to find
-    // out whether we actually have a recording (as we well might not, if we just opened the talking book
-    // tool and haven't recorded anything yet). So we just try to play it and see what happens.
-    // The optional param tells Bloom not to report an error if the file isn't found, and is ignored in
-    // other contexts.
-    player.setAttribute(
-        "src",
-        url + "?nocache=" + new Date().getTime() + "&optional=true",
-    );
+    // If we preloaded this audio segment, reuse the same URL so the browser can serve the
+    // already-downloaded response from its cache instead of issuing a new request.
+    let src: string;
+    if (preloadedAudioId === currentAudioId && preloadedAudioSrc) {
+        src = preloadedAudioSrc;
+        clearPreloadedAudio();
+    } else {
+        // because this code is meant to work in both Bloom and BloomPlayer, we can't call a Bloom API to find
+        // out whether we actually have a recording (as we well might not, if we just opened the talking book
+        // tool and haven't recorded anything yet). So we just try to play it and see what happens.
+        // The optional param tells Bloom not to report an error if the file isn't found, and is ignored in
+        // other contexts.
+        src = url + "?nocache=" + new Date().getTime() + "&optional=true";
+    }
+    player.setAttribute("src", src);
 }
 
 function currentAudioUrl(id: string): string {
@@ -1454,6 +1558,74 @@ function playAllVideoInternal(
         hideVideoError(video);
         hideVideoAutoplayBlockedHint(video);
         setCurrentPlaybackMode(PlaybackMode.VideoPlaying);
+        // Always play each queued video from the beginning.
+        // Without this, a previously played element may remain at end-of-stream
+        // and fail to raise the expected ended event for sequencing.
+        video.currentTime = 0;
+        // Note that we get a new instance of this for each recursive call to playAllVideoInternal.
+        // It serves to make sure we don't try to advance twice if we get both an ended and a pause event.
+        let advanced = false;
+        let watchdogTimerId: number | undefined;
+        const watchdogGraceMs = 250;
+        const clearWatchdog = () => {
+            if (watchdogTimerId !== undefined) {
+                window.clearTimeout(watchdogTimerId);
+                watchdogTimerId = undefined;
+            }
+        };
+        const scheduleWatchdogFromDuration = () => {
+            const duration = video.duration;
+            if (!Number.isFinite(duration) || duration <= 0) {
+                return;
+            }
+            clearWatchdog();
+            const playbackRate =
+                Number.isFinite(video.playbackRate) && video.playbackRate > 0
+                    ? video.playbackRate
+                    : 1;
+            const timeoutMs = Math.max(
+                duration * 1000 / playbackRate + watchdogGraceMs,
+                watchdogGraceMs,
+            );
+            watchdogTimerId = window.setTimeout(() => {
+                advanceToNextVideo();
+            }, timeoutMs);
+        };
+        const advanceToNextVideo = () => {
+            if (advanced) {
+                return;
+            }
+            advanced = true;
+            clearWatchdog();
+            video.removeEventListener("ended", endedHandler);
+            video.removeEventListener("pause", pauseHandler);
+            video.removeEventListener("loadedmetadata", metadataHandler);
+            if (generation !== playAllVideoGeneration) {
+                return;
+            }
+            playAllVideoInternal(elements.slice(1), then, generation);
+        };
+        const endedHandler = () => {
+            advanceToNextVideo();
+        };
+        // In some environments, playback can pause at the end without raising ended.
+        // Treat that as completion so a short first video can't stall the sequence.
+        const pauseHandler = () => {
+            const duration = video.duration;
+            if (!Number.isFinite(duration) || duration <= 0) {
+                return;
+            }
+            if (video.currentTime >= duration - 0.05) {
+                advanceToNextVideo();
+            }
+        };
+        const metadataHandler = () => {
+            scheduleWatchdogFromDuration();
+        };
+        // Register ended before play() so we don't miss extremely fast end transitions.
+        video.addEventListener("ended", endedHandler, { once: true });
+        video.addEventListener("pause", pauseHandler);
+        video.addEventListener("loadedmetadata", metadataHandler);
         const promise = video.play();
         promise
             .then(() => {
@@ -1462,34 +1634,16 @@ function playAllVideoInternal(
                 }
                 transientVideoRetryCounts.delete(video);
                 hideVideoAutoplayBlockedHint(video);
-                // The promise resolves when the video starts playing. We want to know when it ends.
-                // Note: in Bloom Desktop, sometimes this event does not fire normally, even when the video is
-                // played to the end.  I have not figured out why. It may be something to do with how we are
-                // trimming the videos.
-                // In Bloom Desktop, this is worked around by raising the ended event when we detect that it has
-                // paused past the end point in resetToStartAfterPlayingToEndPoint.
-                // In BloomPlayer,I don't think this is a problem. Videos are trimmed when published, so we always
-                // play to the real end (unless the user pauses). So one way or another, we should get the ended
-                // event.
-                video.addEventListener(
-                    "ended",
-                    () => {
-                        if (generation !== playAllVideoGeneration) {
-                            return;
-                        }
-                        playAllVideoInternal(
-                            elements.slice(1),
-                            then,
-                            generation,
-                        );
-                    },
-                    { once: true },
-                );
+                scheduleWatchdogFromDuration();
             })
             .catch((reason) => {
                 if (generation !== playAllVideoGeneration) {
                     return;
                 }
+                clearWatchdog();
+                video.removeEventListener("ended", endedHandler);
+                video.removeEventListener("pause", pauseHandler);
+                video.removeEventListener("loadedmetadata", metadataHandler);
                 if (reason?.name === "NotAllowedError") {
                     console.debug(
                         "Video autoplay blocked until user interaction",