@readium/navigator 2.7.0-alpha.3 → 2.7.0-alpha.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@readium/navigator",
-  "version": "2.7.0-alpha.3",
+  "version": "2.7.0-alpha.4",
   "type": "module",
   "description": "Next generation SDK for publications in Web Apps",
   "author": "readium",

package/src/audio/AudioNavigator.ts

@@ -1,7 +1,7 @@
 import { Link, Locator, LocatorLocations, Publication, Timeline, TimelineItem } from "@readium/shared";
 import { MediaNavigator, IContentProtectionConfig, IKeyboardPeripheralsConfig, KeyboardPeripheralEventData } from "../Navigator.ts";
 import { Configurable } from "../preferences/Configurable.ts";
-import { WebAudioEngine, PlaybackState } from "./engine/index.ts";
+import { WebAudioEngine, PlaybackState, AudioMseLoaderFactory } from "./engine/index.ts";
 import {
     AudioPreferences,
     AudioDefaults,
@@ -69,6 +69,26 @@ export interface AudioNavigatorConfiguration {
     defaults: IAudioDefaults;
     contentProtection?: IAudioContentProtectionConfig;
     keyboardPeripherals?: IKeyboardPeripheralsConfig;
+
+    /**
+     * Called with the persistent playback element before
+     * the first src is assigned, so the host can prepare it. Use for MSE/EME
+     * setups. When it returns a promise, loading of
+     * the initial track (and prefetching of adjacent ones) is deferred until
+     * the promise settles; a rejection is forwarded to the error listener and
+     * loading proceeds anyway.
+     */
+    mediaElementSetup?: (element: HTMLMediaElement) => void | Promise<void>;
+
+    /**
+     * When provided, media bytes reach the playback element through Media
+     * Source Extensions instead of direct `src` assignment: the engine
+     * creates one loader per track via this factory and the loader owns all
+     * fetching. Supply one for e.g. EME encrypted audio, since browsers have
+     * poor support for encrypted audio directly through `src`. The container
+     * handling (WebM, fMP4, …) is the loader implementation's concern.
+     */
+    mseLoaderFactory?: AudioMseLoaderFactory;
 }
 
 export class AudioNavigator extends MediaNavigator implements Configurable<AudioSettings, AudioPreferences> {
@@ -94,6 +114,8 @@ export class AudioNavigator extends MediaNavigator implements Configurable<Audio
     /** True while a track transition is in progress; suppresses spurious mid-navigation events. */
     private _isNavigating: boolean = false;
     private _isStalled: boolean = false;
+    /** Set by destroy(); stops the deferred initial load from touching a dead instance. */
+    private _destroyed: boolean = false;
     private _stalledWatchdog: ReturnType<typeof setInterval> | null = null;
     private _stalledCheckTime: number = 0;
 
@@ -131,9 +153,19 @@ export class AudioNavigator extends MediaNavigator implements Configurable<Audio
         }
 
         const initialHref = this.currentLocation.href.split("#")[0];
-        const trackIndex = this.hrefToTrackIndex(initialHref);
+        let trackIndex = this.hrefToTrackIndex(initialHref);
         if (trackIndex === -1) {
-            throw new Error(`AudioNavigator: initial href "${ initialHref }" not found in reading order`);
+            // Progression-only locators (e.g. restored from an OPDS progression)
+            // Resolve totalProgression against the
+            // cumulative track durations instead of failing outright.
+            const totalProgression = this.currentLocation.locations?.totalProgression;
+            if (totalProgression !== undefined) {
+                const resolved = this.locatorFromTotalProgression(totalProgression);
+                this.currentLocation = resolved.locator;
+                trackIndex = resolved.trackIndex;
+            } else {
+                throw new Error(`AudioNavigator: initial href "${ initialHref }" not found in reading order`);
+            }
         }
         const initialTime = this.currentLocation.locations?.time() || 0;
 
@@ -145,7 +177,8 @@ export class AudioNavigator extends MediaNavigator implements Configurable<Audio
                 } as PlaybackState,
                 playWhenReady: false,
                 index: trackIndex
-            }
+            },
+            mseLoaderFactory: configuration.mseLoaderFactory,
         });
 
         this.pool = new AudioPoolManager(audioEngine, publication, configuration.contentProtection);
@@ -188,6 +221,28 @@ export class AudioNavigator extends MediaNavigator implements Configurable<Audio
         this.setupEventListeners();
 
         this._isNavigating = true;
+
+        const startInitialLoad = () => this.startInitialLoad(trackIndex, initialTime);
+        if (configuration.mediaElementSetup) {
+            // Defer the initial load until the host has prepared the element
+            // (e.g. attached MediaKeys for EME) so no media data is fetched or
+            // decoded before protection is in place.
+            Promise.resolve()
+                .then(() => configuration.mediaElementSetup!(this.pool.audioEngine.getMediaElement()))
+                .catch((error) => { this.listeners.error(error, this.currentLocator); })
+                .then(startInitialLoad);
+        } else {
+            startInitialLoad();
+        }
+    }
+
+    /** Sets the initial track on the primary element and seeks to the starting position. */
+    private startInitialLoad(trackIndex: number, initialTime: number): void {
+        // The navigator may have been destroyed while an async
+        // mediaElementSetup was pending (React StrictMode does this in dev) —
+        // don't start loading media on the orphaned element.
+        if (this._destroyed) return;
+
         this.pool.setCurrentAudio(trackIndex, "forward");
 
         // applyPreferences() must come after setCurrentAudio() so that the src
@@ -289,6 +344,47 @@ export class AudioNavigator extends MediaNavigator implements Configurable<Audio
         return this.hrefToTrackIndex(this.currentLocation.href);
     }
 
+    /**
+     * Builds a locator for an overall publication progression (0–1) by walking
+     * the cumulative track durations. Falls back to the start of the first
+     * track when the reading order carries no duration metadata.
+     */
+    private locatorFromTotalProgression(totalProgression: number): { locator: Locator, trackIndex: number } {
+        const items = this.pub.readingOrder.items;
+        const durations = items.map(link => link.duration ?? 0);
+        const totalDuration = durations.reduce((sum, d) => sum + d, 0);
+
+        let trackIndex = 0;
+        let time = 0;
+        if (totalDuration > 0) {
+            let remaining = Math.min(Math.max(totalProgression, 0), 1) * totalDuration;
+            for (let i = 0; i < durations.length; i++) {
+                trackIndex = i;
+                if (remaining <= durations[i] || i === durations.length - 1) {
+                    time = remaining;
+                    break;
+                }
+                remaining -= durations[i];
+            }
+        }
+
+        const link = items[trackIndex];
+        return {
+            trackIndex,
+            locator: new Locator({
+                href: link.href,
+                type: link.type || "", // Should have a mimetype
+                title: link.title,
+                locations: new LocatorLocations({
+                    position: trackIndex + 1,
+                    progression: durations[trackIndex] > 0 ? time / durations[trackIndex] : 0,
+                    totalProgression: Math.min(Math.max(totalProgression, 0), 1),
+                    fragments: [`t=${ time }`]
+                })
+            })
+        };
+    }
+
     get currentLocator(): Locator {
         return this.currentLocation;
     }
@@ -384,7 +480,11 @@ export class AudioNavigator extends MediaNavigator implements Configurable<Audio
                 fragments: [`t=${this.duration}`]
             }));
             this.listeners.trackEnded(this.currentLocator);
-            if (!this.canGoForward) return;
+            if (!this.canGoForward) {
+                // Set final progress in audiobook
+                this.listeners.positionChanged(this.currentLocator);
+                return;
+            }
             await this.nextTrack();
             if (this._settings.autoPlay) this.play();
         });
@@ -698,6 +798,7 @@ export class AudioNavigator extends MediaNavigator implements Configurable<Audio
     }
 
     destroy(): void {
+        this._destroyed = true;
         this.stopPositionPolling();
         this._stopStalledWatchdog();
         this.destroyMediaSession();

package/src/audio/AudioPoolManager.ts

@@ -38,19 +38,20 @@ export class AudioPoolManager {
         return supported;
     }
 
-    private pickPlayableHref(link: Link): string {
+    private pickPlayableHref(link: Link): { href: string, type?: string } {
         const base = this._publication.baseURL;
         const candidates = [link, ...(link.alternates?.items ?? [])];
-        let best: { href: string; confidence: "probably" | "maybe" } | undefined;
+        let best: { href: string; type?: string; confidence: "probably" | "maybe" } | undefined;
         for (const candidate of candidates) {
             if (!candidate.type) continue;
             const confidence = this._supportedAudioTypes.get(candidate.type);
             if (!confidence) continue;
             const href = candidate.toURL(base) ?? candidate.href;
-            if (confidence === "probably") return href;
-            if (!best) best = { href, confidence };
+            if (confidence === "probably") return { href, type: candidate.type };
+            if (!best) best = { href, type: candidate.type, confidence };
         }
-        return best?.href ?? (link.toURL(base) ?? link.href);
+        if (best) return { href: best.href, type: best.type };
+        return { href: link.toURL(base) ?? link.href, type: link.type };
     }
 
     get audioEngine(): WebAudioEngine {
@@ -84,12 +85,16 @@ export class AudioPoolManager {
      * The current track is excluded — the primary engine element represents it.
      */
     private update(currentIndex: number): void {
+        // Progressive preload elements would feed encrypted media through the
+        // broken src= demuxer path — the MSE loader owns fetching instead
+        if (this._audioEngine.usesMse) return;
+
         const items = this._publication.readingOrder.items;
         const keep = new Set<string>();
 
         for (let j = 0; j < items.length; j++) {
             if (j === currentIndex) continue; // primary element handles the current track
-            const href = this.pickPlayableHref(items[j]);
+            const { href } = this.pickPlayableHref(items[j]);
             if (j >= currentIndex - LOWER_BOUNDARY && j <= currentIndex + LOWER_BOUNDARY) {
                 this.ensure(href);
                 keep.add(href);
@@ -117,8 +122,8 @@ export class AudioPoolManager {
      * session and any Web Audio graph connections across track changes.
      */
     setCurrentAudio(currentIndex: number, _direction: 'forward' | 'backward'): void {
-        const href = this.pickPlayableHref(this._publication.readingOrder.items[currentIndex]);
-        this.audioEngine.changeSrc(href);
+        const { href, type } = this.pickPlayableHref(this._publication.readingOrder.items[currentIndex]);
+        this.audioEngine.changeSrc(href, type);
 
         // Discard any pool entry for this href — the primary element owns it now
         if (this.pool.has(href)) {
@@ -134,6 +139,7 @@ export class AudioPoolManager {
 
     destroy(): void {
         this.audioEngine.stop();
+        this.audioEngine.destroy();
         for (const [, element] of this.pool) {
             element.removeAttribute("src");
             element.load();

package/src/audio/engine/AudioEngine.ts

@@ -30,6 +30,37 @@ export interface Playback {
     buffered?: number;
 }
 
+/**
+ * A consumer-supplied loader that feeds one resource to the playback element
+ * through Media Source Extensions. The engine constructs one per track via
+ * the [AudioMseLoaderFactory] and calls [start] immediately; the loader owns
+ * the element's `src` (a MediaSource object URL), all fetching, buffering and
+ * seek servicing until [destroy] is called. Container specifics (WebM, fMP4,
+ * byte ranges vs. segments…) are entirely up to the implementation.
+ */
+export interface AudioMseLoader {
+    /** Attaches to the element and begins streaming. Called exactly once. */
+    start(): void;
+
+    /**
+     * Cancels in-flight fetches, detaches element listeners and releases the
+     * MediaSource object URL. The element itself must be left intact — the
+     * engine reuses it for the next track.
+     */
+    destroy(): void;
+}
+
+/**
+ * Creates the [AudioMseLoader] for a resource. [mimeType] is the resource's
+ * media type from the manifest, when known — needed to configure the
+ * SourceBuffer.
+ */
+export type AudioMseLoaderFactory = (
+    element: HTMLMediaElement,
+    href: string,
+    mimeType?: string,
+) => AudioMseLoader;
+
 /**
  * An audio engine that plays audio resources from a publication.
  * @playback - The current [Playback] state.
@@ -58,8 +89,10 @@ export interface AudioEngine {
      * Changes the src of the primary media element without swapping it,
      * preserving the RemotePlayback session and all attached event listeners.
      * @param href The URL of the new audio resource.
+     * @param mimeType The media type of the resource, when known. Required for
+     * MSE-based loading to configure the SourceBuffer.
      */
-    changeSrc(href: string): void;
+    changeSrc(href: string, mimeType?: string): void;
     
     /**
      * Plays the current audio resource.

package/src/audio/engine/WebAudioEngine.ts

@@ -2,6 +2,8 @@
 
 import {
   AudioEngine,
+  AudioMseLoader,
+  AudioMseLoaderFactory,
   Playback,
 } from "./AudioEngine.ts";
 import { PreservePitchWorklet } from "./PreservePitchWorklet.ts";
@@ -25,6 +27,10 @@ export class WebAudioEngine implements AudioEngine {
   private isStoppedValue: boolean = false;
   private worklet: PreservePitchWorklet | null = null;
   private webAudioActive: boolean = false;
+  private readonly mseLoaderFactory: AudioMseLoaderFactory | null;
+  private mseLoader: AudioMseLoader | null = null;
+  /** The resource href currently loaded (element src is a blob: URL in MSE mode). */
+  private currentHref: string = "";
 
   private readonly boundOnCanPlayThrough = this.onCanPlayThrough.bind(this);
   private readonly boundOnTimeUpdate = this.onTimeUpdate.bind(this);
@@ -42,8 +48,9 @@ export class WebAudioEngine implements AudioEngine {
   private readonly boundOnPause = this.onPause.bind(this);
   private readonly boundOnProgress = this.onProgress.bind(this);
 
-  constructor(values: { playback: Playback }) {
+  constructor(values: { playback: Playback, mseLoaderFactory?: AudioMseLoaderFactory }) {
     this.playback = values.playback;
+    this.mseLoaderFactory = values.mseLoaderFactory ?? null;
 
     // crossOrigin is set lazily in activateWebAudio() only when the worklet is needed
     this.mediaElement = document.createElement("audio");
@@ -386,6 +393,19 @@ export class WebAudioEngine implements AudioEngine {
   private async activateWebAudio(): Promise<void> {
     if (this.webAudioActive) return;
 
+    if (this.usesMse) {
+      // blob: MediaSource URLs are same-origin — no CORS reload required
+      this.sourceNode = new MediaElementAudioSourceNode(this.getOrCreateAudioContext(), { mediaElement: this.mediaElement });
+      const audioContext = this.getOrCreateAudioContext();
+      this.gainNode = audioContext.createGain();
+      this.gainNode.gain.value = this.mediaElement.volume;
+      this.mediaElement.volume = 1;
+      this.sourceNode.connect(this.gainNode);
+      this.gainNode.connect(audioContext.destination);
+      this.webAudioActive = true;
+      return;
+    }
+
     const src = this.mediaElement.src;
     if (!src) return;
 
@@ -469,6 +489,11 @@ export class WebAudioEngine implements AudioEngine {
     return this.webAudioActive;
   }
 
+  /** True when this engine streams media through Media Source Extensions. */
+  public get usesMse(): boolean {
+    return this.mseLoaderFactory !== null;
+  }
+
   /**
    * Tears down the Web Audio graph and restores the media element to standalone
    * playback. Safe to call even if Web Audio was never activated.
@@ -499,10 +524,11 @@ export class WebAudioEngine implements AudioEngine {
    * the required headers), the graph is torn down and the src is reloaded
    * without CORS so playback continues — just without pitch correction.
    */
-  public changeSrc(href: string): void {
-    if (this.mediaElement.src === href) {
+  public changeSrc(href: string, mimeType?: string): void {
+    if (this.currentHref === href || this.mediaElement.src === href) {
       return;
     }
+    this.currentHref = href;
     this.mediaElement.pause();
     this.isPlayingValue = false;
     this.isPausedValue = false;
@@ -510,6 +536,16 @@ export class WebAudioEngine implements AudioEngine {
     this.isLoadingValue = true;
     this.isEndedValue = false;
 
+    if (this.mseLoaderFactory) {
+      // MSE path: swap loaders on the persistent element. The blob: object
+      // URL is same-origin, so none of the crossOrigin reload handling below
+      // applies here — the Web Audio graph (if active) keeps working.
+      this.mseLoader?.destroy();
+      this.mseLoader = this.mseLoaderFactory(this.mediaElement, href, mimeType);
+      this.mseLoader.start();
+      return;
+    }
+
     if (this.webAudioActive) {
       this.mediaElement.crossOrigin = "anonymous";
       this.mediaElement.src = href;
@@ -542,4 +578,13 @@ export class WebAudioEngine implements AudioEngine {
   public getMediaElement(): HTMLMediaElement {
     return this.mediaElement;
   }
+
+  /**
+   * Releases loader resources (in-flight fetches, MediaSource object URL).
+   * The media element itself is left intact.
+   */
+  public destroy(): void {
+    this.mseLoader?.destroy();
+    this.mseLoader = null;
+  }
 }

package/types/src/audio/AudioNavigator.d.ts

@@ -1,6 +1,7 @@
 import { Link, Locator, Publication, Timeline, TimelineItem } from "@readium/shared";
 import { MediaNavigator, IContentProtectionConfig, IKeyboardPeripheralsConfig, KeyboardPeripheralEventData } from "../Navigator.ts";
 import { Configurable } from "../preferences/Configurable.ts";
+import { AudioMseLoaderFactory } from "./engine/index.ts";
 import { AudioPreferences, AudioSettings, AudioPreferencesEditor, IAudioPreferences, IAudioDefaults } from "./preferences/index.ts";
 import { ContextMenuEvent, SuspiciousActivityEvent } from "@readium/navigator-html-injectables";
 export interface AudioMetadata {
@@ -35,6 +36,24 @@ export interface AudioNavigatorConfiguration {
     defaults: IAudioDefaults;
     contentProtection?: IAudioContentProtectionConfig;
     keyboardPeripherals?: IKeyboardPeripheralsConfig;
+    /**
+     * Called with the persistent playback element before
+     * the first src is assigned, so the host can prepare it. Use for MSE/EME
+     * setups. When it returns a promise, loading of
+     * the initial track (and prefetching of adjacent ones) is deferred until
+     * the promise settles; a rejection is forwarded to the error listener and
+     * loading proceeds anyway.
+     */
+    mediaElementSetup?: (element: HTMLMediaElement) => void | Promise<void>;
+    /**
+     * When provided, media bytes reach the playback element through Media
+     * Source Extensions instead of direct `src` assignment: the engine
+     * creates one loader per track via this factory and the loader owns all
+     * fetching. Supply one for e.g. EME encrypted audio, since browsers have
+     * poor support for encrypted audio directly through `src`. The container
+     * handling (WebM, fMP4, …) is the loader implementation's concern.
+     */
+    mseLoaderFactory?: AudioMseLoaderFactory;
 }
 export declare class AudioNavigator extends MediaNavigator implements Configurable<AudioSettings, AudioPreferences> {
     private readonly pub;
@@ -58,9 +77,13 @@ export declare class AudioNavigator extends MediaNavigator implements Configurab
     /** True while a track transition is in progress; suppresses spurious mid-navigation events. */
     private _isNavigating;
     private _isStalled;
+    /** Set by destroy(); stops the deferred initial load from touching a dead instance. */
+    private _destroyed;
     private _stalledWatchdog;
     private _stalledCheckTime;
     constructor(publication: Publication, listeners: AudioNavigatorListeners, initialPosition?: Locator, configuration?: AudioNavigatorConfiguration);
+    /** Sets the initial track on the primary element and seeks to the starting position. */
+    private startInitialLoad;
     get settings(): AudioSettings;
     get preferencesEditor(): AudioPreferencesEditor;
     submitPreferences(preferences: AudioPreferences): Promise<void>;
@@ -73,6 +96,12 @@ export declare class AudioNavigator extends MediaNavigator implements Configurab
     private hrefToTrackIndex;
     /** Current track index derived from the current location's href. */
     private currentTrackIndex;
+    /**
+     * Builds a locator for an overall publication progression (0–1) by walking
+     * the cumulative track durations. Falls back to the start of the first
+     * track when the reading order carries no duration metadata.
+     */
+    private locatorFromTotalProgression;
     get currentLocator(): Locator;
     get isPlaying(): boolean;
     get isPaused(): boolean;

package/types/src/audio/engine/AudioEngine.d.ts

@@ -27,6 +27,30 @@ export interface Playback {
     offset?: number;
     buffered?: number;
 }
+/**
+ * A consumer-supplied loader that feeds one resource to the playback element
+ * through Media Source Extensions. The engine constructs one per track via
+ * the [AudioMseLoaderFactory] and calls [start] immediately; the loader owns
+ * the element's `src` (a MediaSource object URL), all fetching, buffering and
+ * seek servicing until [destroy] is called. Container specifics (WebM, fMP4,
+ * byte ranges vs. segments…) are entirely up to the implementation.
+ */
+export interface AudioMseLoader {
+    /** Attaches to the element and begins streaming. Called exactly once. */
+    start(): void;
+    /**
+     * Cancels in-flight fetches, detaches element listeners and releases the
+     * MediaSource object URL. The element itself must be left intact — the
+     * engine reuses it for the next track.
+     */
+    destroy(): void;
+}
+/**
+ * Creates the [AudioMseLoader] for a resource. [mimeType] is the resource's
+ * media type from the manifest, when known — needed to configure the
+ * SourceBuffer.
+ */
+export type AudioMseLoaderFactory = (element: HTMLMediaElement, href: string, mimeType?: string) => AudioMseLoader;
 /**
  * An audio engine that plays audio resources from a publication.
  * @playback - The current [Playback] state.
@@ -52,8 +76,10 @@ export interface AudioEngine {
      * Changes the src of the primary media element without swapping it,
      * preserving the RemotePlayback session and all attached event listeners.
      * @param href The URL of the new audio resource.
+     * @param mimeType The media type of the resource, when known. Required for
+     * MSE-based loading to configure the SourceBuffer.
      */
-    changeSrc(href: string): void;
+    changeSrc(href: string, mimeType?: string): void;
     /**
      * Plays the current audio resource.
      */

package/types/src/audio/engine/WebAudioEngine.d.ts

@@ -1,4 +1,4 @@
-import { AudioEngine, Playback } from "./AudioEngine.ts";
+import { AudioEngine, AudioMseLoaderFactory, Playback } from "./AudioEngine.ts";
 type EventCallback = (data: any) => void;
 export declare class WebAudioEngine implements AudioEngine {
     readonly playback: Playback;
@@ -16,6 +16,10 @@ export declare class WebAudioEngine implements AudioEngine {
     private isStoppedValue;
     private worklet;
     private webAudioActive;
+    private readonly mseLoaderFactory;
+    private mseLoader;
+    /** The resource href currently loaded (element src is a blob: URL in MSE mode). */
+    private currentHref;
     private readonly boundOnCanPlayThrough;
     private readonly boundOnTimeUpdate;
     private readonly boundOnError;
@@ -33,6 +37,7 @@ export declare class WebAudioEngine implements AudioEngine {
     private readonly boundOnProgress;
     constructor(values: {
         playback: Playback;
+        mseLoaderFactory?: AudioMseLoaderFactory;
     });
     /**
      * Adds an event listener to the audio engine.
@@ -132,6 +137,8 @@ export declare class WebAudioEngine implements AudioEngine {
      */
     private activateWebAudio;
     get isWebAudioActive(): boolean;
+    /** True when this engine streams media through Media Source Extensions. */
+    get usesMse(): boolean;
     /**
      * Tears down the Web Audio graph and restores the media element to standalone
      * playback. Safe to call even if Web Audio was never activated.
@@ -145,10 +152,15 @@ export declare class WebAudioEngine implements AudioEngine {
      * the required headers), the graph is torn down and the src is reloaded
      * without CORS so playback continues — just without pitch correction.
      */
-    changeSrc(href: string): void;
+    changeSrc(href: string, mimeType?: string): void;
     /**
      * Returns the HTML media element used for playback.
      */
     getMediaElement(): HTMLMediaElement;
+    /**
+     * Releases loader resources (in-flight fetches, MediaSource object URL).
+     * The media element itself is left intact.
+     */
+    destroy(): void;
 }
 export {};