hls.js 1.6.0-beta.2.0.canary.10924 → 1.6.0-beta.2.0.canary.10926

package/src/controller/gap-controller.ts

@@ -1,44 +1,149 @@
 import { State } from './base-stream-controller';
 import { ErrorDetails, ErrorTypes } from '../errors';
 import { Events } from '../events';
+import TaskLoop from '../task-loop';
 import { PlaylistLevelType } from '../types/loader';
 import { BufferHelper } from '../utils/buffer-helper';
-import { Logger } from '../utils/logger';
+import {
+  addEventListener,
+  removeEventListener,
+} from '../utils/event-listener-helper';
+import type { InFlightData } from './base-stream-controller';
+import type { InFlightFragments } from '../hls';
 import type Hls from '../hls';
 import type { FragmentTracker } from './fragment-tracker';
-import type { Fragment } from '../loader/fragment';
-import type { LevelDetails } from '../loader/level-details';
+import type { Fragment, MediaFragment } from '../loader/fragment';
+import type { SourceBufferName } from '../types/buffer';
+import type {
+  BufferAppendedData,
+  MediaAttachedData,
+  MediaDetachingData,
+} from '../types/events';
 import type { BufferInfo } from '../utils/buffer-helper';
 
 export const MAX_START_GAP_JUMP = 2.0;
 export const SKIP_BUFFER_HOLE_STEP_SECONDS = 0.1;
 export const SKIP_BUFFER_RANGE_START = 0.05;
+const TICK_INTERVAL = 100;
 
-export default class GapController extends Logger {
-  private media: HTMLMediaElement | null = null;
-  private fragmentTracker: FragmentTracker | null = null;
+export default class GapController extends TaskLoop {
   private hls: Hls | null = null;
+  private fragmentTracker: FragmentTracker | null = null;
+  private media: HTMLMediaElement | null = null;
+  private mediaSource?: MediaSource;
+
   private nudgeRetry: number = 0;
   private stallReported: boolean = false;
   private stalled: number | null = null;
   private moved: boolean = false;
   private seeking: boolean = false;
+  private buffered: Partial<Record<SourceBufferName, TimeRanges>> = {};
+
+  private lastCurrentTime: number = 0;
   public ended: number = 0;
   public waiting: number = 0;
 
-  constructor(
-    media: HTMLMediaElement,
-    fragmentTracker: FragmentTracker,
-    hls: Hls,
-  ) {
+  constructor(hls: Hls, fragmentTracker: FragmentTracker) {
     super('gap-controller', hls.logger);
-    this.media = media;
-    this.fragmentTracker = fragmentTracker;
     this.hls = hls;
+    this.fragmentTracker = fragmentTracker;
+    this.registerListeners();
+  }
+
+  private registerListeners() {
+    const { hls } = this;
+    if (hls) {
+      hls.on(Events.MEDIA_ATTACHED, this.onMediaAttached, this);
+      hls.on(Events.MEDIA_DETACHING, this.onMediaDetaching, this);
+      hls.on(Events.BUFFER_APPENDED, this.onBufferAppended, this);
+    }
+  }
+
+  private unregisterListeners() {
+    const { hls } = this;
+    if (hls) {
+      hls.off(Events.MEDIA_ATTACHED, this.onMediaAttached, this);
+      hls.off(Events.MEDIA_DETACHING, this.onMediaDetaching, this);
+      hls.off(Events.BUFFER_APPENDED, this.onBufferAppended, this);
+    }
   }
 
   public destroy() {
+    super.destroy();
+    this.unregisterListeners();
     this.media = this.hls = this.fragmentTracker = null;
+    this.mediaSource = undefined;
+  }
+
+  private onMediaAttached(
+    event: Events.MEDIA_ATTACHED,
+    data: MediaAttachedData,
+  ) {
+    this.setInterval(TICK_INTERVAL);
+    this.mediaSource = data.mediaSource;
+    const media = (this.media = data.media);
+    addEventListener(media, 'playing', this.onMediaPlaying);
+    addEventListener(media, 'waiting', this.onMediaWaiting);
+    addEventListener(media, 'ended', this.onMediaEnded);
+  }
+
+  private onMediaDetaching(
+    event: Events.MEDIA_DETACHING,
+    data: MediaDetachingData,
+  ) {
+    this.clearInterval();
+    const { media } = this;
+    if (media) {
+      removeEventListener(media, 'playing', this.onMediaPlaying);
+      removeEventListener(media, 'waiting', this.onMediaWaiting);
+      removeEventListener(media, 'ended', this.onMediaEnded);
+      this.media = null;
+    }
+    this.mediaSource = undefined;
+  }
+
+  private onBufferAppended(
+    event: Events.BUFFER_APPENDED,
+    data: BufferAppendedData,
+  ) {
+    this.buffered = data.timeRanges;
+  }
+
+  private onMediaPlaying = () => {
+    this.ended = 0;
+    this.waiting = 0;
+  };
+
+  private onMediaWaiting = () => {
+    if (this.media?.seeking) {
+      return;
+    }
+    this.waiting = self.performance.now();
+    this.tick();
+  };
+
+  private onMediaEnded = () => {
+    if (this.hls) {
+      // ended is set when triggering MEDIA_ENDED so that we do not trigger it again on stall or on tick with media.ended
+      this.ended = this.media?.currentTime || 1;
+      this.hls.trigger(Events.MEDIA_ENDED, {
+        stalled: false,
+      });
+    }
+  };
+
+  public get hasBuffered(): boolean {
+    return Object.keys(this.buffered).length > 0;
+  }
+
+  public tick() {
+    if (!this.media?.readyState || !this.hasBuffered) {
+      return;
+    }
+
+    const currentTime = this.media.currentTime;
+    this.poll(currentTime, this.lastCurrentTime);
+    this.lastCurrentTime = currentTime;
   }
 
   /**
@@ -47,20 +152,20 @@ export default class GapController extends Logger {
    *
    * @param lastCurrentTime - Previously read playhead position
    */
-  public poll(
-    lastCurrentTime: number,
-    activeFrag: Fragment | null,
-    levelDetails: LevelDetails | undefined,
-    state: string,
-  ) {
+  public poll(currentTime: number, lastCurrentTime: number) {
+    const config = this.hls?.config;
+    if (!config) {
+      return;
+    }
     const { media, stalled } = this;
-
     if (!media) {
       return;
     }
-    const { currentTime, seeking } = media;
+    const { seeking } = media;
     const seeked = this.seeking && !seeking;
     const beginSeek = !this.seeking && seeking;
+    const pausedEndedOrHalted =
+      (media.paused && !seeking) || media.ended || media.playbackRate === 0;
 
     this.seeking = seeking;
 
@@ -72,6 +177,14 @@ export default class GapController extends Logger {
       this.moved = true;
       if (!seeking) {
         this.nudgeRetry = 0;
+        // When crossing between buffered video time ranges, but not audio, flush pipeline with seek (Chrome)
+        if (
+          config.nudgeOnVideoHole &&
+          !pausedEndedOrHalted &&
+          currentTime > lastCurrentTime
+        ) {
+          this.nudgeOnVideoHole(currentTime, lastCurrentTime);
+        }
       }
       if (this.waiting === 0) {
         this.stallResolved(currentTime);
@@ -88,7 +201,7 @@ export default class GapController extends Logger {
     }
 
     // The playhead should not be moving
-    if ((media.paused && !seeking) || media.ended || media.playbackRate === 0) {
+    if (pausedEndedOrHalted) {
       this.nudgeRetry = 0;
       this.stallResolved(currentTime);
       // Fire MEDIA_ENDED to workaround event not being dispatched by browser
@@ -106,28 +219,36 @@ export default class GapController extends Logger {
       return;
     }
 
+    // Resolve stalls at buffer holes using the main buffer, whose ranges are the intersections of the A/V sourcebuffers
     const bufferInfo = BufferHelper.bufferInfo(media, currentTime, 0);
     const nextStart = bufferInfo.nextStart || 0;
     const fragmentTracker = this.fragmentTracker;
 
-    if (seeking && fragmentTracker) {
+    if (seeking && fragmentTracker && this.hls) {
+      // Is there a fragment loading/parsing/appending before currentTime?
+      const inFlightDependency = getInFlightDependency(
+        this.hls.inFlightFragments,
+        currentTime,
+      );
+
       // Waiting for seeking in a buffered range to complete
       const hasEnoughBuffer = bufferInfo.len > MAX_START_GAP_JUMP;
       // Next buffered range is too far ahead to jump to while still seeking
-      const noBufferGap =
+      const noBufferHole =
         !nextStart ||
-        (activeFrag && activeFrag.start <= currentTime) ||
+        inFlightDependency ||
         (nextStart - currentTime > MAX_START_GAP_JUMP &&
           !fragmentTracker.getPartialFragment(currentTime));
-      if (hasEnoughBuffer || noBufferGap) {
+      if (hasEnoughBuffer || noBufferHole) {
         return;
       }
-      // Reset moved state when seeking to a point in or before a gap
+      // Reset moved state when seeking to a point in or before a gap/hole
       this.moved = false;
     }
 
     // Skip start gaps if we haven't played, but the last poll detected the start of a stall
     // The addition poll gives the browser a chance to jump the gap for us
+    const levelDetails = this.hls?.latestLevelDetails;
     if (!this.moved && this.stalled !== null && fragmentTracker) {
       // There is no playable buffer (seeked, waiting for buffer)
       const isBuffered = bufferInfo.len > 0;
@@ -155,10 +276,6 @@ export default class GapController extends Logger {
     }
 
     // Start tracking stall time
-    const config = this.hls?.config;
-    if (!config) {
-      return;
-    }
     const detectStallWithCurrentTimeMs = config.detectStallWithCurrentTimeMs;
     const tnow = self.performance.now();
     const tWaiting = this.waiting;
@@ -180,7 +297,7 @@ export default class GapController extends Logger {
     ) {
       // Dispatch MEDIA_ENDED when media.ended/ended event is not signalled at end of stream
       if (
-        state === State.ENDED &&
+        this.mediaSource?.readyState === 'ended' &&
         !levelDetails?.live &&
         Math.abs(currentTime - (levelDetails?.edge || 0)) < 1
       ) {
@@ -215,7 +332,7 @@ export default class GapController extends Logger {
       // The playhead is now moving, but was previously stalled
       if (this.stallReported) {
         const stalledDuration = self.performance.now() - stalled;
-        this.warn(
+        this.log(
           `playback not stuck anymore @${currentTime}, after ${Math.round(
             stalledDuration,
           )}ms`,
@@ -227,6 +344,81 @@ export default class GapController extends Logger {
     }
   }
 
+  private nudgeOnVideoHole(currentTime: number, lastCurrentTime: number) {
+    // Chrome will play one second past a hole in video buffered time ranges without rendering any video from the subsequent range and then stall as long as audio is buffered:
+    // https://github.com/video-dev/hls.js/issues/5631
+    // https://issues.chromium.org/issues/40280613#comment10
+    // Detect the potential for this situation and proactively seek to flush the video pipeline once the playhead passes the start of the video hole.
+    // When there are audio and video buffers and currentTime is past the end of the first video buffered range...
+    const videoSourceBuffered = this.buffered.video;
+    if (
+      this.hls &&
+      this.media &&
+      this.fragmentTracker &&
+      this.buffered.audio?.length &&
+      videoSourceBuffered &&
+      videoSourceBuffered.length > 1 &&
+      currentTime > videoSourceBuffered.end(0)
+    ) {
+      // and audio is buffered at the playhead
+      const audioBufferInfo = BufferHelper.bufferedInfo(
+        BufferHelper.timeRangesToArray(this.buffered.audio),
+        currentTime,
+        0,
+      );
+      if (audioBufferInfo.len > 1 && lastCurrentTime >= audioBufferInfo.start) {
+        const videoTimes = BufferHelper.timeRangesToArray(videoSourceBuffered);
+        const lastBufferedIndex = BufferHelper.bufferedInfo(
+          videoTimes,
+          lastCurrentTime,
+          0,
+        ).bufferedIndex;
+        // nudge when crossing into another video buffered range (hole).
+        if (
+          lastBufferedIndex > -1 &&
+          lastBufferedIndex < videoTimes.length - 1
+        ) {
+          const bufferedIndex = BufferHelper.bufferedInfo(
+            videoTimes,
+            currentTime,
+            0,
+          ).bufferedIndex;
+          const holeStart = videoTimes[lastBufferedIndex].end;
+          const holeEnd = videoTimes[lastBufferedIndex + 1].start;
+          if (
+            (bufferedIndex === -1 || bufferedIndex > lastBufferedIndex) &&
+            holeEnd - holeStart < 1 && // `maxBufferHole` may be too small and setting it to 0 should not disable this feature
+            currentTime - holeStart < 2
+          ) {
+            const error = new Error(
+              `nudging playhead to flush pipeline after video hole. currentTime: ${currentTime} hole: ${holeStart} -> ${holeEnd} buffered index: ${bufferedIndex}`,
+            );
+            this.warn(error.message);
+            // Magic number to flush the pipeline without interuption to audio playback:
+            this.media.currentTime += 0.000001;
+            const frag =
+              this.fragmentTracker.getPartialFragment(currentTime) || undefined;
+            const bufferInfo = BufferHelper.bufferInfo(
+              this.media,
+              currentTime,
+              0,
+            );
+            this.hls.trigger(Events.ERROR, {
+              type: ErrorTypes.MEDIA_ERROR,
+              details: ErrorDetails.BUFFER_SEEK_OVER_HOLE,
+              fatal: false,
+              error,
+              reason: error.message,
+              frag,
+              buffer: bufferInfo.len,
+              bufferInfo,
+            });
+          }
+        }
+      }
+    }
+  }
+
   /**
    * Detects and attempts to fix known buffer stalling issues.
    * @param bufferInfo - The properties of the current buffer.
@@ -268,7 +460,8 @@ export default class GapController extends Logger {
         bufferInfo.len > config.maxBufferHole) ||
         (bufferInfo.nextStart &&
           bufferInfo.nextStart - currentTime < config.maxBufferHole)) &&
-      stalledDurationMs > config.highBufferWatchdogPeriod * 1000
+      (stalledDurationMs > config.highBufferWatchdogPeriod * 1000 ||
+        this.waiting)
     ) {
       this.warn('Trying to nudge playhead over buffer-hole');
       // Try to nudge currentTime over a buffer hole if we've been stalling for the configured amount of seconds
@@ -310,7 +503,7 @@ export default class GapController extends Logger {
    * @param partial - The partial fragment found at the current time (where playback is stalling).
    * @private
    */
-  private _trySkipBufferHole(partial: Fragment | null): number {
+  private _trySkipBufferHole(partial: MediaFragment | null): number {
     const { fragmentTracker, media } = this;
     const config = this.hls?.config;
     if (!media || !fragmentTracker || !config) {
@@ -322,7 +515,7 @@ export default class GapController extends Logger {
     const bufferInfo = BufferHelper.bufferInfo(media, currentTime, 0);
     const startTime =
       currentTime < bufferInfo.start ? bufferInfo.start : bufferInfo.nextStart;
-    if (startTime) {
+    if (startTime && this.hls) {
       const bufferStarved = bufferInfo.len <= config.maxBufferHole;
       const waiting =
         bufferInfo.len > 0 && bufferInfo.len < 1 && media.readyState < 3;
@@ -348,6 +541,19 @@ export default class GapController extends Logger {
                 PlaylistLevelType.MAIN,
               );
             if (startProvisioned) {
+              // Do not seek when selected variant playlist is unloaded
+              if (!this.hls.loadLevelObj?.details) {
+                return 0;
+              }
+              // Do not seek when required fragments are inflight or appending
+              const inFlightDependency = getInFlightDependency(
+                this.hls.inFlightFragments,
+                startTime,
+              );
+              if (inFlightDependency) {
+                return 0;
+              }
+              // Do not seek if we can't walk tracked fragments to end of gap
               let moreToLoad = false;
               let pos = startProvisioned.end;
               while (pos < startTime) {
@@ -374,7 +580,7 @@ export default class GapController extends Logger {
         );
         this.moved = true;
         media.currentTime = targetTime;
-        if (partial && !partial.gap && this.hls) {
+        if (!partial?.gap) {
           const error = new Error(
             `fragment loaded with buffer holes, seeking from ${currentTime} to ${targetTime}`,
           );
@@ -384,7 +590,7 @@ export default class GapController extends Logger {
             fatal: false,
             error,
             reason: error.message,
-            frag: partial,
+            frag: partial || undefined,
             buffer: bufferInfo.len,
             bufferInfo,
           });
@@ -440,3 +646,32 @@ export default class GapController extends Logger {
     }
   }
 }
+
+function getInFlightDependency(
+  inFlightFragments: InFlightFragments,
+  currentTime: number,
+): Fragment | null {
+  const main = inFlight(inFlightFragments.main);
+  if (main && main.start <= currentTime) {
+    return main;
+  }
+  const audio = inFlight(inFlightFragments.audio);
+  if (audio && audio.start <= currentTime) {
+    return audio;
+  }
+  return null;
+}
+
+function inFlight(inFlightData: InFlightData | undefined): Fragment | null {
+  if (!inFlightData) {
+    return null;
+  }
+  switch (inFlightData.state) {
+    case State.IDLE:
+    case State.STOPPED:
+    case State.ENDED:
+    case State.ERROR:
+      return null;
+  }
+  return inFlightData.frag;
+}

package/src/controller/interstitials-controller.ts

@@ -22,6 +22,10 @@ import {
   TimelineOccupancy,
 } from '../loader/interstitial-event';
 import { BufferHelper } from '../utils/buffer-helper';
+import {
+  addEventListener,
+  removeEventListener,
+} from '../utils/event-listener-helper';
 import { hash } from '../utils/hash';
 import { Logger } from '../utils/logger';
 import { isCompatibleTrackChange } from '../utils/mediasource-helper';
@@ -226,17 +230,17 @@ export default class InterstitialsController
   }
 
   private onDestroying() {
-    const media = this.primaryMedia;
+    const media = this.primaryMedia || this.media;
     if (media) {
       this.removeMediaListeners(media);
     }
   }
 
   private removeMediaListeners(media: HTMLMediaElement) {
-    media.removeEventListener('play', this.onPlay);
-    media.removeEventListener('pause', this.onPause);
-    media.removeEventListener('seeking', this.onSeeking);
-    media.removeEventListener('timeupdate', this.onTimeupdate);
+    removeEventListener(media, 'play', this.onPlay);
+    removeEventListener(media, 'pause', this.onPause);
+    removeEventListener(media, 'seeking', this.onSeeking);
+    removeEventListener(media, 'timeupdate', this.onTimeupdate);
   }
 
   private onMediaAttaching(
@@ -244,11 +248,10 @@ export default class InterstitialsController
     data: MediaAttachingData,
   ) {
     const media = (this.media = data.media);
-    this.removeMediaListeners(media);
-    media.addEventListener('seeking', this.onSeeking);
-    media.addEventListener('timeupdate', this.onTimeupdate);
-    media.addEventListener('play', this.onPlay);
-    media.addEventListener('pause', this.onPause);
+    addEventListener(media, 'seeking', this.onSeeking);
+    addEventListener(media, 'timeupdate', this.onTimeupdate);
+    addEventListener(media, 'play', this.onPlay);
+    addEventListener(media, 'pause', this.onPause);
   }
 
   private onMediaAttached(
@@ -1918,7 +1921,7 @@ Schedule: ${scheduleItems.map((seg) => segmentToString(seg))}`,
     const userConfig = primary.userConfig;
     let videoPreference = userConfig.videoPreference;
     const currentLevel =
-      primary.levels[primary.loadLevel] || primary.levels[primary.currentLevel];
+      primary.loadLevelObj || primary.levels[primary.currentLevel];
     if (videoPreference || currentLevel) {
       videoPreference = Object.assign({}, videoPreference);
       if (currentLevel.videoCodec) {

package/src/controller/level-controller.ts

@@ -389,6 +389,10 @@ export default class LevelController extends BasePlaylistController {
     return this._levels;
   }
 
+  get loadLevelObj(): Level | null {
+    return this.currentLevel;
+  }
+
   get level(): number {
     return this.currentLevelIndex;
   }