@zenvor/hls.js 1.0.0

package/src/remux/mp4-remuxer.ts

@@ -0,0 +1,1261 @@
+import AAC from './aac-helper';
+import MP4 from './mp4-generator';
+import { ErrorDetails, ErrorTypes } from '../errors';
+import { Events } from '../events';
+import { PlaylistLevelType } from '../types/loader';
+import { type ILogger, Logger } from '../utils/logger';
+import {
+  timestampToString,
+  toMsFromMpegTsClock,
+} from '../utils/timescale-conversion';
+import type { HlsConfig } from '../config';
+import type { HlsEventEmitter } from '../events';
+import type { SourceBufferName } from '../types/buffer';
+import type {
+  AudioSample,
+  DemuxedAudioTrack,
+  DemuxedMetadataTrack,
+  DemuxedUserdataTrack,
+  DemuxedVideoTrack,
+  VideoSample,
+} from '../types/demuxer';
+import type {
+  InitSegmentData,
+  Mp4Sample,
+  RemuxedMetadata,
+  RemuxedTrack,
+  RemuxedUserdata,
+  Remuxer,
+  RemuxerResult,
+} from '../types/remuxer';
+import type { TrackSet } from '../types/track';
+import type { TypeSupported } from '../utils/codecs';
+import type {
+  RationalTimestamp,
+  TimestampOffset,
+} from '../utils/timescale-conversion';
+
+const MAX_SILENT_FRAME_DURATION = 10 * 1000; // 10 seconds
+const AAC_SAMPLES_PER_FRAME = 1024;
+const MPEG_AUDIO_SAMPLE_PER_FRAME = 1152;
+const AC3_SAMPLES_PER_FRAME = 1536;
+
+let chromeVersion: number | null = null;
+let safariWebkitVersion: number | null = null;
+
+function createMp4Sample(
+  isKeyframe: boolean,
+  duration: number,
+  size: number,
+  cts: number,
+): Mp4Sample {
+  return {
+    duration,
+    size,
+    cts,
+    flags: {
+      isLeading: 0,
+      isDependedOn: 0,
+      hasRedundancy: 0,
+      degradPrio: 0,
+      dependsOn: isKeyframe ? 2 : 1,
+      isNonSync: isKeyframe ? 0 : 1,
+    },
+  };
+}
+export default class MP4Remuxer extends Logger implements Remuxer {
+  private readonly observer: HlsEventEmitter;
+  private readonly config: HlsConfig;
+  private readonly typeSupported: TypeSupported;
+  private ISGenerated: boolean = false;
+  private _initPTS: TimestampOffset | null = null;
+  private _initDTS: TimestampOffset | null = null;
+  private nextVideoTs: number | null = null;
+  private nextAudioTs: number | null = null;
+  private videoSampleDuration: number | null = null;
+  private isAudioContiguous: boolean = false;
+  private isVideoContiguous: boolean = false;
+  private videoTrackConfig?: {
+    width?: number;
+    height?: number;
+    pixelRatio?: [number, number];
+  };
+
+  constructor(
+    observer: HlsEventEmitter,
+    config: HlsConfig,
+    typeSupported: TypeSupported,
+    logger: ILogger,
+  ) {
+    super('mp4-remuxer', logger);
+    this.observer = observer;
+    this.config = config;
+    this.typeSupported = typeSupported;
+    this.ISGenerated = false;
+
+    if (chromeVersion === null) {
+      const userAgent = navigator.userAgent || '';
+      const result = userAgent.match(/Chrome\/(\d+)/i);
+      chromeVersion = result ? parseInt(result[1]) : 0;
+    }
+    if (safariWebkitVersion === null) {
+      const result = navigator.userAgent.match(/Safari\/(\d+)/i);
+      safariWebkitVersion = result ? parseInt(result[1]) : 0;
+    }
+  }
+
+  destroy() {
+    // @ts-ignore
+    this.config = this.videoTrackConfig = this._initPTS = this._initDTS = null;
+  }
+
+  resetTimeStamp(defaultTimeStamp: TimestampOffset | null) {
+    const initPTS = this._initPTS;
+    if (
+      !initPTS ||
+      !defaultTimeStamp ||
+      defaultTimeStamp.trackId !== initPTS.trackId ||
+      defaultTimeStamp.baseTime !== initPTS.baseTime ||
+      defaultTimeStamp.timescale !== initPTS.timescale
+    ) {
+      this.log(
+        `Reset initPTS: ${initPTS ? timestampToString(initPTS) : initPTS} > ${defaultTimeStamp ? timestampToString(defaultTimeStamp) : defaultTimeStamp}`,
+      );
+    }
+
+    this._initPTS = this._initDTS = defaultTimeStamp;
+  }
+
+  resetNextTimestamp() {
+    this.log('reset next timestamp');
+    this.isVideoContiguous = false;
+    this.isAudioContiguous = false;
+  }
+
+  resetInitSegment() {
+    this.log('ISGenerated flag reset');
+    this.ISGenerated = false;
+    this.videoTrackConfig = undefined;
+  }
+
+  getVideoStartPts(videoSamples: VideoSample[]) {
+    // Get the minimum PTS value relative to the first sample's PTS, normalized for 33-bit wrapping
+    let rolloverDetected = false;
+    const firstPts = videoSamples[0].pts;
+    const startPTS = videoSamples.reduce((minPTS, sample) => {
+      let pts = sample.pts;
+      let delta = pts - minPTS;
+      if (delta < -4294967296) {
+        // 2^32, see PTSNormalize for reasoning, but we're hitting a rollover here, and we don't want that to impact the timeOffset calculation
+        rolloverDetected = true;
+        pts = normalizePts(pts, firstPts);
+        delta = pts - minPTS;
+      }
+      if (delta > 0) {
+        return minPTS;
+      }
+      return pts;
+    }, firstPts);
+    if (rolloverDetected) {
+      this.debug('PTS rollover detected');
+    }
+    return startPTS;
+  }
+
+  remux(
+    audioTrack: DemuxedAudioTrack,
+    videoTrack: DemuxedVideoTrack,
+    id3Track: DemuxedMetadataTrack,
+    textTrack: DemuxedUserdataTrack,
+    timeOffset: number,
+    accurateTimeOffset: boolean,
+    flush: boolean,
+    playlistType: PlaylistLevelType,
+  ): RemuxerResult {
+    let video: RemuxedTrack | undefined;
+    let audio: RemuxedTrack | undefined;
+    let initSegment: InitSegmentData | undefined;
+    let text: RemuxedUserdata | undefined;
+    let id3: RemuxedMetadata | undefined;
+    let independent: boolean | undefined;
+    let audioTimeOffset = timeOffset;
+    let videoTimeOffset = timeOffset;
+
+    // If we're remuxing audio and video progressively, wait until we've received enough samples for each track before proceeding.
+    // This is done to synchronize the audio and video streams. We know if the current segment will have samples if the "pid"
+    // parameter is greater than -1. The pid is set when the PMT is parsed, which contains the tracks list.
+    // However, if the initSegment has already been generated, or we've reached the end of a segment (flush),
+    // then we can remux one track without waiting for the other.
+    const hasAudio = audioTrack.pid > -1;
+    const hasVideo = videoTrack.pid > -1;
+    const length = videoTrack.samples.length;
+    const enoughAudioSamples = audioTrack.samples.length > 0;
+    const enoughVideoSamples = (flush && length > 0) || length > 1;
+    const canRemuxAvc =
+      ((!hasAudio || enoughAudioSamples) &&
+        (!hasVideo || enoughVideoSamples)) ||
+      this.ISGenerated ||
+      flush;
+
+    if (canRemuxAvc) {
+      if (this.ISGenerated) {
+        const config = this.videoTrackConfig;
+        if (
+          (config &&
+            (videoTrack.width !== config.width ||
+              videoTrack.height !== config.height ||
+              videoTrack.pixelRatio?.[0] !== config.pixelRatio?.[0] ||
+              videoTrack.pixelRatio?.[1] !== config.pixelRatio?.[1])) ||
+          (!config && enoughVideoSamples) ||
+          (this.nextAudioTs === null && enoughAudioSamples)
+        ) {
+          this.resetInitSegment();
+        }
+      }
+      if (!this.ISGenerated) {
+        initSegment = this.generateIS(
+          audioTrack,
+          videoTrack,
+          timeOffset,
+          accurateTimeOffset,
+        );
+      }
+
+      const isVideoContiguous = this.isVideoContiguous;
+      let firstKeyFrameIndex = -1;
+      let firstKeyFramePTS;
+
+      if (enoughVideoSamples) {
+        firstKeyFrameIndex = findKeyframeIndex(videoTrack.samples);
+        if (!isVideoContiguous && this.config.forceKeyFrameOnDiscontinuity) {
+          independent = true;
+          if (firstKeyFrameIndex > 0) {
+            this.warn(
+              `Dropped ${firstKeyFrameIndex} out of ${length} video samples due to a missing keyframe`,
+            );
+            const startPTS = this.getVideoStartPts(videoTrack.samples);
+            videoTrack.samples = videoTrack.samples.slice(firstKeyFrameIndex);
+            videoTrack.dropped += firstKeyFrameIndex;
+            videoTimeOffset +=
+              (videoTrack.samples[0].pts - startPTS) /
+              videoTrack.inputTimeScale;
+            firstKeyFramePTS = videoTimeOffset;
+          } else if (firstKeyFrameIndex === -1) {
+            this.warn(`No keyframe found out of ${length} video samples`);
+            independent = false;
+          }
+        }
+      }
+
+      if (this.ISGenerated) {
+        if (enoughAudioSamples && enoughVideoSamples) {
+          // timeOffset is expected to be the offset of the first timestamp of this fragment (first DTS)
+          // if first audio DTS is not aligned with first video DTS then we need to take that into account
+          // when providing timeOffset to remuxAudio / remuxVideo. if we don't do that, there might be a permanent / small
+          // drift between audio and video streams
+          const startPTS = this.getVideoStartPts(videoTrack.samples);
+          const tsDelta =
+            normalizePts(audioTrack.samples[0].pts, startPTS) - startPTS;
+          const audiovideoTimestampDelta = tsDelta / videoTrack.inputTimeScale;
+          audioTimeOffset += Math.max(0, audiovideoTimestampDelta);
+          videoTimeOffset += Math.max(0, -audiovideoTimestampDelta);
+        }
+
+        // Purposefully remuxing audio before video, so that remuxVideo can use nextAudioPts, which is calculated in remuxAudio.
+        if (enoughAudioSamples) {
+          // if initSegment was generated without audio samples, regenerate it again
+          if (!audioTrack.samplerate) {
+            this.warn('regenerate InitSegment as audio detected');
+            initSegment = this.generateIS(
+              audioTrack,
+              videoTrack,
+              timeOffset,
+              accurateTimeOffset,
+            );
+          }
+          audio = this.remuxAudio(
+            audioTrack,
+            audioTimeOffset,
+            this.isAudioContiguous,
+            accurateTimeOffset,
+            hasVideo ||
+              enoughVideoSamples ||
+              playlistType === PlaylistLevelType.AUDIO
+              ? videoTimeOffset
+              : undefined,
+          );
+          if (enoughVideoSamples) {
+            const audioTrackLength = audio ? audio.endPTS - audio.startPTS : 0;
+            // if initSegment was generated without video samples, regenerate it again
+            if (!videoTrack.inputTimeScale) {
+              this.warn('regenerate InitSegment as video detected');
+              initSegment = this.generateIS(
+                audioTrack,
+                videoTrack,
+                timeOffset,
+                accurateTimeOffset,
+              );
+            }
+            video = this.remuxVideo(
+              videoTrack,
+              videoTimeOffset,
+              isVideoContiguous,
+              audioTrackLength,
+            );
+          }
+        } else if (enoughVideoSamples) {
+          video = this.remuxVideo(
+            videoTrack,
+            videoTimeOffset,
+            isVideoContiguous,
+            0,
+          );
+        }
+        if (video) {
+          video.firstKeyFrame = firstKeyFrameIndex;
+          video.independent = firstKeyFrameIndex !== -1;
+          video.firstKeyFramePTS = firstKeyFramePTS;
+        }
+      }
+    }
+
+    // Allow ID3 and text to remux, even if more audio/video samples are required
+    if (this.ISGenerated && this._initPTS && this._initDTS) {
+      if (id3Track.samples.length) {
+        id3 = flushTextTrackMetadataCueSamples(
+          id3Track,
+          timeOffset,
+          this._initPTS,
+          this._initDTS,
+        );
+      }
+
+      if (textTrack.samples.length) {
+        text = flushTextTrackUserdataCueSamples(
+          textTrack,
+          timeOffset,
+          this._initPTS,
+        );
+      }
+    }
+
+    return {
+      audio,
+      video,
+      initSegment,
+      independent,
+      text,
+      id3,
+    };
+  }
+
+  computeInitPts(
+    basetime: number,
+    timescale: number,
+    presentationTime: number,
+    type: 'audio' | 'video',
+  ): number {
+    const offset = Math.round(presentationTime * timescale);
+    let timestamp = normalizePts(basetime, offset);
+    if (timestamp < offset + timescale) {
+      this.log(
+        `Adjusting PTS for rollover in timeline near ${(offset - timestamp) / timescale} ${type}`,
+      );
+      while (timestamp < offset + timescale) {
+        timestamp += 8589934592;
+      }
+    }
+    return timestamp - offset;
+  }
+
+  generateIS(
+    audioTrack: DemuxedAudioTrack,
+    videoTrack: DemuxedVideoTrack,
+    timeOffset: number,
+    accurateTimeOffset: boolean,
+  ): InitSegmentData | undefined {
+    const audioSamples = audioTrack.samples;
+    const videoSamples = videoTrack.samples;
+    const typeSupported = this.typeSupported;
+    const tracks: TrackSet = {};
+    const _initPTS = this._initPTS;
+    let computePTSDTS = !_initPTS || accurateTimeOffset;
+    let container = 'audio/mp4';
+    let initPTS: number | undefined;
+    let initDTS: number | undefined;
+    let timescale: number | undefined;
+    let trackId: number = -1;
+
+    if (computePTSDTS) {
+      initPTS = initDTS = Infinity;
+    }
+
+    if (audioTrack.config && audioSamples.length) {
+      // let's use audio sampling rate as MP4 time scale.
+      // rationale is that there is a integer nb of audio frames per audio sample (1024 for AAC)
+      // using audio sampling rate here helps having an integer MP4 frame duration
+      // this avoids potential rounding issue and AV sync issue
+      audioTrack.timescale = audioTrack.samplerate;
+      switch (audioTrack.segmentCodec) {
+        case 'mp3':
+          if (typeSupported.mpeg) {
+            // Chrome and Safari
+            container = 'audio/mpeg';
+            audioTrack.codec = '';
+          } else if (typeSupported.mp3) {
+            // Firefox
+            audioTrack.codec = 'mp3';
+          }
+          break;
+
+        case 'ac3':
+          audioTrack.codec = 'ac-3';
+          break;
+      }
+      tracks.audio = {
+        id: 'audio',
+        container: container,
+        codec: audioTrack.codec,
+        initSegment:
+          audioTrack.segmentCodec === 'mp3' && typeSupported.mpeg
+            ? new Uint8Array(0)
+            : MP4.initSegment([audioTrack]),
+        metadata: {
+          channelCount: audioTrack.channelCount,
+        },
+      };
+      if (computePTSDTS) {
+        trackId = audioTrack.id;
+        timescale = audioTrack.inputTimeScale;
+        if (!_initPTS || timescale !== _initPTS.timescale) {
+          // remember first PTS of this demuxing context. for audio, PTS = DTS
+          initPTS = initDTS = this.computeInitPts(
+            audioSamples[0].pts,
+            timescale,
+            timeOffset,
+            'audio',
+          );
+        } else {
+          computePTSDTS = false;
+        }
+      }
+    }
+
+    if (videoTrack.sps && videoTrack.pps && videoSamples.length) {
+      // let's use input time scale as MP4 video timescale
+      // we use input time scale straight away to avoid rounding issues on frame duration / cts computation
+      videoTrack.timescale = videoTrack.inputTimeScale;
+      tracks.video = {
+        id: 'main',
+        container: 'video/mp4',
+        codec: videoTrack.codec,
+        initSegment: MP4.initSegment([videoTrack]),
+        metadata: {
+          width: videoTrack.width,
+          height: videoTrack.height,
+        },
+      };
+      if (computePTSDTS) {
+        trackId = videoTrack.id;
+        timescale = videoTrack.inputTimeScale;
+        if (!_initPTS || timescale !== _initPTS.timescale) {
+          const basePTS = this.getVideoStartPts(videoSamples);
+          const baseDTS = normalizePts(videoSamples[0].dts, basePTS);
+          const videoInitDTS = this.computeInitPts(
+            baseDTS,
+            timescale,
+            timeOffset,
+            'video',
+          );
+          const videoInitPTS = this.computeInitPts(
+            basePTS,
+            timescale,
+            timeOffset,
+            'video',
+          );
+          initDTS = Math.min(initDTS as number, videoInitDTS);
+          initPTS = Math.min(initPTS as number, videoInitPTS);
+        } else {
+          computePTSDTS = false;
+        }
+      }
+      this.videoTrackConfig = {
+        width: videoTrack.width,
+        height: videoTrack.height,
+        pixelRatio: videoTrack.pixelRatio,
+      };
+    }
+
+    if (Object.keys(tracks).length) {
+      this.ISGenerated = true;
+      if (computePTSDTS) {
+        if (_initPTS) {
+          this.warn(
+            `Timestamps at playlist time: ${accurateTimeOffset ? '' : '~'}${timeOffset} ${initPTS! / timescale!} != initPTS: ${_initPTS.baseTime / _initPTS.timescale} (${_initPTS.baseTime}/${_initPTS.timescale}) trackId: ${_initPTS.trackId}`,
+          );
+        }
+        this.log(
+          `Found initPTS at playlist time: ${timeOffset} offset: ${initPTS! / timescale!} (${initPTS}/${timescale}) trackId: ${trackId}`,
+        );
+        this._initPTS = {
+          baseTime: initPTS as number,
+          timescale: timescale as number,
+          trackId: trackId as number,
+        };
+        this._initDTS = {
+          baseTime: initDTS as number,
+          timescale: timescale as number,
+          trackId: trackId as number,
+        };
+      } else {
+        initPTS = timescale = undefined;
+      }
+
+      return {
+        tracks,
+        initPTS,
+        timescale,
+        trackId,
+      };
+    }
+  }
+
+  remuxVideo(
+    track: DemuxedVideoTrack,
+    timeOffset: number,
+    contiguous: boolean,
+    audioTrackLength: number,
+  ): RemuxedTrack | undefined {
+    const timeScale: number = track.inputTimeScale;
+    const inputSamples: Array<VideoSample> = track.samples;
+    const outputSamples: Array<Mp4Sample> = [];
+    const nbSamples = inputSamples.length;
+    const initPTS = this._initPTS as RationalTimestamp;
+    const initTime = (initPTS.baseTime * timeScale) / initPTS.timescale;
+    let nextVideoTs = this.nextVideoTs;
+    let offset = 8;
+    let mp4SampleDuration = this.videoSampleDuration;
+    let firstDTS;
+    let lastDTS;
+    let minPTS: number = Number.POSITIVE_INFINITY;
+    let maxPTS: number = Number.NEGATIVE_INFINITY;
+    let sortSamples = false;
+
+    // if parsed fragment is contiguous with last one, let's use last DTS value as reference
+    if (!contiguous || nextVideoTs === null) {
+      const pts = initTime + timeOffset * timeScale;
+      const cts =
+        inputSamples[0].pts -
+        normalizePts(inputSamples[0].dts, inputSamples[0].pts);
+      if (
+        chromeVersion &&
+        nextVideoTs !== null &&
+        Math.abs(pts - cts - (nextVideoTs + initTime)) < 15000
+      ) {
+        // treat as contigous to adjust samples that would otherwise produce video buffer gaps in Chrome
+        contiguous = true;
+      } else {
+        // if not contiguous, let's use target timeOffset
+        nextVideoTs = pts - cts - initTime;
+      }
+    }
+
+    // PTS is coded on 33bits, and can loop from -2^32 to 2^32
+    // PTSNormalize will make PTS/DTS value monotonic, we use last known DTS value as reference value
+    const nextVideoPts = nextVideoTs + initTime;
+    for (let i = 0; i < nbSamples; i++) {
+      const sample = inputSamples[i];
+      sample.pts = normalizePts(sample.pts, nextVideoPts);
+      sample.dts = normalizePts(sample.dts, nextVideoPts);
+      if (sample.dts < inputSamples[i > 0 ? i - 1 : i].dts) {
+        sortSamples = true;
+      }
+    }
+
+    // sort video samples by DTS then PTS then demux id order
+    if (sortSamples) {
+      inputSamples.sort(function (a, b) {
+        const deltadts = a.dts - b.dts;
+        const deltapts = a.pts - b.pts;
+        return deltadts || deltapts;
+      });
+    }
+
+    // Get first/last DTS
+    firstDTS = inputSamples[0].dts;
+    lastDTS = inputSamples[inputSamples.length - 1].dts;
+
+    // Sample duration (as expected by trun MP4 boxes), should be the delta between sample DTS
+    // set this constant duration as being the avg delta between consecutive DTS.
+    const inputDuration = lastDTS - firstDTS;
+    const averageSampleDuration = inputDuration
+      ? Math.round(inputDuration / (nbSamples - 1))
+      : mp4SampleDuration || track.inputTimeScale / 30;
+
+    // if fragment are contiguous, detect hole/overlapping between fragments
+    if (contiguous) {
+      // check timestamp continuity across consecutive fragments (this is to remove inter-fragment gap/hole)
+      const delta = firstDTS - nextVideoPts;
+      const foundHole = delta > averageSampleDuration;
+      const foundOverlap = delta < -1;
+      if (foundHole || foundOverlap) {
+        if (foundHole) {
+          this.warn(
+            `${(track.segmentCodec || '').toUpperCase()}: ${toMsFromMpegTsClock(
+              delta,
+              true,
+            )} ms (${delta}dts) hole between fragments detected at ${timeOffset.toFixed(
+              3,
+            )}`,
+          );
+        } else {
+          this.warn(
+            `${(track.segmentCodec || '').toUpperCase()}: ${toMsFromMpegTsClock(
+              -delta,
+              true,
+            )} ms (${delta}dts) overlapping between fragments detected at ${timeOffset.toFixed(
+              3,
+            )}`,
+          );
+        }
+        if (
+          !foundOverlap ||
+          nextVideoPts >= inputSamples[0].pts ||
+          chromeVersion
+        ) {
+          firstDTS = nextVideoPts;
+          const firstPTS = inputSamples[0].pts - delta;
+          if (foundHole) {
+            inputSamples[0].dts = firstDTS;
+            inputSamples[0].pts = firstPTS;
+          } else {
+            let isPTSOrderRetained = true;
+            for (let i = 0; i < inputSamples.length; i++) {
+              if (inputSamples[i].dts > firstPTS && isPTSOrderRetained) {
+                break;
+              }
+
+              const prevPTS = inputSamples[i].pts;
+              inputSamples[i].dts -= delta;
+              inputSamples[i].pts -= delta;
+
+              // check to see if this sample's PTS order has changed
+              // relative to the next one
+              if (i < inputSamples.length - 1) {
+                const nextSamplePTS = inputSamples[i + 1].pts;
+                const currentSamplePTS = inputSamples[i].pts;
+
+                const currentOrder = nextSamplePTS <= currentSamplePTS;
+                const prevOrder = nextSamplePTS <= prevPTS;
+
+                isPTSOrderRetained = currentOrder == prevOrder;
+              }
+            }
+          }
+          this.log(
+            `Video: Initial PTS/DTS adjusted: ${toMsFromMpegTsClock(
+              firstPTS,
+              true,
+            )}/${toMsFromMpegTsClock(
+              firstDTS,
+              true,
+            )}, delta: ${toMsFromMpegTsClock(delta, true)} ms`,
+          );
+        }
+      }
+    }
+
+    firstDTS = Math.max(0, firstDTS);
+
+    let nbNalu = 0;
+    let naluLen = 0;
+    let dtsStep = firstDTS;
+    for (let i = 0; i < nbSamples; i++) {
+      // compute total/avc sample length and nb of NAL units
+      const sample = inputSamples[i];
+      const units = sample.units;
+      const nbUnits = units.length;
+      let sampleLen = 0;
+      for (let j = 0; j < nbUnits; j++) {
+        sampleLen += units[j].data.length;
+      }
+
+      naluLen += sampleLen;
+      nbNalu += nbUnits;
+      sample.length = sampleLen;
+
+      // ensure sample monotonic DTS
+      if (sample.dts < dtsStep) {
+        sample.dts = dtsStep;
+        dtsStep += (averageSampleDuration / 4) | 0 || 1;
+      } else {
+        dtsStep = sample.dts;
+      }
+
+      minPTS = Math.min(sample.pts, minPTS);
+      maxPTS = Math.max(sample.pts, maxPTS);
+    }
+    lastDTS = inputSamples[nbSamples - 1].dts;
+
+    /* concatenate the video data and construct the mdat in place
+      (need 8 more bytes to fill length and mpdat type) */
+    const mdatSize = naluLen + 4 * nbNalu + 8;
+    let mdat;
+    try {
+      mdat = new Uint8Array(mdatSize);
+    } catch (err) {
+      this.observer.emit(Events.ERROR, Events.ERROR, {
+        type: ErrorTypes.MUX_ERROR,
+        details: ErrorDetails.REMUX_ALLOC_ERROR,
+        fatal: false,
+        error: err,
+        bytes: mdatSize,
+        reason: `fail allocating video mdat ${mdatSize}`,
+      });
+      return;
+    }
+    const view = new DataView(mdat.buffer);
+    view.setUint32(0, mdatSize);
+    mdat.set(MP4.types.mdat, 4);
+
+    let stretchedLastFrame = false;
+    let minDtsDelta = Number.POSITIVE_INFINITY;
+    let minPtsDelta = Number.POSITIVE_INFINITY;
+    let maxDtsDelta = Number.NEGATIVE_INFINITY;
+    let maxPtsDelta = Number.NEGATIVE_INFINITY;
+    for (let i = 0; i < nbSamples; i++) {
+      const VideoSample = inputSamples[i];
+      const VideoSampleUnits = VideoSample.units;
+      let mp4SampleLength = 0;
+      // convert NALU bitstream to MP4 format (prepend NALU with size field)
+      for (let j = 0, nbUnits = VideoSampleUnits.length; j < nbUnits; j++) {
+        const unit = VideoSampleUnits[j];
+        const unitData = unit.data;
+        const unitDataLen = unit.data.byteLength;
+        view.setUint32(offset, unitDataLen);
+        offset += 4;
+        mdat.set(unitData, offset);
+        offset += unitDataLen;
+        mp4SampleLength += 4 + unitDataLen;
+      }
+
+      // expected sample duration is the Decoding Timestamp diff of consecutive samples
+      let ptsDelta;
+      if (i < nbSamples - 1) {
+        mp4SampleDuration = inputSamples[i + 1].dts - VideoSample.dts;
+        ptsDelta = inputSamples[i + 1].pts - VideoSample.pts;
+      } else {
+        const config = this.config;
+        const lastFrameDuration =
+          i > 0
+            ? VideoSample.dts - inputSamples[i - 1].dts
+            : averageSampleDuration;
+        ptsDelta =
+          i > 0
+            ? VideoSample.pts - inputSamples[i - 1].pts
+            : averageSampleDuration;
+        if (config.stretchShortVideoTrack && this.nextAudioTs !== null) {
+          // In some cases, a segment's audio track duration may exceed the video track duration.
+          // Since we've already remuxed audio, and we know how long the audio track is, we look to
+          // see if the delta to the next segment is longer than maxBufferHole.
+          // If so, playback would potentially get stuck, so we artificially inflate
+          // the duration of the last frame to minimize any potential gap between segments.
+          const gapTolerance = Math.floor(config.maxBufferHole * timeScale);
+          const deltaToFrameEnd =
+            (audioTrackLength
+              ? minPTS + audioTrackLength * timeScale
+              : this.nextAudioTs + initTime) - VideoSample.pts;
+          if (deltaToFrameEnd > gapTolerance) {
+            // We subtract lastFrameDuration from deltaToFrameEnd to try to prevent any video
+            // frame overlap. maxBufferHole should be >> lastFrameDuration anyway.
+            mp4SampleDuration = deltaToFrameEnd - lastFrameDuration;
+            if (mp4SampleDuration < 0) {
+              mp4SampleDuration = lastFrameDuration;
+            } else {
+              stretchedLastFrame = true;
+            }
+            this.log(
+              `It is approximately ${
+                deltaToFrameEnd / 90
+              } ms to the next segment; using duration ${
+                mp4SampleDuration / 90
+              } ms for the last video frame.`,
+            );
+          } else {
+            mp4SampleDuration = lastFrameDuration;
+          }
+        } else {
+          mp4SampleDuration = lastFrameDuration;
+        }
+      }
+      const compositionTimeOffset = Math.round(
+        VideoSample.pts - VideoSample.dts,
+      );
+      minDtsDelta = Math.min(minDtsDelta, mp4SampleDuration);
+      maxDtsDelta = Math.max(maxDtsDelta, mp4SampleDuration);
+      minPtsDelta = Math.min(minPtsDelta, ptsDelta);
+      maxPtsDelta = Math.max(maxPtsDelta, ptsDelta);
+
+      outputSamples.push(
+        createMp4Sample(
+          VideoSample.key,
+          mp4SampleDuration,
+          mp4SampleLength,
+          compositionTimeOffset,
+        ),
+      );
+    }
+
+    if (outputSamples.length) {
+      if (chromeVersion) {
+        if (chromeVersion < 70) {
+          // Chrome workaround, mark first sample as being a Random Access Point (keyframe) to avoid sourcebuffer append issue
+          // https://code.google.com/p/chromium/issues/detail?id=229412
+          const flags = outputSamples[0].flags;
+          flags.dependsOn = 2;
+          flags.isNonSync = 0;
+        }
+      } else if (safariWebkitVersion) {
+        // Fix for "CNN special report, with CC" in test-streams (Safari browser only)
+        // Ignore DTS when frame durations are irregular. Safari MSE does not handle this leading to gaps.
+        if (
+          maxPtsDelta - minPtsDelta < maxDtsDelta - minDtsDelta &&
+          averageSampleDuration / maxDtsDelta < 0.025 &&
+          outputSamples[0].cts === 0
+        ) {
+          this.warn(
+            'Found irregular gaps in sample duration. Using PTS instead of DTS to determine MP4 sample duration.',
+          );
+          let dts = firstDTS;
+          for (let i = 0, len = outputSamples.length; i < len; i++) {
+            const nextDts = dts + outputSamples[i].duration;
+            const pts = dts + outputSamples[i].cts;
+            if (i < len - 1) {
+              const nextPts = nextDts + outputSamples[i + 1].cts;
+              outputSamples[i].duration = nextPts - pts;
+            } else {
+              outputSamples[i].duration = i
+                ? outputSamples[i - 1].duration
+                : averageSampleDuration;
+            }
+            outputSamples[i].cts = 0;
+            dts = nextDts;
+          }
+        }
+      }
+    }
+    // next AVC/HEVC sample DTS should be equal to last sample DTS + last sample duration (in PES timescale)
+    mp4SampleDuration =
+      stretchedLastFrame || !mp4SampleDuration
+        ? averageSampleDuration
+        : mp4SampleDuration;
+    const endDTS = lastDTS + mp4SampleDuration;
+    this.nextVideoTs = nextVideoTs = endDTS - initTime;
+    this.videoSampleDuration = mp4SampleDuration;
+    this.isVideoContiguous = true;
+    const moof = MP4.moof(
+      track.sequenceNumber++,
+      firstDTS,
+      Object.assign(track, {
+        samples: outputSamples,
+      }),
+    );
+    const type: SourceBufferName = 'video';
+    const data = {
+      data1: moof,
+      data2: mdat,
+      startPTS: (minPTS - initTime) / timeScale,
+      endPTS: (maxPTS + mp4SampleDuration - initTime) / timeScale,
+      startDTS: (firstDTS - initTime) / timeScale,
+      endDTS: nextVideoTs / timeScale,
+      type,
+      hasAudio: false,
+      hasVideo: true,
+      nb: outputSamples.length,
+      dropped: track.dropped,
+    };
+    track.samples = [];
+    track.dropped = 0;
+    return data;
+  }
+
+  getSamplesPerFrame(track: DemuxedAudioTrack) {
+    switch (track.segmentCodec) {
+      case 'mp3':
+        return MPEG_AUDIO_SAMPLE_PER_FRAME;
+      case 'ac3':
+        return AC3_SAMPLES_PER_FRAME;
+      default:
+        return AAC_SAMPLES_PER_FRAME;
+    }
+  }
+
+  remuxAudio(
+    track: DemuxedAudioTrack,
+    timeOffset: number,
+    contiguous: boolean,
+    accurateTimeOffset: boolean,
+    videoTimeOffset?: number,
+  ): RemuxedTrack | undefined {
+    const inputTimeScale: number = track.inputTimeScale;
+    const mp4timeScale: number = track.samplerate
+      ? track.samplerate
+      : inputTimeScale;
+    const scaleFactor: number = inputTimeScale / mp4timeScale;
+    const mp4SampleDuration: number = this.getSamplesPerFrame(track);
+    const inputSampleDuration: number = mp4SampleDuration * scaleFactor;
+    const initPTS = this._initPTS as RationalTimestamp;
+    const rawMPEG: boolean =
+      track.segmentCodec === 'mp3' && this.typeSupported.mpeg;
+    const outputSamples: Array<Mp4Sample> = [];
+    const alignedWithVideo = videoTimeOffset !== undefined;
+
+    let inputSamples: Array<AudioSample> = track.samples;
+    let offset: number = rawMPEG ? 0 : 8;
+    let nextAudioTs: number = this.nextAudioTs || -1;
+
+    // window.audioSamples ? window.audioSamples.push(inputSamples.map(s => s.pts)) : (window.audioSamples = [inputSamples.map(s => s.pts)]);
+
+    // for audio samples, also consider consecutive fragments as being contiguous (even if a level switch occurs),
+    // for sake of clarity:
+    // consecutive fragments are frags with
+    //  - less than 100ms gaps between new time offset (if accurate) and next expected PTS OR
+    //  - less than 20 audio frames distance
+    // contiguous fragments are consecutive fragments from same quality level (same level, new SN = old SN + 1)
+    // this helps ensuring audio continuity
+    // and this also avoids audio glitches/cut when switching quality, or reporting wrong duration on first audio frame
+    const initTime = (initPTS.baseTime * inputTimeScale) / initPTS.timescale;
+    const timeOffsetMpegTS = initTime + timeOffset * inputTimeScale;
+    this.isAudioContiguous = contiguous =
+      contiguous ||
+      ((inputSamples.length &&
+        nextAudioTs > 0 &&
+        ((accurateTimeOffset &&
+          Math.abs(timeOffsetMpegTS - (nextAudioTs + initTime)) < 9000) ||
+          Math.abs(
+            normalizePts(inputSamples[0].pts, timeOffsetMpegTS) -
+              (nextAudioTs + initTime),
+          ) <
+            20 * inputSampleDuration)) as boolean);
+
+    // compute normalized PTS
+    inputSamples.forEach(function (sample) {
+      sample.pts = normalizePts(sample.pts, timeOffsetMpegTS);
+    });
+
+    if (!contiguous || nextAudioTs < 0) {
+      const sampleCount = inputSamples.length;
+      // filter out sample with negative PTS that are not playable anyway
+      // if we don't remove these negative samples, they will shift all audio samples forward.
+      // leading to audio overlap between current / next fragment
+      inputSamples = inputSamples.filter((sample) => sample.pts >= 0);
+
+      if (sampleCount !== inputSamples.length) {
+        this.warn(
+          `Removed ${inputSamples.length - sampleCount} of ${sampleCount} samples (initPTS ${initTime} / ${inputTimeScale})`,
+        );
+      }
+
+      // in case all samples have negative PTS, and have been filtered out, return now
+      if (!inputSamples.length) {
+        return;
+      }
+
+      if (videoTimeOffset === 0) {
+        // Set the start to match video so that start gaps larger than inputSampleDuration are filled with silence
+        nextAudioTs = 0;
+      } else if (accurateTimeOffset && !alignedWithVideo) {
+        // When not seeking, not live, and LevelDetails.PTSKnown, use fragment start as predicted next audio PTS
+        nextAudioTs = Math.max(0, timeOffsetMpegTS - initTime);
+      } else {
+        // if frags are not contiguous and if we cant trust time offset, let's use first sample PTS as next audio PTS
+        nextAudioTs = inputSamples[0].pts - initTime;
+      }
+    }
+
+    // If the audio track is missing samples, the frames seem to get "left-shifted" within the
+    // resulting mp4 segment, causing sync issues and leaving gaps at the end of the audio segment.
+    // In an effort to prevent this from happening, we inject frames here where there are gaps.
+    // When possible, we inject a silent frame; when that's not possible, we duplicate the last
+    // frame.
+
+    if (track.segmentCodec === 'aac') {
+      const maxAudioFramesDrift = this.config.maxAudioFramesDrift;
+      for (
+        let i = 0, nextPts = nextAudioTs + initTime;
+        i < inputSamples.length;
+        i++
+      ) {
+        // First, let's see how far off this frame is from where we expect it to be
+        const sample = inputSamples[i];
+        const pts = sample.pts;
+        const delta = pts - nextPts;
+        const duration = Math.abs((1000 * delta) / inputTimeScale);
+
+        // When remuxing with video, if we're overlapping by more than a duration, drop this sample to stay in sync
+        if (
+          delta <= -maxAudioFramesDrift * inputSampleDuration &&
+          alignedWithVideo
+        ) {
+          if (i === 0) {
+            this.warn(
+              `Audio frame @ ${(pts / inputTimeScale).toFixed(
+                3,
+              )}s overlaps marker by ${Math.round(
+                (1000 * delta) / inputTimeScale,
+              )} ms.`,
+            );
+            this.nextAudioTs = nextAudioTs = pts - initTime;
+            nextPts = pts;
+          }
+        } // eslint-disable-line brace-style
+
+        // Insert missing frames if:
+        // 1: We're more than maxAudioFramesDrift frame away
+        // 2: Not more than MAX_SILENT_FRAME_DURATION away
+        // 3: currentTime (aka nextPtsNorm) is not 0
+        // 4: remuxing with video (videoTimeOffset !== undefined)
+        else if (
+          delta >= maxAudioFramesDrift * inputSampleDuration &&
+          duration < MAX_SILENT_FRAME_DURATION &&
+          alignedWithVideo
+        ) {
+          let missing = Math.round(delta / inputSampleDuration);
+          // Adjust nextPts so that silent samples are aligned with media pts. This will prevent media samples from
+          // later being shifted if nextPts is based on timeOffset and delta is not a multiple of inputSampleDuration.
+          nextPts = pts - missing * inputSampleDuration;
+          while (nextPts < 0 && missing && inputSampleDuration) {
+            missing--;
+            nextPts += inputSampleDuration;
+          }
+          if (i === 0) {
+            this.nextAudioTs = nextAudioTs = nextPts - initTime;
+          }
+          this.warn(
+            `Injecting ${missing} audio frames @ ${(
+              (nextPts - initTime) /
+              inputTimeScale
+            ).toFixed(3)}s due to ${Math.round(
+              (1000 * delta) / inputTimeScale,
+            )} ms gap.`,
+          );
+          for (let j = 0; j < missing; j++) {
+            let fillFrame = AAC.getSilentFrame(
+              track.parsedCodec || track.manifestCodec || track.codec,
+              track.channelCount,
+            );
+            if (!fillFrame) {
+              this.log(
+                'Unable to get silent frame for given audio codec; duplicating last frame instead.',
+              );
+              fillFrame = sample.unit.subarray();
+            }
+            inputSamples.splice(i, 0, {
+              unit: fillFrame,
+              pts: nextPts,
+            });
+            nextPts += inputSampleDuration;
+            i++;
+          }
+        }
+        sample.pts = nextPts;
+        nextPts += inputSampleDuration;
+      }
+    }
+    let firstPTS: number | null = null;
+    let lastPTS: number | null = null;
+    let mdat: any;
+    let mdatSize: number = 0;
+    let sampleLength: number = inputSamples.length;
+    while (sampleLength--) {
+      mdatSize += inputSamples[sampleLength].unit.byteLength;
+    }
+    for (let j = 0, nbSamples = inputSamples.length; j < nbSamples; j++) {
+      const audioSample = inputSamples[j];
+      const unit = audioSample.unit;
+      let pts = audioSample.pts;
+      if (lastPTS !== null) {
+        // If we have more than one sample, set the duration of the sample to the "real" duration; the PTS diff with
+        // the previous sample
+        const prevSample = outputSamples[j - 1];
+        prevSample.duration = Math.round((pts - lastPTS) / scaleFactor);
+      } else {
+        if (contiguous && track.segmentCodec === 'aac') {
+          // set PTS/DTS to expected PTS/DTS
+          pts = nextAudioTs + initTime;
+        }
+        // remember first PTS of our audioSamples
+        firstPTS = pts;
+        if (mdatSize > 0) {
+          /* concatenate the audio data and construct the mdat in place
+            (need 8 more bytes to fill length and mdat type) */
+          mdatSize += offset;
+          try {
+            mdat = new Uint8Array(mdatSize);
+          } catch (err) {
+            this.observer.emit(Events.ERROR, Events.ERROR, {
+              type: ErrorTypes.MUX_ERROR,
+              details: ErrorDetails.REMUX_ALLOC_ERROR,
+              fatal: false,
+              error: err,
+              bytes: mdatSize,
+              reason: `fail allocating audio mdat ${mdatSize}`,
+            });
+            return;
+          }
+          if (!rawMPEG) {
+            const view = new DataView(mdat.buffer);
+            view.setUint32(0, mdatSize);
+            mdat.set(MP4.types.mdat, 4);
+          }
+        } else {
+          // no audio samples
+          return;
+        }
+      }
+      mdat.set(unit, offset);
+      const unitLen = unit.byteLength;
+      offset += unitLen;
+      // Default the sample's duration to the computed mp4SampleDuration, which will either be 1024 for AAC or 1152 for MPEG
+      // In the case that we have 1 sample, this will be the duration. If we have more than one sample, the duration
+      // becomes the PTS diff with the previous sample
+      outputSamples.push(createMp4Sample(true, mp4SampleDuration, unitLen, 0));
+      lastPTS = pts;
+    }
+
+    // We could end up with no audio samples if all input samples were overlapping with the previously remuxed ones
+    const nbSamples = outputSamples.length;
+    if (!nbSamples) {
+      return;
+    }
+
+    // The next audio sample PTS should be equal to last sample PTS + duration
+    const lastSample = outputSamples[outputSamples.length - 1];
+    nextAudioTs = (lastPTS as number) - initTime;
+    this.nextAudioTs = nextAudioTs + scaleFactor * lastSample.duration;
+
+    // Set the track samples from inputSamples to outputSamples before remuxing
+    const moof = rawMPEG
+      ? new Uint8Array(0)
+      : MP4.moof(
+          track.sequenceNumber++,
+          firstPTS! / scaleFactor,
+          Object.assign({}, track, { samples: outputSamples }),
+        );
+
+    // Clear the track samples. This also clears the samples array in the demuxer, since the reference is shared
+    track.samples = [];
+    const start = (firstPTS! - initTime) / inputTimeScale;
+    const end = this.nextAudioTs / inputTimeScale;
+    const type: SourceBufferName = 'audio';
+    const audioData = {
+      data1: moof,
+      data2: mdat,
+      startPTS: start,
+      endPTS: end,
+      startDTS: start,
+      endDTS: end,
+      type,
+      hasAudio: true,
+      hasVideo: false,
+      nb: nbSamples,
+    };
+
+    this.isAudioContiguous = true;
+    return audioData;
+  }
+}
+
+export function normalizePts(value: number, reference: number | null): number {
+  let offset;
+  if (reference === null) {
+    return value;
+  }
+
+  if (reference < value) {
+    // - 2^33
+    offset = -8589934592;
+  } else {
+    // + 2^33
+    offset = 8589934592;
+  }
+  /* PTS is 33bit (from 0 to 2^33 -1)
+    if diff between value and reference is bigger than half of the amplitude (2^32) then it means that
+    PTS looping occured. fill the gap */
+  while (Math.abs(value - reference) > 4294967296) {
+    value += offset;
+  }
+
+  return value;
+}
+
+function findKeyframeIndex(samples: Array<VideoSample>): number {
+  for (let i = 0; i < samples.length; i++) {
+    if (samples[i].key) {
+      return i;
+    }
+  }
+  return -1;
+}
+
+export function flushTextTrackMetadataCueSamples(
+  track: DemuxedMetadataTrack,
+  timeOffset: number,
+  initPTS: TimestampOffset,
+  initDTS: TimestampOffset,
+): RemuxedMetadata | undefined {
+  const length = track.samples.length;
+  if (!length) {
+    return;
+  }
+  const inputTimeScale = track.inputTimeScale;
+  for (let index = 0; index < length; index++) {
+    const sample = track.samples[index];
+    // setting id3 pts, dts to relative time
+    // using this._initPTS and this._initDTS to calculate relative time
+    sample.pts =
+      normalizePts(
+        sample.pts - (initPTS.baseTime * inputTimeScale) / initPTS.timescale,
+        timeOffset * inputTimeScale,
+      ) / inputTimeScale;
+    sample.dts =
+      normalizePts(
+        sample.dts - (initDTS.baseTime * inputTimeScale) / initDTS.timescale,
+        timeOffset * inputTimeScale,
+      ) / inputTimeScale;
+  }
+  const samples = track.samples;
+  track.samples = [];
+  return {
+    samples,
+  };
+}
+
+export function flushTextTrackUserdataCueSamples(
+  track: DemuxedUserdataTrack,
+  timeOffset: number,
+  initPTS: RationalTimestamp,
+): RemuxedUserdata | undefined {
+  const length = track.samples.length;
+  if (!length) {
+    return;
+  }
+
+  const inputTimeScale = track.inputTimeScale;
+  for (let index = 0; index < length; index++) {
+    const sample = track.samples[index];
+    // setting text pts, dts to relative time
+    // using this._initPTS and this._initDTS to calculate relative time
+    sample.pts =
+      normalizePts(
+        sample.pts - (initPTS.baseTime * inputTimeScale) / initPTS.timescale,
+        timeOffset * inputTimeScale,
+      ) / inputTimeScale;
+  }
+  track.samples.sort((a, b) => a.pts - b.pts);
+  const samples = track.samples;
+  track.samples = [];
+  return {
+    samples,
+  };
+}