hls.js 1.5.14-0.canary.10559 → 1.5.15

package/src/demux/video/avc-video-parser.ts

@@ -1,22 +1,26 @@
 import BaseVideoParser from './base-video-parser';
-import type {
+import {
   DemuxedVideoTrack,
   DemuxedUserdataTrack,
+  VideoSampleUnit,
 } from '../../types/demuxer';
-import { parseSEIMessageFromNALu } from '../../utils/mp4-tools';
-
-import type { PES } from '../tsdemuxer';
-
+import {
+  appendUint8Array,
+  parseSEIMessageFromNALu,
+} from '../../utils/mp4-tools';
 import ExpGolomb from './exp-golomb';
+import type { PES } from '../tsdemuxer';
 
 class AvcVideoParser extends BaseVideoParser {
-  public parsePES(
+  public parseAVCPES(
     track: DemuxedVideoTrack,
     textTrack: DemuxedUserdataTrack,
     pes: PES,
-    endOfSegment: boolean,
+    last: boolean,
+    duration: number,
   ) {
-    const units = this.parseNALu(track, pes.data, endOfSegment);
+    const units = this.parseAVCNALu(track, pes.data);
+    const debug = false;
     let VideoSample = this.VideoSample;
     let push: boolean;
     let spsfound = false;
@@ -31,6 +35,7 @@ class AvcVideoParser extends BaseVideoParser {
         false,
         pes.pts,
         pes.dts,
+        '',
       );
     }
 
@@ -44,7 +49,7 @@ class AvcVideoParser extends BaseVideoParser {
           // only check slice type to detect KF in case SPS found in same packet (any keyframe is preceded by SPS ...)
           if (spsfound && data.length > 4) {
             // retrieve slice type by parsing beginning of NAL unit (follow H264 spec, slice_header definition) to detect keyframe embedded in NDR
-            const sliceType = this.readSliceType(data);
+            const sliceType = new ExpGolomb(data).readSliceType();
             // 2 : I slice, 4 : SI slice, 7 : I slice, 9: SI slice
             // SI slice : A slice that is coded using intra prediction only and using quantisation of the prediction samples.
             // An SI slice can be coded such that its decoded samples can be constructed identically to an SP slice.
@@ -73,8 +78,14 @@ class AvcVideoParser extends BaseVideoParser {
               true,
               pes.pts,
               pes.dts,
+              '',
             );
           }
+
+          if (debug) {
+            VideoSample.debug += 'NDR ';
+          }
+
           VideoSample.frame = true;
           VideoSample.key = iskey;
 
@@ -94,15 +105,23 @@ class AvcVideoParser extends BaseVideoParser {
               true,
               pes.pts,
               pes.dts,
+              '',
             );
           }
 
+          if (debug) {
+            VideoSample.debug += 'IDR ';
+          }
+
           VideoSample.key = true;
           VideoSample.frame = true;
           break;
         // SEI
         case 6: {
           push = true;
+          if (debug && VideoSample) {
+            VideoSample.debug += 'SEI ';
+          }
           parseSEIMessageFromNALu(
             unit.data,
             1,
@@ -115,8 +134,13 @@ class AvcVideoParser extends BaseVideoParser {
         case 7: {
           push = true;
           spsfound = true;
+          if (debug && VideoSample) {
+            VideoSample.debug += 'SPS ';
+          }
           const sps = unit.data;
-          const config = this.readSPS(sps);
+          const expGolombDecoder = new ExpGolomb(sps);
+          const config = expGolombDecoder.readSPS();
+
           if (
             !track.sps ||
             track.width !== config.width ||
@@ -128,6 +152,7 @@ class AvcVideoParser extends BaseVideoParser {
             track.height = config.height;
             track.pixelRatio = config.pixelRatio;
             track.sps = [sps];
+            track.duration = duration;
             const codecarray = sps.subarray(1, 4);
             let codecstring = 'avc1.';
             for (let i = 0; i < 3; i++) {
@@ -140,11 +165,15 @@ class AvcVideoParser extends BaseVideoParser {
             }
             track.codec = codecstring;
           }
+
           break;
         }
         // PPS
         case 8:
           push = true;
+          if (debug && VideoSample) {
+            VideoSample.debug += 'PPS ';
+          }
 
           track.pps = [unit.data];
 
@@ -153,17 +182,16 @@ class AvcVideoParser extends BaseVideoParser {
         case 9:
           push = true;
           track.audFound = true;
-          if (VideoSample?.frame) {
+          if (VideoSample) {
             this.pushAccessUnit(VideoSample, track);
-            VideoSample = null;
-          }
-          if (!VideoSample) {
-            VideoSample = this.VideoSample = this.createVideoSample(
-              false,
-              pes.pts,
-              pes.dts,
-            );
           }
+
+          VideoSample = this.VideoSample = this.createVideoSample(
+            false,
+            pes.pts,
+            pes.dts,
+            debug ? 'AUD ' : '',
+          );
           break;
         // Filler Data
         case 12:
@@ -171,6 +199,9 @@ class AvcVideoParser extends BaseVideoParser {
           break;
         default:
           push = false;
+          if (VideoSample) {
+            VideoSample.debug += 'unknown NAL ' + unit.type + ' ';
+          }
 
           break;
       }
@@ -180,223 +211,129 @@ class AvcVideoParser extends BaseVideoParser {
       }
     });
     // if last PES packet, push samples
-    if (endOfSegment && VideoSample) {
+    if (last && VideoSample) {
       this.pushAccessUnit(VideoSample, track);
       this.VideoSample = null;
     }
   }
 
-  protected getNALuType(data: Uint8Array, offset: number): number {
-    return data[offset] & 0x1f;
-  }
-
-  readSliceType(data: Uint8Array) {
-    const eg = new ExpGolomb(data);
-    // skip NALu type
-    eg.readUByte();
-    // discard first_mb_in_slice
-    eg.readUEG();
-    // return slice_type
-    return eg.readUEG();
-  }
+  private parseAVCNALu(
+    track: DemuxedVideoTrack,
+    array: Uint8Array,
+  ): Array<{
+    data: Uint8Array;
+    type: number;
+    state?: number;
+  }> {
+    const len = array.byteLength;
+    let state = track.naluState || 0;
+    const lastState = state;
+    const units: VideoSampleUnit[] = [];
+    let i = 0;
+    let value: number;
+    let overflow: number;
+    let unitType: number;
+    let lastUnitStart = -1;
+    let lastUnitType: number = 0;
+    // logger.log('PES:' + Hex.hexDump(array));
 
-  /**
-   * The scaling list is optionally transmitted as part of a sequence parameter
-   * set and is not relevant to transmuxing.
-   * @param count the number of entries in this scaling list
-   * @see Recommendation ITU-T H.264, Section 7.3.2.1.1.1
-   */
-  skipScalingList(count: number, reader: ExpGolomb): void {
-    let lastScale = 8;
-    let nextScale = 8;
-    let deltaScale;
-    for (let j = 0; j < count; j++) {
-      if (nextScale !== 0) {
-        deltaScale = reader.readEG();
-        nextScale = (lastScale + deltaScale + 256) % 256;
-      }
-      lastScale = nextScale === 0 ? lastScale : nextScale;
+    if (state === -1) {
+      // special use case where we found 3 or 4-byte start codes exactly at the end of previous PES packet
+      lastUnitStart = 0;
+      // NALu type is value read from offset 0
+      lastUnitType = array[0] & 0x1f;
+      state = 0;
+      i = 1;
     }
-  }
-
-  /**
-   * Read a sequence parameter set and return some interesting video
-   * properties. A sequence parameter set is the H264 metadata that
-   * describes the properties of upcoming video frames.
-   * @returns an object with configuration parsed from the
-   * sequence parameter set, including the dimensions of the
-   * associated video frames.
-   */
-  readSPS(sps: Uint8Array): {
-    width: number;
-    height: number;
-    pixelRatio: [number, number];
-  } {
-    const eg = new ExpGolomb(sps);
-    let frameCropLeftOffset = 0;
-    let frameCropRightOffset = 0;
-    let frameCropTopOffset = 0;
-    let frameCropBottomOffset = 0;
-    let numRefFramesInPicOrderCntCycle;
-    let scalingListCount;
-    let i;
-    const readUByte = eg.readUByte.bind(eg);
-    const readBits = eg.readBits.bind(eg);
-    const readUEG = eg.readUEG.bind(eg);
-    const readBoolean = eg.readBoolean.bind(eg);
-    const skipBits = eg.skipBits.bind(eg);
-    const skipEG = eg.skipEG.bind(eg);
-    const skipUEG = eg.skipUEG.bind(eg);
-    const skipScalingList = this.skipScalingList.bind(this);
 
-    readUByte();
-    const profileIdc = readUByte(); // profile_idc
-    readBits(5); // profileCompat constraint_set[0-4]_flag, u(5)
-    skipBits(3); // reserved_zero_3bits u(3),
-    readUByte(); // level_idc u(8)
-    skipUEG(); // seq_parameter_set_id
-    // some profiles have more optional data we don't need
-    if (
-      profileIdc === 100 ||
-      profileIdc === 110 ||
-      profileIdc === 122 ||
-      profileIdc === 244 ||
-      profileIdc === 44 ||
-      profileIdc === 83 ||
-      profileIdc === 86 ||
-      profileIdc === 118 ||
-      profileIdc === 128
-    ) {
-      const chromaFormatIdc = readUEG();
-      if (chromaFormatIdc === 3) {
-        skipBits(1);
-      } // separate_colour_plane_flag
+    while (i < len) {
+      value = array[i++];
+      // optimization. state 0 and 1 are the predominant case. let's handle them outside of the switch/case
+      if (!state) {
+        state = value ? 0 : 1;
+        continue;
+      }
+      if (state === 1) {
+        state = value ? 0 : 2;
+        continue;
+      }
+      // here we have state either equal to 2 or 3
+      if (!value) {
+        state = 3;
+      } else if (value === 1) {
+        overflow = i - state - 1;
+        if (lastUnitStart >= 0) {
+          const unit: VideoSampleUnit = {
+            data: array.subarray(lastUnitStart, overflow),
+            type: lastUnitType,
+          };
+          // logger.log('pushing NALU, type/size:' + unit.type + '/' + unit.data.byteLength);
+          units.push(unit);
+        } else {
+          // lastUnitStart is undefined => this is the first start code found in this PES packet
+          // first check if start code delimiter is overlapping between 2 PES packets,
+          // ie it started in last packet (lastState not zero)
+          // and ended at the beginning of this PES packet (i <= 4 - lastState)
+          const lastUnit = this.getLastNalUnit(track.samples);
+          if (lastUnit) {
+            if (lastState && i <= 4 - lastState) {
+              // start delimiter overlapping between PES packets
+              // strip start delimiter bytes from the end of last NAL unit
+              // check if lastUnit had a state different from zero
+              if (lastUnit.state) {
+                // strip last bytes
+                lastUnit.data = lastUnit.data.subarray(
+                  0,
+                  lastUnit.data.byteLength - lastState,
+                );
+              }
+            }
+            // If NAL units are not starting right at the beginning of the PES packet, push preceding data into previous NAL unit.
 
-      skipUEG(); // bit_depth_luma_minus8
-      skipUEG(); // bit_depth_chroma_minus8
-      skipBits(1); // qpprime_y_zero_transform_bypass_flag
-      if (readBoolean()) {
-        // seq_scaling_matrix_present_flag
-        scalingListCount = chromaFormatIdc !== 3 ? 8 : 12;
-        for (i = 0; i < scalingListCount; i++) {
-          if (readBoolean()) {
-            // seq_scaling_list_present_flag[ i ]
-            if (i < 6) {
-              skipScalingList(16, eg);
-            } else {
-              skipScalingList(64, eg);
+            if (overflow > 0) {
+              // logger.log('first NALU found with overflow:' + overflow);
+              lastUnit.data = appendUint8Array(
+                lastUnit.data,
+                array.subarray(0, overflow),
+              );
+              lastUnit.state = 0;
             }
           }
         }
+        // check if we can read unit type
+        if (i < len) {
+          unitType = array[i] & 0x1f;
+          // logger.log('find NALU @ offset:' + i + ',type:' + unitType);
+          lastUnitStart = i;
+          lastUnitType = unitType;
+          state = 0;
+        } else {
+          // not enough byte to read unit type. let's read it on next PES parsing
+          state = -1;
+        }
+      } else {
+        state = 0;
       }
     }
-    skipUEG(); // log2_max_frame_num_minus4
-    const picOrderCntType = readUEG();
-    if (picOrderCntType === 0) {
-      readUEG(); // log2_max_pic_order_cnt_lsb_minus4
-    } else if (picOrderCntType === 1) {
-      skipBits(1); // delta_pic_order_always_zero_flag
-      skipEG(); // offset_for_non_ref_pic
-      skipEG(); // offset_for_top_to_bottom_field
-      numRefFramesInPicOrderCntCycle = readUEG();
-      for (i = 0; i < numRefFramesInPicOrderCntCycle; i++) {
-        skipEG();
-      } // offset_for_ref_frame[ i ]
+    if (lastUnitStart >= 0 && state >= 0) {
+      const unit: VideoSampleUnit = {
+        data: array.subarray(lastUnitStart, len),
+        type: lastUnitType,
+        state: state,
+      };
+      units.push(unit);
+      // logger.log('pushing NALU, type/size/state:' + unit.type + '/' + unit.data.byteLength + '/' + state);
     }
-    skipUEG(); // max_num_ref_frames
-    skipBits(1); // gaps_in_frame_num_value_allowed_flag
-    const picWidthInMbsMinus1 = readUEG();
-    const picHeightInMapUnitsMinus1 = readUEG();
-    const frameMbsOnlyFlag = readBits(1);
-    if (frameMbsOnlyFlag === 0) {
-      skipBits(1);
-    } // mb_adaptive_frame_field_flag
-
-    skipBits(1); // direct_8x8_inference_flag
-    if (readBoolean()) {
-      // frame_cropping_flag
-      frameCropLeftOffset = readUEG();
-      frameCropRightOffset = readUEG();
-      frameCropTopOffset = readUEG();
-      frameCropBottomOffset = readUEG();
-    }
-    let pixelRatio: [number, number] = [1, 1];
-    if (readBoolean()) {
-      // vui_parameters_present_flag
-      if (readBoolean()) {
-        // aspect_ratio_info_present_flag
-        const aspectRatioIdc = readUByte();
-        switch (aspectRatioIdc) {
-          case 1:
-            pixelRatio = [1, 1];
-            break;
-          case 2:
-            pixelRatio = [12, 11];
-            break;
-          case 3:
-            pixelRatio = [10, 11];
-            break;
-          case 4:
-            pixelRatio = [16, 11];
-            break;
-          case 5:
-            pixelRatio = [40, 33];
-            break;
-          case 6:
-            pixelRatio = [24, 11];
-            break;
-          case 7:
-            pixelRatio = [20, 11];
-            break;
-          case 8:
-            pixelRatio = [32, 11];
-            break;
-          case 9:
-            pixelRatio = [80, 33];
-            break;
-          case 10:
-            pixelRatio = [18, 11];
-            break;
-          case 11:
-            pixelRatio = [15, 11];
-            break;
-          case 12:
-            pixelRatio = [64, 33];
-            break;
-          case 13:
-            pixelRatio = [160, 99];
-            break;
-          case 14:
-            pixelRatio = [4, 3];
-            break;
-          case 15:
-            pixelRatio = [3, 2];
-            break;
-          case 16:
-            pixelRatio = [2, 1];
-            break;
-          case 255: {
-            pixelRatio = [
-              (readUByte() << 8) | readUByte(),
-              (readUByte() << 8) | readUByte(),
-            ];
-            break;
-          }
-        }
+    // no NALu found
+    if (units.length === 0) {
+      // append pes.data to previous NAL unit
+      const lastUnit = this.getLastNalUnit(track.samples);
+      if (lastUnit) {
+        lastUnit.data = appendUint8Array(lastUnit.data, array);
       }
     }
-    return {
-      width: Math.ceil(
-        (picWidthInMbsMinus1 + 1) * 16 -
-          frameCropLeftOffset * 2 -
-          frameCropRightOffset * 2,
-      ),
-      height:
-        (2 - frameMbsOnlyFlag) * (picHeightInMapUnitsMinus1 + 1) * 16 -
-        (frameMbsOnlyFlag ? 2 : 4) *
-          (frameCropTopOffset + frameCropBottomOffset),
-      pixelRatio: pixelRatio,
-    };
+    track.naluState = state;
+    return units;
   }
 }
 

package/src/demux/video/base-video-parser.ts

@@ -1,20 +1,19 @@
 import type { ParsedVideoSample } from '../tsdemuxer';
-import type {
+import {
   DemuxedVideoTrack,
-  DemuxedUserdataTrack,
   VideoSample,
   VideoSampleUnit,
 } from '../../types/demuxer';
-import type { PES } from '../tsdemuxer';
-import { appendUint8Array } from '../../utils/mp4-tools';
+import { logger } from '../../utils/logger';
 
-abstract class BaseVideoParser {
+class BaseVideoParser {
   protected VideoSample: ParsedVideoSample | null = null;
 
   protected createVideoSample(
     key: boolean,
     pts: number | undefined,
     dts: number | undefined,
+    debug: string,
   ): ParsedVideoSample {
     return {
       key,
@@ -22,6 +21,7 @@ abstract class BaseVideoParser {
       pts,
       dts,
       units: [],
+      debug,
       length: 0,
     };
   }
@@ -63,135 +63,11 @@ abstract class BaseVideoParser {
       }
       videoTrack.samples.push(VideoSample as VideoSample);
     }
-  }
-
-  abstract parsePES(
-    track: DemuxedVideoTrack,
-    textTrack: DemuxedUserdataTrack,
-    pes: PES,
-    last: boolean,
-  );
-
-  protected abstract getNALuType(data: Uint8Array, offset: number): number;
-
-  protected parseNALu(
-    track: DemuxedVideoTrack,
-    array: Uint8Array,
-    endOfSegment: boolean,
-  ): Array<{
-    data: Uint8Array;
-    type: number;
-    state?: number;
-  }> {
-    const len = array.byteLength;
-    let state = track.naluState || 0;
-    const lastState = state;
-    const units: VideoSampleUnit[] = [];
-    let i = 0;
-    let value: number;
-    let overflow: number;
-    let unitType: number;
-    let lastUnitStart = -1;
-    let lastUnitType: number = 0;
-    // logger.log('PES:' + Hex.hexDump(array));
-
-    if (state === -1) {
-      // special use case where we found 3 or 4-byte start codes exactly at the end of previous PES packet
-      lastUnitStart = 0;
-      // NALu type is value read from offset 0
-      lastUnitType = this.getNALuType(array, 0);
-      state = 0;
-      i = 1;
-    }
-
-    while (i < len) {
-      value = array[i++];
-      // optimization. state 0 and 1 are the predominant case. let's handle them outside of the switch/case
-      if (!state) {
-        state = value ? 0 : 1;
-        continue;
-      }
-      if (state === 1) {
-        state = value ? 0 : 2;
-        continue;
-      }
-      // here we have state either equal to 2 or 3
-      if (!value) {
-        state = 3;
-      } else if (value === 1) {
-        overflow = i - state - 1;
-        if (lastUnitStart >= 0) {
-          const unit: VideoSampleUnit = {
-            data: array.subarray(lastUnitStart, overflow),
-            type: lastUnitType,
-          };
-          // logger.log('pushing NALU, type/size:' + unit.type + '/' + unit.data.byteLength);
-          units.push(unit);
-        } else {
-          // lastUnitStart is undefined => this is the first start code found in this PES packet
-          // first check if start code delimiter is overlapping between 2 PES packets,
-          // ie it started in last packet (lastState not zero)
-          // and ended at the beginning of this PES packet (i <= 4 - lastState)
-          const lastUnit = this.getLastNalUnit(track.samples);
-          if (lastUnit) {
-            if (lastState && i <= 4 - lastState) {
-              // start delimiter overlapping between PES packets
-              // strip start delimiter bytes from the end of last NAL unit
-              // check if lastUnit had a state different from zero
-              if (lastUnit.state) {
-                // strip last bytes
-                lastUnit.data = lastUnit.data.subarray(
-                  0,
-                  lastUnit.data.byteLength - lastState,
-                );
-              }
-            }
-            // If NAL units are not starting right at the beginning of the PES packet, push preceding data into previous NAL unit.
-
-            if (overflow > 0) {
-              // logger.log('first NALU found with overflow:' + overflow);
-              lastUnit.data = appendUint8Array(
-                lastUnit.data,
-                array.subarray(0, overflow),
-              );
-              lastUnit.state = 0;
-            }
-          }
-        }
-        // check if we can read unit type
-        if (i < len) {
-          unitType = this.getNALuType(array, i);
-          // logger.log('find NALU @ offset:' + i + ',type:' + unitType);
-          lastUnitStart = i;
-          lastUnitType = unitType;
-          state = 0;
-        } else {
-          // not enough byte to read unit type. let's read it on next PES parsing
-          state = -1;
-        }
-      } else {
-        state = 0;
-      }
-    }
-    if (lastUnitStart >= 0 && state >= 0) {
-      const unit: VideoSampleUnit = {
-        data: array.subarray(lastUnitStart, len),
-        type: lastUnitType,
-        state: state,
-      };
-      units.push(unit);
-      // logger.log('pushing NALU, type/size/state:' + unit.type + '/' + unit.data.byteLength + '/' + state);
-    }
-    // no NALu found
-    if (units.length === 0) {
-      // append pes.data to previous NAL unit
-      const lastUnit = this.getLastNalUnit(track.samples);
-      if (lastUnit) {
-        lastUnit.data = appendUint8Array(lastUnit.data, array);
-      }
+    if (VideoSample.debug.length) {
+      logger.log(
+        VideoSample.pts + '/' + VideoSample.dts + ':' + VideoSample.debug,
+      );
     }
-    track.naluState = state;
-    return units;
   }
 }