jmuxer 2.0.7 → 2.1.0

package/src/parsers/h264.js

@@ -1,56 +1,42 @@
 import { ExpGolomb } from '../util/exp-golomb.js';
-import { NALU } from '../util/nalu.js';
 import * as debug from '../util/debug';
 
+// spec https://www.itu.int/rec/T-REC-H.264/
+
 export class H264Parser {
 
     static extractNALu(buffer) {
         let i = 0,
             length = buffer.byteLength,
-            value,
-            state = 0,
             result = [],
-            left,
-            lastIndex = 0;
+            lastIndex = 0,
+            zeroCount = 0;
 
         while (i < length) {
-            value = buffer[i++];
-            // finding 3 or 4-byte start codes (00 00 01 OR 00 00 00 01)
-            switch (state) {
-                case 0:
-                    if (value === 0) {
-                        state = 1;
-                    }
-                    break;
-                case 1:
-                    if (value === 0) {
-                        state = 2;
-                    } else {
-                        state = 0;
-                    }
-                    break;
-                case 2:
-                case 3:
-                    if (value === 0) {
-                        state = 3;
-                    } else if (value === 1 && i < length) {
-                        if (lastIndex != i - state -1) {
-                            result.push(buffer.subarray(lastIndex, i - state -1));
-                        }
-                        lastIndex = i;
-                        state = 0;
-                    } else {
-                        state = 0;
-                    }
-                    break;
-                default:
-                    break;
+            let value = buffer[i++];
+
+            if (value === 0) {
+                zeroCount++;
+            } else if (value === 1 && zeroCount >= 2) {
+                let startCodeLength = zeroCount + 1;
+
+                if (lastIndex !== i - startCodeLength) {
+                    result.push(buffer.subarray(lastIndex, i - startCodeLength));
+                }
+
+                lastIndex = i;
+                zeroCount = 0;
+            } else {
+                zeroCount = 0;
             }
         }
 
+        // Remaining data after last start code
+        let left = null;
         if (lastIndex < length) {
             left = buffer.subarray(lastIndex, length);
         }
+
         return [result, left];
     }
 
@@ -246,76 +232,95 @@ export class H264Parser {
             height: ((2 - frameMbsOnlyFlag) * (picHeightInMapUnitsMinus1 + 1) * 16) - ((frameMbsOnlyFlag ? 2 : 4) * (frameCropTopOffset + frameCropBottomOffset)),
         };
     }
-    static parseHeader(unit) {
-        let decoder = new ExpGolomb(unit.getPayload());
-        // skip NALu type
-        decoder.readUByte();
-        unit.isfmb = decoder.readUEG() === 0;
-        unit.stype = decoder.readUEG();
+    
+}
+
+export class NALU264 {
+    static get NDR() { return 1; }
+    static get IDR() { return 5; }
+    static get SEI() { return 6; }
+    static get SPS() { return 7; }
+    static get PPS() { return 8; }
+    static get AUD() { return 9; }
+
+    static get TYPES() {
+        return {
+            [NALU264.IDR]: 'IDR',
+            [NALU264.SEI]: 'SEI',
+            [NALU264.SPS]: 'SPS',
+            [NALU264.PPS]: 'PPS',
+            [NALU264.NDR]: 'NDR',
+            [NALU264.AUD]: 'AUD',
+        };
     }
-    constructor(remuxer) {
-        this.remuxer = remuxer;
-        this.track = remuxer.mp4track;
+
+    constructor(data) {
+        this.payload = data;
+        this.nri = (this.payload[0] & 0x60) >> 5; // nal_ref_idc
+        this.nalUnitType = this.payload[0] & 0x1f;
+        this._sliceType = null;
+        this._isFirstSlice = false;
     }
 
-    parseSPS(sps) {
-        var config = H264Parser.readSPS(new Uint8Array(sps));
+    toString() {
+        return `${NALU264.TYPES[this.type()] || 'UNKNOWN'}: NRI: ${this.getNri()}`;
+    }
 
-        this.track.fps = config.fps;
-        this.track.width = config.width;
-        this.track.height = config.height;
-        this.track.sps = [new Uint8Array(sps)];
-        this.track.codec = 'avc1.';
+    getNri() {
+        return this.nri;
+    }
 
-        let codecarray = new DataView(sps.buffer, sps.byteOffset + 1, 4);
-        for (let i = 0; i < 3; ++i) {
-            var h = codecarray.getUint8(i).toString(16);
-            if (h.length < 2) {
-                h = '0' + h;
-            }
-            this.track.codec += h;
-        }
+    type() {
+        return this.nalUnitType;
     }
 
-    parsePPS(pps) {
-        this.track.pps = [new Uint8Array(pps)];
+    get isKeyframe() {
+        return this.nalUnitType === NALU264.IDR;
     }
 
-    parseNAL(unit) {
-        if (!unit) return false;
+    get isVCL() {
+        return this.nalUnitType == NALU264.IDR || this.nalUnitType == NALU264.NDR;
+    }
 
-        let push = false;
-        switch (unit.type()) {
-            case NALU.IDR:
-            case NALU.NDR:
-                push = true;
-                break;
-            case NALU.PPS:
-                if (!this.track.pps) {
-                    this.parsePPS(unit.getPayload());
-                    if (!this.remuxer.readyToDecode && this.track.pps && this.track.sps) {
-                        this.remuxer.readyToDecode = true;
-                    }
-                }
-                push = true;
-                break;
-            case NALU.SPS:
-                if (!this.track.sps) {
-                    this.parseSPS(unit.getPayload());
-                    if (!this.remuxer.readyToDecode && this.track.pps && this.track.sps) {
-                        this.remuxer.readyToDecode = true;
-                    }
-                }
-                push = true;
-                break;
-            case NALU.AUD:
-                debug.log('AUD - ignoing');
-                break;
-            case NALU.SEI:
-                debug.log('SEI - ignoing');
-                break;
-            default:
+    parseHeader() {
+        let decoder = new ExpGolomb(this.getPayload());
+        // skip NALu type
+        decoder.readUByte();
+        this._isFirstSlice = decoder.readUEG() === 0;
+        this._sliceType = decoder.readUEG();
+    }
+
+    get isFirstSlice() {
+        if (!this._isFirstSlice) {
+            this.parseHeader();
         }
-        return push;
+        return this._isFirstSlice;
+    }
+
+    get sliceType() {
+        if (!this._sliceType) {
+            this.parseHeader();
+        }
+        return this._sliceType;
+    }
+    
+    getPayload() {
+        return this.payload;
+    }
+
+    getPayloadSize() {
+        return this.payload.byteLength;
+    }
+
+    getSize() {
+        return 4 + this.getPayloadSize();
+    }
+
+    getData() {
+        const result = new Uint8Array(this.getSize());
+        const view = new DataView(result.buffer);
+        view.setUint32(0, this.getSize() - 4);
+        result.set(this.getPayload(), 4);
+        return result;
     }
 }

package/src/parsers/h265.js

@@ -0,0 +1,303 @@
+import { ExpGolomb } from '../util/exp-golomb.js';
+import * as debug from '../util/debug';
+
+// spec https://www.itu.int/rec/T-REC-H.265/
+
+export class H265Parser {
+
+    static extractNALu(buffer) {
+        let i = 0,
+            length = buffer.byteLength,
+            result = [],
+            lastIndex = 0,
+            zeroCount = 0;
+
+        while (i < length) {
+            let value = buffer[i++];
+
+            if (value === 0) {
+                zeroCount++;
+            } else if (value === 1 && zeroCount >= 2) {
+                let startCodeLength = zeroCount + 1;
+
+                if (lastIndex !== i - startCodeLength) {
+                    result.push(buffer.subarray(lastIndex, i - startCodeLength));
+                }
+
+                lastIndex = i;
+                zeroCount = 0;
+            } else {
+                zeroCount = 0;
+            }
+        }
+
+        // Remaining data after last start code
+        let left = null;
+        if (lastIndex < length) {
+            left = buffer.subarray(lastIndex, length);
+        }
+
+        return [result, left];
+    }
+
+    static removeEmulationPreventionBytes(nal) {
+        const rbsp = [];
+        let zeroCount = 0;
+
+        for (let i = 0; i < nal.length; i++) {
+            const value = nal[i];
+
+            if (zeroCount === 2 && value === 0x03) {
+                // Skip this emulation prevention byte
+                zeroCount = 0;
+                continue;
+            }
+
+            rbsp.push(value);
+
+            if (value === 0x00) {
+                zeroCount++;
+            } else {
+                zeroCount = 0;
+            }
+        }
+
+        return new Uint8Array(rbsp);
+    }
+
+
+    /**
+     * Read a sequence parameter set and return some interesting video
+     * properties. A sequence parameter set is the H265 metadata that
+     * describes the properties of upcoming video frames.
+     * @param data {Uint8Array} the bytes of a sequence parameter set
+     * @return {object} an object with configuration parsed from the
+     * sequence parameter set, including the dimensions of the
+     * associated video frames.
+     */
+    static readSPS(data) {
+        let decoder = new ExpGolomb(data);
+
+        // Skip NALU header (2 bytes for HEVC)
+        decoder.readUByte();
+        decoder.readUByte();
+
+        decoder.readBits(4); // sps_video_parameter_set_id
+        decoder.readBits(3); // sps_max_sub_layers_minus1
+        decoder.readBits(1); // sps_temporal_id_nesting_flag
+
+        // --- profile_tier_level() ---
+        let profile_space = decoder.readBits(2);
+        let tier_flag = decoder.readBits(1);
+        let profile_idc = decoder.readBits(5);
+
+        let profile_compatibility_flags = decoder.readUInt(); // 32 bits
+        let constraint_indicator_flags = new Uint8Array(6);
+        for (let i = 0; i < 6; i++) {
+            constraint_indicator_flags[i] = decoder.readUByte();
+        }
+
+        let level_idc = decoder.readUByte();
+
+        decoder.readUEG(); // seq_parameter_set_id
+
+        let chroma_format_idc = decoder.readUEG();
+        if (chroma_format_idc === 3) {
+            decoder.readBits(1); // separate_colour_plane_flag
+        }
+
+        let pic_width_in_luma_samples = decoder.readUEG();
+        let pic_height_in_luma_samples = decoder.readUEG();
+
+        // Cropping
+        let conformance_window_flag = decoder.readBoolean();
+        let conf_win_left_offset = 0, conf_win_right_offset = 0, conf_win_top_offset = 0, conf_win_bottom_offset = 0;
+        if (conformance_window_flag) {
+            conf_win_left_offset = decoder.readUEG();
+            conf_win_right_offset = decoder.readUEG();
+            conf_win_top_offset = decoder.readUEG();
+            conf_win_bottom_offset = decoder.readUEG();
+        }
+
+        let fps = null;
+        let vui_parameters_present_flag = decoder.readBoolean();
+        if (vui_parameters_present_flag) {
+            let aspect_ratio_info_present_flag = decoder.readBoolean();
+            if (aspect_ratio_info_present_flag) {
+                let aspect_ratio_idc = decoder.readUByte();
+                if (aspect_ratio_idc === 255) { // Extended_SAR
+                    decoder.readUShort(); // sar_width
+                    decoder.readUShort(); // sar_height
+                }
+            }
+            let overscan_info_present_flag = decoder.readBoolean();
+            if (overscan_info_present_flag) {
+                decoder.readBoolean(); // overscan_appropriate_flag
+            }
+            let video_signal_type_present_flag = decoder.readBoolean();
+            if (video_signal_type_present_flag) {
+                decoder.readBits(3); // video_format
+                decoder.readBoolean(); // video_full_range_flag
+                let colour_description_present_flag = decoder.readBoolean();
+                if (colour_description_present_flag) {
+                    decoder.readUByte(); // colour_primaries
+                    decoder.readUByte(); // transfer_characteristics
+                    decoder.readUByte(); // matrix_coeffs
+                }
+            }
+            let chroma_loc_info_present_flag = decoder.readBoolean();
+            if (chroma_loc_info_present_flag) {
+                decoder.readUEG(); // chroma_sample_loc_type_top_field
+                decoder.readUEG(); // chroma_sample_loc_type_bottom_field
+            }
+            decoder.readBoolean(); // neutral_chroma_indication_flag
+            decoder.readBoolean(); // field_seq_flag
+            decoder.readBoolean(); // frame_field_info_present_flag
+
+            // Timing info
+            let timing_info_present_flag = decoder.readBoolean();
+            if (timing_info_present_flag) {
+                let num_units_in_tick = decoder.readUInt();
+                let time_scale = decoder.readUInt();
+                decoder.readBoolean(); // poc_proportional_to_timing_flag
+                if (num_units_in_tick) {
+                    fps = time_scale / (2 * num_units_in_tick);
+                }
+            }
+        }
+
+        // Final width/height
+        let sub_width_c = (chroma_format_idc === 1 || chroma_format_idc === 2) ? 2 : 1;
+        let sub_height_c = (chroma_format_idc === 1) ? 2 : 1;
+        let width = pic_width_in_luma_samples - sub_width_c * (conf_win_right_offset + conf_win_left_offset);
+        let height = pic_height_in_luma_samples - sub_height_c * (conf_win_top_offset + conf_win_bottom_offset);
+
+        return {
+            width,
+            height,
+            profile_space,
+            tier_flag,
+            profile_idc,
+            profile_compatibility_flags,
+            constraint_indicator_flags,
+            level_idc,
+            chroma_format_idc,
+            fps,
+        };
+    }
+
+}
+
+export class NALU265 {
+    static get TRAIL_N()     { return  0; }
+    static get TRAIL_R()     { return  1; }
+    static get IDR_W_RADL()  { return 19; }
+    static get IDR_N_LP()    { return 20; }
+    static get CRA()         { return 21; }
+    static get VPS()         { return 32; }
+    static get SPS()         { return 33; }
+    static get PPS()         { return 34; }
+    static get AUD()         { return 35; }
+    static get SEI()         { return 39; }
+    static get SEI2()        { return 40; }
+
+    static get TYPES() {
+        return {
+            [NALU265.TRAIL_N]:     'TRAIL_N',
+            [NALU265.TRAIL_R]:     'TRAIL_R',
+            [NALU265.IDR_W_RADL]:  'IDR',
+            [NALU265.IDR_N_LP]:    'IDR2',
+            [NALU265.CRA]:         'CRA',
+            [NALU265.VPS]:         'VPS',
+            [NALU265.SPS]:         'SPS',
+            [NALU265.PPS]:         'PPS',
+            [NALU265.AUD]:         'AUD',
+            [NALU265.SEI]:         'SEI',
+            [NALU265.SEI2]:        'SEI2',
+        };
+    }
+
+    constructor(data) {
+        this.payload = data;
+        this.nalUnitType = (data[0] & 0b01111110) >> 1;
+        this.nuhLayerId = ((data[0] & 0b00000001) << 5) | ((data[1] & 0b11111000) >> 3);
+        this.nuhTemporalIdPlus1 = data[1] & 0b00000111;
+        this._isFirstSlice = null;
+        this._sliceType = null;
+    }
+
+    toString() {
+        return `${NALU265.TYPES[this.type()] || 'UNKNOWN (' + this.type() + ')'}: Layer: ${this.nuhLayerId}, Temporal Id: ${this.nuhTemporalIdPlus1}`;
+    }
+
+    type() {
+        return this.nalUnitType;
+    }
+
+    get isKeyframe() {
+        return [
+            NALU265.IDR_W_RADL,
+            NALU265.IDR_N_LP,
+            NALU265.CRA
+        ].includes(this.nalUnitType);
+    }
+
+    get isVCL() {
+        return this.nalUnitType <=31;
+    }
+
+    parseHeader() {
+        let decoder = new ExpGolomb(this.getPayload());
+        // skip NALu type
+        decoder.readUByte();
+        decoder.readUByte();
+
+        // first_slice_segment_in_pic_flag
+        this._isFirstSlice = decoder.readBoolean();
+
+        // if NALU is not IDR/CRA/other IRAP, next comes no_output_of_prior_pics_flag
+        if (this.isKeyframe) {
+            decoder.readBits(1); // no_output_of_prior_pics_flag
+        }
+
+        // slice_pic_parameter_set_id
+        decoder.readUEG();
+
+        // slice_type (only for some NALU types, but useful for debugging)
+        this._sliceType = decoder.readUEG();
+    }
+
+    get isFirstSlice() {
+        if (!this._isFirstSlice) {
+            this.parseHeader();
+        }
+        return this._isFirstSlice;
+    }
+
+    get sliceType() {
+        if (!this._sliceType) {
+            this.parseHeader();
+        }
+        return this._sliceType;
+    }
+    
+    getPayload() {
+        return this.payload;
+    }
+
+    getPayloadSize() {
+        return this.payload.byteLength;
+    }
+
+    getSize() {
+        return 4 + this.getPayloadSize();
+    }
+
+    getData() {
+        const result = new Uint8Array(this.getSize());
+        const view = new DataView(result.buffer);
+        view.setUint32(0, this.getSize() - 4);
+        result.set(this.getPayload(), 4);
+        return result;
+    }
+}

package/src/remuxer/aac.js

@@ -4,9 +4,11 @@ import { BaseRemuxer } from './base.js';
 
 export class AACRemuxer extends BaseRemuxer {
 
-    constructor(timescale) {
-        super();
+    constructor(timescale, duration, frameDuration) {
+        super('AACRemuxer');
+        this.frameDuration = frameDuration;
         this.readyToDecode = false;
+        this.header = null;
         this.nextDts = 0;
         this.dts = 0;
         this.mp4track = {
@@ -16,17 +18,17 @@ export class AACRemuxer extends BaseRemuxer {
             len: 0,
             fragmented: true,
             timescale: timescale,
-            duration: timescale,
+            duration: duration,
             samples: [],
             config: '',
             codec: '',
         };
         this.samples = [];
-        this.aac = new AACParser(this);
     }
 
     resetTrack() {
         this.readyToDecode = false;
+        this.header = null;
         this.mp4track.codec = '';
         this.mp4track.channelCount = '';
         this.mp4track.config = '';
@@ -35,6 +37,39 @@ export class AACRemuxer extends BaseRemuxer {
         this.dts = 0;
     }
 
+    feed(data, duration) {
+        const { valid, header, slices } = AACParser.extractAAC(data);
+        if (!this.header) this.header = header;
+        if (valid && slices.length > 0) {
+            this.remux(this.getAudioFrames(slices, duration));
+            return true;
+        } else {
+            debug.error('Failed to extract audio data from:', data);
+            this.dispatch('outOfData');
+            return false;
+        }
+    }
+
+    getAudioFrames(aacFrames, duration) {
+        let frames = [],
+            fd = 0,
+            tt = 0;
+
+        for (let units of aacFrames) {
+            frames.push({ units });
+        }
+        fd = duration ? duration / frames.length | 0 : this.frameDuration;
+        tt = duration ? (duration - (fd * frames.length)) : 0;
+        frames.map((frame) => {
+            frame.duration = fd;
+            if (tt > 0) {
+                frame.duration++;
+                tt--;
+            }
+        });
+        return frames;
+    }
+
     remux(frames) {
         if (frames.length > 0) {
             for (let i = 0; i < frames.length; i++) {
@@ -48,7 +83,7 @@ export class AACRemuxer extends BaseRemuxer {
                 });
                 this.mp4track.len += size;
                 if (!this.readyToDecode) {
-                    this.aac.setAACConfig();
+                    this.setAACConfig();
                 }
             }
         }
@@ -103,7 +138,28 @@ export class AACRemuxer extends BaseRemuxer {
         return new Uint8Array(payload.buffer, 0, this.mp4track.len);
     }
 
-    getAacParser() {
-        return this.aac;
+    setAACConfig() {
+        let objectType,
+            sampleIndex,
+            channelCount,
+            config = new Uint8Array(2);
+
+        if (!this.header) return;
+            
+        objectType = ((this.header[2] & 0xC0) >>> 6) + 1;
+        sampleIndex = ((this.header[2] & 0x3C) >>> 2);
+        channelCount = ((this.header[2] & 0x01) << 2);
+        channelCount |= ((this.header[3] & 0xC0) >>> 6);
+
+        /* refer to http://wiki.multimedia.cx/index.php?title=MPEG-4_Audio#Audio_Specific_Config */
+        config[0] = objectType << 3;
+        config[0] |= (sampleIndex & 0x0E) >> 1;
+        config[1] |= (sampleIndex & 0x01) << 7;
+        config[1] |= channelCount << 3;
+
+        this.mp4track.codec = 'mp4a.40.' + objectType;
+        this.mp4track.channelCount = channelCount;
+        this.mp4track.config = config;
+        this.readyToDecode = true;
     }
 }

package/src/remuxer/base.js

@@ -1,7 +1,8 @@
 import * as debug from '../util/debug';
+import Event from '../util/event';
 
 let track_id = 1;
-export class BaseRemuxer {
+export class BaseRemuxer extends Event {
 
     static getTrackID() {
         return track_id++;