jmuxer 2.0.7 → 2.1.0

package/package.json

@@ -1,11 +1,13 @@
 {
   "name": "jmuxer",
-  "version": "2.0.7",
+  "version": "2.1.0",
   "description": "jMuxer - a simple javascript mp4 muxer for non-standard streaming communications protocol",
   "main": "dist/jmuxer.min.js",
   "scripts": {
     "build": "rollup -c",
-    "dev": "rollup -c -w",
+    "watch": "rollup -c -w",
+    "server": "cd example && nodemon server.mjs",
+    "dev": "concurrently --names \"BUILD,SERVER\" -c \"bgBlue.bold,bgMagenta.bold\" \"npm run watch\" \"npm run server\"",
     "pro": "NODE_ENV=production rollup -c",
     "lint": "eslint  -c .eslintrc.json ./src/",
     "test": "karma start karma.conf.js"
@@ -42,17 +44,20 @@
     "@rollup/pluginutils": "^4.0.0",
     "caniuse-lite": "^1.0.30001223",
     "chai": "^4.2.0",
+    "concurrently": "^9.2.1",
     "eslint": "^8.2.0",
+    "express": "^5.1.0",
+    "express-ws": "^5.0.2",
     "karma": "^6.3.2",
     "karma-chai": "^0.1.0",
     "karma-chrome-launcher": "^3.1.0",
     "karma-mocha": "^2.0.1",
     "karma-rollup-preprocessor": "^7.0.5",
-    "mocha": "^9.1.3",
-    "patch-package": "^6.4.7",
+    "mocha": "^11.7.2",
+    "msgpack-lite": "^0.1.26",
+    "nodemon": "^3.1.10",
     "path": "^0.12.7",
     "rollup": "^2.59.0",
-    "rollup-plugin-terser": "^7.0.2",
-    "ws": "^8.2.3"
+    "rollup-plugin-terser": "^7.0.2"
   }
 }

package/src/controller/remux.js

@@ -2,38 +2,49 @@ import * as debug from '../util/debug';
 import { MP4 } from '../util/mp4-generator.js';
 import { AACRemuxer } from '../remuxer/aac.js';
 import { H264Remuxer } from '../remuxer/h264.js';
+import { H265Remuxer } from '../remuxer/h265.js';
 import { appendByteArray, secToTime } from '../util/utils.js';
 import Event from '../util/event';
 
 export default class RemuxController extends Event {
 
-    constructor(env) {
+    constructor(env, live, videoCodec, frameDuration) {
         super('remuxer');
+        this.videoCodec = videoCodec;
+        this.frameDuration = frameDuration;
         this.initialized = false;
-        this.trackTypes = [];
         this.tracks = {};
         this.seq = 1;
         this.env = env;
         this.timescale = 1000;
-        this.mediaDuration = 0;
-        this.aacParser = null;
+        this.mediaDuration = live ? 0xffffffff : 0;
     }
 
     addTrack(type) {
         if (type === 'video' || type === 'both') {
-            this.tracks.video = new H264Remuxer(this.timescale);
-            this.trackTypes.push('video');
+            if (this.videoCodec == 'H265') {
+                this.tracks.video = new H265Remuxer(this.timescale, this.mediaDuration, this.frameDuration);
+            } else {
+                this.tracks.video = new H264Remuxer(this.timescale, this.mediaDuration, this.frameDuration);
+            }
+            this.tracks.video.on('outOfData', () => {
+                this.dispatch('missingVideoFrames');
+            });
+            this.tracks.video.on('keyframePosition', (time) => {
+                this.dispatch('keyframePosition', time);
+            });
         }
         if (type === 'audio' || type === 'both') {
-            const aacRemuxer = new AACRemuxer(this.timescale);
-            this.aacParser = aacRemuxer.getAacParser();
+            const aacRemuxer = new AACRemuxer(this.timescale, this.mediaDuration, this.frameDuration);
             this.tracks.audio = aacRemuxer;
-            this.trackTypes.push('audio');
+            this.tracks.video.on('outOfData', () => {
+                this.dispatch('missingAudioFrames');
+            });
         }
     }
 
     reset() {
-        for (let type of this.trackTypes) {
+        for (const type in this.tracks) {
             this.tracks[type].resetTrack();
         }
         this.initialized = false;
@@ -46,41 +57,40 @@ export default class RemuxController extends Event {
 
     flush() {
         if (!this.initialized) {
-            if (this.isReady()) {
-                this.dispatch('ready');
-                this.initSegment();
-                this.initialized = true;
-                this.flush();
-            }
-        } else {
-            for (let type of this.trackTypes) {
-                let track = this.tracks[type];
-                let pay = track.getPayload();
-                if (pay && pay.byteLength) {
-                    const moof = MP4.moof(this.seq, track.dts, track.mp4track);
-                    const mdat = MP4.mdat(pay);
-                    let payload = appendByteArray(moof, mdat);
-                    let data = {
-                        type: type,
-                        payload: payload,
-                        dts: track.dts
-                    };
-                    if (type === 'video') {
-                        data.fps = track.mp4track.fps;
-                    }
-                    this.dispatch('buffer', data);
-                    let duration = secToTime(track.dts / this.timescale);
-                    debug.log(`put segment (${type}): dts: ${track.dts} frames: ${track.mp4track.samples.length} second: ${duration}`);
-                    track.flush();
-                    this.seq++;
+            if (!this.isReady()) return;
+
+            this.dispatch('ready');
+            this.initSegment();
+            this.initialized = true;
+        }
+
+        for (const type in this.tracks) {
+            let track = this.tracks[type];
+            let pay = track.getPayload();
+            if (pay && pay.byteLength) {
+                const moof = MP4.moof(this.seq, track.dts, track.mp4track);
+                const mdat = MP4.mdat(pay);
+                let payload = appendByteArray(moof, mdat);
+                let data = {
+                    type: type,
+                    payload: payload,
+                    dts: track.dts
+                };
+                if (type === 'video') {
+                    data.fps = track.mp4track.fps;
                 }
+                this.dispatch('buffer', data);
+                let duration = secToTime(track.dts / this.timescale);
+                debug.log(`put segment (${type}): dts: ${track.dts} frames: ${track.mp4track.samples.length} second: ${duration}`);
+                track.flush();
+                this.seq++;
             }
         }
     }
 
     initSegment() {
         let tracks = [];
-        for (let type of this.trackTypes) {
+        for (const type in this.tracks) {
             let track = this.tracks[type];
             if (this.env == 'browser') {
                 let data = {
@@ -103,20 +113,27 @@ export default class RemuxController extends Event {
     }
 
     isReady() {
-        for (let type of this.trackTypes) {
+        for (const type in this.tracks) {
             if (!this.tracks[type].readyToDecode || !this.tracks[type].samples.length) return false;
         }
         return true;
     }
 
-    remux(data) {
-        for (let type of this.trackTypes) {
-            let frames = data[type];
-            if (type === 'audio' && this.tracks.video && !this.tracks.video.readyToDecode) continue; /* if video is present, don't add audio until video get ready */
-            if (frames.length > 0) {
-                this.tracks[type].remux(frames);
-            }
+    feed(data) {
+        let remux = false;
+        
+        if (data.video && this.tracks.video) {
+            remux |= this.tracks.video.feed(data.video, data.duration, data.compositionTimeOffset);
         }
+        if (data.audio && this.tracks.audio) {
+            remux |= this.tracks.audio.feed(data.audio, data.duration);
+        }
+
+        if (!remux) {
+            debug.error('Input object must have video and/or audio property. Make sure it is a valid typed array');
+            return;
+        }
+
         this.flush();
     }
 }

package/src/jmuxer.js

@@ -1,8 +1,4 @@
 import * as debug from './util/debug';
-import { NALU } from './util/nalu.js';
-import { appendByteArray } from './util/utils.js';
-import { H264Parser } from './parsers/h264.js';
-import { AACParser } from './parsers/aac.js';
 import Event from './util/event';
 import RemuxController from './controller/remux.js';
 import BufferController from './controller/buffer.js';
@@ -19,6 +15,7 @@ export default class JMuxer extends Event {
         let defaults = {
             node: '',
             mode: 'both', // both, audio, video
+            videoCodec: 'H264', // H264, H265
             flushingTime: 500,
             maxDelay: 500,
             clearBuffer: true,
@@ -28,20 +25,24 @@ export default class JMuxer extends Event {
             onReady: function() {}, // function called when MSE is ready to accept frames
             onData: function() {}, // function called when data is ready to be sent
             onError: function() {}, // function called when jmuxer encounters any buffer related errors
+            onUnsupportedCodec: function() {}, // function called when a codec is not supported by the browser
             onMissingVideoFrames: function () {}, // function called when jmuxer encounters any missing video frames
             onMissingAudioFrames: function () {}, // function called when jmuxer encounters any missing audio frames
+            onKeyframePosition: function () {}, // function called when a keyframe is detected thus the provided time is seekable
+            onLoggerLog: console.log,
+            onLoggerErr: console.error,
         };
         this.options = Object.assign({}, defaults, options);
         this.env = typeof process === 'object' && typeof window === 'undefined' ? 'node' : 'browser';
         if (this.options.debug) {
-            debug.setLogger();
+            debug.setLogger(this.options.onLoggerLog, this.options.onLoggerErr);
         }
 
         if (!this.options.fps) {
             this.options.fps = 30;
         }
         this.frameDuration = (1000 / this.options.fps) | 0;
-        this.remuxController = new RemuxController(this.env);
+        this.remuxController = new RemuxController(this.env, options.live, this.options.videoCodec, this.frameDuration);
         this.remuxController.addTrack(this.options.mode);
 
         this.initData();
@@ -52,12 +53,34 @@ export default class JMuxer extends Event {
             this.remuxController.on('ready', this.createBuffer.bind(this));
             this.initBrowser();
         }
+
+        this.remuxController.on('missingVideoFrames', () => {
+            if (typeof this.options.onMissingVideoFrames === 'function') {
+                this.options.onMissingVideoFrames.call(null);
+            }
+        });
+        this.remuxController.on('missingAudioFrames', () => {
+            if (typeof this.options.onMissingAudioFrames === 'function') {
+                this.options.onMissingAudioFrames.call(null);
+            }
+        });
+        if (this.clearBuffer) {
+            // this is used to know when keyframes are,
+            // to essentially know which specific times are seekable
+            this.remuxController.on('keyframePosition', time => {
+                this.kfPosition.push(time);
+            });
+        }
+        if (typeof this.options.onKeyframePosition === 'function') {
+            this.remuxController.on('keyframePosition', time => {
+                this.options.onKeyframePosition.call(null, time);
+            });
+        }
     }
 
     initData() {
         this.lastCleaningTime = Date.now();
         this.kfPosition = [];
-        this.kfCounter = 0;
         this.pendingUnits = {};
         this.remainingData = new Uint8Array();
         this.startInterval();
@@ -134,143 +157,11 @@ export default class JMuxer extends Event {
     }
 
     feed(data) {
-        let remux = false,
-            slices,
-            left,
-            duration,
-            chunks = {
-                video: [],
-                audio: []
-            };
-
         if (!data || !this.remuxController) return;
-        duration = data.duration ? parseInt(data.duration) : 0;
+        
+        data.duration = data.duration ? parseInt(data.duration) : 0;
 
-        if (data.video) {
-            data.video = appendByteArray(this.remainingData, data.video);
-            [slices, left] = H264Parser.extractNALu(data.video);
-            this.remainingData = left || new Uint8Array();
-
-            if (slices.length > 0) {
-                chunks.video = this.getVideoFrames(slices, duration, data.compositionTimeOffset);
-                remux = true;
-            } else {
-                debug.error('Failed to extract any NAL units from video data:', left);
-                if (typeof this.options.onMissingVideoFrames === 'function') {
-                    this.options.onMissingVideoFrames.call(null, data);
-                }
-                return;
-            }
-        }
-        if (data.audio) {
-            slices = AACParser.extractAAC(data.audio);
-            if (slices.length > 0) {
-                chunks.audio = this.getAudioFrames(slices, duration);
-                remux = true;
-            } else {
-                debug.error('Failed to extract audio data from:', data.audio);
-                if (typeof this.options.onMissingAudioFrames === 'function') {
-                    this.options.onMissingAudioFrames.call(null, data);
-                }
-                return;
-            }
-        }
-        if (!remux) {
-            debug.error('Input object must have video and/or audio property. Make sure it is a valid typed array');
-            return;
-        }
-        this.remuxController.remux(chunks);
-    }
-
-    getVideoFrames(nalus, duration, compositionTimeOffset) {
-        let units = [],
-            frames = [],
-            fd = 0,
-            tt = 0,
-            keyFrame = false,
-            vcl = false;
-        if (this.pendingUnits.units) {
-            units = this.pendingUnits.units;
-            vcl = this.pendingUnits.vcl;
-            keyFrame = this.pendingUnits.keyFrame;
-            this.pendingUnits = {};
-        }
-        for (let nalu of nalus) {
-            let unit = new NALU(nalu);
-            if (unit.type() === NALU.IDR || unit.type() === NALU.NDR) {
-                H264Parser.parseHeader(unit);
-            }
-            if (units.length && vcl && (unit.isfmb || !unit.isvcl)) {
-                frames.push({
-                    units,
-                    keyFrame
-                });
-                units = [];
-                keyFrame = false;
-                vcl = false;
-            }
-            units.push(unit);
-            keyFrame = keyFrame || unit.isKeyframe();
-            vcl = vcl || unit.isvcl;
-        }
-        if (units.length) {
-            // lets keep indecisive nalus as pending in case of fixed fps
-            if (!duration) {
-                this.pendingUnits = {
-                    units,
-                    keyFrame,
-                    vcl
-                };
-            }
-            else if (vcl) {
-                frames.push({
-                    units,
-                    keyFrame
-                });
-            } else {
-                let last = frames.length - 1;
-                if (last >= 0) {
-                    frames[last].units = frames[last].units.concat(units);
-                }
-            }
-        }
-        fd = duration ? duration / frames.length | 0 : this.frameDuration;
-        tt = duration ? (duration - (fd * frames.length)) : 0;
-
-        frames.map((frame) => {
-            frame.duration = fd;
-            frame.compositionTimeOffset = compositionTimeOffset;
-            if (tt > 0) {
-                frame.duration++;
-                tt--;
-            }
-            this.kfCounter++;
-            if (frame.keyFrame && this.options.clearBuffer) {
-                this.kfPosition.push((this.kfCounter * fd) / 1000);
-            }
-        });
-        debug.log(`jmuxer: No. of frames of the last chunk: ${frames.length}`);
-        return frames;
-    }
-
-    getAudioFrames(aacFrames, duration) {
-        let frames = [],
-            fd = 0,
-            tt = 0;
-
-        for (let units of aacFrames) {
-            frames.push({ units });
-        }
-        fd = duration ? duration / frames.length | 0 : this.frameDuration;
-        tt = duration ? (duration - (fd * frames.length)) : 0;
-        frames.map((frame) => {
-            frame.duration = fd;
-            if (tt > 0) {
-                frame.duration++;
-                tt--;
-            }
-        });
-        return frames;
+        this.remuxController.feed(data);
     }
 
     destroy() {
@@ -324,7 +215,10 @@ export default class JMuxer extends Event {
         for (let type in this.remuxController.tracks) {
             let track = this.remuxController.tracks[type];
             if (!JMuxer.isSupported(`${type}/mp4; codecs="${track.mp4track.codec}"`)) {
-                debug.error('Browser does not support codec');
+                debug.error(`Browser does not support codec: ${type}/mp4; codecs="${track.mp4track.codec}"`);
+                if (typeof this.options.onUnsupportedCodec === 'function') {
+                    this.options.onUnsupportedCodec.call(null, track.mp4track.codec);
+                }
                 return false;
             }
             let sb = this.mediaSource.addSourceBuffer(`${type}/mp4; codecs="${track.mp4track.codec}"`);
@@ -353,7 +247,8 @@ export default class JMuxer extends Event {
         if (this.node.buffered && this.node.buffered.length > 0 && !this.node.seeking) {
             const end = this.node.buffered.end(0);
             if (end - this.node.currentTime > (this.options.maxDelay / 1000)) {
-                console.log('delay');
+                debug.log('delay');
+                if (this.node.paused) this.node.play().catch(debug.error);
                 this.node.currentTime = end - 0.001;
             }
         }

package/src/parsers/aac.js

@@ -1,7 +1,6 @@
 import * as debug from '../util/debug';
 
 export class AACParser {
-    static aacHeader;
 
     static get samplingRateMap() {
         return [96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000, 7350];
@@ -22,56 +21,30 @@ export class AACParser {
     static extractAAC(buffer) {
         let i = 0,
             length = buffer.byteLength,
-            result = [],
+            slices = [],
             headerLength,
             frameLength;
 
         if (!AACParser.isAACPattern(buffer)) {
             debug.error('Invalid ADTS audio format');
-            return result;
+            return {
+                valid: false,
+            };
         }
         headerLength = AACParser.getHeaderLength(buffer);
-        if (!AACParser.aacHeader) {
-            AACParser.aacHeader = buffer.subarray(0, headerLength);
-        }
+        const header = buffer.subarray(0, headerLength);
 
         while (i < length) {
             frameLength = AACParser.getFrameLength(buffer);
-            result.push(buffer.subarray(headerLength, frameLength));
+            slices.push(buffer.subarray(headerLength, frameLength));
             buffer = buffer.slice(frameLength);
             i += frameLength;
         }
-        return result;
-    }
-
-    constructor(remuxer) {
-        this.remuxer = remuxer;
-        this.track = remuxer.mp4track;
+        return {
+            valid: true,
+            header,
+            slices,
+        };
     }
 
-    setAACConfig() {
-        let objectType,
-            sampleIndex,
-            channelCount,
-            config = new Uint8Array(2),
-            headerData = AACParser.aacHeader;
-
-        if (!headerData) return;
-            
-        objectType = ((headerData[2] & 0xC0) >>> 6) + 1;
-        sampleIndex = ((headerData[2] & 0x3C) >>> 2);
-        channelCount = ((headerData[2] & 0x01) << 2);
-        channelCount |= ((headerData[3] & 0xC0) >>> 6);
-
-        /* refer to http://wiki.multimedia.cx/index.php?title=MPEG-4_Audio#Audio_Specific_Config */
-        config[0] = objectType << 3;
-        config[0] |= (sampleIndex & 0x0E) >> 1;
-        config[1] |= (sampleIndex & 0x01) << 7;
-        config[1] |= channelCount << 3;
-
-        this.track.codec = 'mp4a.40.' + objectType;
-        this.track.channelCount = channelCount;
-        this.track.config = config;
-        this.remuxer.readyToDecode = true;
-    }
 }