@havue/solutions 1.0.0 → 1.1.1

package/dist/types/ws-video-manager/src/render/index.d.ts

@@ -37,21 +37,30 @@ export type RenderEvents = {
 export declare class Render extends EventBus<RenderEvents> {
     /** video元素 */
     private _videoEl;
-    /** pixi.js 实例 */
     /** mp4box 实例 */
     private _mp4box;
-    /** 接收到的socket消息 视频数据buffer数组 */
-    private _bufsQueue;
+    /** mp4box onFragment获取的视频数据buffer数组 */
+    private _audioBufsQueue;
+    private _videoBufsQueue;
     /** MediaSource 实例 */
     private _mediaSource;
     /** SourceBuffer 实例 */
-    private _sourceBuffer;
+    private _audioSourceBuffer;
+    private _videoSourceBuffer;
+    private _audioTrackId;
+    private _videoTrackId;
     /** 用于MediaSource的mimeType */
     private _mimeType;
+    private _audioMimeType;
+    private _videoMimeType;
     /** 是否暂停播放 */
     private _paused;
     private _options;
     private _cacheAnimationID;
+    /** fmp4初始化片段是否已经添加 */
+    private _isAudioInitSegmentAdded;
+    private _isVideoInitSegmentAdded;
+    private _offset;
     constructor(options?: Partial<RenderConstructorOptionType>);
     get muted(): boolean;
     set muted(val: boolean);
@@ -72,6 +81,7 @@ export declare class Render extends EventBus<RenderEvents> {
      * @param info mp4box解析信息
      */
     private _onMp4boxReady;
+    private _onSegment;
     /**
      * 初始化视频元素
      */
@@ -86,6 +96,7 @@ export declare class Render extends EventBus<RenderEvents> {
      * @returns
      */
     private _setupMSE;
+    private _setupSourceBuffer;
     /**
      * 将_bufsQueue中的数据添加到SourceBuffer中
      * @returns
@@ -98,6 +109,7 @@ export declare class Render extends EventBus<RenderEvents> {
     /** 重置解析的视频mime type */
     resetMimeType(): void;
     private destroyMediaSource;
+    destroyMp4box(): void;
     /**
      * 销毁
      */

package/dist/types/ws-video-manager/src/render/mp4box.d.ts

@@ -0,0 +1,2 @@
+import MP4Box from 'mp4box';
+export default MP4Box;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@havue/solutions",
-  "version": "1.0.0",
+  "version": "1.1.1",
   "description": "Javascript class for special scene",
   "keywords": [
     "havue",
@@ -17,9 +17,9 @@
   },
   "dependencies": {
     "lodash": "^4.17.21",
-    "@havue/shared": "^1.0.0",
-    "@havue/bc-connect": "^1.0.0",
-    "@havue/ws-video-manager": "^1.0.0"
+    "@havue/shared": "^1.1.1",
+    "@havue/ws-video-manager": "^1.1.1",
+    "@havue/bc-connect": "^1.1.1"
   },
   "publishConfig": {
     "access": "public",

package/ws-video-manager/dist/types/src/render/index.d.ts

@@ -37,21 +37,30 @@ export type RenderEvents = {
 export declare class Render extends EventBus<RenderEvents> {
     /** video元素 */
     private _videoEl;
-    /** pixi.js 实例 */
     /** mp4box 实例 */
     private _mp4box;
-    /** 接收到的socket消息 视频数据buffer数组 */
-    private _bufsQueue;
+    /** mp4box onFragment获取的视频数据buffer数组 */
+    private _audioBufsQueue;
+    private _videoBufsQueue;
     /** MediaSource 实例 */
     private _mediaSource;
     /** SourceBuffer 实例 */
-    private _sourceBuffer;
+    private _audioSourceBuffer;
+    private _videoSourceBuffer;
+    private _audioTrackId;
+    private _videoTrackId;
     /** 用于MediaSource的mimeType */
     private _mimeType;
+    private _audioMimeType;
+    private _videoMimeType;
     /** 是否暂停播放 */
     private _paused;
     private _options;
     private _cacheAnimationID;
+    /** fmp4初始化片段是否已经添加 */
+    private _isAudioInitSegmentAdded;
+    private _isVideoInitSegmentAdded;
+    private _offset;
     constructor(options?: Partial<RenderConstructorOptionType>);
     get muted(): boolean;
     set muted(val: boolean);
@@ -72,6 +81,7 @@ export declare class Render extends EventBus<RenderEvents> {
      * @param info mp4box解析信息
      */
     private _onMp4boxReady;
+    private _onSegment;
     /**
      * 初始化视频元素
      */
@@ -86,6 +96,7 @@ export declare class Render extends EventBus<RenderEvents> {
      * @returns
      */
     private _setupMSE;
+    private _setupSourceBuffer;
     /**
      * 将_bufsQueue中的数据添加到SourceBuffer中
      * @returns
@@ -98,6 +109,7 @@ export declare class Render extends EventBus<RenderEvents> {
     /** 重置解析的视频mime type */
     resetMimeType(): void;
     private destroyMediaSource;
+    destroyMp4box(): void;
     /**
      * 销毁
      */

package/ws-video-manager/dist/types/src/render/mp4box.d.ts

@@ -0,0 +1,2 @@
+import MP4Box from 'mp4box';
+export default MP4Box;

package/ws-video-manager/dist/ws-video-manager.mjs

@@ -7235,6 +7235,57 @@ var mp4box_all = {};
   }
 })(mp4box_all);
 const MP4Box = /* @__PURE__ */ getDefaultExportFromCjs(mp4box_all);
+function ExtendMp4box() {
+  const MP4BoxFile = MP4Box.createFile().constructor;
+  MP4BoxFile.prototype.removeUsedSamples = function(id) {
+    const track = this.getTrackById(id);
+    const samples = track.samples;
+    const lastSample = samples[samples.length - 1];
+    lastSample.data = null;
+    lastSample.description = null;
+    lastSample.alreadyRead = 0;
+    track.samples = [];
+    track.samples.push(lastSample);
+    track.nextSample = track.samples.length;
+    this.boxes = [];
+    this.mdats = [];
+    this.moofs = [];
+    this.lastMoofIndex = 0;
+  };
+  MP4BoxFile.prototype.destroy = function() {
+    if (this.stream) {
+      this.stream.buffers = [];
+      this.stream.bufferIndex = -1;
+      this.stream = null;
+    }
+    this.boxes = [];
+    this.mdats = [];
+    this.moofs = [];
+    this.isProgressive = false;
+    this.moovStartFound = false;
+    this.onMoovStart = null;
+    this.moovStartSent = false;
+    this.onReady = null;
+    this.readySent = false;
+    this.onSegment = null;
+    this.onSamples = null;
+    this.onError = null;
+    this.sampleListBuilt = false;
+    this.fragmentedTracks = [];
+    this.extractedTracks = [];
+    this.isFragmentationInitialized = false;
+    this.sampleProcessingStarted = false;
+    this.nextMoofNumber = 0;
+    this.itemListBuilt = false;
+    this.onSidx = null;
+    this.sidxSent = false;
+    this.moov = null;
+    this.ftyp = null;
+    this.items = [];
+    this.entity_groups = [];
+  };
+}
+ExtendMp4box();
 const WS_VIDEO_RENDER_DEFAULT_OPTIONS = Object.freeze({
   liveMaxLatency: 0.3,
   maxCacheBufByte: 200 * 1024,
@@ -7263,24 +7314,33 @@ class Render extends EventBus {
     super();
     /** video元素 */
     __publicField(this, "_videoEl");
-    /** pixi.js 实例 */
-    // private _pixiApp: Application | null = null
     /** mp4box 实例 */
     __publicField(this, "_mp4box", MP4Box.createFile());
-    /** 接收到的socket消息 视频数据buffer数组 */
-    __publicField(this, "_bufsQueue", []);
+    /** mp4box onFragment获取的视频数据buffer数组 */
+    __publicField(this, "_audioBufsQueue", []);
+    __publicField(this, "_videoBufsQueue", []);
     /** MediaSource 实例 */
     __publicField(this, "_mediaSource");
     /** SourceBuffer 实例 */
-    __publicField(this, "_sourceBuffer");
+    __publicField(this, "_audioSourceBuffer");
+    __publicField(this, "_videoSourceBuffer");
+    __publicField(this, "_audioTrackId");
+    __publicField(this, "_videoTrackId");
     /** 用于MediaSource的mimeType */
     __publicField(this, "_mimeType", "");
+    __publicField(this, "_audioMimeType", "");
+    __publicField(this, "_videoMimeType", "");
     /** 是否暂停播放 */
     __publicField(this, "_paused", false);
     __publicField(this, "_options");
     __publicField(this, "_cacheAnimationID");
+    /** fmp4初始化片段是否已经添加 */
+    __publicField(this, "_isAudioInitSegmentAdded", false);
+    __publicField(this, "_isVideoInitSegmentAdded", false);
+    __publicField(this, "_offset", 0);
     this._options = options ? Object.assign({}, WS_VIDEO_RENDER_DEFAULT_OPTIONS, options) : WS_VIDEO_RENDER_DEFAULT_OPTIONS;
     this._mp4box.onReady = this._onMp4boxReady.bind(this);
+    this._mp4box.onSegment = this._onSegment.bind(this);
     this._setupVideo();
   }
   get muted() {
@@ -7318,33 +7378,15 @@ class Render extends EventBus {
    * @param buf
    */
   appendMediaBuffer(bufs) {
-    var _a;
     if (this._paused) {
       return;
     }
-    if (!this._sourceBuffer) {
-      const buf = bufs[0];
-      buf.fileStart = 0;
-      this._mp4box.appendBuffer(buf);
-    }
-    this._bufsQueue.push(...bufs);
-    if (this._sourceBuffer && !((_a = this._videoEl) == null ? void 0 : _a.paused) && this._bufsQueue.length > 2) {
-      const len = this._bufsQueue.length;
-      const maxTotal = this._options.maxCacheBufByte;
-      let lastIndex = len - 1;
-      let total = 0;
-      for (let i2 = len - 1; i2 > 0; i2--) {
-        total += this._bufsQueue[i2].byteLength;
-        lastIndex = i2;
-        if (total >= maxTotal) {
-          this._bufsQueue = this._bufsQueue.slice(lastIndex);
-          break;
-        }
-      }
-    }
-    this._cacheAnimationID && cancelAnimationFrame(this._cacheAnimationID);
-    this._cacheAnimationID = void 0;
-    this._cache();
+    bufs.forEach((b) => {
+      b.fileStart = this._offset;
+      this._offset += b.byteLength;
+      this._mp4box.appendBuffer(b);
+    });
+    return;
   }
   /**
    * mp4box解析完成
@@ -7352,7 +7394,6 @@ class Render extends EventBus {
    */
   _onMp4boxReady(info) {
     console.log("onMp4boxReady", info);
-    this._mp4box.flush();
     if (!info.isFragmented) {
       console.error("not fragmented mp4");
       return;
@@ -7364,10 +7405,66 @@ class Render extends EventBus {
         width,
         height
       });
+      const audioTrack = info.audioTracks[0];
+      const videoTrack = info.videoTracks[0];
+      if (audioTrack) {
+        this._audioMimeType = `audio/mp4; codecs="${audioTrack.codec}"`;
+        this._audioTrackId = audioTrack.id;
+        this._mp4box.setSegmentOptions(audioTrack.id, void 0, {
+          nbSamples: 100
+        });
+      }
+      if (videoTrack) {
+        this._videoMimeType = `video/mp4; codecs="${videoTrack.codec}"`;
+        this._videoTrackId = videoTrack.id;
+        this._mp4box.setSegmentOptions(videoTrack.id, void 0, {
+          nbSamples: 100
+        });
+      }
+      const initSegments = this._mp4box.initializeSegmentation();
+      for (const seg of initSegments) {
+        if (audioTrack && seg.id === audioTrack.id) {
+          this._audioBufsQueue.push(seg.buffer);
+        } else if (videoTrack && seg.id === videoTrack.id) {
+          this._videoBufsQueue.push(seg.buffer);
+        }
+      }
     } catch (error) {
       console.error(error);
     }
     this._setupMSE();
+    this._mp4box.start();
+  }
+  _onSegment(id, __, buffer, sampleNumber) {
+    var _a;
+    const isAudio = id === this._audioTrackId;
+    const isVideo = id === this._videoTrackId;
+    const bufQueue = isAudio ? this._audioBufsQueue : isVideo ? this._videoBufsQueue : null;
+    if (!bufQueue) {
+      return;
+    }
+    const sourceBuffer = isVideo ? this._videoSourceBuffer : this._audioSourceBuffer;
+    bufQueue.push(buffer);
+    this._mp4box.releaseUsedSamples(id, sampleNumber);
+    this._mp4box.removeUsedSamples(id);
+    const segmentAdded = isAudio ? this._isAudioInitSegmentAdded : this._isVideoInitSegmentAdded;
+    if (segmentAdded && sourceBuffer && !((_a = this._videoEl) == null ? void 0 : _a.paused) && bufQueue.length > 2) {
+      const len = bufQueue.length;
+      const maxTotal = this._options.maxCacheBufByte;
+      let lastIndex = len - 1;
+      let total = 0;
+      for (let i2 = len - 1; i2 > 0; i2--) {
+        total += bufQueue[i2].byteLength;
+        lastIndex = i2;
+        if (total >= maxTotal) {
+          bufQueue.splice(0, lastIndex);
+          break;
+        }
+      }
+    }
+    this._cacheAnimationID && cancelAnimationFrame(this._cacheAnimationID);
+    this._cacheAnimationID = void 0;
+    this._cache(isVideo);
   }
   /**
    * 初始化视频元素
@@ -7452,30 +7549,45 @@ class Render extends EventBus {
     URL.revokeObjectURL(this._videoEl.src);
     this._videoEl.src = URL.createObjectURL(this._mediaSource);
     this._mediaSource.addEventListener("sourceopen", () => {
-      const sourceBuffer = this._sourceBuffer = this._mediaSource.addSourceBuffer(this._mimeType);
-      sourceBuffer.mode = "sequence";
-      sourceBuffer.onupdateend = () => {
-        var _a, _b, _c;
-        if (!this._videoEl || !sourceBuffer || ((_a = this._mediaSource) == null ? void 0 : _a.readyState) !== "open" || ![...this._mediaSource.sourceBuffers].includes(sourceBuffer)) {
-          return;
-        }
-        const currentTime = this._videoEl.currentTime;
-        if (sourceBuffer.buffered.length > 0) {
-          let bufferedLen = sourceBuffer.buffered.length;
-          const needDelBuf = bufferedLen > 1;
-          if (needDelBuf && currentTime) {
-            const lastIndex = bufferedLen - 1;
-            if (currentTime < sourceBuffer.buffered.start(lastIndex)) {
-              this._videoEl.currentTime = sourceBuffer.buffered.start(lastIndex);
-            }
-            const delBufEnd = sourceBuffer.buffered.end(lastIndex - 1);
-            if (!this._sourceBuffer.updating && currentTime > delBufEnd) {
-              (_b = this._sourceBuffer) == null ? void 0 : _b.remove(0, delBufEnd);
-            }
+      if (!this._mediaSource) {
+        return;
+      }
+      if (this._audioMimeType) {
+        this._audioSourceBuffer = this._mediaSource.addSourceBuffer(this._audioMimeType);
+        this._audioSourceBuffer.mode = "sequence";
+        this._setupSourceBuffer(this._audioSourceBuffer, false);
+      }
+      if (this._videoMimeType) {
+        this._videoSourceBuffer = this._mediaSource.addSourceBuffer(this._videoMimeType);
+        this._videoSourceBuffer.mode = "sequence";
+        this._setupSourceBuffer(this._videoSourceBuffer, true);
+      }
+    });
+  }
+  _setupSourceBuffer(sourceBuffer, isVideo = false) {
+    sourceBuffer.onupdateend = () => {
+      var _a;
+      if (!this._videoEl || !sourceBuffer || ((_a = this._mediaSource) == null ? void 0 : _a.readyState) !== "open" || ![...this._mediaSource.sourceBuffers].includes(sourceBuffer)) {
+        return;
+      }
+      const currentTime = this._videoEl.currentTime;
+      if (sourceBuffer.buffered.length > 0) {
+        let bufferedLen = sourceBuffer.buffered.length;
+        const needDelBuf = bufferedLen > 1;
+        if (needDelBuf && currentTime) {
+          const lastIndex = bufferedLen - 1;
+          if (currentTime < sourceBuffer.buffered.start(lastIndex)) {
+            this._videoEl.currentTime = sourceBuffer.buffered.start(lastIndex);
           }
-          bufferedLen = sourceBuffer.buffered.length;
-          const start = sourceBuffer.buffered.start(bufferedLen - 1);
-          const end = sourceBuffer.buffered.end(bufferedLen - 1);
+          const delBufEnd = sourceBuffer.buffered.end(lastIndex - 1);
+          if (!sourceBuffer.updating && currentTime > delBufEnd) {
+            sourceBuffer == null ? void 0 : sourceBuffer.remove(0, delBufEnd);
+          }
+        }
+        bufferedLen = sourceBuffer.buffered.length;
+        const start = sourceBuffer.buffered.start(bufferedLen - 1);
+        const end = sourceBuffer.buffered.end(bufferedLen - 1);
+        if (isVideo) {
           if (!currentTime && start) {
             this._videoEl.currentTime = start;
           }
@@ -7485,33 +7597,35 @@ class Render extends EventBus {
               this._videoEl.currentTime = end - 0.1;
             }
           }
-          if (!this._sourceBuffer.updating && currentTime - start > this._options.maxCache) {
-            (_c = this._sourceBuffer) == null ? void 0 : _c.remove(0, currentTime - this._options.maxCache / 2);
-          }
         }
-      };
-    });
+        if (!sourceBuffer.updating && currentTime - start > this._options.maxCache) {
+          sourceBuffer == null ? void 0 : sourceBuffer.remove(0, currentTime - this._options.maxCache / 2);
+        }
+      }
+    };
   }
   /**
    * 将_bufsQueue中的数据添加到SourceBuffer中
    * @returns
    */
-  _cache() {
+  _cache(isVideo = false) {
     if (!this._videoEl) {
       return;
     }
-    if (!this._mediaSource || !this._sourceBuffer || this._sourceBuffer.updating || !this._bufsQueue.length || this._mediaSource.readyState !== "open") {
-      this._cacheAnimationID === void 0 && (this._cacheAnimationID = requestAnimationFrame(() => this._cache()));
+    const queue = isVideo ? this._videoBufsQueue : this._audioBufsQueue;
+    const sourceBuffer = isVideo ? this._videoSourceBuffer : this._audioSourceBuffer;
+    if (!this._mediaSource || !sourceBuffer || sourceBuffer.updating || !queue.length || this._mediaSource.readyState !== "open") {
+      this._cacheAnimationID === void 0 && (this._cacheAnimationID = requestAnimationFrame(() => this._cache(isVideo)));
       return;
     }
     if (this._videoEl.error) {
       this._setupMSE();
-      return this._cacheAnimationID === void 0 && (this._cacheAnimationID = requestAnimationFrame(() => this._cache()));
+      return this._cacheAnimationID === void 0 && (this._cacheAnimationID = requestAnimationFrame(() => this._cache(isVideo)));
     }
     this._cacheAnimationID = void 0;
     let frame;
-    if (this._bufsQueue.length > 1) {
-      const freeBuffer = this._bufsQueue.splice(0, this._bufsQueue.length);
+    if (queue.length > 1) {
+      const freeBuffer = queue.splice(0, queue.length);
       const length = freeBuffer.map((e) => e.byteLength).reduce((a, b) => a + b, 0);
       const buffer = new Uint8Array(length);
       let offset = 0;
@@ -7522,10 +7636,15 @@ class Render extends EventBus {
       }
       frame = buffer;
     } else {
-      frame = new Uint8Array(this._bufsQueue.shift() || []);
+      frame = new Uint8Array(queue.shift() || []);
     }
     if (frame) {
-      this._sourceBuffer.appendBuffer(frame);
+      if (isVideo) {
+        !this._isVideoInitSegmentAdded && (this._isVideoInitSegmentAdded = true);
+      } else {
+        !this._isAudioInitSegmentAdded && (this._isAudioInitSegmentAdded = true);
+      }
+      sourceBuffer.appendBuffer(frame);
     }
   }
   /**
@@ -7539,13 +7658,14 @@ class Render extends EventBus {
   }
   /** 重置解析的视频mime type */
   resetMimeType() {
+    this.destroyMp4box();
     this.destroyMediaSource();
     if (this._videoEl) {
       this._videoEl.src = "";
     }
     this._mp4box = MP4Box.createFile();
     this._mp4box.onReady = this._onMp4boxReady.bind(this);
-    this._bufsQueue.length = 0;
+    this._mp4box.onSegment = this._onSegment.bind(this);
   }
   destroyMediaSource() {
     var _a;
@@ -7555,21 +7675,41 @@ class Render extends EventBus {
         this._videoEl.src = "";
       }
       if (this._mediaSource.readyState === "open") {
-        if (this._sourceBuffer) {
-          this._sourceBuffer.abort();
-          this._mediaSource.removeSourceBuffer(this._sourceBuffer);
+        if (this._audioSourceBuffer) {
+          this._audioSourceBuffer.abort();
+          this._mediaSource.removeSourceBuffer(this._audioSourceBuffer);
+        }
+        if (this._videoSourceBuffer) {
+          this._videoSourceBuffer.abort();
+          this._mediaSource.removeSourceBuffer(this._videoSourceBuffer);
         }
         this._mediaSource.endOfStream();
       }
       this._mediaSource = void 0;
-      this._sourceBuffer = void 0;
+      this._audioSourceBuffer = void 0;
+      this._videoSourceBuffer = void 0;
     }
   }
+  destroyMp4box() {
+    this._audioTrackId = void 0;
+    this._videoTrackId = void 0;
+    this._mimeType = "";
+    this._audioMimeType = "";
+    this._videoMimeType = "";
+    this._isAudioInitSegmentAdded = false;
+    this._isVideoInitSegmentAdded = false;
+    this._audioBufsQueue.length = 0;
+    this._videoBufsQueue.length = 0;
+    this._offset = 0;
+    this._mp4box.stop();
+    this._mp4box.flush();
+    this._mp4box.destroy();
+    this._mp4box = null;
+  }
   /**
    * 销毁
    */
   destroy() {
-    this._bufsQueue = [];
     if (this._videoEl) {
       this._videoEl.pause();
       this._videoEl.currentTime = 0;
@@ -7580,9 +7720,9 @@ class Render extends EventBus {
       this._videoEl.src = "";
       this._videoEl = void 0;
     }
-    this._mimeType = "";
     this._cacheAnimationID && cancelAnimationFrame(this._cacheAnimationID);
     this._cacheAnimationID = void 0;
+    this.destroyMp4box();
     this.destroyMediaSource();
   }
 }