@scrypted/prebuffer-mixin 0.1.227 → 0.1.228

package/src/main.ts

@@ -1,11 +1,12 @@
 
 import { AutoenableMixinProvider } from '@scrypted/common/src/autoenable-mixin-provider';
+import { getH264EncoderArgs, LIBX264_ENCODER_TITLE } from '@scrypted/common/src/ffmpeg-hardware-acceleration';
 import { startFFMPegFragmentedMP4Session } from '@scrypted/common/src/ffmpeg-mp4-parser-session';
 import { handleRebroadcasterClient, ParserOptions, ParserSession, setupActivityTimer, startParserSession } from '@scrypted/common/src/ffmpeg-rebroadcast';
 import { closeQuiet, createBindZero, listenZeroSingleClient } from '@scrypted/common/src/listen-cluster';
-import { ffmpegLogInitialOutput, safeKillFFmpeg } from '@scrypted/common/src/media-helpers';
+import { safeKillFFmpeg } from '@scrypted/common/src/media-helpers';
 import { readLength } from '@scrypted/common/src/read-stream';
-import { createRtspParser, H264_NAL_TYPE_IDR, findH264NaluType, RtspClient, RtspServer, RTSP_FRAME_MAGIC, parseSemicolonDelimited } from '@scrypted/common/src/rtsp-server';
+import { createRtspParser, findH264NaluType, H264_NAL_TYPE_IDR, RtspClient, RtspServer, RTSP_FRAME_MAGIC } from '@scrypted/common/src/rtsp-server';
 import { addTrackControls, parseSdp } from '@scrypted/common/src/sdp-utils';
 import { StorageSettings } from '@scrypted/common/src/settings';
 import { SettingsMixinDeviceBase, SettingsMixinDeviceOptions } from "@scrypted/common/src/settings-mixin";
@@ -18,7 +19,6 @@ import net from 'net';
 import { Duplex } from 'stream';
 import { connectRFC4571Parser, RtspChannelCodecMapping, startRFC4571Parser } from './rfc4571';
 import { createStreamSettings, getPrebufferedStreams } from './stream-settings';
-import { getH264EncoderArgs, LIBX264_ENCODER_TITLE } from '@scrypted/common/src/ffmpeg-hardware-acceleration';
 
 const { mediaManager, log, systemManager, deviceManager } = sdk;
 
@@ -44,9 +44,8 @@ const VALID_AUDIO_CONFIGS = [
   TRANSCODE_AUDIO,
 ];
 
-interface PrebufferStreamChunk {
-  chunk: StreamChunk;
-  time: number;
+interface PrebufferStreamChunk extends StreamChunk {
+  time?: number;
 }
 
 interface Prebuffers {
@@ -70,7 +69,6 @@ class PrebufferSession {
   parsers: { [container: string]: StreamParser };
   sdp: Promise<string>;
 
-  detectedIdrInterval: number;
   audioDisabled = false;
 
   mixinDevice: VideoCamera & VideoCameraConfiguration;
@@ -107,6 +105,38 @@ class PrebufferSession {
     }
   }
 
+  getDetectedIdrInterval() {
+    const durations: number[] = [];
+    if (this.prebuffers.mp4.length) {
+      let last: number;
+
+      for (const chunk of this.prebuffers.mp4) {
+        if (chunk.type === 'mdat') {
+          if (last)
+            durations.push(chunk.time - last);
+          last = chunk.time;
+        }
+      }
+    }
+    else if (this.prebuffers.rtsp.length) {
+      let last: number;
+
+      for (const chunk of this.prebuffers.rtsp) {
+        if (findH264NaluType(chunk, H264_NAL_TYPE_IDR)) {
+          if (last)
+            durations.push(chunk.time - last);
+          last = chunk.time;
+        }
+      }
+    }
+
+    if (!durations.length)
+      return;
+
+    const total = durations.reduce((prev, current) => prev + current, 0);
+    return total / durations.length;
+  }
+
   get maxBitrate() {
     let ret = parseInt(this.storage.getItem(this.maxBitrateKey));
     if (!ret) {
@@ -229,7 +259,7 @@ class PrebufferSession {
     const { muxingMp4, rtspMode, defaultMode } = this.getRebroadcastMode();
     for (const prebuffer of (muxingMp4 ? this.prebuffers.mp4 : this.prebuffers.rtsp)) {
       start = start || prebuffer.time;
-      for (const chunk of prebuffer.chunk.chunks) {
+      for (const chunk of prebuffer.chunks) {
         total += chunk.byteLength;
       }
     }
@@ -327,6 +357,7 @@ class PrebufferSession {
         ? `${session.inputVideoResolution?.width}x${session.inputVideoResolution?.height}`
         : 'unknown';
 
+        const idrInterval = this.getDetectedIdrInterval();
       settings.push(
         {
           key: 'detectedResolution',
@@ -350,7 +381,7 @@ class PrebufferSession {
           title: 'Detected Keyframe Interval',
           description: "Configuring your camera to 4 seconds is recommended (IDR aka Frame Interval = FPS * 4 seconds).",
           readonly: true,
-          value: (this.detectedIdrInterval || 0) / 1000 || 'unknown',
+          value: (idrInterval || 0) / 1000 || 'unknown',
         },
       );
     }
@@ -684,7 +715,7 @@ class PrebufferSession {
             handleRTSP: async () => {
               await rtspClient.readMessage();
             },
-            onLoop: async () => {
+            onLoop: () => {
               if (rtspClient.needKeepAlive) {
                 rtspClient.needKeepAlive = false;
                 rtspClient.writeGetParameter();
@@ -842,50 +873,22 @@ class PrebufferSession {
 
     for (const container of PrebufferParserValues) {
       let shifts = 0;
+      let prebufferContainer: PrebufferStreamChunk[] = this.prebuffers[container];
 
-      let prevIdr: number;
-
-      const updateIdr = (now: number) => {
-        if (prevIdr) {
-          const sendEvent = typeof this.detectedIdrInterval !== 'number';
-          this.detectedIdrInterval = now - prevIdr;
-          // only on the first idr update should we send a settings refresh.
-          if (sendEvent)
-            deviceManager.onMixinEvent(this.mixin.id, this.mixin.mixinProviderNativeId, ScryptedInterface.Settings, undefined);
-        }
-        prevIdr = now;
-      }
-
-      this.detectedIdrInterval = undefined;
-      session.on(container, (chunk: StreamChunk) => {
-        const prebufferContainer: PrebufferStreamChunk[] = this.prebuffers[container];
+      session.on(container, (chunk: PrebufferStreamChunk) => {
         const now = Date.now();
 
-        // this is only valid for mp4, so its no op for everything else
-        // used to detect idr interval.
-        if (chunk.type === 'mdat') {
-          updateIdr(now);
-        }
-        else if (chunk.type === 'h264') {
-          if (findH264NaluType(chunk, H264_NAL_TYPE_IDR)) {
-            // only update the rtsp computed idr once a minute.
-            // per packet bitscan is not great.
-            updateIdr(now);
-          }
-        }
-
-        prebufferContainer.push({
-          time: now,
-          chunk,
-        });
+        chunk.time = now;
+        prebufferContainer.push(chunk);
 
         while (prebufferContainer.length && prebufferContainer[0].time < now - prebufferDurationMs) {
           prebufferContainer.shift();
           shifts++;
         }
 
-        if (shifts > 1000) {
-          this.prebuffers[container] = prebufferContainer.slice();
+        if (shifts > 100000) {
+          prebufferContainer = prebufferContainer.slice();
+          this.prebuffers[container] = prebufferContainer;
           shifts = 0;
         }
       });
@@ -976,17 +979,17 @@ class PrebufferSession {
 
         const prebufferContainer: PrebufferStreamChunk[] = this.prebuffers[container];
         if (container !== 'rtsp') {
-          for (const prebuffer of prebufferContainer) {
-            if (prebuffer.time < now - requestedPrebuffer)
+          for (const chunk of prebufferContainer) {
+            if (chunk.time < now - requestedPrebuffer)
               continue;
 
-            safeWriteData(prebuffer.chunk);
+            safeWriteData(chunk);
           }
         }
         else {
           // for some reason this doesn't work as well as simply guessing and dumping.
           const parser = this.parsers[container];
-          const availablePrebuffers = parser.findSyncFrame(prebufferContainer.filter(pb => pb.time >= now - requestedPrebuffer).map(pb => pb.chunk));
+          const availablePrebuffers = parser.findSyncFrame(prebufferContainer.filter(pb => pb.time >= now - requestedPrebuffer));
           for (const prebuffer of availablePrebuffers) {
             safeWriteData(prebuffer);
           }
@@ -1012,10 +1015,11 @@ class PrebufferSession {
 
     const session = await this.parserSessionPromise;
 
+    const idrInterval = this.getDetectedIdrInterval();
     let requestedPrebuffer = options?.prebuffer;
     if (requestedPrebuffer == null) {
       // get into the general area of finding a sync frame.
-      requestedPrebuffer = Math.max(4000, (this.detectedIdrInterval || 4000)) * 1.5;
+      requestedPrebuffer = Math.max(4000, (idrInterval || 4000)) * 1.5;
     }
 
     const { rtspMode } = this.getRebroadcastMode();
@@ -1026,7 +1030,7 @@ class PrebufferSession {
     // If a mp4 prebuffer was explicitly requested, but an mp4 prebuffer is not available (rtsp mode),
     // rewind a little bit earlier to gaurantee a valid full segment of that length is sent.
     if (options?.prebuffer && container !== 'mp4' && options?.container === 'mp4') {
-      requestedPrebuffer += (this.detectedIdrInterval || 4000) * 1.5;
+      requestedPrebuffer += (idrInterval || 4000) * 1.5;
     }
 
     const mediaStreamOptions: ResponseMediaStreamOptions = session.negotiateMediaStream(options);
@@ -1114,7 +1118,7 @@ class PrebufferSession {
     for (const prebuffer of prebufferContainer) {
       if (prebuffer.time < now - requestedPrebuffer)
         continue;
-      for (const chunk of prebuffer.chunk.chunks) {
+      for (const chunk of prebuffer.chunks) {
         available += chunk.length;
       }
     }
@@ -1510,7 +1514,8 @@ class RebroadcastPlugin extends AutoenableMixinProvider implements MixinProvider
         await server.handlePlayback();
         const session = await prebufferSession.parserSessionPromise;
 
-        const requestedPrebuffer = Math.max(4000, (prebufferSession.detectedIdrInterval || 4000)) * 1.5;
+        const idrInterval = prebufferSession.getDetectedIdrInterval();
+        const requestedPrebuffer = Math.max(4000, (idrInterval || 4000)) * 1.5;
 
         prebufferSession.handleRebroadcasterClient({
           isActiveClient: true,

package/src/rfc4571.ts

@@ -23,7 +23,7 @@ export type RtspChannelCodecMapping = { [key: number]: string };
 export async function startRFC4571Parser(console: Console, socket: Readable, sdp: string, mediaStreamOptions: ResponseMediaStreamOptions, options?: ParserOptions<"rtsp">, rtspOptions?: {
     channelMap: RtspChannelCodecMapping,
     handleRTSP: () => Promise<void>,
-    onLoop?: () => Promise<void>,
+    onLoop?: () => void,
 }): Promise<ParserSession<"rtsp">> {
     let isActive = true;
     const events = new EventEmitter();
@@ -85,73 +85,78 @@ export async function startRFC4571Parser(console: Console, socket: Readable, sdp
     (async () => {
         const headerLength = rtspOptions?.channelMap ? 4 : 2;
         const offset = rtspOptions?.channelMap ? 2 : 0;
-        const skipHeader = async (header: Buffer) => {
+        const skipHeader = (header: Buffer, resumeRead: () => void) => {
             if (header.toString() !== 'RTSP')
                 return false;
             socket.unshift(header);
-            await rtspOptions.handleRTSP();
+            rtspOptions.handleRTSP().then(resumeRead);
             return true;
         }
-        for await (let { header, data } of read16BELengthLoop(socket, headerLength, offset, skipHeader)) {
-            let type: string;
-
-            await rtspOptions?.onLoop?.();
-
-            if (rtspOptions?.channelMap) {
-                const channel = header.readUInt8(1);
-                type = rtspOptions?.channelMap[channel];
-            }
-            else {
-                const pt = data[1] & 0x7f;
-
-                const prefix = Buffer.alloc(2);
-                prefix[0] = RTSP_FRAME_MAGIC;
-                if (pt === audioPt) {
-                    prefix[1] = 0;
-                }
-                else if (pt === videoPt) {
-                    prefix[1] = 2;
+        await read16BELengthLoop(socket, {
+            headerLength,
+            offset,
+            skipHeader,
+            callback: (header, data) => {
+                let type: string;
+
+                rtspOptions?.onLoop?.();
+
+                if (rtspOptions?.channelMap) {
+                    const channel = header.readUInt8(1);
+                    type = rtspOptions?.channelMap[channel];
                 }
-                header = Buffer.concat([prefix, header]);
+                else {
+                    const pt = data[1] & 0x7f;
 
-                if (pt === audioPt)
-                    type = inputAudioCodec;
-                else if (pt === videoPt)
-                    type = inputVideoCodec;
-            }
+                    const prefix = Buffer.alloc(2);
+                    prefix[0] = RTSP_FRAME_MAGIC;
+                    if (pt === audioPt) {
+                        prefix[1] = 0;
+                    }
+                    else if (pt === videoPt) {
+                        prefix[1] = 2;
+                    }
+                    header = Buffer.concat([prefix, header]);
 
-            const chunk: StreamChunk = {
-                startStream,
-                chunks: [header, data],
-                type,
-            };
+                    if (pt === audioPt)
+                        type = inputAudioCodec;
+                    else if (pt === videoPt)
+                        type = inputVideoCodec;
+                }
 
-            if (!inputVideoResolution) {
-                const sps = findH264NaluType(chunk, H264_NAL_TYPE_SPS);
-                if (sps) {
-                    try {
-                        const parsedSps = spsParse(sps);
-                        inputVideoResolution = getSpsResolution(parsedSps);
-                        console.log(inputVideoResolution);
-                        console.log('parsed bitstream sps', parsedSps);
-                    }
-                    catch (e) {
-                        console.warn('sps parsing failed');
-                        inputVideoResolution = {
-                            width: NaN,
-                            height: NaN,
+                const chunk: StreamChunk = {
+                    startStream,
+                    chunks: [header, data],
+                    type,
+                };
+
+                if (!inputVideoResolution) {
+                    const sps = findH264NaluType(chunk, H264_NAL_TYPE_SPS);
+                    if (sps) {
+                        try {
+                            const parsedSps = spsParse(sps);
+                            inputVideoResolution = getSpsResolution(parsedSps);
+                            console.log(inputVideoResolution);
+                            console.log('parsed bitstream sps', parsedSps);
+                        }
+                        catch (e) {
+                            console.warn('sps parsing failed');
+                            inputVideoResolution = {
+                                width: NaN,
+                                height: NaN,
+                            }
                         }
                     }
                 }
-            }
 
-            events.emit('rtsp', chunk);
-            resetActivityTimer();
-        }
+                events.emit('rtsp', chunk);
+                resetActivityTimer();
+            }
+        });
     })()
-    .catch(e => {
-        throw e;
-    })
+        .catch(e => {
+            throw e;
+        })
         .finally(kill);