@scrypted/prebuffer-mixin 0.1.257 → 0.1.260

package/src/main.ts

@@ -2,20 +2,19 @@
 import { AutoenableMixinProvider } from '@scrypted/common/src/autoenable-mixin-provider';
 import { getH264EncoderArgs, LIBX264_ENCODER_TITLE } from '@scrypted/common/src/ffmpeg-hardware-acceleration';
 import { handleRebroadcasterClient, ParserOptions, ParserSession, startParserSession } from '@scrypted/common/src/ffmpeg-rebroadcast';
-import { closeQuiet, createBindZero, listenZeroSingleClient } from '@scrypted/common/src/listen-cluster';
+import { closeQuiet, listenZeroSingleClient } from '@scrypted/common/src/listen-cluster';
 import { readLength } from '@scrypted/common/src/read-stream';
-import { createRtspParser, findH264NaluType, H264_NAL_TYPE_IDR, H264_NAL_TYPE_SEI, parseSemicolonDelimited, RtspClient, RtspServer, RTSP_FRAME_MAGIC } from '@scrypted/common/src/rtsp-server';
+import { createRtspParser, findH264NaluType, H264_NAL_TYPE_IDR, H264_NAL_TYPE_SEI, RtspServer } from '@scrypted/common/src/rtsp-server';
 import { addTrackControls, parseSdp } from '@scrypted/common/src/sdp-utils';
 import { StorageSettings } from '@scrypted/common/src/settings';
 import { SettingsMixinDeviceBase, SettingsMixinDeviceOptions } from "@scrypted/common/src/settings-mixin";
-import { sleep } from '@scrypted/common/src/sleep';
 import { createFragmentedMp4Parser, createMpegTsParser, StreamChunk, StreamParser } from '@scrypted/common/src/stream-parser';
 import sdk, { BufferConverter, DeviceProvider, FFmpegInput, MediaObject, MediaStreamOptions, MixinProvider, RequestMediaStreamOptions, ResponseMediaStreamOptions, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, Setting, Settings, SettingValue, VideoCamera, VideoCameraConfiguration } from '@scrypted/sdk';
 import crypto from 'crypto';
-import dgram from 'dgram';
 import net from 'net';
 import { Duplex } from 'stream';
-import { connectRFC4571Parser, requeueRtspVideoData, RtspChannelCodecMapping, startRFC4571Parser } from './rfc4571';
+import { connectRFC4571Parser, startRFC4571Parser } from './rfc4571';
+import { startRtspSession } from './rtsp-session';
 import { createStreamSettings, getPrebufferedStreams } from './stream-settings';
 import { getTranscodeMixinProviderId, REBROADCAST_MIXIN_INTERFACE_TOKEN, TranscodeMixinProvider, TRANSCODE_MIXIN_PROVIDER_NATIVE_ID } from './transcode-settings';
 
@@ -267,7 +266,7 @@ class PrebufferSession {
 
     return {
       rtspMode: mode?.startsWith('RTSP'),
-      muxingMp4: !rtspMode || mode?.includes('MP4'),
+      muxingMp4: !rtspMode,
     };
   }
 
@@ -306,7 +305,7 @@ class PrebufferSession {
       }
     );
 
-    const addFFmpegSettings = () => {
+    const addFFmpegAudioSettings = () => {
       settings.push(
         {
           title: 'Audio Codec Transcoding',
@@ -322,6 +321,11 @@ class PrebufferSession {
             TRANSCODE_AUDIO_DESCRIPTION,
           ],
         },
+      );
+    };
+
+    const addFFmpegInputSettings = () => {
+      settings.push(
         {
           title: 'FFmpeg Input Arguments Prefix',
           group,
@@ -337,7 +341,9 @@ class PrebufferSession {
           combobox: true,
         },
       )
-    };
+    }
+
+    let usingFFmpeg = muxingMp4;
 
     if (this.canUseRtspParser(this.advertisedMediaStreamOptions)) {
       const canUseScryptedParser = rtspMode;
@@ -349,13 +355,15 @@ class PrebufferSession {
         SCRYPTED_PARSER_UDP,
       ] : [];
 
+      const currentParser = this.storage.getItem(this.rtspParserKey) || STRING_DEFAULT;
+
       settings.push(
         {
           key: this.rtspParserKey,
           group,
           title: 'RTSP Parser',
           description: `The RTSP Parser used to read the stream. The default is "${defaultValue}" for this container.`,
-          value: this.storage.getItem(this.rtspParserKey) || STRING_DEFAULT,
+          value: currentParser,
           choices: [
             STRING_DEFAULT,
             ...scryptedOptions,
@@ -364,9 +372,17 @@ class PrebufferSession {
           ],
         }
       );
+
+      if (!(currentParser === STRING_DEFAULT ? defaultValue : currentParser).includes('Scrypted')) {
+        usingFFmpeg = true;
+      }
     }
-    else {
-      addFFmpegSettings();
+
+    if (muxingMp4) {
+      addFFmpegAudioSettings();
+    }
+    if (usingFFmpeg) {
+      addFFmpegInputSettings();
     }
 
     if (session) {
@@ -672,154 +688,12 @@ class PrebufferSession {
       }
 
       if (usingScryptedParser) {
-        this.console.log('bypassing ffmpeg: using scrypted rtsp/rfc4571 parser');
-        const rtspClient = new RtspClient(ffmpegInput.url, this.console);
-
-        let servers: dgram.Socket[] = [];
-        const cleanupServers = () => {
-          for (const server of servers) {
-            closeQuiet(server);
-          }
-        }
-
-        try {
-          rtspClient.requestTimeout = 10000;
-          await rtspClient.options();
-          const sdpResponse = await rtspClient.describe();
-          const contentBase = sdpResponse.headers['content-base'];
-          if (contentBase) {
-            const url = new URL(contentBase);
-            const existing = new URL(rtspClient.url);
-            url.username = existing.username;
-            url.password = existing.password;
-            rtspClient.url = url.toString();
-          }
-          let sdp = sdpResponse.body.toString().trim();
-          this.console.log('sdp', sdp);
-
-          const parsedSdp = parseSdp(sdp);
-          let channel = 0;
-          const mapping: RtspChannelCodecMapping = {};
-          const useUdp = parser === SCRYPTED_PARSER_UDP;
-          let udpSessionTimeout: number;
-          const checkUdpSessionTimeout = (headers: { [key: string]: string }) => {
-            if (useUdp && headers.session && !udpSessionTimeout) {
-              const sessionDict = parseSemicolonDelimited(headers.session);
-              udpSessionTimeout = parseInt(sessionDict['timeout']);
-            }
-          }
-
-          const doSetup = async (control: string, codec: string) => {
-            let setupChannel = channel;
-            let udp: dgram.Socket;
-            if (useUdp) {
-              const rtspChannel = channel;
-              const { port, server } = await createBindZero();
-              udp = server;
-              servers.push(server);
-              setupChannel = port;
-              server.on('message', data => {
-                const prefix = Buffer.alloc(4);
-                prefix.writeUInt8(RTSP_FRAME_MAGIC, 0);
-                prefix.writeUInt8(rtspChannel, 1);
-                prefix.writeUInt16BE(data.length, 2);
-                const chunk: StreamChunk = {
-                  chunks: [prefix, data],
-                  type: codec,
-                };
-                session?.emit('rtsp', chunk);
-                session?.resetActivityTimer?.();
-              })
-            }
-            const setupResult = await rtspClient.setup(setupChannel, control, useUdp);
-            checkUdpSessionTimeout(setupResult.headers);
-
-            if (udp) {
-              const punch = Buffer.alloc(1);
-              const transport = setupResult.headers['transport'];
-              const match = transport.match(/.*?server_port=([0-9]+)-([0-9]+)/);
-              const [_, rtp, rtcp] = match;
-              const { hostname } = new URL(rtspClient.url);
-              udp.send(punch, parseInt(rtp), hostname)
-
-              mapping[channel] = codec;
-            }
-            else {
-              if (setupResult.interleaved)
-                mapping[setupResult.interleaved.begin] = codec;
-              else
-                mapping[channel] = codec;
-            }
-
-            channel += 2;
-          }
-
-          let setupVideoSection = false;
-
-          parsedSdp.msections = parsedSdp.msections.filter(section => {
-            if (section.type === 'video') {
-              if (setupVideoSection) {
-                this.console.warn('additional video section found. skipping.');
-                return false;
-              }
-              setupVideoSection = true;
-            }
-            else if (section.type !== 'audio') {
-              this.console.warn('unknown section', section.type);
-              return false;
-            }
-            else if (audioSoftMuted) {
-              return false;
-            }
-            return true;
-          });
-
-          for (const section of parsedSdp.msections) {
-            await doSetup(section.control, section.codec)
-          }
-
-          // sdp may contain multiple audio/video sections. take only the first video section.
-          sdp = [...parsedSdp.header.lines, ...parsedSdp.msections.map(msection => msection.lines).flat()].join('\r\n');
-
-          this.sdp = Promise.resolve(sdp);
-
-          const play = await rtspClient.play();
-          checkUdpSessionTimeout(play.headers);
-
-          requeueRtspVideoData(rtspClient);
-
-          session = startRFC4571Parser(this.console, rtspClient.client, sdp, ffmpegInput.mediaStreamOptions, rbo, {
-            channelMap: mapping,
-            rtspClient,
-            udpSessionTimeout,
-          });
-          const sessionKill = session.kill.bind(session);
-          let issuedTeardown = false;
-          session.kill = async () => {
-            try {
-              cleanupServers();
-              // issue a teardown to upstream to close gracefully but don't rely on it responding.
-              if (!issuedTeardown) {
-                issuedTeardown = true;
-                rtspClient.writeTeardown();
-              }
-              await sleep(500);
-            }
-            finally {
-              rtspClient.client.destroy();
-              sessionKill();
-            }
-          }
-          if (!session.isActive)
-            throw new Error('parser was killed before rtsp client started');
-
-          rtspClient.client.on('close', () => session.kill());
-        }
-        catch (e) {
-          cleanupServers();
-          rtspClient.client.destroy();
-          throw e;
-        }
+        session = await startRtspSession(this.console, ffmpegInput.url, ffmpegInput.mediaStreamOptions, {
+          useUdp: parser === SCRYPTED_PARSER_UDP,
+          audioSoftMuted,
+          rtspRequestTimeout: 10000,
+        });
+        this.sdp = session.sdp.then(buffers => Buffer.concat(buffers).toString());
       }
       else {
         if (parser === FFMPEG_PARSER_UDP)
@@ -851,8 +725,10 @@ class PrebufferSession {
           return;
 
         let { isDefault } = this.getParser(rtspMode, sessionMso);
-        if (!isDefault)
+        if (!isDefault) {
+          this.console.warn('SEI packet detected while operating with Scrypted Parser. If there are issues streaming, consider using the Default parser.');
           return;
+        }
         this.console.warn('SEI packet detected while operating with Scrypted Parser as default. Restarting rebroadcast.');
         session.kill();
         this.startPrebufferSession();
@@ -1002,6 +878,7 @@ class PrebufferSession {
   }
 
   async handleRebroadcasterClient(options: {
+    requestedContainer: string,
     isActiveClient: boolean,
     container: PrebufferParsers,
     session: ParserSession<PrebufferParsers>,
@@ -1049,7 +926,16 @@ class PrebufferSession {
         session.once('killed', cleanup);
 
         const prebufferContainer: PrebufferStreamChunk[] = this.prebuffers[container];
-        if (container !== 'rtsp') {
+        // if the requested container or the source container is not rtsp, use an exact seek.
+        // this works better when the requested container is mp4, and rtsp is the source.
+        // if starting on a sync frame, ffmpeg will skip the first segment while initializing
+        // on live sources like rtsp. the buffer before the sync frame stream will be enough
+        // for ffmpeg to analyze and start up in time for the sync frame.
+        // may be worth considering playing with a few other things to avoid this:
+        // mpeg-ts as a container (would need to write a muxer)
+        // specifying the buffer before the sync frame with probesize.
+        if (container !== 'rtsp'
+          || (options?.requestedContainer && options?.requestedContainer !== 'rtsp')) {
           for (const chunk of prebufferContainer) {
             if (chunk.time < now - requestedPrebuffer)
               continue;
@@ -1097,17 +983,11 @@ class PrebufferSession {
       requestedPrebuffer = idrInterval * 1.5;
     }
 
-    const { rtspMode } = this.getRebroadcastContainer();
+    const { rtspMode, muxingMp4 } = this.getRebroadcastContainer();
     const defaultContainer = rtspMode ? 'rtsp' : 'mpegts';
 
     let container: PrebufferParsers = this.parsers[options?.container] ? options?.container as PrebufferParsers : defaultContainer;
 
-    // If a mp4 prebuffer was explicitly requested, but an mp4 prebuffer is not available (rtsp mode),
-    // rewind a little bit earlier to gaurantee a valid full segment of that length is sent.
-    if (requestedPrebuffer && container !== 'mp4' && options?.container === 'mp4') {
-      requestedPrebuffer += idrInterval * 1.5;
-    }
-
     const mediaStreamOptions: ResponseMediaStreamOptions = session.negotiateMediaStream(options);
     let sdp = await this.sdp;
 
@@ -1118,7 +998,11 @@ class PrebufferSession {
 
     if (container === 'rtsp') {
       const parsedSdp = parseSdp(sdp);
-      parsedSdp.msections = parsedSdp.msections.filter(msection => msection.codec === mediaStreamOptions.video?.codec || msection.codec === mediaStreamOptions.audio?.codec);
+      const videoSection = parsedSdp.msections.find(msection => msection.codec && msection.codec === mediaStreamOptions.video?.codec) || parsedSdp.msections.find(msection => msection.type === 'video');
+      let audioSection = parsedSdp.msections.find(msection => msection.codec && msection.codec === mediaStreamOptions.audio?.codec) || parsedSdp.msections.find(msection => msection.type === 'audio');
+      if (mediaStreamOptions.audio === null)
+        audioSection = undefined;
+      parsedSdp.msections = parsedSdp.msections.filter(msection => msection === videoSection || msection === audioSection);
       const filterPrebufferAudio = options?.prebuffer === undefined;
       const videoCodec = parsedSdp.msections.find(msection => msection.type === 'video')?.codec;
       sdp = parsedSdp.toSdp();
@@ -1164,6 +1048,7 @@ class PrebufferSession {
     const isActiveClient = options?.refresh !== false;
 
     this.handleRebroadcasterClient({
+      requestedContainer: options?.container,
       isActiveClient,
       container,
       requestedPrebuffer,
@@ -1179,7 +1064,7 @@ class PrebufferSession {
     if (this.audioDisabled) {
       mediaStreamOptions.audio = null;
     }
-    else if (reencodeAudio) {
+    else if (reencodeAudio && muxingMp4) {
       mediaStreamOptions.audio = {
         codec: 'aac',
         encoder: 'aac',
@@ -1621,6 +1506,7 @@ export class RebroadcastPlugin extends AutoenableMixinProvider implements MixinP
         const requestedPrebuffer = Math.max(4000, (idrInterval || 4000)) * 1.5;
 
         prebufferSession.handleRebroadcasterClient({
+          requestedContainer: 'rtsp',
           isActiveClient: true,
           container: 'rtsp',
           session,

package/src/rfc4571.ts

@@ -1,14 +1,59 @@
 import { cloneDeep } from "@scrypted/common/src/clone-deep";
 import { ParserOptions, ParserSession, setupActivityTimer } from "@scrypted/common/src/ffmpeg-rebroadcast";
-import { read16BELengthLoop, readLength } from "@scrypted/common/src/read-stream";
-import { findH264NaluType, H264_NAL_TYPE_SPS, RtspClient, RTSP_FRAME_MAGIC } from "@scrypted/common/src/rtsp-server";
+import { read16BELengthLoop } from "@scrypted/common/src/read-stream";
+import { findH264NaluType, H264_NAL_TYPE_SPS, RTSP_FRAME_MAGIC } from "@scrypted/common/src/rtsp-server";
 import { parseSdp } from "@scrypted/common/src/sdp-utils";
 import { sleep } from "@scrypted/common/src/sleep";
 import { StreamChunk } from "@scrypted/common/src/stream-parser";
-import { ResponseMediaStreamOptions } from "@scrypted/sdk";
+import { MediaStreamOptions, ResponseMediaStreamOptions } from "@scrypted/sdk";
+import { parse as spsParse } from "h264-sps-parser";
 import net from 'net';
 import { EventEmitter, Readable } from "stream";
-import { getSpsResolution, parse as spsParse } from "./sps-parser";
+import { getSpsResolution } from "./sps-resolution";
+
+export function negotiateMediaStream(sdp: string, mediaStreamOptions: MediaStreamOptions, inputVideoCodec: string, inputAudioCodec: string, requestMediaStream: MediaStreamOptions) {
+    const parsedSdp = parseSdp(sdp);
+    const ret: ResponseMediaStreamOptions = cloneDeep(mediaStreamOptions) || {
+        id: undefined,
+        name: undefined,
+    };
+
+    // if the source doesn't provide a video codec, dummy one up
+    if (ret.video === undefined)
+        ret.video = {};
+
+    // the requests does not want video
+    if (requestMediaStream?.video === null)
+        ret.video = null;
+
+    if (ret.video)
+        ret.video.codec = inputVideoCodec;
+
+    // some rtsp like unifi offer alternate audio tracks (aac and opus).
+    if (requestMediaStream?.audio?.codec && requestMediaStream?.audio?.codec !== inputAudioCodec) {
+        const alternateAudio = parsedSdp.msections.find(msection => msection.type === 'audio' && msection.codec === requestMediaStream?.audio?.codec);
+        if (alternateAudio) {
+            ret.audio = {
+                codec: requestMediaStream?.audio?.codec,
+            };
+
+            return ret;
+        }
+    }
+
+    // reported codecs may be wrong/cached/etc, so before blindly copying the audio codec info,
+    // verify what was found.
+    if (ret?.audio?.codec === inputAudioCodec) {
+        ret.audio = mediaStreamOptions?.audio;
+    }
+    else {
+        ret.audio = {
+            codec: inputAudioCodec,
+        }
+    }
+
+    return ret;
+}
 
 export function connectRFC4571Parser(url: string) {
     const u = new URL(url);
@@ -19,21 +64,7 @@ export function connectRFC4571Parser(url: string) {
     return socket;
 }
 
-export type RtspChannelCodecMapping = { [key: number]: string };
-
-const RTSP_BUFFER = Buffer.from('RTSP');
-
-export function requeueRtspVideoData(rtspClient: RtspClient) {
-    const videoData = rtspClient.rfc4571.read();
-    if (videoData)
-      rtspClient.client.unshift(videoData);
-}
-
-export function startRFC4571Parser(console: Console, socket: Readable, sdp: string, mediaStreamOptions: ResponseMediaStreamOptions, options?: ParserOptions<"rtsp">, rtspOptions?: {
-    channelMap: RtspChannelCodecMapping,
-    rtspClient: RtspClient,
-    udpSessionTimeout: number,
-}): ParserSession<"rtsp"> {
+export function startRFC4571Parser(console: Console, socket: Readable, sdp: string, mediaStreamOptions: ResponseMediaStreamOptions, options?: ParserOptions<"rtsp">): ParserSession<"rtsp"> {
     let isActive = true;
     const events = new EventEmitter();
     // need this to prevent kill from throwing due to uncaught Error during cleanup
@@ -98,65 +129,27 @@ export function startRFC4571Parser(console: Console, socket: Readable, sdp: stri
         // don't start parsing until next tick, to prevent missed packets.
         await sleep(0);
 
-        if (rtspOptions?.udpSessionTimeout) {
-            while (true) {
-                await sleep(rtspOptions.udpSessionTimeout * 1000 - 5000);
-                await rtspOptions.rtspClient.getParameter();
-            }
-        }
-
-        const headerLength = rtspOptions?.channelMap ? 4 : 2;
-        const offset = rtspOptions?.channelMap ? 2 : 0;
-        const skipHeader = (header: Buffer, resumeRead: () => void) => {
-            if (!rtspOptions?.rtspClient.needKeepAlive)
-                return false;
-
-            socket.unshift(header);
-            rtspOptions.rtspClient.needKeepAlive = false;
-            rtspOptions.rtspClient.getParameter().then(() => {
-                requeueRtspVideoData(rtspOptions.rtspClient);
-                resumeRead();
-            })
-            .catch(e => {
-                console.error('error during RTSP keepalive', e);
-                kill();
-            });
-            return true;
-        }
         await read16BELengthLoop(socket, {
-            headerLength,
-            offset,
-            skipHeader,
+            headerLength: 2,
+            skipHeader: undefined,
             callback: (header, data) => {
                 let type: string;
+                const pt = data[1] & 0x7f;
 
-                if (rtspOptions?.channelMap) {
-                    const channel = header.readUInt8(1);
-                    type = rtspOptions?.channelMap[channel];
-                    if (!type) {
-                        const rtpType = rtspOptions?.channelMap[channel - 1];
-                        if (rtpType)
-                            type = `rtcp-${rtpType}`;
-                    }
+                const prefix = Buffer.alloc(2);
+                prefix[0] = RTSP_FRAME_MAGIC;
+                if (pt === audioPt) {
+                    prefix[1] = 0;
                 }
-                else {
-                    const pt = data[1] & 0x7f;
-
-                    const prefix = Buffer.alloc(2);
-                    prefix[0] = RTSP_FRAME_MAGIC;
-                    if (pt === audioPt) {
-                        prefix[1] = 0;
-                    }
-                    else if (pt === videoPt) {
-                        prefix[1] = 2;
-                    }
-                    header = Buffer.concat([prefix, header]);
-
-                    if (pt === audioPt)
-                        type = inputAudioCodec;
-                    else if (pt === videoPt)
-                        type = inputVideoCodec;
+                else if (pt === videoPt) {
+                    prefix[1] = 2;
                 }
+                header = Buffer.concat([prefix, header]);
+
+                if (pt === audioPt)
+                    type = inputAudioCodec;
+                else if (pt === videoPt)
+                    type = inputVideoCodec;
 
                 const chunk: StreamChunk = {
                     chunks: [header, data],
@@ -209,46 +202,7 @@ export function startRFC4571Parser(console: Console, socket: Readable, sdp: stri
         killed,
         resetActivityTimer,
         negotiateMediaStream: (requestMediaStream) => {
-            const ret: ResponseMediaStreamOptions = cloneDeep(mediaStreamOptions) || {
-                id: undefined,
-                name: undefined,
-            };
-
-            // if the source doesn't provide a video codec, dummy one up
-            if (ret.video === undefined)
-                ret.video = {};
-
-            // the requests does not want video
-            if (requestMediaStream?.video === null)
-                ret.video = null;
-
-            if (ret.video)
-                ret.video.codec = inputVideoCodec;
-
-            // some rtsp like unifi offer alternate audio tracks (aac and opus).
-            if (requestMediaStream?.audio?.codec && requestMediaStream?.audio?.codec !== inputAudioCodec) {
-                const alternateAudio = parsedSdp.msections.find(msection => msection.type === 'audio' && msection.codec === requestMediaStream?.audio?.codec);
-                if (alternateAudio) {
-                    ret.audio = {
-                        codec: requestMediaStream?.audio?.codec,
-                    };
-
-                    return ret;
-                }
-            }
-
-            // reported codecs may be wrong/cached/etc, so before blindly copying the audio codec info,
-            // verify what was found.
-            if (ret?.audio?.codec === inputAudioCodec) {
-                ret.audio = mediaStreamOptions?.audio;
-            }
-            else {
-                ret.audio = {
-                    codec: inputAudioCodec,
-                }
-            }
-
-            return ret;
+            return negotiateMediaStream(sdp, mediaStreamOptions, inputVideoCodec, inputAudioCodec, requestMediaStream);
         },
         emit(container: 'rtsp', chunk: StreamChunk) {
             events.emit(container, chunk);
@@ -267,4 +221,4 @@ export function startRFC4571Parser(console: Console, socket: Readable, sdp: stri
             return this;
         }
     }
-}
+}