@scrypted/prebuffer-mixin 0.1.258 → 0.1.261

package/src/main.ts

@@ -2,20 +2,19 @@
 import { AutoenableMixinProvider } from '@scrypted/common/src/autoenable-mixin-provider';
 import { getH264EncoderArgs, LIBX264_ENCODER_TITLE } from '@scrypted/common/src/ffmpeg-hardware-acceleration';
 import { handleRebroadcasterClient, ParserOptions, ParserSession, startParserSession } from '@scrypted/common/src/ffmpeg-rebroadcast';
-import { closeQuiet, createBindZero, listenZeroSingleClient } from '@scrypted/common/src/listen-cluster';
+import { closeQuiet, listenZeroSingleClient } from '@scrypted/common/src/listen-cluster';
 import { readLength } from '@scrypted/common/src/read-stream';
-import { createRtspParser, findH264NaluType, H264_NAL_TYPE_IDR, H264_NAL_TYPE_SEI, parseSemicolonDelimited, RtspClient, RtspServer, RTSP_FRAME_MAGIC } from '@scrypted/common/src/rtsp-server';
+import { createRtspParser, findH264NaluType, H264_NAL_TYPE_IDR, H264_NAL_TYPE_SEI, RtspServer } from '@scrypted/common/src/rtsp-server';
 import { addTrackControls, parseSdp } from '@scrypted/common/src/sdp-utils';
 import { StorageSettings } from '@scrypted/common/src/settings';
 import { SettingsMixinDeviceBase, SettingsMixinDeviceOptions } from "@scrypted/common/src/settings-mixin";
-import { sleep } from '@scrypted/common/src/sleep';
 import { createFragmentedMp4Parser, createMpegTsParser, StreamChunk, StreamParser } from '@scrypted/common/src/stream-parser';
 import sdk, { BufferConverter, DeviceProvider, FFmpegInput, MediaObject, MediaStreamOptions, MixinProvider, RequestMediaStreamOptions, ResponseMediaStreamOptions, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, Setting, Settings, SettingValue, VideoCamera, VideoCameraConfiguration } from '@scrypted/sdk';
 import crypto from 'crypto';
-import dgram from 'dgram';
 import net from 'net';
 import { Duplex } from 'stream';
-import { connectRFC4571Parser, requeueRtspVideoData, RtspChannelCodecMapping, startRFC4571Parser } from './rfc4571';
+import { connectRFC4571Parser, startRFC4571Parser } from './rfc4571';
+import { startRtspSession } from './rtsp-session';
 import { createStreamSettings, getPrebufferedStreams } from './stream-settings';
 import { getTranscodeMixinProviderId, REBROADCAST_MIXIN_INTERFACE_TOKEN, TranscodeMixinProvider, TRANSCODE_MIXIN_PROVIDER_NATIVE_ID } from './transcode-settings';
 
@@ -267,7 +266,7 @@ class PrebufferSession {
 
     return {
       rtspMode: mode?.startsWith('RTSP'),
-      muxingMp4: !rtspMode || mode?.includes('MP4'),
+      muxingMp4: !rtspMode,
     };
   }
 
@@ -306,7 +305,7 @@ class PrebufferSession {
       }
     );
 
-    const addFFmpegSettings = () => {
+    const addFFmpegAudioSettings = () => {
       settings.push(
         {
           title: 'Audio Codec Transcoding',
@@ -322,6 +321,11 @@ class PrebufferSession {
             TRANSCODE_AUDIO_DESCRIPTION,
           ],
         },
+      );
+    };
+
+    const addFFmpegInputSettings = () => {
+      settings.push(
         {
           title: 'FFmpeg Input Arguments Prefix',
           group,
@@ -337,7 +341,9 @@ class PrebufferSession {
           combobox: true,
         },
       )
-    };
+    }
+
+    let usingFFmpeg = muxingMp4;
 
     if (this.canUseRtspParser(this.advertisedMediaStreamOptions)) {
       const canUseScryptedParser = rtspMode;
@@ -349,13 +355,15 @@ class PrebufferSession {
         SCRYPTED_PARSER_UDP,
       ] : [];
 
+      const currentParser = this.storage.getItem(this.rtspParserKey) || STRING_DEFAULT;
+
       settings.push(
         {
           key: this.rtspParserKey,
           group,
           title: 'RTSP Parser',
           description: `The RTSP Parser used to read the stream. The default is "${defaultValue}" for this container.`,
-          value: this.storage.getItem(this.rtspParserKey) || STRING_DEFAULT,
+          value: currentParser,
           choices: [
             STRING_DEFAULT,
             ...scryptedOptions,
@@ -364,9 +372,17 @@ class PrebufferSession {
           ],
         }
       );
+
+      if (!(currentParser === STRING_DEFAULT ? defaultValue : currentParser).includes('Scrypted')) {
+        usingFFmpeg = true;
+      }
     }
-    else {
-      addFFmpegSettings();
+
+    if (muxingMp4) {
+      addFFmpegAudioSettings();
+    }
+    if (usingFFmpeg) {
+      addFFmpegInputSettings();
     }
 
     if (session) {
@@ -666,160 +682,23 @@ class PrebufferSession {
       let { parser, isDefault } = this.getParser(rtspMode, sessionMso);
       usingScryptedParser = parser === SCRYPTED_PARSER_TCP || parser === SCRYPTED_PARSER_UDP;
       if (isDefault && (parser === SCRYPTED_PARSER_TCP || parser === SCRYPTED_PARSER_UDP) && this.getLastH264Probe().seiDetected) {
-        this.console.warn('SEI packet detected was in video stream, the Default Scrypted RTSP Parser will not be used. Falling back to FFmpeg. This can be overriden by setting the RTSP Parser to Scrypted.');
-        usingScryptedParser = false;
-        parser = FFMPEG_PARSER_TCP;
+        if (sessionMso.tool === 'scrypted') {
+          this.console.warn('SEI packet detected was in video stream, but stream is marked safe by Scrypted. The Default Scrypted RTSP Parser will  be used. This can be overriden by setting the RTSP Parser to Scrypted.');
+        }
+        else {
+          this.console.warn('SEI packet detected was in video stream, the Default Scrypted RTSP Parser will not be used. Falling back to FFmpeg. This can be overriden by setting the RTSP Parser to Scrypted.');
+          usingScryptedParser = false;
+          parser = FFMPEG_PARSER_TCP;
+        }
       }
 
       if (usingScryptedParser) {
-        this.console.log('bypassing ffmpeg: using scrypted rtsp/rfc4571 parser');
-        const rtspClient = new RtspClient(ffmpegInput.url, this.console);
-
-        let servers: dgram.Socket[] = [];
-        const cleanupServers = () => {
-          for (const server of servers) {
-            closeQuiet(server);
-          }
-        }
-
-        try {
-          rtspClient.requestTimeout = 10000;
-          await rtspClient.options();
-          const sdpResponse = await rtspClient.describe();
-          const contentBase = sdpResponse.headers['content-base'];
-          if (contentBase) {
-            const url = new URL(contentBase);
-            const existing = new URL(rtspClient.url);
-            url.username = existing.username;
-            url.password = existing.password;
-            rtspClient.url = url.toString();
-          }
-          let sdp = sdpResponse.body.toString().trim();
-          this.console.log('sdp', sdp);
-
-          const parsedSdp = parseSdp(sdp);
-          let channel = 0;
-          const mapping: RtspChannelCodecMapping = {};
-          const useUdp = parser === SCRYPTED_PARSER_UDP;
-          let udpSessionTimeout: number;
-          const checkUdpSessionTimeout = (headers: { [key: string]: string }) => {
-            if (useUdp && headers.session && !udpSessionTimeout) {
-              const sessionDict = parseSemicolonDelimited(headers.session);
-              udpSessionTimeout = parseInt(sessionDict['timeout']);
-            }
-          }
-
-          const doSetup = async (control: string, codec: string) => {
-            let setupChannel = channel;
-            let udp: dgram.Socket;
-            if (useUdp) {
-              const rtspChannel = channel;
-              const { port, server } = await createBindZero();
-              udp = server;
-              servers.push(server);
-              setupChannel = port;
-              server.on('message', data => {
-                const prefix = Buffer.alloc(4);
-                prefix.writeUInt8(RTSP_FRAME_MAGIC, 0);
-                prefix.writeUInt8(rtspChannel, 1);
-                prefix.writeUInt16BE(data.length, 2);
-                const chunk: StreamChunk = {
-                  chunks: [prefix, data],
-                  type: codec,
-                };
-                session?.emit('rtsp', chunk);
-                session?.resetActivityTimer?.();
-              })
-            }
-            const setupResult = await rtspClient.setup(setupChannel, control, useUdp);
-            checkUdpSessionTimeout(setupResult.headers);
-
-            if (udp) {
-              const punch = Buffer.alloc(1);
-              const transport = setupResult.headers['transport'];
-              const match = transport.match(/.*?server_port=([0-9]+)-([0-9]+)/);
-              const [_, rtp, rtcp] = match;
-              const { hostname } = new URL(rtspClient.url);
-              udp.send(punch, parseInt(rtp), hostname)
-
-              mapping[channel] = codec;
-            }
-            else {
-              if (setupResult.interleaved)
-                mapping[setupResult.interleaved.begin] = codec;
-              else
-                mapping[channel] = codec;
-            }
-
-            channel += 2;
-          }
-
-          let setupVideoSection = false;
-
-          parsedSdp.msections = parsedSdp.msections.filter(section => {
-            if (section.type === 'video') {
-              if (setupVideoSection) {
-                this.console.warn('additional video section found. skipping.');
-                return false;
-              }
-              setupVideoSection = true;
-            }
-            else if (section.type !== 'audio') {
-              this.console.warn('unknown section', section.type);
-              return false;
-            }
-            else if (audioSoftMuted) {
-              return false;
-            }
-            return true;
-          });
-
-          for (const section of parsedSdp.msections) {
-            await doSetup(section.control, section.codec)
-          }
-
-          // sdp may contain multiple audio/video sections. take only the first video section.
-          sdp = [...parsedSdp.header.lines, ...parsedSdp.msections.map(msection => msection.lines).flat()].join('\r\n');
-
-          this.sdp = Promise.resolve(sdp);
-
-          const play = await rtspClient.play();
-          checkUdpSessionTimeout(play.headers);
-
-          requeueRtspVideoData(rtspClient);
-
-          session = startRFC4571Parser(this.console, rtspClient.client, sdp, ffmpegInput.mediaStreamOptions, rbo, {
-            channelMap: mapping,
-            rtspClient,
-            udpSessionTimeout,
-          });
-          const sessionKill = session.kill.bind(session);
-          let issuedTeardown = false;
-          session.kill = async () => {
-            try {
-              cleanupServers();
-              // issue a teardown to upstream to close gracefully but don't rely on it responding.
-              if (!issuedTeardown) {
-                issuedTeardown = true;
-                rtspClient.writeTeardown();
-              }
-              await sleep(500);
-            }
-            finally {
-              rtspClient.client.destroy();
-              sessionKill();
-            }
-          }
-          if (!session.isActive)
-            throw new Error('parser was killed before rtsp client started');
-
-          rtspClient.client.on('close', () => session.kill());
-        }
-        catch (e) {
-          cleanupServers();
-          rtspClient.client.destroy();
-          throw e;
-        }
+        session = await startRtspSession(this.console, ffmpegInput.url, ffmpegInput.mediaStreamOptions, {
+          useUdp: parser === SCRYPTED_PARSER_UDP,
+          audioSoftMuted,
+          rtspRequestTimeout: 10000,
+        });
+        this.sdp = session.sdp.then(buffers => Buffer.concat(buffers).toString());
       }
       else {
         if (parser === FFMPEG_PARSER_UDP)
@@ -1004,6 +883,7 @@ class PrebufferSession {
   }
 
   async handleRebroadcasterClient(options: {
+    requestedContainer: string,
     isActiveClient: boolean,
     container: PrebufferParsers,
     session: ParserSession<PrebufferParsers>,
@@ -1051,7 +931,16 @@ class PrebufferSession {
         session.once('killed', cleanup);
 
         const prebufferContainer: PrebufferStreamChunk[] = this.prebuffers[container];
-        if (container !== 'rtsp') {
+        // if the requested container or the source container is not rtsp, use an exact seek.
+        // this works better when the requested container is mp4, and rtsp is the source.
+        // if starting on a sync frame, ffmpeg will skip the first segment while initializing
+        // on live sources like rtsp. the buffer before the sync frame stream will be enough
+        // for ffmpeg to analyze and start up in time for the sync frame.
+        // may be worth considering playing with a few other things to avoid this:
+        // mpeg-ts as a container (would need to write a muxer)
+        // specifying the buffer before the sync frame with probesize.
+        if (container !== 'rtsp'
+          || (options?.requestedContainer && options?.requestedContainer !== 'rtsp')) {
           for (const chunk of prebufferContainer) {
             if (chunk.time < now - requestedPrebuffer)
               continue;
@@ -1099,17 +988,11 @@ class PrebufferSession {
       requestedPrebuffer = idrInterval * 1.5;
     }
 
-    const { rtspMode } = this.getRebroadcastContainer();
+    const { rtspMode, muxingMp4 } = this.getRebroadcastContainer();
     const defaultContainer = rtspMode ? 'rtsp' : 'mpegts';
 
     let container: PrebufferParsers = this.parsers[options?.container] ? options?.container as PrebufferParsers : defaultContainer;
 
-    // If a mp4 prebuffer was explicitly requested, but an mp4 prebuffer is not available (rtsp mode),
-    // rewind a little bit earlier to gaurantee a valid full segment of that length is sent.
-    if (requestedPrebuffer && container !== 'mp4' && options?.container === 'mp4') {
-      requestedPrebuffer += idrInterval * 1.5;
-    }
-
     const mediaStreamOptions: ResponseMediaStreamOptions = session.negotiateMediaStream(options);
     let sdp = await this.sdp;
 
@@ -1120,7 +1003,11 @@ class PrebufferSession {
 
     if (container === 'rtsp') {
       const parsedSdp = parseSdp(sdp);
-      parsedSdp.msections = parsedSdp.msections.filter(msection => msection.codec === mediaStreamOptions.video?.codec || msection.codec === mediaStreamOptions.audio?.codec);
+      const videoSection = parsedSdp.msections.find(msection => msection.codec && msection.codec === mediaStreamOptions.video?.codec) || parsedSdp.msections.find(msection => msection.type === 'video');
+      let audioSection = parsedSdp.msections.find(msection => msection.codec && msection.codec === mediaStreamOptions.audio?.codec) || parsedSdp.msections.find(msection => msection.type === 'audio');
+      if (mediaStreamOptions.audio === null)
+        audioSection = undefined;
+      parsedSdp.msections = parsedSdp.msections.filter(msection => msection === videoSection || msection === audioSection);
       const filterPrebufferAudio = options?.prebuffer === undefined;
       const videoCodec = parsedSdp.msections.find(msection => msection.type === 'video')?.codec;
       sdp = parsedSdp.toSdp();
@@ -1166,6 +1053,7 @@ class PrebufferSession {
     const isActiveClient = options?.refresh !== false;
 
     this.handleRebroadcasterClient({
+      requestedContainer: options?.container,
       isActiveClient,
       container,
       requestedPrebuffer,
@@ -1181,7 +1069,7 @@ class PrebufferSession {
     if (this.audioDisabled) {
       mediaStreamOptions.audio = null;
     }
-    else if (reencodeAudio) {
+    else if (reencodeAudio && muxingMp4) {
       mediaStreamOptions.audio = {
         codec: 'aac',
         encoder: 'aac',
@@ -1623,6 +1511,7 @@ export class RebroadcastPlugin extends AutoenableMixinProvider implements MixinP
         const requestedPrebuffer = Math.max(4000, (idrInterval || 4000)) * 1.5;
 
         prebufferSession.handleRebroadcasterClient({
+          requestedContainer: 'rtsp',
           isActiveClient: true,
           container: 'rtsp',
           session,

package/src/rfc4571.ts

@@ -1,14 +1,59 @@
 import { cloneDeep } from "@scrypted/common/src/clone-deep";
 import { ParserOptions, ParserSession, setupActivityTimer } from "@scrypted/common/src/ffmpeg-rebroadcast";
-import { read16BELengthLoop, readLength } from "@scrypted/common/src/read-stream";
-import { findH264NaluType, H264_NAL_TYPE_SPS, RtspClient, RTSP_FRAME_MAGIC } from "@scrypted/common/src/rtsp-server";
+import { read16BELengthLoop } from "@scrypted/common/src/read-stream";
+import { findH264NaluType, H264_NAL_TYPE_SPS, RTSP_FRAME_MAGIC } from "@scrypted/common/src/rtsp-server";
 import { parseSdp } from "@scrypted/common/src/sdp-utils";
 import { sleep } from "@scrypted/common/src/sleep";
 import { StreamChunk } from "@scrypted/common/src/stream-parser";
-import { ResponseMediaStreamOptions } from "@scrypted/sdk";
+import { MediaStreamOptions, ResponseMediaStreamOptions } from "@scrypted/sdk";
+import { parse as spsParse } from "h264-sps-parser";
 import net from 'net';
 import { EventEmitter, Readable } from "stream";
-import { getSpsResolution, parse as spsParse } from "./sps-parser";
+import { getSpsResolution } from "./sps-resolution";
+
+export function negotiateMediaStream(sdp: string, mediaStreamOptions: MediaStreamOptions, inputVideoCodec: string, inputAudioCodec: string, requestMediaStream: MediaStreamOptions) {
+    const parsedSdp = parseSdp(sdp);
+    const ret: ResponseMediaStreamOptions = cloneDeep(mediaStreamOptions) || {
+        id: undefined,
+        name: undefined,
+    };
+
+    // if the source doesn't provide a video codec, dummy one up
+    if (ret.video === undefined)
+        ret.video = {};
+
+    // the requests does not want video
+    if (requestMediaStream?.video === null)
+        ret.video = null;
+
+    if (ret.video)
+        ret.video.codec = inputVideoCodec;
+
+    // some rtsp like unifi offer alternate audio tracks (aac and opus).
+    if (requestMediaStream?.audio?.codec && requestMediaStream?.audio?.codec !== inputAudioCodec) {
+        const alternateAudio = parsedSdp.msections.find(msection => msection.type === 'audio' && msection.codec === requestMediaStream?.audio?.codec);
+        if (alternateAudio) {
+            ret.audio = {
+                codec: requestMediaStream?.audio?.codec,
+            };
+
+            return ret;
+        }
+    }
+
+    // reported codecs may be wrong/cached/etc, so before blindly copying the audio codec info,
+    // verify what was found.
+    if (ret?.audio?.codec === inputAudioCodec) {
+        ret.audio = mediaStreamOptions?.audio;
+    }
+    else {
+        ret.audio = {
+            codec: inputAudioCodec,
+        }
+    }
+
+    return ret;
+}
 
 export function connectRFC4571Parser(url: string) {
     const u = new URL(url);
@@ -19,21 +64,7 @@ export function connectRFC4571Parser(url: string) {
     return socket;
 }
 
-export type RtspChannelCodecMapping = { [key: number]: string };
-
-const RTSP_BUFFER = Buffer.from('RTSP');
-
-export function requeueRtspVideoData(rtspClient: RtspClient) {
-    const videoData = rtspClient.rfc4571.read();
-    if (videoData)
-      rtspClient.client.unshift(videoData);
-}
-
-export function startRFC4571Parser(console: Console, socket: Readable, sdp: string, mediaStreamOptions: ResponseMediaStreamOptions, options?: ParserOptions<"rtsp">, rtspOptions?: {
-    channelMap: RtspChannelCodecMapping,
-    rtspClient: RtspClient,
-    udpSessionTimeout: number,
-}): ParserSession<"rtsp"> {
+export function startRFC4571Parser(console: Console, socket: Readable, sdp: string, mediaStreamOptions: ResponseMediaStreamOptions, options?: ParserOptions<"rtsp">): ParserSession<"rtsp"> {
     let isActive = true;
     const events = new EventEmitter();
     // need this to prevent kill from throwing due to uncaught Error during cleanup
@@ -98,65 +129,27 @@ export function startRFC4571Parser(console: Console, socket: Readable, sdp: stri
         // don't start parsing until next tick, to prevent missed packets.
         await sleep(0);
 
-        if (rtspOptions?.udpSessionTimeout) {
-            while (true) {
-                await sleep(rtspOptions.udpSessionTimeout * 1000 - 5000);
-                await rtspOptions.rtspClient.getParameter();
-            }
-        }
-
-        const headerLength = rtspOptions?.channelMap ? 4 : 2;
-        const offset = rtspOptions?.channelMap ? 2 : 0;
-        const skipHeader = (header: Buffer, resumeRead: () => void) => {
-            if (!rtspOptions?.rtspClient.needKeepAlive)
-                return false;
-
-            socket.unshift(header);
-            rtspOptions.rtspClient.needKeepAlive = false;
-            rtspOptions.rtspClient.getParameter().then(() => {
-                requeueRtspVideoData(rtspOptions.rtspClient);
-                resumeRead();
-            })
-            .catch(e => {
-                console.error('error during RTSP keepalive', e);
-                kill();
-            });
-            return true;
-        }
         await read16BELengthLoop(socket, {
-            headerLength,
-            offset,
-            skipHeader,
+            headerLength: 2,
+            skipHeader: undefined,
             callback: (header, data) => {
                 let type: string;
+                const pt = data[1] & 0x7f;
 
-                if (rtspOptions?.channelMap) {
-                    const channel = header.readUInt8(1);
-                    type = rtspOptions?.channelMap[channel];
-                    if (!type) {
-                        const rtpType = rtspOptions?.channelMap[channel - 1];
-                        if (rtpType)
-                            type = `rtcp-${rtpType}`;
-                    }
+                const prefix = Buffer.alloc(2);
+                prefix[0] = RTSP_FRAME_MAGIC;
+                if (pt === audioPt) {
+                    prefix[1] = 0;
                 }
-                else {
-                    const pt = data[1] & 0x7f;
-
-                    const prefix = Buffer.alloc(2);
-                    prefix[0] = RTSP_FRAME_MAGIC;
-                    if (pt === audioPt) {
-                        prefix[1] = 0;
-                    }
-                    else if (pt === videoPt) {
-                        prefix[1] = 2;
-                    }
-                    header = Buffer.concat([prefix, header]);
-
-                    if (pt === audioPt)
-                        type = inputAudioCodec;
-                    else if (pt === videoPt)
-                        type = inputVideoCodec;
+                else if (pt === videoPt) {
+                    prefix[1] = 2;
                 }
+                header = Buffer.concat([prefix, header]);
+
+                if (pt === audioPt)
+                    type = inputAudioCodec;
+                else if (pt === videoPt)
+                    type = inputVideoCodec;
 
                 const chunk: StreamChunk = {
                     chunks: [header, data],
@@ -209,46 +202,7 @@ export function startRFC4571Parser(console: Console, socket: Readable, sdp: stri
         killed,
         resetActivityTimer,
         negotiateMediaStream: (requestMediaStream) => {
-            const ret: ResponseMediaStreamOptions = cloneDeep(mediaStreamOptions) || {
-                id: undefined,
-                name: undefined,
-            };
-
-            // if the source doesn't provide a video codec, dummy one up
-            if (ret.video === undefined)
-                ret.video = {};
-
-            // the requests does not want video
-            if (requestMediaStream?.video === null)
-                ret.video = null;
-
-            if (ret.video)
-                ret.video.codec = inputVideoCodec;
-
-            // some rtsp like unifi offer alternate audio tracks (aac and opus).
-            if (requestMediaStream?.audio?.codec && requestMediaStream?.audio?.codec !== inputAudioCodec) {
-                const alternateAudio = parsedSdp.msections.find(msection => msection.type === 'audio' && msection.codec === requestMediaStream?.audio?.codec);
-                if (alternateAudio) {
-                    ret.audio = {
-                        codec: requestMediaStream?.audio?.codec,
-                    };
-
-                    return ret;
-                }
-            }
-
-            // reported codecs may be wrong/cached/etc, so before blindly copying the audio codec info,
-            // verify what was found.
-            if (ret?.audio?.codec === inputAudioCodec) {
-                ret.audio = mediaStreamOptions?.audio;
-            }
-            else {
-                ret.audio = {
-                    codec: inputAudioCodec,
-                }
-            }
-
-            return ret;
+            return negotiateMediaStream(sdp, mediaStreamOptions, inputVideoCodec, inputAudioCodec, requestMediaStream);
         },
         emit(container: 'rtsp', chunk: StreamChunk) {
             events.emit(container, chunk);
@@ -267,4 +221,4 @@ export function startRFC4571Parser(console: Console, socket: Readable, sdp: stri
             return this;
         }
     }
-}
+}