@scrypted/prebuffer-mixin 0.1.224 → 0.1.227

package/src/main.ts

@@ -2,21 +2,23 @@
 import { AutoenableMixinProvider } from '@scrypted/common/src/autoenable-mixin-provider';
 import { startFFMPegFragmentedMP4Session } from '@scrypted/common/src/ffmpeg-mp4-parser-session';
 import { handleRebroadcasterClient, ParserOptions, ParserSession, setupActivityTimer, startParserSession } from '@scrypted/common/src/ffmpeg-rebroadcast';
-import { closeQuiet, listenZeroSingleClient } from '@scrypted/common/src/listen-cluster';
-import { safeKillFFmpeg } from '@scrypted/common/src/media-helpers';
+import { closeQuiet, createBindZero, listenZeroSingleClient } from '@scrypted/common/src/listen-cluster';
+import { ffmpegLogInitialOutput, safeKillFFmpeg } from '@scrypted/common/src/media-helpers';
 import { readLength } from '@scrypted/common/src/read-stream';
-import { createRtspParser, RtspClient, RtspServer } from '@scrypted/common/src/rtsp-server';
-import { addTrackControls, parsePayloadTypes, parseSdp } from '@scrypted/common/src/sdp-utils';
+import { createRtspParser, H264_NAL_TYPE_IDR, findH264NaluType, RtspClient, RtspServer, RTSP_FRAME_MAGIC, parseSemicolonDelimited } from '@scrypted/common/src/rtsp-server';
+import { addTrackControls, parseSdp } from '@scrypted/common/src/sdp-utils';
 import { StorageSettings } from '@scrypted/common/src/settings';
 import { SettingsMixinDeviceBase, SettingsMixinDeviceOptions } from "@scrypted/common/src/settings-mixin";
 import { sleep } from '@scrypted/common/src/sleep';
 import { createFragmentedMp4Parser, createMpegTsParser, parseMp4StreamChunks, StreamChunk, StreamParser } from '@scrypted/common/src/stream-parser';
-import sdk, { BufferConverter, FFMpegInput, MediaObject, MediaStreamOptions, MixinProvider, RequestMediaStreamOptions, ResponseMediaStreamOptions, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, Setting, Settings, SettingValue, VideoCamera, VideoCameraConfiguration } from '@scrypted/sdk';
+import sdk, { BufferConverter, FFmpegInput, MediaObject, MediaStreamOptions, MixinProvider, RequestMediaStreamOptions, ResponseMediaStreamOptions, ScryptedDeviceType, ScryptedInterface, ScryptedMimeTypes, Setting, Settings, SettingValue, VideoCamera, VideoCameraConfiguration } from '@scrypted/sdk';
 import crypto from 'crypto';
+import dgram from 'dgram';
 import net from 'net';
 import { Duplex } from 'stream';
-import { connectRFC4571Parser, startRFC4571Parser } from './rfc4571';
+import { connectRFC4571Parser, RtspChannelCodecMapping, startRFC4571Parser } from './rfc4571';
 import { createStreamSettings, getPrebufferedStreams } from './stream-settings';
+import { getH264EncoderArgs, LIBX264_ENCODER_TITLE } from '@scrypted/common/src/ffmpeg-hardware-acceleration';
 
 const { mediaManager, log, systemManager, deviceManager } = sdk;
 
@@ -32,7 +34,8 @@ const COMPATIBLE_AUDIO_CODECS = ['aac', 'mp3', 'mp2', 'opus'];
 const DEFAULT_FFMPEG_INPUT_ARGUMENTS = '-fflags +genpts';
 
 const SCRYPTED_PARSER = 'Scrypted';
-const FFMPEG_PARSER = 'FFmpeg';
+const FFMPEG_PARSER_TCP = 'FFmpeg (TCP)';
+const FFMPEG_PARSER_UDP = 'FFmpeg (UDP)';
 const STRING_DEFAULT = 'Default';
 
 const VALID_AUDIO_CONFIGS = [
@@ -67,8 +70,7 @@ class PrebufferSession {
   parsers: { [container: string]: StreamParser };
   sdp: Promise<string>;
 
-  detectedIdrInterval = 0;
-  prevIdr = 0;
+  detectedIdrInterval: number;
   audioDisabled = false;
 
   mixinDevice: VideoCamera & VideoCameraConfiguration;
@@ -181,18 +183,20 @@ class PrebufferSession {
 
   getParser(rtspMode: boolean, muxingMp4: boolean, mediaStreamOptions: MediaStreamOptions) {
     if (!this.canUseRtspParser(muxingMp4, mediaStreamOptions))
-      return FFMPEG_PARSER;
+      return STRING_DEFAULT;
 
     const defaultValue = rtspMode
       && mediaStreamOptions?.tool === 'scrypted' ?
-      SCRYPTED_PARSER : FFMPEG_PARSER;
+      SCRYPTED_PARSER : STRING_DEFAULT;
     const rtspParser = this.storage.getItem(this.rtspParserKey);
     if (!rtspParser || rtspParser === STRING_DEFAULT)
       return defaultValue;
     if (rtspParser === SCRYPTED_PARSER)
       return SCRYPTED_PARSER;
-    if (rtspParser === FFMPEG_PARSER)
-      return FFMPEG_PARSER;
+    if (rtspParser === FFMPEG_PARSER_TCP)
+      return FFMPEG_PARSER_TCP;
+    if (rtspParser === FFMPEG_PARSER_UDP)
+      return FFMPEG_PARSER_UDP;
     return defaultValue;
   }
 
@@ -232,7 +236,7 @@ class PrebufferSession {
     const elapsed = Date.now() - start;
     const bitrate = Math.round(total / elapsed * 8);
 
-    const group = this.streamName ? `Rebroadcast: ${this.streamName}` : 'Rebroadcast';
+    const group = this.streamName ? `Stream: ${this.streamName}` : 'Stream';
 
     settings.push(
       {
@@ -258,7 +262,6 @@ class PrebufferSession {
           STRING_DEFAULT,
           'MPEG-TS',
           'RTSP',
-          // 'RTSP+MP4',
         ],
         key: this.rebroadcastModeKey,
         value: this.storage.getItem(this.rebroadcastModeKey) || STRING_DEFAULT,
@@ -290,17 +293,21 @@ class PrebufferSession {
       && this.advertisedMediaStreamOptions?.container === 'rtsp') {
 
       const value = this.getParser(rtspMode, muxingMp4, this.advertisedMediaStreamOptions);
+      const defaultValue = rtspMode
+        && this.advertisedMediaStreamOptions?.tool === 'scrypted' ?
+        SCRYPTED_PARSER : 'FFmpeg';
 
       settings.push(
         {
           key: this.rtspParserKey,
           group,
           title: 'RTSP Parser',
-          description: `Experimental: The RTSP Parser used to read the stream. FFmpeg is stable. The Scrypted parser is lower latency. The Scrypted Parser is only available when the Audo Codec is not Transcoding and the Rebroadcast Container is RTSP. The default is "${value}" for this camera.`,
+          description: `Experimental: The RTSP Parser used to read the stream. FFmpeg is stable. The Scrypted parser is lower latency. The Scrypted Parser is only available when the Audo Codec is not Transcoding and the Rebroadcast Container is RTSP. The default is "${defaultValue}" for this camera.`,
           value: this.storage.getItem(this.rtspParserKey) || STRING_DEFAULT,
           choices: [
             STRING_DEFAULT,
-            FFMPEG_PARSER,
+            FFMPEG_PARSER_TCP,
+            FFMPEG_PARSER_UDP,
             SCRYPTED_PARSER,
           ],
         }
@@ -316,6 +323,9 @@ class PrebufferSession {
     }
 
     if (session) {
+      const resolution = session.inputVideoResolution?.width && session.inputVideoResolution?.height
+        ? `${session.inputVideoResolution?.width}x${session.inputVideoResolution?.height}`
+        : 'unknown';
 
       settings.push(
         {
@@ -323,7 +333,7 @@ class PrebufferSession {
           group,
           title: 'Detected Resolution and Bitrate',
           readonly: true,
-          value: `${session?.inputVideoResolution?.[0] || "unknown"} @ ${bitrate || "unknown"} Kb/s`,
+          value: `${resolution} @ ${bitrate || "unknown"} Kb/s`,
           description: 'Configuring your camera to 1920x1080, 2000Kb/S, Variable Bit Rate, is recommended.',
         },
         {
@@ -591,12 +601,12 @@ class PrebufferSession {
       const json = await mediaManager.convertMediaObjectToJSON<any>(mo, 'x-scrypted/x-rfc4571');
       const { url, sdp, mediaStreamOptions } = json;
 
-      session = await startRFC4571Parser(this.console, connectRFC4571Parser(url), sdp, mediaStreamOptions, false, rbo);
+      session = await startRFC4571Parser(this.console, connectRFC4571Parser(url), sdp, mediaStreamOptions, rbo);
       this.sdp = session.sdp.then(buffers => Buffer.concat(buffers).toString());
     }
     else {
       const moBuffer = await mediaManager.convertMediaObjectToBuffer(mo, ScryptedMimeTypes.FFmpegInput);
-      const ffmpegInput = JSON.parse(moBuffer.toString()) as FFMpegInput;
+      const ffmpegInput = JSON.parse(moBuffer.toString()) as FFmpegInput;
       sessionMso = ffmpegInput.mediaStreamOptions;
 
       const parser = this.getParser(rtspMode, muxingMp4, ffmpegInput.mediaStreamOptions);
@@ -604,6 +614,14 @@ class PrebufferSession {
         usingScryptedParser = true;
         this.console.log('bypassing ffmpeg: using scrypted rtsp/rfc4571 parser');
         const rtspClient = new RtspClient(ffmpegInput.url, this.console);
+
+        let servers: dgram.Socket[] = [];
+        const cleanupServers = () => {
+          for (const server of servers) {
+            closeQuiet(server);
+          }
+        }
+
         try {
           rtspClient.requestTimeout = 10000;
           await rtspClient.options();
@@ -612,50 +630,100 @@ class PrebufferSession {
           this.console.log('sdp', sdp);
 
           const parsedSdp = parseSdp(sdp);
-          const audioSection = parsedSdp.msections.find(msection => msection.type === 'audio');
+          let channel = 0;
+          const mapping: RtspChannelCodecMapping = {};
+          const useUdp = false;
+
+          const doSetup = async (control: string, codec: string) => {
+            let setupChannel = channel;
+            if (useUdp) {
+              const rtspChannel = channel;
+              const { port, server } = await createBindZero();
+              servers.push(server);
+              setupChannel = port;
+              server.on('message', data => {
+                const prefix = Buffer.alloc(4);
+                prefix.writeUInt8(RTSP_FRAME_MAGIC, 0);
+                prefix.writeUInt8(rtspChannel, 1);
+                prefix.writeUInt16BE(data.length, 2);
+                const chunk: StreamChunk = {
+                  chunks: [prefix, data],
+                  type: codec,
+                };
+                session?.emit('rtsp', chunk);
+                session?.resetActivityTimer?.();
+              })
+            }
+            await rtspClient.setup(setupChannel, control, useUdp);
+            mapping[channel] = codec;
+            channel += 2;
+          }
+
+          // grab all available audio sections
+          if (!audioSoftMuted) {
+            for (const audioSection of parsedSdp.msections.filter(msection => msection.type === 'audio')) {
+              await doSetup(audioSection.control, audioSection.codec)
+            }
+
+            if (channel === 0)
+              this.console.warn('sdp did not contain audio track and audio was not reported as missing.');
+          }
+
           const videoSection = parsedSdp.msections.find(msection => msection.type === 'video');
 
-          // sdp may contain multiple audio/video sections. take only the first.
-          parsedSdp.msections = parsedSdp.msections.filter(msection => msection === audioSection || msection === videoSection);
+          // sdp may contain multiple audio/video sections. take only the first video section.
+          parsedSdp.msections = parsedSdp.msections.filter(msection => msection === videoSection || msection.type === 'audio');
           sdp = [...parsedSdp.header.lines, ...parsedSdp.msections.map(msection => msection.lines).flat()].join('\r\n');
 
           this.sdp = Promise.resolve(sdp);
-          let channel = 0;
-          if (!audioSoftMuted) {
-            if (audioSection) {
-              await rtspClient.setup(channel, audioSection.control);
-              channel += 2;
-            }
-            else {
-              this.console.warn('sdp did not contain audio track and audio was not reported as missing.');
+          await doSetup(videoSection.control, videoSection.codec);
+          rtspClient.writePlay();
+
+          session = await startRFC4571Parser(this.console, rtspClient.client, sdp, ffmpegInput.mediaStreamOptions, rbo, {
+            channelMap: mapping,
+            handleRTSP: async () => {
+              await rtspClient.readMessage();
+            },
+            onLoop: async () => {
+              if (rtspClient.needKeepAlive) {
+                rtspClient.needKeepAlive = false;
+                rtspClient.writeGetParameter();
+              }
             }
-          }
-          await rtspClient.setup(channel, videoSection.control);
-          await rtspClient.play();
-          session = await startRFC4571Parser(this.console, rtspClient.rfc4571, sdp, ffmpegInput.mediaStreamOptions, true, rbo);
+          });
           const sessionKill = session.kill.bind(session);
           let issuedTeardown = false;
           session.kill = async () => {
-            // issue a teardown to upstream to close gracefully but don't rely on it responding.
-            if (!issuedTeardown) {
-              issuedTeardown = true;
-              rtspClient.teardown().finally(sessionKill);
+            try {
+              cleanupServers();
+              // issue a teardown to upstream to close gracefully but don't rely on it responding.
+              if (!issuedTeardown) {
+                issuedTeardown = true;
+                rtspClient.writeTeardown();
+              }
+              await sleep(500);
+            }
+            finally {
+              rtspClient.client.destroy();
+              sessionKill();
             }
-            await sleep(500);
-            rtspClient.client.destroy();
-            sessionKill();
           }
           if (!session.isActive)
             throw new Error('parser was killed before rtsp client started');
 
-          rtspClient.readLoop().finally(() => session.kill());
+          rtspClient.client.on('close', () => session.kill());
         }
         catch (e) {
+          cleanupServers();
           rtspClient.client.destroy();
           throw e;
         }
       }
       else {
+        if (parser === FFMPEG_PARSER_UDP)
+          ffmpegInput.inputArguments = ['-rtsp_transport', 'udp', '-i', ffmpegInput.url];
+        else if (parser === FFMPEG_PARSER_TCP)
+          ffmpegInput.inputArguments = ['-rtsp_transport', 'tcp', '-i', ffmpegInput.url];
         // create missing pts from dts so mpegts and mp4 muxing does not fail
         const extraInputArguments = this.storage.getItem(this.ffmpegInputArgumentsKey) || DEFAULT_FFMPEG_INPUT_ARGUMENTS;
         ffmpegInput.inputArguments.unshift(...extraInputArguments.split(' '));
@@ -670,9 +738,8 @@ class PrebufferSession {
         id: this.streamId,
         refresh: false,
       })
-        .then(async (stream) => {
+        .then(async (ffmpegInput) => {
           const extraInputArguments = this.storage.getItem(this.ffmpegInputArgumentsKey) || DEFAULT_FFMPEG_INPUT_ARGUMENTS;
-          const ffmpegInput = await mediaManager.convertMediaObjectToJSON<FFMpegInput>(stream, ScryptedMimeTypes.FFmpegInput);
           ffmpegInput.inputArguments.unshift(...extraInputArguments.split(' '));
           const mp4Session = await startFFMPegFragmentedMP4Session(ffmpegInput.inputArguments, acodec, vcodec, this.console);
 
@@ -750,7 +817,7 @@ class PrebufferSession {
           return;
         const mo = await this.mixinDevice.getVideoStream(mso);
         const moBuffer = await mediaManager.convertMediaObjectToBuffer(mo, ScryptedMimeTypes.FFmpegInput);
-        const ffmpegInput = JSON.parse(moBuffer.toString()) as FFMpegInput;
+        const ffmpegInput = JSON.parse(moBuffer.toString()) as FFmpegInput;
         mso = ffmpegInput.mediaStreamOptions;
 
         scheduleRefresh(mso);
@@ -776,33 +843,34 @@ class PrebufferSession {
     for (const container of PrebufferParserValues) {
       let shifts = 0;
 
+      let prevIdr: number;
+
+      const updateIdr = (now: number) => {
+        if (prevIdr) {
+          const sendEvent = typeof this.detectedIdrInterval !== 'number';
+          this.detectedIdrInterval = now - prevIdr;
+          // only on the first idr update should we send a settings refresh.
+          if (sendEvent)
+            deviceManager.onMixinEvent(this.mixin.id, this.mixin.mixinProviderNativeId, ScryptedInterface.Settings, undefined);
+        }
+        prevIdr = now;
+      }
+
+      this.detectedIdrInterval = undefined;
       session.on(container, (chunk: StreamChunk) => {
         const prebufferContainer: PrebufferStreamChunk[] = this.prebuffers[container];
         const now = Date.now();
 
-        const updateIdr = () => {
-          if (this.prevIdr) {
-            const sendEvent = typeof this.detectedIdrInterval !== 'number';
-            this.detectedIdrInterval = now - this.prevIdr;
-            // only on the first idr update should we send a settings refresh.
-            if (sendEvent)
-              deviceManager.onMixinEvent(this.mixin.id, this.mixin.mixinProviderNativeId, ScryptedInterface.Settings, undefined);
-          }
-          this.prevIdr = now;
-        }
-
         // this is only valid for mp4, so its no op for everything else
         // used to detect idr interval.
         if (chunk.type === 'mdat') {
-          updateIdr();
+          updateIdr(now);
         }
-        if (chunk.type === 'rtp-video') {
-          const fragmentType = chunk.chunks[1].readUInt8(12) & 0x1f;
-          const second = chunk.chunks[1].readUInt8(13);
-          const nalType = second & 0x1f;
-          const startBit = second & 0x80;
-          if (((fragmentType === 28 || fragmentType === 29) && nalType === 5 && startBit == 128) || fragmentType == 5) {
-            updateIdr();
+        else if (chunk.type === 'h264') {
+          if (findH264NaluType(chunk, H264_NAL_TYPE_IDR)) {
+            // only update the rtsp computed idr once a minute.
+            // per packet bitscan is not great.
+            updateIdr(now);
           }
         }
 
@@ -865,6 +933,7 @@ class PrebufferSession {
     session: ParserSession<PrebufferParsers>,
     socketPromise: Promise<Duplex>,
     requestedPrebuffer: number,
+    filter?: (chunk: StreamChunk) => StreamChunk,
   }) {
     const { isActiveClient, container, session, socketPromise, requestedPrebuffer } = options;
     if (requestedPrebuffer)
@@ -884,6 +953,11 @@ class PrebufferSession {
         const now = Date.now();
 
         const safeWriteData = (chunk: StreamChunk) => {
+          if (options.filter) {
+            chunk = options.filter(chunk);
+            if (!chunk)
+              return;
+          }
           const buffered = writeData(chunk);
           if (buffered > 100000000) {
             this.console.log('more than 100MB has been buffered, did downstream die? killing connection.', this.streamName);
@@ -901,7 +975,7 @@ class PrebufferSession {
         session.once('killed', cleanup);
 
         const prebufferContainer: PrebufferStreamChunk[] = this.prebuffers[container];
-        if (true) {
+        if (container !== 'rtsp') {
           for (const prebuffer of prebufferContainer) {
             if (prebuffer.time < now - requestedPrebuffer)
               continue;
@@ -930,7 +1004,7 @@ class PrebufferSession {
     })
   }
 
-  async getVideoStream(options?: RequestMediaStreamOptions): Promise<MediaObject> {
+  async getVideoStream(options?: RequestMediaStreamOptions) {
     if (options?.refresh === false && !this.parserSessionPromise)
       throw new Error('Stream is currently unavailable and will not be started for this request. RequestMediaStreamOptions.refresh === false');
 
@@ -955,50 +1029,70 @@ class PrebufferSession {
       requestedPrebuffer += (this.detectedIdrInterval || 4000) * 1.5;
     }
 
-    const createContainerServer = async (container: PrebufferParsers) => {
-
-      let socketPromise: Promise<Duplex>;
-      let containerUrl: string;
-
-      if (container === 'rtsp') {
-        const client = await listenZeroSingleClient();
-        socketPromise = client.clientPromise.then(async (socket) => {
-          let sdp = await this.sdp;
-          sdp = addTrackControls(sdp);
-          const server = new RtspServer(socket, sdp);
-          //server.console = this.console;
-          await server.handlePlayback();
-          return socket;
-        })
-        containerUrl = client.url.replace('tcp://', 'rtsp://');
-      }
-      else {
-        const client = await listenZeroSingleClient();
-        socketPromise = client.clientPromise;
-        containerUrl = `tcp://127.0.0.1:${client.port}`
+    const mediaStreamOptions: ResponseMediaStreamOptions = session.negotiateMediaStream(options);
+    let sdp = await this.sdp;
+
+    let socketPromise: Promise<Duplex>;
+    let url: string;
+    let filter: (chunk: StreamChunk) => StreamChunk;
+    const codecMap = new Map<string, number>();
+
+    if (container === 'rtsp') {
+      const parsedSdp = parseSdp(sdp);
+      if (parsedSdp.msections.length > 2) {
+        parsedSdp.msections = parsedSdp.msections.filter(msection => msection.codec === mediaStreamOptions.video?.codec || msection.codec === mediaStreamOptions.audio?.codec);
+        sdp = parsedSdp.toSdp();
+        filter = chunk => {
+          const channel = codecMap.get(chunk.type);
+          if (channel == undefined)
+            return;
+          const chunks = chunk.chunks.slice();
+          const header = Buffer.from(chunks[0]);
+          header.writeUInt8(channel, 1);
+          chunks[0] = header;
+          return {
+            startStream: chunk.startStream,
+            chunks,
+          }
+        }
       }
 
-      const isActiveClient = options?.refresh !== false;
+      const client = await listenZeroSingleClient();
+      socketPromise = client.clientPromise.then(async (socket) => {
+        sdp = addTrackControls(sdp);
+        const server = new RtspServer(socket, sdp);
+        server.console = this.console;
+        await server.handlePlayback();
+        for (const track of Object.values(server.setupTracks)) {
+          codecMap.set(track.codec, track.destination);
+        }
+        return socket;
+      })
+      url = client.url.replace('tcp://', 'rtsp://');
+    }
+    else {
+      const client = await listenZeroSingleClient();
+      socketPromise = client.clientPromise;
+      url = `tcp://127.0.0.1:${client.port}`
+    }
 
-      this.handleRebroadcasterClient({
-        isActiveClient,
-        container,
-        requestedPrebuffer,
-        socketPromise,
-        session,
-      });
+    mediaStreamOptions.sdp = sdp;
 
-      return containerUrl;
-    }
+    const isActiveClient = options?.refresh !== false;
 
-    const mediaStreamOptions: ResponseMediaStreamOptions = Object.assign({}, session.mediaStreamOptions);
-    mediaStreamOptions.sdp = (await session.sdp)?.toString();
+    this.handleRebroadcasterClient({
+      isActiveClient,
+      container,
+      requestedPrebuffer,
+      socketPromise,
+      session,
+      filter,
+    });
 
     mediaStreamOptions.prebuffer = requestedPrebuffer;
 
     const { reencodeAudio } = this.getAudioConfig();
 
-    let codecCopy = false;
     if (this.audioDisabled) {
       mediaStreamOptions.audio = null;
     }
@@ -1009,33 +1103,9 @@ class PrebufferSession {
         profile: 'aac_low',
       }
     }
-    else {
-      codecCopy = true;
-    }
 
-    if (codecCopy) {
-      // reported codecs may be wrong/cached/etc, so before blindly copying the audio codec info,
-      // verify what was found.
-      if (session?.mediaStreamOptions?.audio?.codec === session?.inputAudioCodec) {
-        mediaStreamOptions.audio = session?.mediaStreamOptions?.audio;
-      }
-      else {
-        mediaStreamOptions.audio = {
-          codec: session?.inputAudioCodec,
-        }
-      }
-    }
-
-    if (!mediaStreamOptions.video)
-      mediaStreamOptions.video = {};
-
-    mediaStreamOptions.video.codec = session.inputVideoCodec;
-
-    if (session.inputVideoResolution?.[2] && session.inputVideoResolution?.[3]) {
-      Object.assign(mediaStreamOptions.video, {
-        width: parseInt(session.inputVideoResolution[2]),
-        height: parseInt(session.inputVideoResolution[3]),
-      });
+    if (session.inputVideoResolution?.width && session.inputVideoResolution?.height) {
+      Object.assign(mediaStreamOptions.video, session.inputVideoResolution);
     }
 
     const now = Date.now();
@@ -1051,8 +1121,7 @@ class PrebufferSession {
 
     const length = Math.max(500000, available).toString();
 
-    const url = await createContainerServer(container);
-    const ffmpegInput: FFMpegInput = {
+    const ffmpegInput: FFmpegInput = {
       url,
       container,
       inputArguments: [
@@ -1064,7 +1133,7 @@ class PrebufferSession {
       mediaStreamOptions,
     }
 
-    return this.mixin.createMediaObject(ffmpegInput, ScryptedMimeTypes.FFmpegInput);
+    return ffmpegInput;
   }
 }
 
@@ -1074,7 +1143,7 @@ class PrebufferMixin extends SettingsMixinDeviceBase<VideoCamera & VideoCameraCo
 
   streamSettings = createStreamSettings(this);
 
-  constructor(public plugin: PrebufferProvider, options: SettingsMixinDeviceOptions<VideoCamera & VideoCameraConfiguration>) {
+  constructor(public plugin: RebroadcastPlugin, options: SettingsMixinDeviceOptions<VideoCamera & VideoCameraConfiguration>) {
     super(options);
 
     this.delayStart();
@@ -1093,43 +1162,71 @@ class PrebufferMixin extends SettingsMixinDeviceBase<VideoCamera & VideoCameraCo
     await this.ensurePrebufferSessions();
 
     let id = options?.id;
-    if (options?.destination) {
-      const msos = await this.mixinDevice.getVideoStreamOptions();
-      let result: {
-        stream: ResponseMediaStreamOptions,
-        isDefault: boolean,
-      };
+    let h264EncoderArguments: string[];
+    let destinationVideoBitrate: number;
 
-      switch (options.destination) {
+    const msos = await this.mixinDevice.getVideoStreamOptions();
+    let result: {
+      stream: ResponseMediaStreamOptions,
+      isDefault: boolean,
+      title: string;
+    };
+
+    const defaultLocalBitrate = 2000000;
+    const defaultLowResolutionBitrate = 512000;
+    if (!id) {
+      switch (options?.destination) {
+        case 'medium-resolution':
         case 'remote':
           result = this.streamSettings.getRemoteStream(msos);
+          destinationVideoBitrate = this.plugin.storageSettings.values.remoteStreamingBitrate;
           break;
         case 'low-resolution':
           result = this.streamSettings.getLowResolutionStream(msos);
+          destinationVideoBitrate = defaultLowResolutionBitrate;
           break;
         case 'local-recorder':
           result = this.streamSettings.getRecordingStream(msos);
+          destinationVideoBitrate = defaultLocalBitrate;
           break;
         case 'remote-recorder':
           result = this.streamSettings.getRemoteRecordingStream(msos);
+          destinationVideoBitrate = defaultLocalBitrate;
           break;
         default:
           result = this.streamSettings.getDefaultStream(msos);
+          destinationVideoBitrate = defaultLocalBitrate;
           break;
       }
 
-      if (!result.isDefault || !id) {
-        id = result.stream.id;
-      }
-      else {
-        this.console.log('Default stream overriden by legacy stream id setting for ', id);
+      id = result.stream.id;
+      this.console.log('Selected stream', result.stream.name);
+      // transcoding video should never happen transparently since it is CPU intensive.
+      // encourage users at every step to configure proper codecs.
+      // for this reason, do not automatically supply h264 encoder arguments
+      // even if h264 is requested, to force a visible failure.
+      if (this.streamSettings.storageSettings.values.transcodeStreams?.includes(result.title)) {
+        h264EncoderArguments = this.plugin.storageSettings.values.h264EncoderArguments?.split(' ');
       }
     }
 
     const session = this.sessions.get(id);
     if (!session)
       return this.mixinDevice.getVideoStream(options);
-    return session.getVideoStream(options);
+
+    const ffmpegInput = await session.getVideoStream(options);
+    ffmpegInput.h264EncoderArguments = h264EncoderArguments;
+    ffmpegInput.destinationVideoBitrate = destinationVideoBitrate;
+
+    if (this.streamSettings.storageSettings.values.missingCodecParameters) {
+      ffmpegInput.h264FilterArguments = ffmpegInput.h264FilterArguments || [];
+      ffmpegInput.h264FilterArguments.push("-bsf:v", "dump_extra");
+    }
+
+    ffmpegInput.videoDecoderArguments = this.streamSettings.storageSettings.values.videoDecoderArguments?.split(' ');
+    return mediaManager.createFFmpegMediaObject(ffmpegInput, {
+      sourceId: this.id,
+    });
   }
 
   async ensurePrebufferSessions() {
@@ -1241,13 +1338,19 @@ class PrebufferMixin extends SettingsMixinDeviceBase<VideoCamera & VideoCameraCo
   }
 
   async putMixinSetting(key: string, value: SettingValue): Promise<void> {
-    const sessions = this.sessions;
-    this.sessions = new Map();
     if (this.streamSettings.storageSettings.settings[key])
       await this.streamSettings.storageSettings.putSetting(key, value);
     else
       this.storage.setItem(key, value?.toString());
 
+    // no prebuffer change necessary if the setting is a transcoding hint.
+    if (this.streamSettings.storageSettings.settings[key]?.group === 'Transcoding')
+      return;
+
+    const sessions = this.sessions;
+    this.sessions = new Map();
+
+    // kill and reinitiate the prebuffers.
     for (const session of sessions.values()) {
       session?.parserSessionPromise?.then(session => session.kill());
     }
@@ -1308,12 +1411,26 @@ function millisUntilMidnight() {
   return (midnight.getTime() - new Date().getTime());
 }
 
-class PrebufferProvider extends AutoenableMixinProvider implements MixinProvider, BufferConverter {
+class RebroadcastPlugin extends AutoenableMixinProvider implements MixinProvider, BufferConverter, Settings {
   storageSettings = new StorageSettings(this, {
     rebroadcastPort: {
       title: 'Rebroadcast Port',
       description: 'The port of the RTSP server that will rebroadcast your streams.',
       type: 'number',
+    },
+    remoteStreamingBitrate: {
+      title: 'Remote Streaming Bitrate',
+      type: 'number',
+      defaultValue: 1000000,
+      description: 'The bitrate to use when remote streaming. This setting will only be used when transcoding or adaptive bitrate is enabled on a camera.',
+    },
+    h264EncoderArguments: {
+      title: 'H264 Encoder Arguments',
+      description: 'FFmpeg arguments used to encode h264 video. This is not camera specific and is used to setup the hardware accelerated encoder on your Scrypted server. This setting will only be used when transcoding is enabled on a camera.',
+      choices: Object.keys(getH264EncoderArgs()),
+      defaultValue: getH264EncoderArgs()[LIBX264_ENCODER_TITLE].join(' '),
+      combobox: true,
+      mapPut: (oldValue, newValue) => getH264EncoderArgs()[newValue]?.join(' ') || newValue || getH264EncoderArgs()[LIBX264_ENCODER_TITLE]?.join(' '),
     }
   });
   rtspServer: net.Server;
@@ -1350,6 +1467,14 @@ class PrebufferProvider extends AutoenableMixinProvider implements MixinProvider
     this.startRtspServer();
   }
 
+  getSettings(): Promise<Setting[]> {
+    return this.storageSettings.getSettings();
+  }
+
+  putSetting(key: string, value: SettingValue): Promise<void> {
+    return this.storageSettings.putSetting(key, value);
+  }
+
   startRtspServer() {
     closeQuiet(this.rtspServer);
 
@@ -1413,13 +1538,19 @@ class PrebufferProvider extends AutoenableMixinProvider implements MixinProvider
     const json = JSON.parse(data.toString());
     const { url, sdp } = json;
 
-    const { audioPayloadTypes, videoPayloadTypes } = parsePayloadTypes(sdp);
+    const parsedSdp = parseSdp(sdp);
+    const trackLookups = new Map<number, string>();
+    for (const msection of parsedSdp.msections) {
+      for (const pt of msection.payloadTypes) {
+        trackLookups.set(pt, msection.control);
+      }
+    }
 
     const u = new URL(url);
     if (!u.protocol.startsWith('tcp'))
       throw new Error('rfc4751 url must be tcp');
     const { clientPromise, url: clientUrl } = await listenZeroSingleClient();
-    const ffmpeg: FFMpegInput = {
+    const ffmpeg: FFmpegInput = {
       url: clientUrl,
       inputArguments: [
         "-rtsp_transport", "tcp",
@@ -1445,19 +1576,15 @@ class PrebufferProvider extends AutoenableMixinProvider implements MixinProvider
         const length = header.readInt16BE(0);
         const data = await readLength(socket, length);
         const pt = data[1] & 0x7f;
-        if (audioPayloadTypes.has(pt)) {
-          rtsp.sendAudio(data, false);
-        }
-        else if (videoPayloadTypes.has(pt)) {
-          rtsp.sendVideo(data, false);
-        }
-        else {
+        const track = trackLookups.get(pt);
+        if (!track) {
           client.destroy();
           socket.destroy();
           throw new Error('unknown payload type ' + pt);
         }
+        rtsp.sendTrack(track, data, false);
       }
-    })
+    });
 
     return Buffer.from(JSON.stringify(ffmpeg));
   }
@@ -1478,7 +1605,7 @@ class PrebufferProvider extends AutoenableMixinProvider implements MixinProvider
       mixinDeviceState,
       mixinProviderNativeId: this.nativeId,
       mixinDeviceInterfaces,
-      group: "Stream Selection",
+      group: "Stream Management",
       groupKey: "prebuffer",
     });
     this.currentMixins.set(mixinDeviceState.id, ret);
@@ -1492,4 +1619,4 @@ class PrebufferProvider extends AutoenableMixinProvider implements MixinProvider
   }
 }
 
-export default new PrebufferProvider();
+export default new RebroadcastPlugin();