@scrypted/prebuffer-mixin 0.1.259 → 0.1.262

package/src/rtsp-session.ts

@@ -0,0 +1,251 @@
+import { ParserSession, setupActivityTimer } from "@scrypted/common/src/ffmpeg-rebroadcast";
+import { closeQuiet, createBindZero } from "@scrypted/common/src/listen-cluster";
+import { parseSemicolonDelimited, RtspClient, RTSP_FRAME_MAGIC } from "@scrypted/common/src/rtsp-server";
+import { parseSdp } from "@scrypted/common/src/sdp-utils";
+import { StreamChunk } from "@scrypted/common/src/stream-parser";
+import { ResponseMediaStreamOptions } from "@scrypted/sdk";
+import dgram from 'dgram';
+import { parse as spsParse } from "h264-sps-parser";
+import { EventEmitter } from "stream";
+import { negotiateMediaStream } from "./rfc4571";
+import { getSpsResolution } from "./sps-resolution";
+
+export type RtspChannelCodecMapping = { [key: number]: string };
+
+
+export async function startRtspSession(console: Console, url: string, mediaStreamOptions: ResponseMediaStreamOptions, options: {
+    useUdp: boolean,
+    audioSoftMuted: boolean,
+    rtspRequestTimeout: number,
+}): Promise<ParserSession<"rtsp">> {
+    let isActive = true;
+    const events = new EventEmitter();
+    // need this to prevent kill from throwing due to uncaught Error during cleanup
+    events.on('error', e => console.error('rebroadcast error', e));
+
+    let servers: dgram.Socket[] = [];
+    const rtspClient = new RtspClient(url, console);
+    rtspClient.requestTimeout = options.rtspRequestTimeout;
+
+    const cleanupSockets = () => {
+        for (const server of servers) {
+            closeQuiet(server);
+        }
+        rtspClient.safeTeardown();
+    }
+
+    let sessionKilled: any;
+    const killed = new Promise<void>(resolve => {
+        sessionKilled = resolve;
+    });
+
+    const kill = () => {
+        if (isActive) {
+            events.emit('killed');
+            events.emit('error', new Error('killed'));
+        }
+        isActive = false;
+        sessionKilled();
+        cleanupSockets();
+    };
+
+    rtspClient.client.on('close', () => {
+        kill();
+    });
+    rtspClient.client.on('error', () => {
+        kill();
+    });
+
+    const { resetActivityTimer } = setupActivityTimer('rtsp', kill, events, options?.rtspRequestTimeout);
+
+    try {
+        await rtspClient.options();
+        const sdpResponse = await rtspClient.describe();
+        const contentBase = sdpResponse.headers['content-base'];
+        if (contentBase) {
+            const url = new URL(contentBase);
+            const existing = new URL(rtspClient.url);
+            url.username = existing.username;
+            url.password = existing.password;
+            rtspClient.url = url.toString();
+        }
+        let sdp = sdpResponse.body.toString().trim();
+        console.log('sdp', sdp);
+
+        const parsedSdp = parseSdp(sdp);
+        let channel = 0;
+        const mapping: RtspChannelCodecMapping = {};
+        let udpSessionTimeout: number;
+        const { useUdp } = options;
+        const checkUdpSessionTimeout = (headers: { [key: string]: string }) => {
+            if (useUdp && headers.session && !udpSessionTimeout) {
+                const sessionDict = parseSemicolonDelimited(headers.session);
+                udpSessionTimeout = parseInt(sessionDict['timeout']);
+            }
+        }
+
+        const doSetup = async (control: string, codec: string) => {
+            let setupChannel = channel;
+            let udp: dgram.Socket;
+            if (useUdp) {
+                const rtspChannel = channel;
+                const { port, server } = await createBindZero();
+                udp = server;
+                servers.push(server);
+                setupChannel = port;
+                server.on('message', data => {
+                    const prefix = Buffer.alloc(4);
+                    prefix.writeUInt8(RTSP_FRAME_MAGIC, 0);
+                    prefix.writeUInt8(rtspChannel, 1);
+                    prefix.writeUInt16BE(data.length, 2);
+                    const chunk: StreamChunk = {
+                        chunks: [prefix, data],
+                        type: codec,
+                    };
+                    events.emit('rtsp', chunk);
+                    resetActivityTimer?.();
+                });
+
+                const setupResult = await rtspClient.setup({
+                    path: control,
+                    type: 'udp',
+                    port: setupChannel,
+                });
+                checkUdpSessionTimeout(setupResult.headers);
+
+                const punch = Buffer.alloc(1);
+                const transport = setupResult.headers['transport'];
+                const match = transport.match(/.*?server_port=([0-9]+)-([0-9]+)/);
+                const [_, rtp, rtcp] = match;
+                const { hostname } = new URL(rtspClient.url);
+                udp.send(punch, parseInt(rtp), hostname)
+
+                mapping[channel] = codec;
+            }
+            else {
+                const setupResult = await rtspClient.setup({
+                    path: control,
+                    type: 'tcp',
+                    port: setupChannel,
+                    onRtp: (header, data) => {
+                        const chunk: StreamChunk = {
+                            chunks: [header, data],
+                            type: codec,
+                        };
+                        events.emit('rtsp', chunk);
+                        resetActivityTimer?.();
+                    },
+                });
+
+                if (setupResult.interleaved)
+                    mapping[setupResult.interleaved.begin] = codec;
+                else
+                    mapping[channel] = codec;
+            }
+
+            channel += 2;
+        }
+
+        let setupVideoSection = false;
+
+        parsedSdp.msections = parsedSdp.msections.filter(section => {
+            if (section.type === 'video') {
+                if (setupVideoSection) {
+                    console.warn('additional video section found. skipping.');
+                    return false;
+                }
+                setupVideoSection = true;
+            }
+            else if (section.type !== 'audio') {
+                console.warn('unknown section', section.type);
+                return false;
+            }
+            else if (options.audioSoftMuted) {
+                return false;
+            }
+            return true;
+        });
+
+        for (const section of parsedSdp.msections) {
+            await doSetup(section.control, section.codec)
+        }
+
+        // sdp may contain multiple audio/video sections. take only the first video section.
+        sdp = [...parsedSdp.header.lines, ...parsedSdp.msections.map(msection => msection.lines).flat()].join('\r\n');
+
+        await rtspClient.play();
+
+        // don't start parsing until next tick when this function returns to allow
+        // event handlers to be set prior to parsing.
+        process.nextTick(() => rtspClient.readLoop().finally(kill));
+
+        // this return block is intentional, to ensure that the remaining code happens sync.
+        return (() => {
+            const audioSection = parsedSdp.msections.find(msection => msection.type === 'audio');
+            const videoSection = parsedSdp.msections.find(msection => msection.type === 'video');
+
+            const inputAudioCodec = audioSection?.codec;
+            const inputVideoCodec = videoSection.codec;
+
+            let inputVideoResolution: {
+                width: number;
+                height: number;
+            };
+
+            const sprop = videoSection
+                ?.fmtp?.[0]?.parameters?.['sprop-parameter-sets'];
+            const sdpSps = sprop?.split(',')?.[0];
+            // const sdpPps = sprop?.split(',')?.[1];
+
+            if (sdpSps) {
+                try {
+                    const sps = Buffer.from(sdpSps, 'base64');
+                    const parsedSps = spsParse(sps);
+                    inputVideoResolution = getSpsResolution(parsedSps);
+                    console.log('parsed sdp sps', parsedSps);
+                }
+                catch (e) {
+                    console.warn('sdp sps parsing failed');
+                }
+            }
+
+            return {
+                sdp: Promise.resolve([Buffer.from(sdp)]),
+                inputAudioCodec,
+                inputVideoCodec,
+                get inputVideoResolution() {
+                    return inputVideoResolution;
+                },
+                get isActive() { return isActive },
+                kill() {
+                    kill();
+                },
+                killed,
+                resetActivityTimer,
+                negotiateMediaStream: (requestMediaStream) => {
+                    return negotiateMediaStream(sdp, mediaStreamOptions, inputVideoCodec, inputAudioCodec, requestMediaStream);
+                },
+                emit(container: 'rtsp', chunk: StreamChunk) {
+                    events.emit(container, chunk);
+                    return this;
+                },
+                on(event: string, cb: any) {
+                    events.on(event, cb);
+                    return this;
+                },
+                once(event: any, cb: any) {
+                    events.once(event, cb);
+                    return this;
+                },
+                removeListener(event, cb) {
+                    events.removeListener(event, cb);
+                    return this;
+                }
+            }
+        })();
+    }
+    catch (e) {
+        cleanupSockets();
+        throw e;
+    }
+}

package/src/sps-resolution.ts

@@ -0,0 +1,50 @@
+import { SPSInfo } from "h264-sps-parser";
+
+export function getSpsResolution(sps: SPSInfo) {
+    let SubWidthC: number;
+    let SubHeightC: number;
+
+    if (sps.chroma_format_idc == 0 && sps.color_plane_flag == 0) { //monochrome
+        SubWidthC = SubHeightC = 0;
+    }
+    else if (sps.chroma_format_idc == 1 && sps.color_plane_flag == 0) { //4:2:0 
+        SubWidthC = SubHeightC = 2;
+    }
+    else if (sps.chroma_format_idc == 2 && sps.color_plane_flag == 0) { //4:2:2 
+        SubWidthC = 2;
+        SubHeightC = 1;
+    }
+    else if (sps.chroma_format_idc == 3) { //4:4:4
+        if (sps.color_plane_flag == 0) {
+            SubWidthC = SubHeightC = 1;
+        }
+        else if (sps.color_plane_flag == 1) {
+            SubWidthC = SubHeightC = 0;
+        }
+    }
+
+    let PicWidthInMbs = sps.pic_width_in_mbs;
+
+    let PicHeightInMapUnits = sps.pic_height_in_map_units;
+    let FrameHeightInMbs = (2 - sps.frame_mbs_only_flag) * PicHeightInMapUnits;
+
+    let crop_left = 0;
+    let crop_right = 0;
+    let crop_top = 0;
+    let crop_bottom = 0;
+
+    if (sps.frame_cropping_flag) {
+        crop_left = sps.frame_cropping.left;
+        crop_right = sps.frame_cropping.right;
+        crop_top = sps.frame_cropping.top;
+        crop_bottom = sps.frame_cropping.bottom;
+    }
+
+    let width = PicWidthInMbs * 16 - SubWidthC * (crop_left + crop_right);
+    let height = FrameHeightInMbs * 16 - SubHeightC * (2 - sps.frame_mbs_only_flag) * (crop_top + crop_bottom);
+
+    return {
+        width,
+        height,
+    };
+}

package/src/stream-settings.ts

@@ -1,6 +1,6 @@
 import { getH264DecoderArgs } from "@scrypted/common/src/ffmpeg-hardware-acceleration";
 import { StorageSetting, StorageSettings } from "@scrypted/common/src/settings";
-import sdk, { MixinDeviceBase, ResponseMediaStreamOptions, VideoCamera } from "@scrypted/sdk";
+import { MixinDeviceBase, ResponseMediaStreamOptions, VideoCamera } from "@scrypted/sdk";
 import { getTranscodeMixinProviderId } from "./transcode-settings";
 
 

package/src/sps-parser/bitstream.ts

@@ -1,75 +0,0 @@
-function ExpGolomInit(view: DataView, bitoffset: number): { zeros: number; skip: number; byt: number; byteoffset: number } {
-    let bit = 0;
-    let byteoffset = bitoffset >> 3;
-    let skip = bitoffset & 7;
-    let zeros = -1;
-  
-    let byt = view.getUint8(byteoffset) << skip;
-    do {
-      bit = byt & 0x80;
-      byt <<= 1;
-      zeros++;
-      skip++;
-      if (skip === 8) {
-        skip = 0;
-        byteoffset++;
-        byt = view.getUint8(byteoffset);
-      }
-    } while (!bit);
-  
-    return { zeros, skip, byt, byteoffset };
-  }
-  
-  export class Bitstream {
-    public bitoffset = 0;
-      constructor (public view: DataView) {}
-  
-    ExpGolomb(): number {
-      const { view } = this;
-      let {
-        zeros, skip, byt, byteoffset,
-      } = ExpGolomInit(view, this.bitoffset);
-      
-      let code = 1;
-      while (zeros > 0) {
-        code = (code << 1) | ((byt & 0x80) >>> 7);
-        byt <<= 1;
-        skip++;
-        zeros--;
-        if (skip === 8) {
-          skip = 0;
-          byteoffset++;
-          byt = view.getUint8(byteoffset);
-        }
-      }
-      
-      this.bitoffset = (byteoffset << 3) | skip;
-      return code - 1;
-    }
-  
-    SignedExpGolomb(): number {
-      const code = this.ExpGolomb();
-      return code & 1 ? (code + 1) >>> 1 : -(code >>> 1);
-    }
-  
-    readBit(): 0 | 1 {
-      const skip = this.bitoffset & 7;
-      const byteoffset = this.bitoffset >> 3;
-      this.bitoffset++;
-      return ((this.view.getUint8(byteoffset) >> (7 - skip)) & 1) as 0|1;
-    }
-    
-    readByte(): number {
-      const skip = this.bitoffset & 7;
-      const byteoffset = this.bitoffset >> 3;
-      this.bitoffset += 8;
-  
-      const high = this.view.getUint8(byteoffset);
-      if (skip === 0) return high;
-  
-      const low = this.view.getUint8(byteoffset + 1);
-  
-      return (high << skip) | (low >> (8 - skip));
-    }
-  }
-  

package/src/sps-parser/index.ts

@@ -1,237 +0,0 @@
-import { Bitstream } from "./bitstream";
-import { getVUIParams, VUIParams } from "./vui";
-
-type FrameCropping = {
-  left: number;
-  right: number;
-  top: number;
-  bottom: number;
-};
-
-export type SPSInfo = {
-  sps_id: number;
-  profile_idc: number;
-  level_idc: number;
-  profile_compatibility: number;
-  frame_mbs_only_flag: 0|1;
-  pic_width_in_mbs: number;
-  pic_height_in_map_units: number;
-  frame_cropping_flag: 0|1;
-  frame_cropping: FrameCropping;
-
-  chroma_format_idc: number;
-  bit_depth_luma: number;
-  bit_depth_chroma: number;
-  color_plane_flag: 0|1;
-  qpprime_y_zero_transform_bypass_flag: 0|1;
-  seq_scaling_matrix_present_flag: 0|1;
-  seq_scaling_matrix: number[][];
-  log2_max_frame_num: number;
-  pic_order_cnt_type: number;
-  delta_pic_order_always_zero_flag: 0|1;
-  offset_for_non_ref_pic: number;
-  offset_for_top_to_bottom_field: number;
-  offset_for_ref_frame: number[];
-  log2_max_pic_order_cnt_lsb: number;
-
-  max_num_ref_frames: number;
-  gaps_in_frame_num_value_allowed_flag: 0|1;
-  mb_adaptive_frame_field_flag: 0|1;
-  direct_8x8_inference_flag: 0|1;
-  vui_parameters_present_flag: 0|1;
-  vui_parameters: VUIParams;
-};
-
-function scaling_list(stream: Bitstream, sizeOfScalingList: number): number[] {
-  let lastScale = 8;
-  let nextScale = 8;
-  const scaling_list = [];
-  for (let j = 0; j < sizeOfScalingList; j++) {
-    if (nextScale !== 0) {
-      const deltaScale = stream.SignedExpGolomb();
-      nextScale = (lastScale + deltaScale + 256) % 256;
-    }
-    if (nextScale) { lastScale = nextScale; }
-    scaling_list.push(nextScale);
-  }
-  return scaling_list;
-}
-
-function getFrameCropping(flag: 0|1, stream: Bitstream): FrameCropping {
-  if (!flag) return { left: 0, right: 0, top: 0, bottom: 0 };
-
-  const left = stream.ExpGolomb();
-  const right = stream.ExpGolomb();
-  const top = stream.ExpGolomb();
-  const bottom = stream.ExpGolomb();
-  return { left, right, top, bottom };
-}
-
-export function parse(nalu: Uint8Array): SPSInfo {
-  if ((nalu[0] & 0x1F) !== 7) throw new Error("Not an SPS unit");
- 
-  const stream = new Bitstream(new DataView(nalu.buffer, nalu.byteOffset + 4));
-
-  const profile_idc = nalu[1];
-  const profile_compatibility = nalu[2];
-  const level_idc = nalu[3];
-  const sps_id = stream.ExpGolomb();
-
-  let chroma_format_idc = 1;
-  let bit_depth_luma = 0;
-  let bit_depth_chroma = 0;
-  let color_plane_flag: 0|1 = 0;
-  let qpprime_y_zero_transform_bypass_flag: 0|1 = 0;
-  let seq_scaling_matrix_present_flag: 0|1 = 0;
-
-  const seq_scaling_matrix = [];
-
-  if (	profile_idc === 100 || profile_idc === 110 ||
-    profile_idc === 122 || profile_idc === 244 || profile_idc === 44 ||
-    profile_idc === 83  || profile_idc === 86  || profile_idc === 118 ||
-    profile_idc === 128 ) {
-    chroma_format_idc = stream.ExpGolomb();
-    let limit = 8;
-    if (chroma_format_idc === 3) {
-      limit = 12;
-      color_plane_flag = stream.readBit();
-    }
-    bit_depth_luma = stream.ExpGolomb() + 8;
-    bit_depth_chroma = stream.ExpGolomb() + 8;
-    qpprime_y_zero_transform_bypass_flag = stream.readBit();
-    seq_scaling_matrix_present_flag = stream.readBit();
-    if (seq_scaling_matrix_present_flag) {
-      let i = 0;
-      for (; i < 6; i++) {
-        if (stream.readBit()) { //seq_scaling_list_present_flag
-          seq_scaling_matrix.push(scaling_list(stream, 16));
-        }
-      }
-      for (; i < limit; i++) {
-        if (stream.readBit()) { //seq_scaling_list_present_flag
-          seq_scaling_matrix.push(scaling_list(stream, 64));
-        }
-      }
-    }
-  }
-
-  const log2_max_frame_num = stream.ExpGolomb() + 4;
-  const pic_order_cnt_type = stream.ExpGolomb();
-
-  let delta_pic_order_always_zero_flag: 0|1 = 0;
-  let offset_for_non_ref_pic = 0;
-  let offset_for_top_to_bottom_field = 0;
-
-  const offset_for_ref_frame = [];
-
-  let log2_max_pic_order_cnt_lsb = 0;
-  if (pic_order_cnt_type === 0) {
-    log2_max_pic_order_cnt_lsb = stream.ExpGolomb() + 4;
-  } else if (pic_order_cnt_type === 1) {
-    delta_pic_order_always_zero_flag = stream.readBit();
-    offset_for_non_ref_pic = stream.SignedExpGolomb();
-    offset_for_top_to_bottom_field = stream.SignedExpGolomb();
-    const num_ref_frames_in_pic_order_cnt_cycle = stream.SignedExpGolomb();
-    for (let i = 0; i < num_ref_frames_in_pic_order_cnt_cycle; i++) {
-      offset_for_ref_frame.push(stream.SignedExpGolomb());
-    }
-  }
-
-  const max_num_ref_frames = stream.ExpGolomb();
-  const gaps_in_frame_num_value_allowed_flag = stream.readBit();
-  const pic_width_in_mbs = stream.ExpGolomb() + 1;
-  const pic_height_in_map_units = stream.ExpGolomb() + 1;
-  const frame_mbs_only_flag = stream.readBit();
-  let mb_adaptive_frame_field_flag: 0|1 = 0;
-  if (!frame_mbs_only_flag) {
-    mb_adaptive_frame_field_flag = stream.readBit();
-   }
-
-  const direct_8x8_inference_flag = stream.readBit();
-  const frame_cropping_flag = stream.readBit();
-  const frame_cropping = getFrameCropping(frame_cropping_flag, stream);
-
-  const vui_parameters_present_flag = stream.readBit();
-  const vui_parameters = getVUIParams(vui_parameters_present_flag, stream);
-
-  return {
-    sps_id,
-    profile_compatibility,
-    profile_idc,
-    level_idc,
-    chroma_format_idc,
-    bit_depth_luma,
-    bit_depth_chroma,
-    color_plane_flag,
-    qpprime_y_zero_transform_bypass_flag,
-    seq_scaling_matrix_present_flag,
-    seq_scaling_matrix,
-    log2_max_frame_num,
-    pic_order_cnt_type,
-    delta_pic_order_always_zero_flag,
-    offset_for_non_ref_pic,
-    offset_for_top_to_bottom_field,
-    offset_for_ref_frame,
-    log2_max_pic_order_cnt_lsb,
-    max_num_ref_frames,
-    gaps_in_frame_num_value_allowed_flag,
-    pic_width_in_mbs,
-    pic_height_in_map_units,
-    frame_mbs_only_flag,
-    mb_adaptive_frame_field_flag,
-    direct_8x8_inference_flag,
-    frame_cropping_flag,
-    frame_cropping,
-    vui_parameters_present_flag,
-    vui_parameters,
-  };
-}
-
-export function getSpsResolution(sps: SPSInfo) {
-    let SubWidthC: number;
-    let SubHeightC: number;
-
-    if (sps.chroma_format_idc == 0 && sps.color_plane_flag == 0) { //monochrome
-        SubWidthC = SubHeightC = 0;
-    }
-    else if (sps.chroma_format_idc == 1 && sps.color_plane_flag == 0) { //4:2:0 
-        SubWidthC = SubHeightC = 2;
-    }
-    else if (sps.chroma_format_idc == 2 && sps.color_plane_flag == 0) { //4:2:2 
-        SubWidthC = 2;
-        SubHeightC = 1;
-    }
-    else if (sps.chroma_format_idc == 3) { //4:4:4
-        if (sps.color_plane_flag == 0) {
-            SubWidthC = SubHeightC = 1;
-        }
-        else if (sps.color_plane_flag == 1) {
-            SubWidthC = SubHeightC = 0;
-        }
-    }
-
-    let PicWidthInMbs = sps.pic_width_in_mbs;
-
-    let PicHeightInMapUnits = sps.pic_height_in_map_units;
-    let FrameHeightInMbs = (2 - sps.frame_mbs_only_flag) * PicHeightInMapUnits;
-
-    let crop_left = 0;
-    let crop_right = 0;
-    let crop_top = 0;
-    let crop_bottom = 0;
-
-    if (sps.frame_cropping_flag) {
-        crop_left = sps.frame_cropping.left;
-        crop_right = sps.frame_cropping.right;
-        crop_top = sps.frame_cropping.top;
-        crop_bottom = sps.frame_cropping.bottom;
-    }
-
-    let width = PicWidthInMbs * 16 - SubWidthC * (crop_left + crop_right);
-    let height = FrameHeightInMbs * 16 - SubHeightC * (2 - sps.frame_mbs_only_flag) * (crop_top + crop_bottom);
-
-    return {
-        width,
-        height,
-    };
-}