@camera.ui/server 0.0.48 → 0.0.50-internal.0

package/src/camera/streaming/peer-connection.ts

@@ -0,0 +1,273 @@
+import { Subscribed } from '@camera.ui/common';
+import { ReplaySubject, Subject, interval, merge } from 'rxjs';
+import { MediaStreamTrack, RTCPeerConnection, RTCRtpCodecParameters } from 'werift';
+
+import type { BasicPeerConnection, IceServer, LoggerService } from '@camera.ui/types';
+import type { ConnectionState, RTCIceCandidate, RTCIceServer, RTCSessionDescription, RtcpPacket, RtpPacket } from 'werift';
+import type { CameraDevice } from '../device.js';
+
+export function convertIceServers(iceServers: IceServer[]): RTCIceServer[] {
+  const rtcIceServers: RTCIceServer[] = [];
+
+  iceServers.forEach((iceServer) => {
+    iceServer.urls.forEach((url) => {
+      rtcIceServers.push({
+        urls: url,
+        username: iceServer.username,
+        credential: iceServer.credential,
+      });
+    });
+  });
+
+  return rtcIceServers;
+}
+
+export class WeriftPeerConnection extends Subscribed implements BasicPeerConnection {
+  public onAudioRtp = new Subject<RtpPacket>();
+  public onAudioRtcp = new Subject<RtcpPacket>();
+  public onVideoRtp = new Subject<RtpPacket>();
+  public onVideoRtcp = new Subject<RtcpPacket>();
+  public onIceCandidate = new Subject<RTCIceCandidate>();
+  public onConnectionState = new ReplaySubject<ConnectionState>(1);
+  public returnAudioTrack = new MediaStreamTrack({ kind: 'audio' });
+
+  #onRequestKeyFrame = new Subject<void>();
+
+  #cameraDevice: CameraDevice;
+  #logger: LoggerService;
+
+  #pc: RTCPeerConnection;
+
+  get #logPrefix(): string {
+    return `[${this.#cameraDevice.name}]`;
+  }
+
+  constructor(cameraDevice: CameraDevice, logger: LoggerService) {
+    super();
+
+    this.#cameraDevice = cameraDevice;
+    this.#logger = logger;
+
+    const pc = (this.#pc = this.#createPeerCoonection());
+
+    pc.addTransceiver(this.returnAudioTrack, {
+      direction: 'sendonly',
+    });
+
+    const audioTransceiver = pc.addTransceiver(this.returnAudioTrack, {
+      direction: 'recvonly',
+    });
+
+    audioTransceiver.onTrack.subscribe((track) => {
+      track.onReceiveRtp.subscribe((rtp) => {
+        this.onAudioRtp.next(rtp);
+      });
+
+      track.onReceiveRtcp.subscribe((rtcp) => {
+        this.onAudioRtcp.next(rtcp);
+      });
+
+      track.onReceiveRtp.once(() => {
+        this.#logger.debug(this.#logPrefix, 'received first audio packet');
+      });
+    });
+
+    const videoTransceiver = pc.addTransceiver('video', {
+      direction: 'recvonly',
+    });
+
+    videoTransceiver.onTrack.subscribe((track) => {
+      track.onReceiveRtp.subscribe((rtp) => {
+        this.onVideoRtp.next(rtp);
+      });
+
+      track.onReceiveRtcp.subscribe((rtcp) => {
+        this.onVideoRtcp.next(rtcp);
+      });
+
+      track.onReceiveRtp.once(() => {
+        this.#logger.debug(this.#logPrefix, 'received first video packet');
+
+        this.addSubscriptions(
+          merge(this.#onRequestKeyFrame, interval(4000)).subscribe(() => {
+            if (track.ssrc) {
+              videoTransceiver.receiver.sendRtcpPLI(track.ssrc).catch((error: any) => this.#logger.error(this.#logPrefix, error));
+            }
+          }),
+        );
+
+        this.requestKeyFrame();
+      });
+    });
+
+    this.#pc.onIceCandidate.subscribe((iceCandidate: RTCIceCandidate | undefined) => {
+      if (iceCandidate && this.#isLocalCandidate(iceCandidate.candidate)) {
+        this.onIceCandidate.next(iceCandidate);
+      }
+    });
+
+    pc.iceConnectionStateChange.subscribe(() => {
+      this.#logger.debug(this.#logPrefix, `iceConnectionStateChange: ${pc.iceConnectionState}`);
+
+      if (pc.iceConnectionState === 'closed') {
+        this.onConnectionState.next('closed');
+      }
+    });
+
+    pc.connectionStateChange.subscribe(() => {
+      this.#logger.debug(this.#logPrefix, `connectionStateChange: ${pc.connectionState}`);
+      this.onConnectionState.next(pc.connectionState);
+    });
+  }
+
+  public async createOffer(): Promise<RTCSessionDescription> {
+    const offer = await this.#pc.createOffer();
+    this.#pc.setLocalDescription(offer);
+    return offer;
+  }
+
+  public async acceptAnswer(answer: { type: 'answer'; sdp: string }): Promise<void> {
+    await this.#pc.setRemoteDescription(answer);
+  }
+
+  public async addIceCandidate(candidate: RTCIceCandidate): Promise<void> {
+    if (this.#isLocalCandidate(candidate.candidate)) {
+      this.#pc.addIceCandidate(candidate);
+    }
+  }
+
+  public requestKeyFrame(): void {
+    this.#onRequestKeyFrame.next();
+  }
+
+  public close(): void {
+    this.#pc.close().catch((error: any) => this.#logger.error(this.#logPrefix, error));
+    this.unsubscribe();
+  }
+
+  #createPeerCoonection(): RTCPeerConnection {
+    const prefferedVideoCodec = ['vp8', 'vp9', 'h264', 'av1'];
+    const prefferedAudioCodec = ['opus', 'g722', 'pcmu', 'pcma'];
+
+    const video: RTCRtpCodecParameters[] = [];
+    const audio: RTCRtpCodecParameters[] = [];
+
+    for (const codec of prefferedVideoCodec) {
+      switch (codec) {
+        case 'vp8':
+          video.push(
+            new RTCRtpCodecParameters({
+              mimeType: 'video/VP8',
+              clockRate: 90000,
+              payloadType: 100,
+              rtcpFeedback: [{ type: 'goog-remb' }, { type: 'ccm', parameter: 'fir' }, { type: 'nack' }, { type: 'nack', parameter: 'pli' }],
+            }),
+          );
+          break;
+        case 'vp9':
+          video.push(
+            new RTCRtpCodecParameters({
+              mimeType: 'video/VP9',
+              clockRate: 90000,
+              payloadType: 101,
+              rtcpFeedback: [{ type: 'goog-remb' }, { type: 'ccm', parameter: 'fir' }, { type: 'nack' }, { type: 'nack', parameter: 'pli' }],
+            }),
+          );
+          break;
+        case 'h264':
+          video.push(
+            new RTCRtpCodecParameters({
+              mimeType: 'video/H264',
+              clockRate: 90000,
+              rtcpFeedback: [{ type: 'transport-cc' }, { type: 'ccm', parameter: 'fir' }, { type: 'nack' }, { type: 'nack', parameter: 'pli' }, { type: 'goog-remb' }],
+              parameters: 'packetization-mode=1;profile-level-id=640029;level-asymmetry-allowed=1',
+            }),
+            new RTCRtpCodecParameters({
+              mimeType: 'video/H265',
+              clockRate: 90000,
+              rtcpFeedback: [{ type: 'transport-cc' }, { type: 'ccm', parameter: 'fir' }, { type: 'nack' }, { type: 'nack', parameter: 'pli' }, { type: 'goog-remb' }],
+              parameters: 'packetization-mode=1;profile-level-id=640029;level-asymmetry-allowed=1',
+            }),
+            new RTCRtpCodecParameters({
+              mimeType: 'video/rtx',
+              clockRate: 90000,
+            }),
+          );
+          break;
+        case 'av1':
+          video.push(
+            new RTCRtpCodecParameters({
+              mimeType: 'video/AV1',
+              clockRate: 90000,
+              payloadType: 102,
+              rtcpFeedback: [{ type: 'goog-remb' }, { type: 'ccm', parameter: 'fir' }, { type: 'nack' }, { type: 'nack', parameter: 'pli' }],
+            }),
+          );
+          break;
+      }
+    }
+
+    for (const codec of prefferedAudioCodec) {
+      switch (codec) {
+        case 'opus':
+          audio.push(
+            new RTCRtpCodecParameters({
+              mimeType: 'audio/opus',
+              clockRate: 48000,
+              payloadType: 101,
+              channels: 2,
+              parameters: 'minptime=10;useinbandfec=1',
+            }),
+          );
+          break;
+        case 'g722':
+          audio.push(
+            new RTCRtpCodecParameters({
+              mimeType: 'audio/G722',
+              clockRate: 8000,
+              channels: 1,
+              payloadType: 9,
+            }),
+          );
+          break;
+        case 'pcmu':
+          audio.push(
+            new RTCRtpCodecParameters({
+              mimeType: 'audio/PCMU',
+              clockRate: 8000,
+              channels: 1,
+              payloadType: 0,
+            }),
+          );
+          break;
+        case 'pcma':
+          audio.push(
+            new RTCRtpCodecParameters({
+              mimeType: 'audio/PCMA',
+              clockRate: 8000,
+              channels: 1,
+              payloadType: 8,
+            }),
+          );
+          break;
+      }
+    }
+
+    return new RTCPeerConnection({
+      codecs: {
+        video,
+        audio,
+      },
+      bundlePolicy: 'max-bundle',
+      iceAdditionalHostAddresses: ['127.0.0.1'],
+      iceUseIpv4: true,
+    });
+  }
+
+  #isLocalCandidate(candidateString: string): boolean {
+    const parts = candidateString.split(' ');
+    const ip = parts[4];
+    const type = parts[7];
+    return ip === '127.0.0.1' || type === 'host';
+  }
+}

package/src/camera/streaming/rtsp-session.ts

@@ -0,0 +1,331 @@
+import {
+  FfmpegProcess,
+  generateAudioStreamInfo,
+  generateSdp,
+  generateVideoStreamInfo,
+  reservePorts,
+  RtpSplitter,
+  transformAudioStreamInfoToFFmpegArgs,
+} from '@camera.ui/common/camera';
+import { getCleanSdp, SubscribedPublic } from '@camera.ui/common/utils';
+import { RTSPClient } from '@camera.ui/rtsp/rtspClient';
+import { ReplaySubject, Subject } from 'rxjs';
+import { take, tap } from 'rxjs/operators';
+import { RtpPacket } from 'werift';
+
+import type {
+  AudioStreamInfo,
+  CameraDeviceSource,
+  CameraInput,
+  FfmpegOptions,
+  LoggerService,
+  ReturnAudioFFmpegOptions,
+  RTSPConnectionOptions,
+  RTSPSession as RTSPSessionInterface,
+  VideoStreamInfo,
+} from '@camera.ui/types';
+import type { CameraDevice } from '../device.js';
+
+export class RTSPSession extends SubscribedPublic implements RTSPSessionInterface {
+  readonly onStarted = new ReplaySubject<void>();
+  readonly onError = new ReplaySubject<Error>();
+  readonly onCallEnded = new ReplaySubject<void>(1);
+  readonly onVideoRtp = new Subject<RtpPacket>();
+  readonly onAudioRtp = new Subject<RtpPacket>();
+
+  readonly #audioSplitter = new RtpSplitter();
+  readonly #videoSplitter = new RtpSplitter();
+  readonly #returnAudioSplitter = new RtpSplitter();
+
+  #hasEnded = false;
+  #cameraDevice: CameraDevice;
+  #logger: LoggerService;
+  #source: CameraDeviceSource;
+  #client: RTSPClient;
+
+  constructor(cameraDevice: CameraDevice, source: CameraInput, options?: RTSPConnectionOptions) {
+    super();
+
+    this.#cameraDevice = cameraDevice;
+    this.#logger = cameraDevice.logger;
+    this.#source = this.#cameraDevice.sources.find((s) => s.name === source.name)!;
+
+    const url = options?.url ?? this.#source.generateRTSPUrl();
+
+    this.#client = new RTSPClient(
+      { url },
+      {
+        // log: (...args) => this.#logger.log(...args),
+        error: (...args) => this.#logger.error(...args),
+        // debug: (...args) => this.#logger.trace(...args),
+        warn: (...args) => this.#logger.warn(...args),
+      },
+    );
+
+    this.#bindConnection();
+  }
+
+  public async startStream(): Promise<void> {
+    this.#hasEnded = false;
+    await this.#client.start();
+  }
+
+  public async startTranscoding(ffmpegOptions: FfmpegOptions): Promise<void> {
+    const transcodeAudioStream = ffmpegOptions.audio !== undefined && ffmpegOptions.audio.length > 0;
+    const transcodeVideoStream = ffmpegOptions.video !== undefined && ffmpegOptions.video.length > 0;
+
+    const loggerPrefix = ffmpegOptions.logPrefix
+      ? `[${ffmpegOptions.logPrefix}] `
+      : transcodeAudioStream && transcodeVideoStream
+        ? '[Transcoding] '
+        : transcodeAudioStream
+          ? '[Audio] '
+          : '[Video] ';
+
+    try {
+      if (!transcodeAudioStream && !transcodeVideoStream) {
+        throw new Error('No transcoding options provided! Can not start transcoding!');
+      }
+
+      await this.startStream();
+
+      const audioTrack = this.#client.getAudioTrack();
+      const videoTrack = this.#client.getVideoTrack();
+
+      if (!audioTrack && !videoTrack) {
+        throw new Error('No audio or video tracks found! Can not start transcoding!');
+      }
+
+      await this.#audioSplitter.prepare('udp4');
+      await this.#videoSplitter.prepare('udp4');
+
+      if (!this.#audioSplitter.port || !this.#videoSplitter.port) {
+        throw new Error('No ports available for audio and video splitter! Can not start transcoding!');
+      }
+
+      const videoPort = await this.#reservePort(1);
+      const audioPort = await this.#reservePort(1);
+
+      const videoStreamInfos = videoTrack ? generateVideoStreamInfo([videoTrack.mediaSource]) : [];
+      const audioStreamInfos = audioTrack ? generateAudioStreamInfo([audioTrack.mediaSource]) : [];
+      const parsedSdp = generateSdp(videoStreamInfos, audioStreamInfos);
+
+      const audioStream: AudioStreamInfo | undefined = audioStreamInfos[0];
+      const videoStream: VideoStreamInfo | undefined = videoStreamInfos[0];
+
+      const audioInputArgs = audioTrack && transcodeAudioStream && audioStream ? ['-acodec', audioStream.ffmpegCodec] : [];
+      const videoInputArgs = videoTrack && transcodeVideoStream && videoStream ? ['-vcodec', videoStream.ffmpegCodec] : [];
+
+      const audioOutputArgs = audioTrack && transcodeAudioStream ? (ffmpegOptions.audio ?? ['-acodec', audioStream.ffmpegCodec]) : [];
+      const videoOutputArgs = videoTrack && transcodeVideoStream ? (ffmpegOptions.video ?? ['-vcodec', videoStream.ffmpegCodec]) : [];
+
+      const inputArgs = ffmpegOptions.input ?? [];
+      const outputArgs = ffmpegOptions.output ?? [];
+
+      const inputSdp = getCleanSdp(parsedSdp, transcodeVideoStream, 'recvonly')
+        .replace(/m=audio \d+/, `m=audio ${audioPort}`)
+        .replace(/m=video \d+/, `m=video ${videoPort}`);
+
+      this.#logger.trace(loggerPrefix + 'Start transcoding with SDP:\n', inputSdp);
+
+      const ff = new FfmpegProcess({
+        ffmpegArgs: [
+          '-hide_banner',
+          '-protocol_whitelist',
+          'pipe,udp,rtp,file,crypto',
+          '-reorder_queue_size',
+          '0',
+          ...audioInputArgs,
+          ...videoInputArgs,
+          '-f',
+          'sdp',
+          ...inputArgs,
+          '-i',
+          'pipe:',
+          ...audioOutputArgs,
+          ...videoOutputArgs,
+          ...outputArgs,
+        ],
+        ffmpegPath: ffmpegOptions.ffmpegPath,
+        startedCallback: () => {
+          this.onStarted.next();
+        },
+        exitCallback: (code, signal) => {
+          if (code !== 0 && code !== 255) {
+            this.#callError(new Error(`FFmpeg exited with code ${code} and signal ${signal}`));
+          } else {
+            this.#callEnded();
+          }
+        },
+        logLabel: loggerPrefix,
+        logger: {
+          error: (...args) => this.#logger.error(...args),
+          trace: (...args) => this.#logger.trace(...args),
+        },
+      });
+
+      if (transcodeAudioStream) {
+        this.addSubscriptions(
+          this.onAudioRtp
+            .pipe(
+              tap((rtp) => {
+                return this.#audioSplitter.send(rtp.serialize(), {
+                  port: audioPort,
+                  address: '127.0.0.1',
+                });
+              }),
+            )
+            .subscribe(),
+        );
+      }
+
+      if (transcodeVideoStream) {
+        this.addSubscriptions(
+          this.onVideoRtp
+            .pipe(
+              tap((rtp) => {
+                return this.#videoSplitter.send(rtp.serialize(), {
+                  port: videoPort,
+                  address: '127.0.0.1',
+                });
+              }),
+            )
+            .subscribe(),
+        );
+      }
+
+      this.onCallEnded.pipe(take(1)).subscribe(() => ff.stop());
+
+      ff.writeStdin(inputSdp);
+    } catch (error: any) {
+      this.stop();
+      throw error;
+    }
+  }
+
+  public async transcodeReturnAudio(ffmpegOptions: ReturnAudioFFmpegOptions): Promise<void> {
+    if (this.#hasEnded) {
+      return;
+    }
+
+    const loggerPrefix = ffmpegOptions.logPrefix ? `[${ffmpegOptions.logPrefix}] ` : '[Return Audio] ';
+
+    try {
+      await this.startStream();
+
+      const returnAudioTrack = this.#client.getBackchannelTrack();
+      const returnAudioStreamInfos = returnAudioTrack ? generateAudioStreamInfo([returnAudioTrack.mediaSource]) : [];
+      const returnAudioStream = returnAudioStreamInfos[0];
+
+      if (!returnAudioStream) {
+        this.stop();
+        throw new Error('No return audio stream found! Can not start talkback!');
+      }
+
+      const audioOutForwarder = new RtpSplitter();
+      await audioOutForwarder.prepare('udp4');
+
+      if (!audioOutForwarder.port) {
+        this.stop();
+        throw new Error('No ports available for audio out splitter! Can not start talkback!');
+      }
+
+      audioOutForwarder.addMessageHandler(({ message }) => {
+        try {
+          const rtp = RtpPacket.deSerialize(message);
+          this.#client.sendAudioRtp(rtp);
+        } catch {
+          //
+        }
+
+        return null;
+      });
+
+      this.#logger.log(loggerPrefix.trim(), returnAudioStream);
+
+      const inputArgs = ffmpegOptions.input ?? [];
+      const returnAudioCodecs = transformAudioStreamInfoToFFmpegArgs(returnAudioStream);
+
+      const ff = new FfmpegProcess({
+        ffmpegArgs: [
+          '-hide_banner',
+          '-protocol_whitelist',
+          'pipe,udp,rtp,file,crypto',
+          '-re',
+          '-i',
+          ...inputArgs,
+          ...returnAudioCodecs,
+          '-flags',
+          '+global_header',
+          '-f',
+          'rtp',
+          `rtp://127.0.0.1:${audioOutForwarder.port}`,
+        ],
+        ffmpegPath: ffmpegOptions.ffmpegPath,
+        startedCallback: () => {
+          this.onStarted.next();
+        },
+        exitCallback: () => this.#callEnded(),
+        logLabel: loggerPrefix,
+        logger: {
+          error: (...args) => this.#logger.error(...args),
+          trace: (...args) => this.#logger.trace(...args),
+        },
+      });
+
+      this.onCallEnded.pipe(take(1)).subscribe(() => ff.stop());
+    } catch (error: any) {
+      this.stop();
+      throw error;
+    }
+  }
+
+  public stop(): void {
+    this.#callEnded();
+  }
+
+  public sendAudioPacket(rtp: RtpPacket): void {
+    if (this.#hasEnded) {
+      return;
+    }
+
+    this.#client.sendAudioRtp(rtp);
+  }
+
+  async #reservePort(bufferPorts = 0): Promise<number> {
+    const ports = await reservePorts({ count: bufferPorts + 1 });
+    return ports[0];
+  }
+
+  #bindConnection(): void {
+    this.#client.onVideoRtp.subscribe(this.onVideoRtp);
+    this.#client.onAudioRtp.subscribe(this.onAudioRtp);
+    this.#client.onClose.subscribe((error) => {
+      if (error) {
+        this.#callError(error);
+      } else {
+        this.#callEnded();
+      }
+    });
+  }
+
+  #callError(error: Error): void {
+    this.onError.next(error);
+    this.#callEnded();
+  }
+
+  #callEnded(): void {
+    if (this.#hasEnded) {
+      return;
+    }
+
+    this.#hasEnded = true;
+
+    this.unsubscribe();
+    this.#client.close();
+    this.onCallEnded.next();
+    this.#audioSplitter.close();
+    this.#videoSplitter.close();
+    this.#returnAudioSplitter.close();
+  }
+}