@camera.ui/server 0.0.50-internal.0 → 0.0.50-internal.1

package/src/camera/streaming/peer-connection.ts

@@ -1,273 +0,0 @@
-import { Subscribed } from '@camera.ui/common';
-import { ReplaySubject, Subject, interval, merge } from 'rxjs';
-import { MediaStreamTrack, RTCPeerConnection, RTCRtpCodecParameters } from 'werift';
-
-import type { BasicPeerConnection, IceServer, LoggerService } from '@camera.ui/types';
-import type { ConnectionState, RTCIceCandidate, RTCIceServer, RTCSessionDescription, RtcpPacket, RtpPacket } from 'werift';
-import type { CameraDevice } from '../device.js';
-
-export function convertIceServers(iceServers: IceServer[]): RTCIceServer[] {
-  const rtcIceServers: RTCIceServer[] = [];
-
-  iceServers.forEach((iceServer) => {
-    iceServer.urls.forEach((url) => {
-      rtcIceServers.push({
-        urls: url,
-        username: iceServer.username,
-        credential: iceServer.credential,
-      });
-    });
-  });
-
-  return rtcIceServers;
-}
-
-export class WeriftPeerConnection extends Subscribed implements BasicPeerConnection {
-  public onAudioRtp = new Subject<RtpPacket>();
-  public onAudioRtcp = new Subject<RtcpPacket>();
-  public onVideoRtp = new Subject<RtpPacket>();
-  public onVideoRtcp = new Subject<RtcpPacket>();
-  public onIceCandidate = new Subject<RTCIceCandidate>();
-  public onConnectionState = new ReplaySubject<ConnectionState>(1);
-  public returnAudioTrack = new MediaStreamTrack({ kind: 'audio' });
-
-  #onRequestKeyFrame = new Subject<void>();
-
-  #cameraDevice: CameraDevice;
-  #logger: LoggerService;
-
-  #pc: RTCPeerConnection;
-
-  get #logPrefix(): string {
-    return `[${this.#cameraDevice.name}]`;
-  }
-
-  constructor(cameraDevice: CameraDevice, logger: LoggerService) {
-    super();
-
-    this.#cameraDevice = cameraDevice;
-    this.#logger = logger;
-
-    const pc = (this.#pc = this.#createPeerCoonection());
-
-    pc.addTransceiver(this.returnAudioTrack, {
-      direction: 'sendonly',
-    });
-
-    const audioTransceiver = pc.addTransceiver(this.returnAudioTrack, {
-      direction: 'recvonly',
-    });
-
-    audioTransceiver.onTrack.subscribe((track) => {
-      track.onReceiveRtp.subscribe((rtp) => {
-        this.onAudioRtp.next(rtp);
-      });
-
-      track.onReceiveRtcp.subscribe((rtcp) => {
-        this.onAudioRtcp.next(rtcp);
-      });
-
-      track.onReceiveRtp.once(() => {
-        this.#logger.debug(this.#logPrefix, 'received first audio packet');
-      });
-    });
-
-    const videoTransceiver = pc.addTransceiver('video', {
-      direction: 'recvonly',
-    });
-
-    videoTransceiver.onTrack.subscribe((track) => {
-      track.onReceiveRtp.subscribe((rtp) => {
-        this.onVideoRtp.next(rtp);
-      });
-
-      track.onReceiveRtcp.subscribe((rtcp) => {
-        this.onVideoRtcp.next(rtcp);
-      });
-
-      track.onReceiveRtp.once(() => {
-        this.#logger.debug(this.#logPrefix, 'received first video packet');
-
-        this.addSubscriptions(
-          merge(this.#onRequestKeyFrame, interval(4000)).subscribe(() => {
-            if (track.ssrc) {
-              videoTransceiver.receiver.sendRtcpPLI(track.ssrc).catch((error: any) => this.#logger.error(this.#logPrefix, error));
-            }
-          }),
-        );
-
-        this.requestKeyFrame();
-      });
-    });
-
-    this.#pc.onIceCandidate.subscribe((iceCandidate: RTCIceCandidate | undefined) => {
-      if (iceCandidate && this.#isLocalCandidate(iceCandidate.candidate)) {
-        this.onIceCandidate.next(iceCandidate);
-      }
-    });
-
-    pc.iceConnectionStateChange.subscribe(() => {
-      this.#logger.debug(this.#logPrefix, `iceConnectionStateChange: ${pc.iceConnectionState}`);
-
-      if (pc.iceConnectionState === 'closed') {
-        this.onConnectionState.next('closed');
-      }
-    });
-
-    pc.connectionStateChange.subscribe(() => {
-      this.#logger.debug(this.#logPrefix, `connectionStateChange: ${pc.connectionState}`);
-      this.onConnectionState.next(pc.connectionState);
-    });
-  }
-
-  public async createOffer(): Promise<RTCSessionDescription> {
-    const offer = await this.#pc.createOffer();
-    this.#pc.setLocalDescription(offer);
-    return offer;
-  }
-
-  public async acceptAnswer(answer: { type: 'answer'; sdp: string }): Promise<void> {
-    await this.#pc.setRemoteDescription(answer);
-  }
-
-  public async addIceCandidate(candidate: RTCIceCandidate): Promise<void> {
-    if (this.#isLocalCandidate(candidate.candidate)) {
-      this.#pc.addIceCandidate(candidate);
-    }
-  }
-
-  public requestKeyFrame(): void {
-    this.#onRequestKeyFrame.next();
-  }
-
-  public close(): void {
-    this.#pc.close().catch((error: any) => this.#logger.error(this.#logPrefix, error));
-    this.unsubscribe();
-  }
-
-  #createPeerCoonection(): RTCPeerConnection {
-    const prefferedVideoCodec = ['vp8', 'vp9', 'h264', 'av1'];
-    const prefferedAudioCodec = ['opus', 'g722', 'pcmu', 'pcma'];
-
-    const video: RTCRtpCodecParameters[] = [];
-    const audio: RTCRtpCodecParameters[] = [];
-
-    for (const codec of prefferedVideoCodec) {
-      switch (codec) {
-        case 'vp8':
-          video.push(
-            new RTCRtpCodecParameters({
-              mimeType: 'video/VP8',
-              clockRate: 90000,
-              payloadType: 100,
-              rtcpFeedback: [{ type: 'goog-remb' }, { type: 'ccm', parameter: 'fir' }, { type: 'nack' }, { type: 'nack', parameter: 'pli' }],
-            }),
-          );
-          break;
-        case 'vp9':
-          video.push(
-            new RTCRtpCodecParameters({
-              mimeType: 'video/VP9',
-              clockRate: 90000,
-              payloadType: 101,
-              rtcpFeedback: [{ type: 'goog-remb' }, { type: 'ccm', parameter: 'fir' }, { type: 'nack' }, { type: 'nack', parameter: 'pli' }],
-            }),
-          );
-          break;
-        case 'h264':
-          video.push(
-            new RTCRtpCodecParameters({
-              mimeType: 'video/H264',
-              clockRate: 90000,
-              rtcpFeedback: [{ type: 'transport-cc' }, { type: 'ccm', parameter: 'fir' }, { type: 'nack' }, { type: 'nack', parameter: 'pli' }, { type: 'goog-remb' }],
-              parameters: 'packetization-mode=1;profile-level-id=640029;level-asymmetry-allowed=1',
-            }),
-            new RTCRtpCodecParameters({
-              mimeType: 'video/H265',
-              clockRate: 90000,
-              rtcpFeedback: [{ type: 'transport-cc' }, { type: 'ccm', parameter: 'fir' }, { type: 'nack' }, { type: 'nack', parameter: 'pli' }, { type: 'goog-remb' }],
-              parameters: 'packetization-mode=1;profile-level-id=640029;level-asymmetry-allowed=1',
-            }),
-            new RTCRtpCodecParameters({
-              mimeType: 'video/rtx',
-              clockRate: 90000,
-            }),
-          );
-          break;
-        case 'av1':
-          video.push(
-            new RTCRtpCodecParameters({
-              mimeType: 'video/AV1',
-              clockRate: 90000,
-              payloadType: 102,
-              rtcpFeedback: [{ type: 'goog-remb' }, { type: 'ccm', parameter: 'fir' }, { type: 'nack' }, { type: 'nack', parameter: 'pli' }],
-            }),
-          );
-          break;
-      }
-    }
-
-    for (const codec of prefferedAudioCodec) {
-      switch (codec) {
-        case 'opus':
-          audio.push(
-            new RTCRtpCodecParameters({
-              mimeType: 'audio/opus',
-              clockRate: 48000,
-              payloadType: 101,
-              channels: 2,
-              parameters: 'minptime=10;useinbandfec=1',
-            }),
-          );
-          break;
-        case 'g722':
-          audio.push(
-            new RTCRtpCodecParameters({
-              mimeType: 'audio/G722',
-              clockRate: 8000,
-              channels: 1,
-              payloadType: 9,
-            }),
-          );
-          break;
-        case 'pcmu':
-          audio.push(
-            new RTCRtpCodecParameters({
-              mimeType: 'audio/PCMU',
-              clockRate: 8000,
-              channels: 1,
-              payloadType: 0,
-            }),
-          );
-          break;
-        case 'pcma':
-          audio.push(
-            new RTCRtpCodecParameters({
-              mimeType: 'audio/PCMA',
-              clockRate: 8000,
-              channels: 1,
-              payloadType: 8,
-            }),
-          );
-          break;
-      }
-    }
-
-    return new RTCPeerConnection({
-      codecs: {
-        video,
-        audio,
-      },
-      bundlePolicy: 'max-bundle',
-      iceAdditionalHostAddresses: ['127.0.0.1'],
-      iceUseIpv4: true,
-    });
-  }
-
-  #isLocalCandidate(candidateString: string): boolean {
-    const parts = candidateString.split(' ');
-    const ip = parts[4];
-    const type = parts[7];
-    return ip === '127.0.0.1' || type === 'host';
-  }
-}

package/src/camera/streaming/rtsp-session.ts

@@ -1,331 +0,0 @@
-import {
-  FfmpegProcess,
-  generateAudioStreamInfo,
-  generateSdp,
-  generateVideoStreamInfo,
-  reservePorts,
-  RtpSplitter,
-  transformAudioStreamInfoToFFmpegArgs,
-} from '@camera.ui/common/camera';
-import { getCleanSdp, SubscribedPublic } from '@camera.ui/common/utils';
-import { RTSPClient } from '@camera.ui/rtsp/rtspClient';
-import { ReplaySubject, Subject } from 'rxjs';
-import { take, tap } from 'rxjs/operators';
-import { RtpPacket } from 'werift';
-
-import type {
-  AudioStreamInfo,
-  CameraDeviceSource,
-  CameraInput,
-  FfmpegOptions,
-  LoggerService,
-  ReturnAudioFFmpegOptions,
-  RTSPConnectionOptions,
-  RTSPSession as RTSPSessionInterface,
-  VideoStreamInfo,
-} from '@camera.ui/types';
-import type { CameraDevice } from '../device.js';
-
-export class RTSPSession extends SubscribedPublic implements RTSPSessionInterface {
-  readonly onStarted = new ReplaySubject<void>();
-  readonly onError = new ReplaySubject<Error>();
-  readonly onCallEnded = new ReplaySubject<void>(1);
-  readonly onVideoRtp = new Subject<RtpPacket>();
-  readonly onAudioRtp = new Subject<RtpPacket>();
-
-  readonly #audioSplitter = new RtpSplitter();
-  readonly #videoSplitter = new RtpSplitter();
-  readonly #returnAudioSplitter = new RtpSplitter();
-
-  #hasEnded = false;
-  #cameraDevice: CameraDevice;
-  #logger: LoggerService;
-  #source: CameraDeviceSource;
-  #client: RTSPClient;
-
-  constructor(cameraDevice: CameraDevice, source: CameraInput, options?: RTSPConnectionOptions) {
-    super();
-
-    this.#cameraDevice = cameraDevice;
-    this.#logger = cameraDevice.logger;
-    this.#source = this.#cameraDevice.sources.find((s) => s.name === source.name)!;
-
-    const url = options?.url ?? this.#source.generateRTSPUrl();
-
-    this.#client = new RTSPClient(
-      { url },
-      {
-        // log: (...args) => this.#logger.log(...args),
-        error: (...args) => this.#logger.error(...args),
-        // debug: (...args) => this.#logger.trace(...args),
-        warn: (...args) => this.#logger.warn(...args),
-      },
-    );
-
-    this.#bindConnection();
-  }
-
-  public async startStream(): Promise<void> {
-    this.#hasEnded = false;
-    await this.#client.start();
-  }
-
-  public async startTranscoding(ffmpegOptions: FfmpegOptions): Promise<void> {
-    const transcodeAudioStream = ffmpegOptions.audio !== undefined && ffmpegOptions.audio.length > 0;
-    const transcodeVideoStream = ffmpegOptions.video !== undefined && ffmpegOptions.video.length > 0;
-
-    const loggerPrefix = ffmpegOptions.logPrefix
-      ? `[${ffmpegOptions.logPrefix}] `
-      : transcodeAudioStream && transcodeVideoStream
-        ? '[Transcoding] '
-        : transcodeAudioStream
-          ? '[Audio] '
-          : '[Video] ';
-
-    try {
-      if (!transcodeAudioStream && !transcodeVideoStream) {
-        throw new Error('No transcoding options provided! Can not start transcoding!');
-      }
-
-      await this.startStream();
-
-      const audioTrack = this.#client.getAudioTrack();
-      const videoTrack = this.#client.getVideoTrack();
-
-      if (!audioTrack && !videoTrack) {
-        throw new Error('No audio or video tracks found! Can not start transcoding!');
-      }
-
-      await this.#audioSplitter.prepare('udp4');
-      await this.#videoSplitter.prepare('udp4');
-
-      if (!this.#audioSplitter.port || !this.#videoSplitter.port) {
-        throw new Error('No ports available for audio and video splitter! Can not start transcoding!');
-      }
-
-      const videoPort = await this.#reservePort(1);
-      const audioPort = await this.#reservePort(1);
-
-      const videoStreamInfos = videoTrack ? generateVideoStreamInfo([videoTrack.mediaSource]) : [];
-      const audioStreamInfos = audioTrack ? generateAudioStreamInfo([audioTrack.mediaSource]) : [];
-      const parsedSdp = generateSdp(videoStreamInfos, audioStreamInfos);
-
-      const audioStream: AudioStreamInfo | undefined = audioStreamInfos[0];
-      const videoStream: VideoStreamInfo | undefined = videoStreamInfos[0];
-
-      const audioInputArgs = audioTrack && transcodeAudioStream && audioStream ? ['-acodec', audioStream.ffmpegCodec] : [];
-      const videoInputArgs = videoTrack && transcodeVideoStream && videoStream ? ['-vcodec', videoStream.ffmpegCodec] : [];
-
-      const audioOutputArgs = audioTrack && transcodeAudioStream ? (ffmpegOptions.audio ?? ['-acodec', audioStream.ffmpegCodec]) : [];
-      const videoOutputArgs = videoTrack && transcodeVideoStream ? (ffmpegOptions.video ?? ['-vcodec', videoStream.ffmpegCodec]) : [];
-
-      const inputArgs = ffmpegOptions.input ?? [];
-      const outputArgs = ffmpegOptions.output ?? [];
-
-      const inputSdp = getCleanSdp(parsedSdp, transcodeVideoStream, 'recvonly')
-        .replace(/m=audio \d+/, `m=audio ${audioPort}`)
-        .replace(/m=video \d+/, `m=video ${videoPort}`);
-
-      this.#logger.trace(loggerPrefix + 'Start transcoding with SDP:\n', inputSdp);
-
-      const ff = new FfmpegProcess({
-        ffmpegArgs: [
-          '-hide_banner',
-          '-protocol_whitelist',
-          'pipe,udp,rtp,file,crypto',
-          '-reorder_queue_size',
-          '0',
-          ...audioInputArgs,
-          ...videoInputArgs,
-          '-f',
-          'sdp',
-          ...inputArgs,
-          '-i',
-          'pipe:',
-          ...audioOutputArgs,
-          ...videoOutputArgs,
-          ...outputArgs,
-        ],
-        ffmpegPath: ffmpegOptions.ffmpegPath,
-        startedCallback: () => {
-          this.onStarted.next();
-        },
-        exitCallback: (code, signal) => {
-          if (code !== 0 && code !== 255) {
-            this.#callError(new Error(`FFmpeg exited with code ${code} and signal ${signal}`));
-          } else {
-            this.#callEnded();
-          }
-        },
-        logLabel: loggerPrefix,
-        logger: {
-          error: (...args) => this.#logger.error(...args),
-          trace: (...args) => this.#logger.trace(...args),
-        },
-      });
-
-      if (transcodeAudioStream) {
-        this.addSubscriptions(
-          this.onAudioRtp
-            .pipe(
-              tap((rtp) => {
-                return this.#audioSplitter.send(rtp.serialize(), {
-                  port: audioPort,
-                  address: '127.0.0.1',
-                });
-              }),
-            )
-            .subscribe(),
-        );
-      }
-
-      if (transcodeVideoStream) {
-        this.addSubscriptions(
-          this.onVideoRtp
-            .pipe(
-              tap((rtp) => {
-                return this.#videoSplitter.send(rtp.serialize(), {
-                  port: videoPort,
-                  address: '127.0.0.1',
-                });
-              }),
-            )
-            .subscribe(),
-        );
-      }
-
-      this.onCallEnded.pipe(take(1)).subscribe(() => ff.stop());
-
-      ff.writeStdin(inputSdp);
-    } catch (error: any) {
-      this.stop();
-      throw error;
-    }
-  }
-
-  public async transcodeReturnAudio(ffmpegOptions: ReturnAudioFFmpegOptions): Promise<void> {
-    if (this.#hasEnded) {
-      return;
-    }
-
-    const loggerPrefix = ffmpegOptions.logPrefix ? `[${ffmpegOptions.logPrefix}] ` : '[Return Audio] ';
-
-    try {
-      await this.startStream();
-
-      const returnAudioTrack = this.#client.getBackchannelTrack();
-      const returnAudioStreamInfos = returnAudioTrack ? generateAudioStreamInfo([returnAudioTrack.mediaSource]) : [];
-      const returnAudioStream = returnAudioStreamInfos[0];
-
-      if (!returnAudioStream) {
-        this.stop();
-        throw new Error('No return audio stream found! Can not start talkback!');
-      }
-
-      const audioOutForwarder = new RtpSplitter();
-      await audioOutForwarder.prepare('udp4');
-
-      if (!audioOutForwarder.port) {
-        this.stop();
-        throw new Error('No ports available for audio out splitter! Can not start talkback!');
-      }
-
-      audioOutForwarder.addMessageHandler(({ message }) => {
-        try {
-          const rtp = RtpPacket.deSerialize(message);
-          this.#client.sendAudioRtp(rtp);
-        } catch {
-          //
-        }
-
-        return null;
-      });
-
-      this.#logger.log(loggerPrefix.trim(), returnAudioStream);
-
-      const inputArgs = ffmpegOptions.input ?? [];
-      const returnAudioCodecs = transformAudioStreamInfoToFFmpegArgs(returnAudioStream);
-
-      const ff = new FfmpegProcess({
-        ffmpegArgs: [
-          '-hide_banner',
-          '-protocol_whitelist',
-          'pipe,udp,rtp,file,crypto',
-          '-re',
-          '-i',
-          ...inputArgs,
-          ...returnAudioCodecs,
-          '-flags',
-          '+global_header',
-          '-f',
-          'rtp',
-          `rtp://127.0.0.1:${audioOutForwarder.port}`,
-        ],
-        ffmpegPath: ffmpegOptions.ffmpegPath,
-        startedCallback: () => {
-          this.onStarted.next();
-        },
-        exitCallback: () => this.#callEnded(),
-        logLabel: loggerPrefix,
-        logger: {
-          error: (...args) => this.#logger.error(...args),
-          trace: (...args) => this.#logger.trace(...args),
-        },
-      });
-
-      this.onCallEnded.pipe(take(1)).subscribe(() => ff.stop());
-    } catch (error: any) {
-      this.stop();
-      throw error;
-    }
-  }
-
-  public stop(): void {
-    this.#callEnded();
-  }
-
-  public sendAudioPacket(rtp: RtpPacket): void {
-    if (this.#hasEnded) {
-      return;
-    }
-
-    this.#client.sendAudioRtp(rtp);
-  }
-
-  async #reservePort(bufferPorts = 0): Promise<number> {
-    const ports = await reservePorts({ count: bufferPorts + 1 });
-    return ports[0];
-  }
-
-  #bindConnection(): void {
-    this.#client.onVideoRtp.subscribe(this.onVideoRtp);
-    this.#client.onAudioRtp.subscribe(this.onAudioRtp);
-    this.#client.onClose.subscribe((error) => {
-      if (error) {
-        this.#callError(error);
-      } else {
-        this.#callEnded();
-      }
-    });
-  }
-
-  #callError(error: Error): void {
-    this.onError.next(error);
-    this.#callEnded();
-  }
-
-  #callEnded(): void {
-    if (this.#hasEnded) {
-      return;
-    }
-
-    this.#hasEnded = true;
-
-    this.unsubscribe();
-    this.#client.close();
-    this.onCallEnded.next();
-    this.#audioSplitter.close();
-    this.#videoSplitter.close();
-    this.#returnAudioSplitter.close();
-  }
-}