@camera.ui/server 0.0.50-internal.0 → 0.0.50-internal.1

package/src/camera/streaming/webrtc-connection.ts

@@ -1,260 +0,0 @@
-import { Subscribed } from '@camera.ui/common';
-import { ReplaySubject, Subject, fromEvent } from 'rxjs';
-import { concatMap } from 'rxjs/operators';
-import { WebSocket } from 'ws';
-
-import { WeriftPeerConnection } from './peer-connection.js';
-
-import type { BasicPeerConnection, CameraInput, LoggerService, WebRTCConnectionOptions } from '@camera.ui/types';
-import type { Observable } from 'rxjs';
-import type { RtpPacket } from 'werift';
-import type { CameraDevice } from '../device.js';
-
-export interface Go2RtcOfferMessage {
-  type: 'webrtc/offer';
-  value: Sdp;
-}
-
-export interface Go2RtcAnswerMessage {
-  type: 'webrtc/answer';
-  value: Sdp;
-}
-
-export interface Go2RtcIceCandidateMessage {
-  type: 'webrtc/candidate';
-  value: IceCandiate;
-}
-
-export type Sdp = string;
-
-export type IceCandiate = string;
-
-export interface ErrorMessage {
-  type: 'error';
-  value: string;
-}
-
-export type Go2RtcIncomingMessage = Go2RtcAnswerMessage | Go2RtcIceCandidateMessage | ErrorMessage;
-
-export type Go2RtcOutgoingMessage = Go2RtcOfferMessage | Go2RtcIceCandidateMessage;
-
-export class WebrtcConnection extends Subscribed {
-  public onCameraConnected!: ReplaySubject<void>;
-  public onCallAnswered!: ReplaySubject<string>;
-  public onCallEnded!: ReplaySubject<void>;
-  public onError!: ReplaySubject<void>;
-  public onMessage!: ReplaySubject<{ method: string }>;
-  public onWsOpen!: Observable<unknown>;
-  public onAudioRtp!: Subject<RtpPacket>;
-  public onVideoRtp!: Subject<RtpPacket>;
-
-  #onOfferSent!: ReplaySubject<void>;
-
-  #pc!: BasicPeerConnection;
-  #ws!: WebSocket;
-
-  #hasEnded = false;
-
-  #cameraDevice: CameraDevice;
-  #source: CameraInput;
-  #logger: LoggerService;
-  #options: WebRTCConnectionOptions;
-
-  get #logPrefix(): string {
-    return `[${this.#cameraDevice.name}]`;
-  }
-
-  constructor(cameraDevice: CameraDevice, source: CameraInput, logger: LoggerService, options: WebRTCConnectionOptions = {}) {
-    super();
-
-    this.#cameraDevice = cameraDevice;
-    this.#source = source;
-    this.#logger = logger;
-    this.#options = options;
-
-    this.#initializeConnection(this.#source);
-  }
-
-  public async upgradeConnection(source: CameraInput): Promise<void> {
-    // Close existing WebSocket and PeerConnection
-    this.unsubscribe();
-    this.#ws.close();
-    this.#pc.close();
-
-    // Reinitialize connection
-    this.#initializeConnection(source);
-  }
-
-  public sendAudioPacket(rtp: RtpPacket): void {
-    if (this.#hasEnded) {
-      return;
-    }
-
-    if (this.#pc instanceof WeriftPeerConnection) {
-      this.#pc.returnAudioTrack.writeRtp(rtp);
-    } else {
-      throw new Error('Cannot send audio packets to a custom peer connection implementation');
-    }
-  }
-
-  public stop(): void {
-    this.#callEnded();
-  }
-
-  public requestKeyFrame(): void {
-    this.#pc.requestKeyFrame?.();
-  }
-
-  #initializeConnection(source: CameraInput) {
-    const cameraSource = this.#cameraDevice.sources.find((s) => s.name === source.name);
-    const webRtcSource = cameraSource?.urls.ws.webrtc;
-
-    if (!webRtcSource) {
-      throw new Error(`Invalid source: ${source.name}`);
-    }
-
-    this.onCameraConnected = new ReplaySubject<void>(1);
-    this.onCallAnswered = new ReplaySubject<string>(1);
-    this.onCallEnded = new ReplaySubject<void>(1);
-    this.onError = new ReplaySubject<void>(1);
-    this.onMessage = new ReplaySubject<{ method: string }>();
-    this.#onOfferSent = new ReplaySubject<void>(1);
-
-    this.#ws = new WebSocket(webRtcSource, {
-      rejectUnauthorized: false,
-    });
-
-    if (this.#options.createPeerConnection) {
-      this.#pc = this.#options.createPeerConnection();
-      this.onAudioRtp = new Subject<RtpPacket>();
-      this.onVideoRtp = new Subject<RtpPacket>();
-    } else {
-      this.#pc = new WeriftPeerConnection(this.#cameraDevice, this.#logger);
-      this.onAudioRtp = (this.#pc as WeriftPeerConnection).onAudioRtp;
-      this.onVideoRtp = (this.#pc as WeriftPeerConnection).onVideoRtp;
-    }
-
-    this.onWsOpen = fromEvent(this.#ws, 'open');
-
-    this.#setupEventListeners();
-  }
-
-  #setupEventListeners() {
-    const onMessage = fromEvent(this.#ws, 'message');
-    const onError = fromEvent(this.#ws, 'error');
-    const onClose = fromEvent(this.#ws, 'close');
-
-    this.addSubscriptions(
-      onMessage
-        .pipe(
-          concatMap((event) => {
-            const message = JSON.parse((event as any).data);
-            this.onMessage.next(message);
-
-            return this.#handleMessage(message).catch((error: any) => {
-              this.onError.next(error);
-              throw error;
-            });
-          }),
-        )
-        .subscribe(),
-
-      onError.subscribe((error: any) => {
-        this.#logger.error(this.#logPrefix, error);
-        this.#callEnded();
-      }),
-
-      onClose.subscribe(() => {
-        this.#callEnded();
-      }),
-
-      this.#cameraDevice.onConnected.subscribe((state) => {
-        if (!state) {
-          this.#callEnded();
-        }
-      }),
-
-      this.#pc.onConnectionState.subscribe((state) => {
-        if (state === 'failed') {
-          this.#logger.error(this.#logPrefix, 'Stream connection failed');
-          this.#callEnded();
-        }
-
-        if (state === 'closed') {
-          this.#logger.debug(this.#logPrefix, 'Stream connection closed');
-          this.#callEnded();
-        }
-      }),
-
-      this.onError.subscribe((error: any) => {
-        this.#logger.error(this.#logPrefix, error);
-        this.#callEnded();
-      }),
-
-      this.onWsOpen.subscribe(() => {
-        this.#initiateCall().catch((error: any) => {
-          this.#logger.error(this.#logPrefix, error);
-          this.#callEnded();
-        });
-      }),
-
-      this.#pc.onIceCandidate.subscribe((iceCandidate) => {
-        this.#sendMessage({
-          type: 'webrtc/candidate',
-          value: iceCandidate.toJSON().candidate || '',
-        });
-      }),
-    );
-  }
-
-  async #initiateCall(): Promise<void> {
-    const { sdp } = await this.#pc.createOffer();
-
-    this.#sendMessage({
-      type: 'webrtc/offer',
-      value: sdp,
-    });
-
-    this.#onOfferSent.next();
-  }
-
-  async #handleMessage(message: Go2RtcIncomingMessage): Promise<void> {
-    switch (message.type) {
-      case 'webrtc/answer':
-        const sdp = message.value;
-        await this.#pc.acceptAnswer({ type: 'answer', sdp });
-        this.onCallAnswered.next(sdp);
-        return;
-      case 'webrtc/candidate':
-        await this.#pc.addIceCandidate({
-          candidate: message.value,
-          sdpMid: '0',
-        });
-        return;
-      case 'error':
-        throw new Error(message.value);
-      default:
-        throw new Error(message);
-    }
-  }
-
-  #sendMessage(message: Go2RtcOutgoingMessage): void {
-    if (this.#hasEnded) {
-      return;
-    }
-
-    this.#ws.send(JSON.stringify(message));
-  }
-
-  #callEnded(): void {
-    if (this.#hasEnded) {
-      return;
-    }
-
-    this.#hasEnded = true;
-    this.unsubscribe();
-    this.onCallEnded.next();
-    this.#pc.close();
-    this.#ws.close();
-  }
-}

package/src/camera/streaming/werift-session.ts

@@ -1,317 +0,0 @@
-import { FfmpegProcess, reservePorts, RtpSplitter, transformAudioStreamInfoToFFmpegArgs } from '@camera.ui/common/camera';
-import { getCleanSdp, isWebRtcCompatibleAudioCodec, isWebRtcCompatibleVideoCodec, SubscribedPublic } from '@camera.ui/common/utils';
-import { firstValueFrom, ReplaySubject, Subject } from 'rxjs';
-import { take, tap } from 'rxjs/operators';
-import { RtpPacket } from 'werift';
-
-import { WebrtcConnection } from './webrtc-connection.js';
-
-import type {
-  CameraInput,
-  CameraSource,
-  FfmpegOptions,
-  LoggerService,
-  ProbeStream,
-  ReturnAudioFFmpegOptions,
-  WebRTCConnectionOptions,
-  WeriftSession as WeriftSessionInterface,
-} from '@camera.ui/types';
-import type { CameraDevice } from '../device.js';
-
-export class WeriftSession extends SubscribedPublic implements WeriftSessionInterface {
-  readonly onStarted = new ReplaySubject<void>();
-  readonly onError = new ReplaySubject<Error>();
-  readonly onCallEnded = new ReplaySubject<void>(1);
-  readonly onVideoRtp = new Subject<RtpPacket>();
-  readonly onAudioRtp = new Subject<RtpPacket>();
-
-  readonly #audioSplitter = new RtpSplitter();
-  readonly #videoSplitter = new RtpSplitter();
-  readonly #returnAudioSplitter = new RtpSplitter();
-
-  #hasEnded = false;
-  #cameraDevice: CameraDevice;
-  #logger: LoggerService;
-  #source: CameraSource;
-  #webrtcConnection: WebrtcConnection;
-
-  constructor(cameraDevice: CameraDevice, source: CameraInput, options: WebRTCConnectionOptions = {}) {
-    super();
-
-    this.#cameraDevice = cameraDevice;
-    this.#logger = cameraDevice.logger;
-    this.#source = this.#cameraDevice.sources.find((s) => s.name === source.name)!;
-    this.#webrtcConnection = new WebrtcConnection(this.#cameraDevice, source, this.#logger, options);
-
-    this.#bindConnection();
-  }
-
-  public async startTranscoding(ffmpegOptions: FfmpegOptions, streamProbe?: ProbeStream): Promise<void> {
-    if (this.#hasEnded) {
-      return;
-    }
-
-    const transcodeAudioStream = ffmpegOptions.audio !== undefined && ffmpegOptions.audio.length > 0;
-    const transcodeVideoStream = ffmpegOptions.video !== undefined && ffmpegOptions.video.length > 0;
-    const loggerPrefix = (ffmpegOptions.logPrefix ?? (transcodeAudioStream && transcodeVideoStream)) ? 'Transcoding' : transcodeAudioStream ? 'Audio' : 'Video';
-
-    try {
-      if (!transcodeAudioStream && !transcodeVideoStream) {
-        throw new Error('No transcoding options provided! Can not start transcoding!');
-      }
-
-      const sdp = await Promise.race([firstValueFrom(this.#webrtcConnection.onCallAnswered), firstValueFrom(this.onCallEnded)]);
-
-      if (!sdp) {
-        throw new Error('No SDP received! Can not start transcoding!');
-      }
-
-      const probeStream = streamProbe ?? (await this.#source.probeStream({ video: true, audio: ['pcma', 'opus'] }));
-      if (!probeStream) {
-        throw new Error('Could not probe stream! Can not start transcoding!');
-      }
-
-      const audioStream = probeStream.audio.filter((audio) => audio.direction === 'sendonly' && isWebRtcCompatibleAudioCodec(audio.codec))[0];
-      const videoStream = probeStream.video.filter((video) => video.direction === 'sendonly' && isWebRtcCompatibleVideoCodec(video.codec))[0];
-
-      if (!audioStream && !videoStream) {
-        throw new Error('No sendonly audio or video streams found! Can not start transcoding!');
-      }
-
-      await this.#audioSplitter.prepare('udp4');
-      await this.#videoSplitter.prepare('udp4');
-
-      if (!this.#audioSplitter.port || !this.#videoSplitter.port) {
-        throw new Error('No ports available for audio or video splitter! Can not start transcoding!');
-      }
-
-      const videoPort = await this.#reservePort(1);
-      const audioPort = await this.#reservePort(1);
-
-      const audioInputArgs = audioStream && transcodeAudioStream && audioStream ? ['-acodec', audioStream.ffmpegCodec] : [];
-      const videoInputArgs = videoStream && transcodeVideoStream && videoStream ? ['-vcodec', videoStream.ffmpegCodec] : [];
-
-      const audioOutputArgs = audioStream && transcodeAudioStream ? (ffmpegOptions.audio ?? ['-acodec', audioStream.ffmpegCodec]) : [];
-      const videoOutputArgs = videoStream && transcodeVideoStream ? (ffmpegOptions.video ?? ['-vcodec', videoStream.ffmpegCodec]) : [];
-
-      const inputArgs = ffmpegOptions.input ?? [];
-      const outputArgs = ffmpegOptions.output ?? [];
-
-      const inputSdp = getCleanSdp(probeStream.sdp, transcodeVideoStream, 'sendonly', true)
-        .replace(/m=audio \d+/, `m=audio ${audioPort}`)
-        .replace(/m=video \d+/, `m=video ${videoPort}`);
-
-      this.#logger.trace('Start transcoding with SDP:\n', inputSdp);
-
-      const ff = new FfmpegProcess({
-        ffmpegArgs: [
-          '-hide_banner',
-          '-protocol_whitelist',
-          'pipe,udp,rtp,file,crypto',
-          '-reorder_queue_size',
-          '0',
-          ...audioInputArgs,
-          ...videoInputArgs,
-          '-f',
-          'sdp',
-          ...inputArgs,
-          '-i',
-          'pipe:',
-          ...audioOutputArgs,
-          ...videoOutputArgs,
-          ...outputArgs,
-        ],
-        ffmpegPath: ffmpegOptions.ffmpegPath,
-        startedCallback: () => {
-          this.onStarted.next();
-        },
-        exitCallback: (code, signal) => {
-          if (code !== 0 && code !== 255) {
-            this.#callError(new Error(`FFmpeg exited with code ${code} and signal ${signal}`));
-          } else {
-            this.#callEnded();
-          }
-        },
-        logLabel: loggerPrefix,
-        logger: {
-          error: (...args) => this.#logger.error(...args),
-          trace: (...args) => this.#logger.trace(...args),
-        },
-      });
-
-      if (transcodeAudioStream) {
-        this.addSubscriptions(
-          this.onAudioRtp
-            .pipe(
-              tap((rtp) => {
-                return this.#audioSplitter.send(rtp.serialize(), {
-                  port: audioPort,
-                  address: '127.0.0.1',
-                });
-              }),
-            )
-            .subscribe(),
-        );
-      }
-
-      if (transcodeVideoStream) {
-        this.addSubscriptions(
-          this.onVideoRtp
-            .pipe(
-              tap((rtp) => {
-                return this.#videoSplitter.send(rtp.serialize(), {
-                  port: videoPort,
-                  address: '127.0.0.1',
-                });
-              }),
-            )
-            .subscribe(),
-        );
-      }
-
-      this.onCallEnded.pipe(take(1)).subscribe(() => ff.stop());
-
-      ff.writeStdin(inputSdp);
-
-      this.requestKeyFrame();
-    } catch (error: any) {
-      this.stop();
-      throw error;
-    }
-  }
-
-  public async transcodeReturnAudio(ffmpegOptions: ReturnAudioFFmpegOptions, streamProbe?: ProbeStream): Promise<void> {
-    if (this.#hasEnded) {
-      return;
-    }
-
-    const loggerPrefix = ffmpegOptions.logPrefix ?? 'Return Audio';
-
-    try {
-      const probeStream = streamProbe ?? (await this.#source.probeStream({ microphone: true }));
-      if (!probeStream) {
-        this.stop();
-        throw new Error('Could not probe stream! Can not start transcoding!');
-      }
-
-      const audioStreams = probeStream.audio.filter((audio) => audio.direction === 'recvonly');
-      const returnAudioStream = audioStreams[0];
-
-      if (!returnAudioStream) {
-        this.stop();
-        throw new Error('No recvonly audio stream found! Can not start transcoding!');
-      }
-
-      const audioOutForwarder = new RtpSplitter();
-      await audioOutForwarder.prepare('udp4');
-
-      if (!audioOutForwarder.port) {
-        this.stop();
-        throw new Error('No ports available for audio out splitter! Can not start transcoding!');
-      }
-
-      audioOutForwarder.addMessageHandler(({ message }) => {
-        const rtp = RtpPacket.deSerialize(message);
-        this.#webrtcConnection.sendAudioPacket(rtp);
-        return null;
-      });
-
-      const inputArgs = ffmpegOptions.input ?? [];
-      const returnAudioCodecs = transformAudioStreamInfoToFFmpegArgs(returnAudioStream);
-
-      const ff = new FfmpegProcess({
-        ffmpegArgs: [
-          '-hide_banner',
-          '-protocol_whitelist',
-          'pipe,udp,rtp,file,crypto',
-          '-re',
-          '-i',
-          ...inputArgs,
-          ...returnAudioCodecs,
-          '-flags',
-          '+global_header',
-          '-f',
-          'rtp',
-          `rtp://127.0.0.1:${audioOutForwarder.port}`,
-        ],
-        ffmpegPath: ffmpegOptions.ffmpegPath,
-        startedCallback: () => {
-          this.onStarted.next();
-        },
-        exitCallback: (code, signal) => {
-          if (code !== 0 && code !== 255) {
-            this.#callError(new Error(`FFmpeg exited with code ${code} and signal ${signal}`));
-          } else {
-            this.#callEnded();
-          }
-        },
-        logLabel: loggerPrefix,
-        logger: {
-          error: (...args) => this.#logger.error(...args),
-          trace: (...args) => this.#logger.trace(...args),
-        },
-      });
-
-      this.onCallEnded.pipe(take(1)).subscribe(() => ff.stop());
-    } catch (error: any) {
-      this.stop();
-      throw error;
-    }
-  }
-
-  public stop(): void {
-    this.#callEnded();
-  }
-
-  public sendAudioPacket(rtp: RtpPacket | Buffer): void {
-    if (this.#hasEnded) {
-      return;
-    }
-
-    this.#webrtcConnection.sendAudioPacket(Buffer.isBuffer(rtp) ? RtpPacket.deSerialize(rtp) : rtp);
-  }
-
-  public requestKeyFrame(): void {
-    this.#webrtcConnection.requestKeyFrame();
-  }
-
-  async #reservePort(bufferPorts = 0): Promise<number> {
-    const ports = await reservePorts({ count: bufferPorts + 1 });
-    return ports[0];
-  }
-
-  #handleVideo(rtp: RtpPacket): void {
-    this.onVideoRtp.next(rtp);
-  }
-
-  #handleAudio(rtp: RtpPacket): void {
-    this.onAudioRtp.next(rtp);
-  }
-
-  #bindConnection(): void {
-    this.addSubscriptions(
-      this.#webrtcConnection.onVideoRtp.subscribe(this.#handleVideo.bind(this)),
-      this.#webrtcConnection.onAudioRtp.subscribe(this.#handleAudio.bind(this)),
-      this.#webrtcConnection.onCallEnded.subscribe(() => this.#callEnded()),
-    );
-  }
-
-  #callError(error: Error): void {
-    this.onError.next(error);
-  }
-
-  #callEnded(): void {
-    if (this.#hasEnded) {
-      return;
-    }
-
-    this.#hasEnded = true;
-
-    this.unsubscribe();
-    this.onCallEnded.next();
-    this.#webrtcConnection.stop();
-    this.#audioSplitter.close();
-    this.#videoSplitter.close();
-    this.#returnAudioSplitter.close();
-  }
-}