@livedigital/client 1.9.0 → 1.10.0

package/src/engine/PeerProducer.ts

@@ -1,5 +1,5 @@
 import { MediaKind } from 'mediasoup-client/lib/RtpParameters';
-import { ProducerData } from '../types/common';
+import { ProducerData, TrackLabel } from '../types/common';
 
 class PeerProducer {
   readonly id: string;
@@ -8,13 +8,19 @@ class PeerProducer {
 
   readonly peerId: string;
 
+  readonly label: TrackLabel;
+
   public score = 10;
 
   constructor(params: ProducerData) {
-    const { id, kind, peerId } = params;
+    const {
+      id, kind, peerId, label,
+    } = params;
+
     this.id = id;
     this.kind = kind;
     this.peerId = peerId;
+    this.label = label;
   }
 }
 

package/src/engine/index.ts

@@ -1,12 +1,18 @@
-import { RtpCapabilities, RtpEncodingParameters } from 'mediasoup-client/lib/RtpParameters';
+import { RtpCapabilities } from 'mediasoup-client/lib/RtpParameters';
+import { VIDEO_CONSTRAINS } from 'constants/videoConstrains';
 import {
   PeerResponse,
   JoinChannelParams,
   SocketResponse,
-  PublishAudioOptions,
-  PublishVideoOptions,
+  CreateCameraVideoTrackOptions,
+  CreateMicrophoneAudioTrackOptions,
+  CreateScreenVideoTrackOptions,
+  CreateScreenAudioTrackOptions,
+  Track,
+  TrackLabel,
+  EndTrackPayload,
+  StartTrackPayload,
 } from '../types/common';
-import { WEBCAM_SIMULCAST_ENCODINGS } from '../constants/simulcastEncodings';
 import EnhancedEventEmitter from '../EnhancedEventEmitter';
 import System from './system';
 import Peer from './Peer';
@@ -84,8 +90,7 @@ class Engine {
 
       this.network.socket.disconnect();
       this.peersRepository.clear();
-      await this.unpublishAudio();
-      await this.unpublishVideo();
+      await this.unpublish();
       await this.network.closeSendTransport();
       await this.network.closeReceiveTransport();
 
@@ -154,253 +159,217 @@ class Engine {
     this.peersRepository.delete(peerId);
   }
 
-  public async publishVideo(track?: MediaStreamTrack, options?: PublishVideoOptions): Promise<void> {
-    if (!track) {
-      if (this.system.isEnableVideoDevicesLock) {
-        this.logger.warn('publishVideo()', { message: 'Video devices is locked' });
-        return;
-      }
-
-      if (!this.initialized) {
-        throw new Error('First you need to connect');
-      }
+  public async pause(track: Track): Promise<void> {
+    const producer = track.getProducer();
+    if (!producer) {
+      this.logger.warn('pause()', { message: 'Track is not published', kind: track.kind });
+      return;
+    }
 
-      if (!this.media.mediasoupDevice.canProduce('video')) {
-        throw new Error('Cant produce video');
-      }
+    try {
+      await this.pauseRemoteProducer(producer.id);
+      producer.pause();
+      this.logger.debug('pause()', { kind: track.kind });
+    } catch (error) {
+      this.logger.error('pause()', { error, kind: track.kind });
+      throw new Error('Can`t pause track');
+    }
+  }
 
-      if (!this.system.availableVideoDevices.length) {
-        throw new Error('No available media devices');
-      }
+  public async resume(track: Track): Promise<void> {
+    const producer = track.getProducer();
+    if (!producer) {
+      this.logger.warn('resume()', { message: 'Track is not published' });
+      return;
+    }
 
-      await this.system.startVideoStream(options?.videoTrackConstraints);
-      this.system.setIsEnableVideoDevicesLock(true);
+    try {
+      await this.resumeRemoteProducer(producer.id);
+      producer.resume();
+      this.logger.debug('resume()', { kind: track.kind });
+    } catch (error) {
+      this.logger.error('resume()', { error, kind: track.kind });
+      throw new Error('Can`t resume track');
     }
+  }
 
+  async createCameraVideoTrack(options?: CreateCameraVideoTrackOptions): Promise<Track | null> {
     try {
-      if (this.media.videoProducer) {
-        this.media.closeVideoProducer();
-        await this.network.closeRemoteProducer(this.media.videoProducer.id);
-      }
+      const track = await this.media.createUserMediaTrack({
+        audio: false,
+        video: {
+          deviceId: options?.deviceId,
+          frameRate: options?.frameRate || 15,
+          width: options?.width || VIDEO_CONSTRAINS.hd.width,
+          height: options?.height || VIDEO_CONSTRAINS.hd.height,
+        },
+      });
 
-      const trackToProduce = track || this.system.videoStream?.getVideoTracks().pop();
-      if (!trackToProduce) {
-        this.logger.warn('publishVideo()', { message: 'No media tracks to publish' });
-        return;
+      track.setLabel(TrackLabel.Camera);
+      if (options?.encoderConfig) {
+        track.setEncoderConfig(options.encoderConfig);
       }
 
-      const codec = this.media.getCodec();
-      const encodings = options?.encodings || WEBCAM_SIMULCAST_ENCODINGS;
-      const codecOptions = options?.codecOptions || {
-        videoGoogleStartBitrate: 1000,
-      };
+      this.logger.debug('createCameraVideoTrack()', { options, track });
+      return track;
+    } catch (error) {
+      this.logger.error('createCameraVideoTrack()', { error, options });
+      return null;
+    }
+  }
 
-      const videoProducer = await this.network.sendTransport?.produce({
-        track: trackToProduce,
-        encodings,
-        codecOptions,
-        codec,
-        stopTracks: true,
-        disableTrackOnPause: false,
-        zeroRtpOnPause: true,
-        appData: {
-          peerId: this.mySocketId,
+  async createMicrophoneAudioTrack(options?: CreateMicrophoneAudioTrackOptions): Promise<Track | null> {
+    try {
+      const track = await this.media.createUserMediaTrack({
+        video: false,
+        audio: {
+          deviceId: options?.deviceId,
         },
       });
 
-      if (!videoProducer) {
-        return;
+      track.setLabel(TrackLabel.Microphone);
+      if (options?.encoderConfig) {
+        track.setEncoderConfig(options.encoderConfig);
       }
 
-      this.media.setVideoProducer(videoProducer);
-      const myPeer = this.peersRepository.get(<string> this.mySocketId);
-      myPeer?.observer.safeEmit(PEER_EVENTS.trackStart, { kind: videoProducer.kind, track: trackToProduce });
-      this.logger.debug('publishVideo()', { codec });
+      this.logger.debug('createMicrophoneAudioTrack()', { options, track });
+      return track;
     } catch (error) {
-      this.logger.error('publishVideo()', { error });
-      throw new Error('Enable video error');
-    } finally {
-      this.system.setIsEnableVideoDevicesLock(false);
+      this.logger.error('createMicrophoneAudioTrack()', { error, options });
+      return null;
     }
   }
 
-  public async publishAudio(track?: MediaStreamTrack, options?: PublishAudioOptions): Promise<void> {
-    if (!track) {
-      if (this.system.isEnableAudioDevicesLock) {
-        this.logger.warn('publishAudio()', { message: 'Audio devices is locked' });
-        return;
-      }
-
-      if (!this.initialized) {
-        throw new Error('First you need to connect');
-      }
+  async createScreenVideoTrack(options?: CreateScreenVideoTrackOptions): Promise<Track | null> {
+    try {
+      const track = await this.media.createDisplayMediaTrack({
+        audio: false,
+        video: {
+          frameRate: options?.frameRate || 30,
+          width: options?.width || VIDEO_CONSTRAINS.fullhd.width,
+          height: options?.height || VIDEO_CONSTRAINS.fullhd.height,
+        },
+      });
 
-      if (!this.system.availableAudioDevices.length) {
-        throw new Error('No available media devices');
-      }
+      track.mediaStreamTrack.addEventListener('ended', () => {
+        this.unpublish(track);
+      });
 
-      if (!this.media.mediasoupDevice.canProduce('audio')) {
-        throw new Error('Cant produce audio');
+      track.setLabel(TrackLabel.ScreenVideo);
+      if (options?.encoderConfig) {
+        track.setEncoderConfig(options.encoderConfig);
       }
 
-      await this.system.startAudioStream(options?.audioTrackConstraints);
-      this.system.setIsEnableAudioDevicesLock(true);
+      this.logger.debug('createScreenVideoTrack()', { options, track });
+      return track;
+    } catch (error) {
+      this.logger.error('createScreenVideoTrack()', { error, options });
+      return null;
     }
+  }
 
-    const codecOptions = options?.codecOptions || {
-      opusFec: true,
-    };
-
+  async createScreenAudioTrack(options?: CreateScreenAudioTrackOptions): Promise<Track | null> {
     try {
-      if (this.media.audioProducer) {
-        this.media.closeAudioProducer();
-        await this.network.closeRemoteProducer(this.media.audioProducer.id);
-      }
+      const track = await this.media.createDisplayMediaTrack({
+        video: false,
+        audio: true,
+      });
 
-      const trackToProduce = track || this.system.audioStream?.getAudioTracks().pop();
-      if (!trackToProduce) {
-        this.logger.warn('publishAudio()', { message: 'No media tracks to publish' });
-        return;
+      track.mediaStreamTrack.addEventListener('ended', () => {
+        this.unpublish(track);
+      });
+
+      track.setLabel(TrackLabel.ScreenAudio);
+      if (options?.encoderConfig) {
+        track.setEncoderConfig(options.encoderConfig);
       }
 
-      const audioProducer = await this.network.sendTransport?.produce({
-        track: trackToProduce,
+      this.logger.debug('createScreenAudioTrack()', { options, track });
+      return track;
+    } catch (error) {
+      this.logger.error('createScreenAudioTrack()', { error, options });
+      return null;
+    }
+  }
+
+  public async publish(tracks: Track | Track[]): Promise<void> {
+    const tracksToPublish = Array.isArray(tracks) ? tracks : [tracks];
+    const actions = tracksToPublish.map(async (track) => {
+      const encodings = 'getEncodings' in track
+        ? track.getEncodings()
+        : undefined;
+
+      const codecOptions = 'getCodecOptions' in track
+        ? track.getCodecOptions()
+        : undefined;
+
+      const producer = await this.network.sendTransport?.produce({
+        track: track.mediaStreamTrack,
+        encodings,
         codecOptions,
+        codec: this.media.getTrackCodec(track),
         stopTracks: true,
-        disableTrackOnPause: false,
+        disableTrackOnPause: true,
         zeroRtpOnPause: true,
         appData: {
           peerId: this.mySocketId,
+          label: track.getLabel(),
         },
       });
 
-      if (!audioProducer) {
+      if (!producer) {
         return;
       }
 
-      this.media.setAudioProducer(audioProducer);
+      track.setProducer(producer);
       const myPeer = this.peersRepository.get(<string> this.mySocketId);
-      myPeer?.observer.safeEmit(PEER_EVENTS.trackStart, { kind: 'audio', track: trackToProduce });
-      this.logger.debug('publishAudio()');
-    } catch (error) {
-      this.logger.error('publishAudio()', { error });
-      throw new Error('Enable audio error');
-    } finally {
-      this.system.setIsEnableAudioDevicesLock(false);
-    }
-  }
+      myPeer?.observer.safeEmit(PEER_EVENTS.trackStart, {
+        producerId: producer.id,
+        track: track.mediaStreamTrack,
+        label: track.getLabel(),
+      } as StartTrackPayload);
 
-  public async unpublishVideo(): Promise<void> {
-    if (!this.media.videoProducer) {
-      this.logger.warn('unpublishVideo()', { message: 'No published video' });
-      return;
-    }
+      this.logger.debug('publish()', { track });
+    });
 
     try {
-      if (this.network.socket.connection?.connected) {
-        await this.network.closeRemoteProducer(this.media.videoProducer.id);
-      }
-
-      this.media.closeVideoProducer();
-      this.system.stopVideoStream();
-      const myPeer = this.peersRepository.get(<string> this.mySocketId);
-      if (myPeer) {
-        myPeer.observer.safeEmit(PEER_EVENTS.trackEnd, { kind: 'video' });
-      }
-
-      this.logger.debug('unpublishVideo()');
+      await Promise.all(actions);
     } catch (error) {
-      this.logger.error('unpublishVideo()', { error });
-      throw new Error('Error unpublish video');
+      this.logger.error('publish()', { error });
     }
   }
 
-  public async unpublishAudio(): Promise<void> {
-    if (!this.media.audioProducer) {
-      this.logger.warn('unpublishAudio()', { message: 'No published audio' });
-      return;
+  public async unpublish(tracks?: Track | Track[]): Promise<void> {
+    let tracksToUnpublish = [];
+    if (!tracks) {
+      tracksToUnpublish = this.media.getAllTracks();
+    } else {
+      tracksToUnpublish = Array.isArray(tracks) ? tracks : [tracks];
     }
 
-    try {
-      if (this.network.socket.connection?.connected) {
-        await this.network.closeRemoteProducer(this.media.audioProducer.id);
+    const actions = tracksToUnpublish.map(async (track) => {
+      const producer = track.getProducer();
+      if (producer && this.network.socket.connection?.connected) {
+        await this.network.closeRemoteProducer(producer.id);
       }
 
-      this.media.closeAudioProducer();
-      this.system.stopAudioStream();
       const myPeer = this.peersRepository.get(<string> this.mySocketId);
       if (myPeer) {
-        myPeer.observer.safeEmit(PEER_EVENTS.trackEnd, { kind: 'audio' });
+        myPeer.observer.safeEmit(PEER_EVENTS.trackEnd, {
+          producerId: track.getProducer()?.id,
+          kind: track.getProducer()?.kind,
+          label: track.getLabel(),
+        } as EndTrackPayload);
       }
 
-      this.logger.debug('unpublishAudio()');
-    } catch (error) {
-      this.logger.error('unpublishAudio()', { error });
-      throw new Error('Error unpublish audio');
-    }
-  }
-
-  public async pauseVideo(): Promise<void> {
-    if (!this.media.videoProducer) {
-      this.logger.warn('pauseVideo()', { message: 'No published video' });
-      return;
-    }
-
-    try {
-      await this.pauseRemoteProducer(this.media.videoProducer.id);
-      this.media.videoProducer.pause();
-      this.logger.debug('pauseVideo()');
-    } catch (error) {
-      this.logger.error('pauseVideo()', { error });
-      throw new Error('Can`t pause video');
-    }
-  }
-
-  public async resumeVideo(): Promise<void> {
-    if (!this.media.videoProducer) {
-      this.logger.warn('resumeVideo()', { message: 'No published video' });
-      return;
-    }
-
-    try {
-      await this.resumeRemoteProducer(this.media.videoProducer.id);
-      this.media.videoProducer.resume();
-      this.logger.debug('resumeVideo()');
-    } catch (error) {
-      this.logger.error('resumeVideo()', { error });
-      throw new Error('Can`t resume video');
-    }
-  }
-
-  public async pauseAudio(): Promise<void> {
-    if (!this.media.audioProducer) {
-      this.logger.warn('pauseAudio()', { message: 'No published audio' });
-      return;
-    }
-
-    try {
-      await this.pauseRemoteProducer(this.media.audioProducer.id);
-      this.media.audioProducer.pause();
-      this.logger.debug('pauseAudio()');
-    } catch (error) {
-      this.logger.error('pauseAudio()', { error });
-      throw new Error('Can`t pause audio');
-    }
-  }
-
-  public async resumeAudio(): Promise<void> {
-    if (!this.media.audioProducer) {
-      this.logger.warn('resumeAudio()', { message: 'No published audio' });
-      return;
-    }
+      await this.media.deleteTrack(track);
+      this.logger.debug('unpublish()', { track });
+    });
 
     try {
-      await this.resumeRemoteProducer(this.media.audioProducer.id);
-      this.media.audioProducer.resume();
-      this.logger.debug('resumeAudio()');
+      await Promise.all(actions);
     } catch (error) {
-      this.logger.error('resumeAudio()', { error });
-      throw new Error('Can`t resume audio');
+      this.logger.error('unpublish()', { error });
     }
   }
 }

package/src/engine/media/Consumer.ts

@@ -4,10 +4,6 @@ class Consumer extends MediasoupConsumer {
   public score = 10;
 
   public producerScore = 10;
-
-  constructor(consumer: MediasoupConsumer) {
-    super(consumer);
-  }
 }
 
 export default Consumer;

package/src/engine/media/VideoConsumer.ts

@@ -1,21 +1,12 @@
-import Consumer from './Consumer';
 import { Consumer as MediasoupConsumer } from 'mediasoup-client/lib/types';
 import { parseScalabilityMode } from 'mediasoup-client';
+import Consumer from './Consumer';
 
 class VideoConsumer extends Consumer {
   public spatialLayers = 0;
 
   public temporalLayers = 0;
 
-  private currentSpatialLayer = 0;
-
-  private currentTemporalLayer = 0;
-
-  currentPreferredLayers: { spatialLayer?: number, temporalLayer?: number } = {
-    spatialLayer: undefined,
-    temporalLayer: undefined,
-  };
-
   constructor(consumer: MediasoupConsumer) {
     super(consumer);
     this.parseScalabilityMode();
@@ -29,14 +20,6 @@ class VideoConsumer extends Consumer {
       this.temporalLayers = temporalLayers;
     }
   }
-
-  setCurrentSpatialLayer(currentSpatialLayer: number): void {
-    this.currentSpatialLayer = currentSpatialLayer;
-  }
-
-  setCurrentTemporalLayer(currentTemporalLayer: number): void {
-    this.currentTemporalLayer = currentTemporalLayer;
-  }
 }
 
 export default VideoConsumer;

package/src/engine/media/index.ts

@@ -1,19 +1,15 @@
 import { Device } from 'mediasoup-client';
-import { Producer } from 'mediasoup-client/lib/Producer';
 import { RtpCapabilities, RtpCodecCapability } from 'mediasoup-client/lib/RtpParameters';
+import VideoTrack from './tracks/VideoTrack';
+import AudioTrack from './tracks/AudioTrack';
+import { Track } from '../../types/common';
 
 class Media {
   public mediasoupDevice: Device;
 
-  public audioProducer?: Producer;
-
-  public videoProducer?: Producer;
-
   public isDeviceLoaded = false;
 
-  public isVideoDisabled = false;
-
-  public isAudioDisabled = false;
+  private tracks: Map<string, Track> = new Map();
 
   constructor() {
     this.mediasoupDevice = new Device();
@@ -26,46 +22,50 @@ class Media {
     }
   }
 
-  getCodec(): RtpCodecCapability | undefined {
+  getTrackCodec(track: Track): RtpCodecCapability | undefined {
+    if (!(track instanceof VideoTrack)) {
+      return undefined;
+    }
+
     if (!this.mediasoupDevice.rtpCapabilities?.codecs) {
       return undefined;
     }
 
-    const preferredCodec = 'h264';
     return this.mediasoupDevice
       .rtpCapabilities
       .codecs
-      .find((c) => c.mimeType.toLowerCase() === `video/${preferredCodec}`);
+      .find((c) => c.mimeType.toLowerCase() === `video/${track.getPreferredCodec()}`);
   }
 
-  setVideoProducer(producer: Producer): void {
-    this.videoProducer = producer;
-    this.isVideoDisabled = false;
-  }
+  private createTrack(stream: MediaStream): Track {
+    const mediaStreamTrack = stream.getTracks()[0];
+
+    const track = mediaStreamTrack.kind === 'audio'
+      ? new AudioTrack(mediaStreamTrack)
+      : new VideoTrack(mediaStreamTrack);
 
-  setAudioProducer(producer: Producer): void {
-    this.audioProducer = producer;
-    this.isAudioDisabled = false;
+    this.tracks.set(track.id, track);
+    return track;
   }
 
-  closeVideoProducer(): void {
-    if (!this.videoProducer) {
-      return;
-    }
+  async createUserMediaTrack(constraints: MediaStreamConstraints): Promise<Track> {
+    const stream = await navigator.mediaDevices.getUserMedia(constraints);
+    return this.createTrack(stream);
+  }
 
-    this.videoProducer.close();
-    this.videoProducer = undefined;
-    this.isVideoDisabled = true;
+  async createDisplayMediaTrack(constraints: MediaStreamConstraints): Promise<Track> {
+    const stream = await navigator.mediaDevices.getDisplayMedia(constraints);
+    return this.createTrack(stream);
   }
 
-  closeAudioProducer(): void {
-    if (!this.audioProducer) {
-      return;
-    }
+  async deleteTrack(track: Track) {
+    track.closeProducer();
+    track.mediaStreamTrack.stop();
+    this.tracks.delete(track.id);
+  }
 
-    this.audioProducer.close();
-    this.audioProducer = undefined;
-    this.isAudioDisabled = true;
+  getAllTracks(): Track[] {
+    return Array.from(this.tracks.values());
   }
 }
 

package/src/engine/media/tracks/AudioTrack.ts

@@ -0,0 +1,18 @@
+import { ProducerCodecOptions } from 'mediasoup-client/lib/Producer';
+import { AudioEncoderConfig } from 'types/common';
+import BaseTrack from './BaseTrack';
+import TrackWithCodecOptions from './TrackWithCodecOptions';
+
+class AudioTrack extends BaseTrack implements TrackWithCodecOptions {
+  getEncoderConfig(): AudioEncoderConfig {
+    return this.encoderConfig;
+  }
+
+  getCodecOptions(): ProducerCodecOptions {
+    return {
+      opusFec: this.getEncoderConfig().enableFec || true,
+    };
+  }
+}
+
+export default AudioTrack;