@livedigital/client 2.0.1 → 2.2.0

package/src/engine/handlers/MediaSoupEventHandler.ts

@@ -2,11 +2,13 @@ import {
   ChangePreferredLayersParams,
   ConsumerScoreChangedPayload,
   ProducerData,
-  ProducerScoreChangedPayload,
+  ProducerRequestMaxSpatialLayer,
+  ProducerScoreChangedPayload, ProducerSetMaxSpatialLayer,
 } from '../../types/common';
 import Engine from '../index';
 import { MEDIASOUP_EVENTS } from '../../constants/events';
 import Logger from '../Logger';
+import VideoTrack from '../media/tracks/VideoTrack';
 
 class MediaSoupEventHandler {
   private readonly engine: Engine;
@@ -108,6 +110,83 @@ class MediaSoupEventHandler {
 
       peer.observer.safeEmit(MEDIASOUP_EVENTS.producerScoreChanged, payload);
     });
+
+    connection.on(MEDIASOUP_EVENTS.producerRequestMaxSpatialLayer, async ({
+      producerId,
+      spatialLayer,
+    }: ProducerRequestMaxSpatialLayer) => {
+      const track = this.engine.media.getAllTracks().find((t) => t.getProducer()?.id === producerId);
+      if (!track) {
+        this.logger.warn('producerRequestMaxSpatialLayer()', { message: 'Producer not found' });
+        return;
+      }
+
+      if (!(track instanceof VideoTrack)) {
+        this.logger.warn('producerRequestMaxSpatialLayer()', { message: 'Wrong producer kind' });
+        return;
+      }
+
+      const currentMaxSpatialLayer = track.getMaxSpatialLayer();
+      if (currentMaxSpatialLayer === spatialLayer) {
+        this.logger.debug('producerRequestMaxSpatialLayer()', {
+          message: 'Skip set max spatial layer',
+          requestedSpatialLayer: spatialLayer,
+          currentMaxSpatialLayer,
+        });
+        return;
+      }
+
+      await track.setMaxSpatialLayer(spatialLayer);
+      await this.engine.network.socket.request(MEDIASOUP_EVENTS.producerSetMaxSpatialLayer, {
+        producerId,
+        maxSpatialLayer: spatialLayer,
+      });
+    });
+
+    connection.on(MEDIASOUP_EVENTS.producerSetMaxSpatialLayer, async ({
+      peerId,
+      producerId,
+      spatialLayer,
+    }: ProducerSetMaxSpatialLayer) => {
+      const peer = this.engine.peers.find((item) => item.id === peerId);
+      if (!peer || peer.isMe) {
+        return;
+      }
+
+      const consumer = peer.getConsumerByProducerId(producerId);
+      if (!consumer) {
+        this.logger.warn('producerSetMaxSpatialLayer()', { message: 'Consumer not found' });
+        return;
+      }
+
+      consumer.setCurrentMaxSpatialLayer(spatialLayer);
+      this.logger.debug('producerSetMaxSpatialLayer()', {
+        producerId,
+        consumerId: consumer.id,
+        spatialLayer,
+      });
+
+      if (consumer.requestedSpatialLayer !== spatialLayer) {
+        this.logger.debug('producerSetMaxSpatialLayer()', {
+          message: 'No need to request new preferred layer',
+          currentMaxSpatialLayer: spatialLayer,
+          requestedMaxSpatialLayer: consumer.requestedSpatialLayer,
+        });
+
+        return;
+      }
+
+      await this.engine.network.socket.request(MEDIASOUP_EVENTS.consumerChangePreferredLayers, {
+        consumerId: consumer.id,
+        spatialLayer,
+      });
+
+      this.logger.debug('requestVideoPreferredLayers()', {
+        producerId,
+        consumerId: consumer.id,
+        spatialLayer,
+      });
+    });
   }
 }
 

package/src/engine/index.ts

@@ -8,7 +8,6 @@ import {
   PeerResponse,
   Role,
   SocketResponse,
-  StartTrackPayload,
   Track,
   TrackLabel,
 } from '../types/common';
@@ -23,6 +22,9 @@ import Logger from './Logger';
 import {
   CHANNEL_EVENTS, CLIENT_EVENTS, MEDIASOUP_EVENTS, PEER_EVENTS, SocketIOEvents,
 } from '../constants/events';
+import VideoTrack from './media/tracks/VideoTrack';
+import AudioTrack from './media/tracks/AudioTrack';
+import PeerTrack from './media/tracks/PeerTrack';
 
 type EngineParams = {
   clientEventEmitter: EnhancedEventEmitter,
@@ -219,8 +221,11 @@ class Engine {
         video: trackParams.videoTrackOptions,
       });
 
-      track.setLabel(TrackLabel.Camera);
-      track.setEncoderConfig(trackParams.encoderConfig);
+      if (track instanceof VideoTrack) {
+        track.setLabel(TrackLabel.Camera);
+        track.setEncoderConfig(trackParams.encoderConfig);
+        track.setTransformParams(trackParams.transformParams);
+      }
 
       this.logger.debug('createCameraVideoTrack()', { trackParams, track });
       return track;
@@ -265,9 +270,10 @@ class Engine {
           this.unpublish(track);
         });
 
-        if (track.kind === 'video') {
+        if (track instanceof VideoTrack) {
           track.setLabel(TrackLabel.ScreenVideo);
           track.setEncoderConfig(videoTrackParams.encoderConfig);
+          track.setTransformParams(videoTrackParams.transformParams);
           this.logger.debug('createScreenMediaTrack()', { trackParams: videoTrackParams, track });
           return;
         }
@@ -303,6 +309,19 @@ class Engine {
         ? track.getCodecOptions()
         : undefined;
 
+      const maxSpatialLayer = encodings && encodings.length >= 1 ? 1 : 0;
+      const appData = {
+        peerId: this.mySocketId,
+        label: track.getLabel(),
+        trackTransformParams: {},
+        maxSpatialLayer: 0,
+      };
+
+      if (track instanceof VideoTrack) {
+        appData.maxSpatialLayer = maxSpatialLayer;
+        appData.trackTransformParams = track.transformParams;
+      }
+
       const producer = await this.network.sendTransport?.produce({
         track: track.mediaStreamTrack,
         encodings,
@@ -311,10 +330,7 @@ class Engine {
         stopTracks: true,
         disableTrackOnPause: true,
         zeroRtpOnPause: true,
-        appData: {
-          peerId: this.mySocketId,
-          label: track.getLabel(),
-        },
+        appData,
       });
 
       if (!producer) {
@@ -322,12 +338,31 @@ class Engine {
       }
 
       track.setProducer(producer);
+
+      if (track instanceof VideoTrack) {
+        await track.setMaxSpatialLayer(maxSpatialLayer);
+        await this.network.socket.request(MEDIASOUP_EVENTS.producerSetMaxSpatialLayer, {
+          producerId: track.getProducer()?.id,
+          maxSpatialLayer,
+        });
+      }
+
+      if (track instanceof AudioTrack) {
+        await track.setPriority('high');
+      }
+
       const myPeer = this.peersRepository.get(<string> this.mySocketId);
-      myPeer?.observer.safeEmit(PEER_EVENTS.trackStart, {
-        producerId: producer.id,
-        track: track.mediaStreamTrack,
+      if (!myPeer) {
+        return;
+      }
+
+      const peerTrack = new PeerTrack({
+        mediaStreamTrack: track.mediaStreamTrack,
         label: track.getLabel(),
-      } as StartTrackPayload);
+        engine: this,
+      });
+      myPeer.tracks.set(peerTrack.label, peerTrack);
+      myPeer.observer.safeEmit(PEER_EVENTS.trackStart, peerTrack);
 
       this.logger.debug('publish()', { track, encodings, codecOptions });
     });

package/src/engine/media/index.ts

@@ -93,6 +93,10 @@ class Media {
         width: { ideal: options?.width || VIDEO_CONSTRAINS.fullhd.width },
         height: { ideal: options?.height || VIDEO_CONSTRAINS.fullhd.height },
       },
+      transformParams: {
+        width: options?.width || VIDEO_CONSTRAINS.hd.width,
+        height: options?.height || VIDEO_CONSTRAINS.hd.height,
+      },
       encoderConfig: options?.encoderConfig || {
         encodings: SCREEN_SHARING_SIMULCAST_ENCODINGS,
       },
@@ -107,6 +111,10 @@ class Media {
         width: { ideal: options?.width || VIDEO_CONSTRAINS.hd.width },
         height: { ideal: options?.height || VIDEO_CONSTRAINS.hd.height },
       },
+      transformParams: {
+        width: options?.width || VIDEO_CONSTRAINS.hd.width,
+        height: options?.height || VIDEO_CONSTRAINS.hd.height,
+      },
       encoderConfig: options?.encoderConfig || {
         encodings: WEBCAM_SIMULCAST_ENCODINGS,
       },

package/src/engine/media/tracks/BaseTrack.ts

@@ -1,6 +1,7 @@
 import { Producer } from 'mediasoup-client/lib/Producer';
 import { MediaKind } from 'mediasoup-client/lib/RtpParameters';
 import { EncoderConfig, TrackLabel } from '../../../types/common';
+import Logger from '../../Logger';
 
 class BaseTrack {
   readonly #mediaStreamTrack: MediaStreamTrack;
@@ -11,6 +12,8 @@ class BaseTrack {
 
   private label: TrackLabel = TrackLabel.Unknown;
 
+  protected logger = new Logger('Track');
+
   constructor(mediaStreamTrack: MediaStreamTrack) {
     this.#mediaStreamTrack = mediaStreamTrack;
   }
@@ -65,6 +68,19 @@ class BaseTrack {
     this.producer?.close();
     this.producer = undefined;
   }
+
+  async setPriority(priority: RTCPriorityType): Promise<void> {
+    if (!this.producer) {
+      this.logger.warn('setPriority()', 'Not published yet');
+      return;
+    }
+
+    await this.producer.setRtpEncodingParameters({ priority });
+    this.logger.debug('setPriority()', {
+      label: this.getLabel(),
+      priority,
+    });
+  }
 }
 
 export default BaseTrack;

package/src/engine/media/tracks/PeerTrack.ts

@@ -0,0 +1,214 @@
+import {
+  PreferredLayersParams, SetConsumerPriorityParams, SocketResponse,
+  SpatialLayerParams,
+  TrackLabel,
+} from '../../../types/common';
+import Logger from '../../Logger';
+import { MEDIASOUP_EVENTS } from '../../../constants/events';
+import PeerConsumer from '../../PeerConsumer';
+import Engine from '../../index';
+
+interface PeerTrackConstructor {
+  mediaStreamTrack: MediaStreamTrack,
+  label: TrackLabel,
+  consumer?: PeerConsumer,
+  engine: Engine,
+}
+
+class PeerTrack {
+  public readonly mediaStreamTrack: MediaStreamTrack;
+
+  public readonly label: TrackLabel = TrackLabel.Unknown;
+
+  readonly consumer?: PeerConsumer;
+
+  readonly #engine: Engine;
+
+  readonly #logger = new Logger('PeerTrack');
+
+  constructor(payload: PeerTrackConstructor) {
+    this.mediaStreamTrack = payload.mediaStreamTrack;
+    this.label = payload.label;
+    this.consumer = payload.consumer;
+    this.#engine = payload.engine;
+  }
+
+  get consumerId(): string | undefined {
+    return this.consumer?.id;
+  }
+
+  get currentSpatialLayer(): number {
+    return this.consumer?.currentSpatialLayer || 0;
+  }
+
+  public close(): void {
+    this.consumer?.close();
+  }
+
+  public async pause(): Promise<void> {
+    if (!this.consumer) {
+      return;
+    }
+
+    if (this.consumer.paused) {
+      this.#logger.warn('pause()', { message: 'Already paused', peer: this, consumer: this.consumer });
+      return;
+    }
+
+    try {
+      await this.pauseRemoteConsumer(this.consumer.id);
+      this.consumer.pause();
+      this.#logger.debug('pause()', { peer: this, consumer: this.consumer });
+    } catch (err) {
+      this.#logger.error('pause()', err);
+    }
+  }
+
+  public async resume(): Promise<void> {
+    if (!this.consumer) {
+      return;
+    }
+
+    if (!this.consumer.paused) {
+      this.#logger.warn('resume()', { message: 'Already playing', peer: this, consumer: this.consumer });
+      return;
+    }
+
+    try {
+      await this.resumeRemoteConsumer(this.consumer.id);
+      this.consumer.resume();
+      this.#logger.debug('resume()', { peer: this });
+    } catch (err) {
+      this.#logger.error('resume()', err);
+    }
+  }
+
+  public async setPriority(priority: number): Promise<void> {
+    if (!this.consumer) {
+      return;
+    }
+
+    await this.setConsumerPriority({ consumerId: this.consumer.id, priority });
+  }
+
+  private async setConsumerPriority(params: SetConsumerPriorityParams): Promise<void> {
+    try {
+      await this.#engine.network.socket.request(MEDIASOUP_EVENTS.setConsumerPriority, params);
+      this.#logger.debug('setConsumerPriority()', { peer: this, params });
+    } catch (err) {
+      this.#logger.error('setConsumerPriority()', { peer: this, params });
+      throw new Error('Can`t change stream priority');
+    }
+  }
+
+  private async pauseRemoteConsumer(consumerId: string): Promise<SocketResponse> {
+    return this.#engine.network.socket.request(MEDIASOUP_EVENTS.pauseConsumer, { consumerId });
+  }
+
+  private async resumeRemoteConsumer(consumerId: string): Promise<SocketResponse> {
+    return this.#engine.network.socket.request(MEDIASOUP_EVENTS.resumeConsumer, { consumerId });
+  }
+
+  public async requestVideoPreferredLayers({ spatialLayer, temporalLayer }: PreferredLayersParams): Promise<void> {
+    if (!this.consumer) {
+      return;
+    }
+
+    try {
+      if (this.consumer.isAudio) {
+        this.#logger.warn('requestVideoPreferredLayers()', { message: 'Only video consumers support' });
+        return;
+      }
+
+      if (spatialLayer === this.consumer.currentSpatialLayer) {
+        return;
+      }
+
+      if (spatialLayer > this.consumer.spatialLayers) {
+        this.#logger.warn('requestVideoPreferredLayers()', { message: 'Invalid spatial layer' });
+        return;
+      }
+
+      if (spatialLayer > this.consumer.currentMaxSpatialLayer) {
+        await this.requestMaxSpatialLayer(this.consumer.id, spatialLayer);
+        return;
+      }
+
+      await this.#engine.network.socket.request(MEDIASOUP_EVENTS.consumerChangePreferredLayers, {
+        consumerId: this.consumer.id,
+        spatialLayer,
+        temporalLayer,
+      });
+
+      this.#logger.debug('requestVideoPreferredLayers()', {
+        peer: this,
+        consumer: this.consumer,
+        spatialLayer,
+        temporalLayer,
+      });
+    } catch (err) {
+      this.#logger.error('requestVideoPreferredLayers()', {
+        peer: this,
+        consumerId: this.consumer.id,
+        error: err,
+      });
+      throw new Error('Error request preferred layers');
+    }
+  }
+
+  public getVideoAvailableLayers(): SpatialLayerParams[] | undefined {
+    if (!this.consumer) {
+      return undefined;
+    }
+
+    if (this.consumer.isAudio) {
+      this.#logger.warn('getVideoAvailableLayers()', { message: 'Only video consumers support' });
+      return undefined;
+    }
+
+    return this.consumer.availableSpatialLayers;
+  }
+
+  private async requestMaxSpatialLayer(consumerId: string, spatialLayer: number): Promise<void> {
+    try {
+      if (!this.consumer) {
+        return;
+      }
+
+      const cantRequest = this.consumer.isAudio
+        || this.consumer.spatialLayers === 0
+        || this.consumer.currentMaxSpatialLayer === spatialLayer
+        || spatialLayer < this.consumer.currentMaxSpatialLayer;
+
+      if (cantRequest) {
+        this.#logger.warn('requestMaxSpatialLayer()', {
+          message: 'Can`t request this spatial layer',
+          consumerId,
+          spatialLayer,
+        });
+
+        return;
+      }
+
+      this.consumer.setRequestedSpatialLayer(spatialLayer);
+      await this.#engine.network.socket.request(MEDIASOUP_EVENTS.producerRequestMaxSpatialLayer, {
+        producerId: this.consumer.producerId,
+        spatialLayer,
+      });
+
+      this.#logger.debug('requestMaxSpatialLayer()', {
+        consumerId,
+        spatialLayer,
+      });
+    } catch (error) {
+      this.#logger.warn('requestMaxSpatialLayer()', {
+        message: 'Error request spatial layer',
+        consumerId,
+        spatialLayer,
+        error,
+      });
+    }
+  }
+}
+
+export default PeerTrack;

package/src/engine/media/tracks/VideoTrack.ts

@@ -7,6 +7,8 @@ import TrackWithCodecOptions from './TrackWithCodecOptions';
 import TrackWithEncodings from './TrackWithEncodings';
 
 class VideoTrack extends BaseTrack implements TrackWithCodecOptions, TrackWithEncodings {
+  public transformParams = {};
+
   getEncoderConfig(): VideoEncoderConfig {
     return this.encoderConfig;
   }
@@ -24,6 +26,34 @@ class VideoTrack extends BaseTrack implements TrackWithCodecOptions, TrackWithEn
   getEncodings(): RtpEncodingParameters[] {
     return this.getEncoderConfig().encodings || WEBCAM_SIMULCAST_ENCODINGS;
   }
+
+  setTransformParams(transformParams: { }) {
+    this.transformParams = transformParams;
+  }
+
+  async setMaxSpatialLayer(spatialLayer: number): Promise<void> {
+    const producer = this.getProducer();
+    if (!producer) {
+      this.logger.warn('setMaxSpatialLayer()', { message: 'Not produced yet' });
+      return;
+    }
+
+    await producer.setMaxSpatialLayer(spatialLayer);
+    this.logger.debug('setMaxSpatialLayer()', {
+      label: this.getLabel(),
+      spatialLayer,
+    });
+  }
+
+  getMaxSpatialLayer(): number | undefined {
+    const producer = this.getProducer();
+    if (!producer) {
+      this.logger.warn('getMaxSpatialLayer()', { message: 'Not produced yet' });
+      return undefined;
+    }
+
+    return producer.maxSpatialLayer;
+  }
 }
 
 export default VideoTrack;

package/src/types/common.ts

@@ -23,11 +23,26 @@ export type ProduceParams = {
   appData: Record<string, unknown>,
 };
 
+export type TrackTransformParams = {
+  width?: number,
+  height?: number,
+};
+
 export type ProducerData = {
   id: string,
   kind: MediaKind,
   peerId: string,
   label: TrackLabel,
+  encodings: RtpEncodingParameters[],
+  trackTransformParams: TrackTransformParams,
+  maxSpatialLayer: number,
+};
+
+export type ConsumerData = {
+  producerData: ProducerData,
+  peerId: string,
+  appId: string,
+  channelId: string,
 };
 
 export type Role = 'audience' | 'host';
@@ -210,4 +225,21 @@ export type CreateVideoTrackParams = {
     frameRate?: { max?: number, min?: number, ideal?: number };
   },
   encoderConfig: VideoEncoderConfig,
+  transformParams: TransformParams,
+};
+
+export type SpatialLayerParams = RtpEncodingParameters & TransformParams;
+
+export type ProducerRequestMaxSpatialLayer = {
+  producerId: string,
+  spatialLayer: number,
+};
+
+export type ProducerSetMaxSpatialLayer = ProducerRequestMaxSpatialLayer & {
+  peerId: string,
+};
+
+export type TransformParams = {
+  width?: number,
+  height?: number,
 };

package/dist/engine/media/Consumer.d.ts

@@ -1,6 +0,0 @@
-import { Consumer as MediasoupConsumer } from 'mediasoup-client/lib/types';
-declare class Consumer extends MediasoupConsumer {
-    score: number;
-    producerScore: number;
-}
-export default Consumer;

package/dist/engine/media/VideoConsumer.d.ts

@@ -1,14 +0,0 @@
-import { Consumer as MediasoupConsumer } from 'mediasoup-client/lib/types';
-import Consumer from './Consumer';
-declare class VideoConsumer extends Consumer {
-    spatialLayers: number;
-    temporalLayers: number;
-    currentSpatialLayer: number;
-    currentTemporalLayer: number;
-    private readonly logger;
-    constructor(consumer: MediasoupConsumer);
-    parseScalabilityMode(): void;
-    setCurrentSpatialLayer(currentSpatialLayer: number): void;
-    setCurrentTemporalLayer(currentTemporalLayer: number): void;
-}
-export default VideoConsumer;

package/src/engine/media/Consumer.ts

@@ -1,9 +0,0 @@
-import { Consumer as MediasoupConsumer } from 'mediasoup-client/lib/types';
-
-class Consumer extends MediasoupConsumer {
-  public score = 10;
-
-  public producerScore = 10;
-}
-
-export default Consumer;

package/src/engine/media/VideoConsumer.ts

@@ -1,48 +0,0 @@
-import { Consumer as MediasoupConsumer } from 'mediasoup-client/lib/types';
-import { parseScalabilityMode } from 'mediasoup-client';
-import Consumer from './Consumer';
-import Logger from '../Logger';
-
-class VideoConsumer extends Consumer {
-  public spatialLayers = 0;
-
-  public temporalLayers = 0;
-
-  public currentSpatialLayer = 0;
-
-  public currentTemporalLayer = 0;
-
-  private readonly logger: Logger;
-
-  constructor(consumer: MediasoupConsumer) {
-    super(consumer);
-    this.logger = new Logger('VideoConsumer');
-    this.parseScalabilityMode();
-  }
-
-  parseScalabilityMode(): void {
-    if (this.rtpParameters.encodings) {
-      const { scalabilityMode } = this.rtpParameters.encodings[0];
-      const { spatialLayers, temporalLayers } = parseScalabilityMode(scalabilityMode);
-      this.spatialLayers = spatialLayers;
-      this.temporalLayers = temporalLayers;
-      this.logger.debug('parseScalabilityMode()', {
-        scalabilityMode,
-        spatialLayers,
-        temporalLayers,
-      });
-    }
-  }
-
-  setCurrentSpatialLayer(currentSpatialLayer: number): void {
-    this.currentSpatialLayer = currentSpatialLayer;
-    this.logger.debug('setCurrentSpatialLayer()', { currentSpatialLayer });
-  }
-
-  setCurrentTemporalLayer(currentTemporalLayer: number): void {
-    this.currentTemporalLayer = currentTemporalLayer;
-    this.logger.debug('setCurrentTemporalLayer()', { currentTemporalLayer });
-  }
-}
-
-export default VideoConsumer;