@livedigital/client 2.0.1 → 2.2.0

package/dist/types/common.d.ts

@@ -15,11 +15,24 @@ export declare type ProduceParams = {
     rtpParameters: RtpParameters;
     appData: Record<string, unknown>;
 };
+export declare type TrackTransformParams = {
+    width?: number;
+    height?: number;
+};
 export declare type ProducerData = {
     id: string;
     kind: MediaKind;
     peerId: string;
     label: TrackLabel;
+    encodings: RtpEncodingParameters[];
+    trackTransformParams: TrackTransformParams;
+    maxSpatialLayer: number;
+};
+export declare type ConsumerData = {
+    producerData: ProducerData;
+    peerId: string;
+    appId: string;
+    channelId: string;
 };
 export declare type Role = 'audience' | 'host';
 export declare type PeerResponse = {
@@ -185,4 +198,17 @@ export declare type CreateVideoTrackParams = {
         };
     };
     encoderConfig: VideoEncoderConfig;
+    transformParams: TransformParams;
+};
+export declare type SpatialLayerParams = RtpEncodingParameters & TransformParams;
+export declare type ProducerRequestMaxSpatialLayer = {
+    producerId: string;
+    spatialLayer: number;
+};
+export declare type ProducerSetMaxSpatialLayer = ProducerRequestMaxSpatialLayer & {
+    peerId: string;
+};
+export declare type TransformParams = {
+    width?: number;
+    height?: number;
 };

package/package.json

@@ -2,7 +2,7 @@
   "name": "@livedigital/client",
   "author": "vlprojects",
   "license": "MIT",
-  "version": "2.0.1",
+  "version": "2.2.0",
   "private": false,
   "bugs": {
     "url": "https://github.com/vlprojects/livedigital-sdk/issues"
@@ -43,7 +43,7 @@
   "dependencies": {
     "axios": "^0.21.4",
     "debug": "^4.3.1",
-    "mediasoup-client": "^3.6.42",
+    "mediasoup-client": "^3.6.50",
     "qs": "^6.9.6",
     "serialize-error": "^7.0.1",
     "socket.io-client": "^4.3.0"

package/src/constants/events.ts

@@ -36,6 +36,8 @@ export const MEDIASOUP_EVENTS = {
   pauseProducer: 'producer.pause',
   resumeProducer: 'producer.resume',
   producerScoreChanged: 'producer.scoreChanged',
+  producerSetMaxSpatialLayer: 'producer.setMaxSpatialLayer',
+  producerRequestMaxSpatialLayer: 'producer.requestMaxSpatialLayer',
   createConsumer: 'consumer.create',
   closeConsumer: 'consumer.close',
   pauseConsumer: 'consumer.pause',

package/src/constants/simulcastEncodings.ts

@@ -1,10 +1,24 @@
 export const WEBCAM_SIMULCAST_ENCODINGS = [
-  { scaleResolutionDownBy: 5, maxBitrate: 200000 },
-  { scaleResolutionDownBy: 4, maxBitrate: 400000 },
-  { scaleResolutionDownBy: 2, maxBitrate: 1500000 },
+  {
+    scaleResolutionDownBy: 4, // 320x180 0.15mbps 12fps by default
+    maxBitrate: 150000,
+    maxFramerate: 12,
+  },
+  {
+    scaleResolutionDownBy: 2, // 640x360 0.35mbps 18fps by default
+    maxBitrate: 350000,
+    maxFramerate: 18,
+  },
+  {
+    scaleResolutionDownBy: 1, // 1280x720 0.9mbps 24fps by default
+    maxBitrate: 900000,
+    maxFramerate: 24,
+  },
 ];
 
 export const SCREEN_SHARING_SIMULCAST_ENCODINGS = [
-  { dtx: true, maxBitrate: 1500000 },
-  { dtx: true, maxBitrate: 6000000 },
+  {
+    maxBitrate: 1000000, // 1920x1080 1mbps 30fps by default
+    dtx: true,
+  },
 ];

package/src/engine/Peer.ts

@@ -1,34 +1,30 @@
 import { ConsumerOptions } from 'mediasoup-client/lib/Consumer';
 import {
   ConsumerScoreChangedPayload,
-  PreferredLayersParams,
   ProducerData,
   ProducerScoreChangedPayload,
-  SetConsumerPriorityParams,
-  SocketResponse,
   ConnectionQuality,
-  StartTrackPayload,
   EndTrackPayload,
   PayloadOfPublishedMedia,
   PayloadOfUnpublishedMedia,
   ChangePreferredLayersPayload,
   Role,
 } from '../types/common';
-import Consumer from './media/Consumer';
-import VideoConsumer from './media/VideoConsumer';
 import EnhancedEventEmitter from '../EnhancedEventEmitter';
 import Engine from './index';
 import { MEDIASOUP_EVENTS, PEER_EVENTS } from '../constants/events';
 import Logger from './Logger';
 import PeerProducer from './PeerProducer';
+import PeerConsumer from './PeerConsumer';
+import PeerTrack from './media/tracks/PeerTrack';
 
 interface PeerConstructor {
   id: string;
   channelIds: string[];
   appId: string;
   producers: ProducerData[],
-  videoConsumer?: VideoConsumer;
-  audioConsumer?: Consumer;
+  videoConsumer?: PeerConsumer;
+  audioConsumer?: PeerConsumer;
   engine: Engine;
   soundLevel?: number;
   loginDate: Date;
@@ -60,7 +56,7 @@ class Peer {
 
   private producers: Map<string, PeerProducer> = new Map();
 
-  private consumers: Map<string, Consumer | VideoConsumer> = new Map();
+  public readonly tracks: Map<string, PeerTrack> = new Map();
 
   private readonly engine: Engine;
 
@@ -128,105 +124,6 @@ class Peer {
     await this.createConsumer(producer);
   }
 
-  public async setMinResolution(consumerId: string): Promise<void> {
-    const consumer = this.consumers.get(consumerId);
-    if (!(consumer instanceof VideoConsumer) || consumer.spatialLayers === 0) {
-      return;
-    }
-
-    await this.changeConsumerPreferredLayers(consumer, { spatialLayer: 0 });
-  }
-
-  public async setMediumResolution(consumerId: string): Promise<void> {
-    const consumer = this.consumers.get(consumerId);
-    if (!(consumer instanceof VideoConsumer) || consumer.spatialLayers === 0) {
-      return;
-    }
-
-    const mediumLayer = consumer.spatialLayers > 2
-      ? consumer.spatialLayers - 2
-      : consumer.spatialLayers;
-    await this.changeConsumerPreferredLayers(consumer, { spatialLayer: mediumLayer });
-  }
-
-  public async setMaxResolution(consumerId: string): Promise<void> {
-    const consumer = this.consumers.get(consumerId);
-    if (!(consumer instanceof VideoConsumer) || consumer.spatialLayers === 0) {
-      return;
-    }
-
-    await this.changeConsumerPreferredLayers(consumer, {
-      spatialLayer: consumer.spatialLayers - 1,
-    });
-  }
-
-  public async pause(consumerId: string): Promise<void> {
-    const consumer = this.consumers.get(consumerId);
-    if (!consumer) {
-      this.logger.warn('pause()', { message: 'This peer has no consumer to pause', peer: this, consumerId });
-      return;
-    }
-
-    if (consumer.paused) {
-      this.logger.warn('pause()', { message: 'Already paused', peer: this, consumerId });
-      return;
-    }
-
-    try {
-      await this.pauseRemoteConsumer(consumer.id);
-      consumer.pause();
-      this.logger.debug('pause()', { peer: this, consumerId });
-    } catch (err) {
-      this.logger.error('pause()', err);
-    }
-  }
-
-  public async resume(consumerId: string): Promise<void> {
-    const consumer = this.consumers.get(consumerId);
-    if (!consumer) {
-      this.logger.warn('resume()', { message: 'This peer has no consumer to resume', peer: this, consumerId });
-      return;
-    }
-
-    if (!consumer.paused) {
-      this.logger.warn('resume()', { message: 'Already playing', peer: this, consumerId });
-      return;
-    }
-
-    try {
-      await this.resumeRemoteConsumer(consumer.id);
-      consumer.resume();
-      this.logger.debug('resume()', { peer: this });
-    } catch (err) {
-      this.logger.error('resume()', err);
-    }
-  }
-
-  public async setPriority(consumerId: string, priority: number): Promise<void> {
-    const consumer = this.consumers.get(consumerId);
-    if (!consumer) {
-      this.logger.warn('setPriority()', {
-        message: 'This peer has no consumer to change priority',
-        peer: this,
-        consumerId,
-      });
-
-      return;
-    }
-
-    await this.setConsumerPriority({ consumerId, priority });
-  }
-
-  private async setConsumerPriority(params: SetConsumerPriorityParams): Promise<void> {
-    try {
-      await this.engine.network.socket.request(MEDIASOUP_EVENTS.setConsumerPriority, params);
-      this.logger.debug('setConsumerPriority()', { peer: this, params });
-    } catch (err) {
-      this.logger.error('setConsumerPriority()', { peer: this, params });
-      throw new Error('Can`t change stream priority');
-    }
-  }
-
   private async createConsumer(producer: PeerProducer): Promise<void> {
     const transport = this.engine.network.receiveTransport;
     if (!transport) {
@@ -250,62 +147,21 @@ class Peer {
       const consumer = await transport.consume(remoteConsumer);
       await this.engine.network.socket.request(MEDIASOUP_EVENTS.resumeConsumer, { consumerId: consumer.id });
 
-      if (producer.kind === 'audio') {
-        this.consumers.set(consumer.id, new Consumer(consumer));
-      }
-
-      if (producer.kind === 'video') {
-        this.consumers.set(consumer.id, new VideoConsumer(consumer));
-      }
-
       this.logger.debug(`Subscribed for ${producer.kind}`, { peer: this });
-      this.observer.safeEmit(PEER_EVENTS.trackStart, {
-        producerId: producer.id,
-        consumerId: consumer.id,
-        track: consumer.track,
+      const track = new PeerTrack({
+        mediaStreamTrack: consumer.track,
         label: producer.label,
-      } as StartTrackPayload);
+        consumer: new PeerConsumer(consumer),
+        engine: this.engine,
+      });
+      this.tracks.set(track.label, track);
+      this.observer.safeEmit(PEER_EVENTS.trackStart, track);
     } catch (error) {
       this.logger.error('createConsumer()', producer, error);
       throw new Error('Error subscribe media');
     }
   }
 
-  private async changeConsumerPreferredLayers(
-    consumer: VideoConsumer,
-    { spatialLayer, temporalLayer }: PreferredLayersParams,
-  ): Promise<void> {
-    try {
-      await this.engine.network.socket.request(MEDIASOUP_EVENTS.consumerChangePreferredLayers, {
-        consumerId: consumer.id,
-        spatialLayer,
-        temporalLayer,
-      });
-
-      this.logger.debug('changeConsumerPreferredLayers()', {
-        peer: this,
-        consumer,
-        spatialLayer,
-        temporalLayer,
-      });
-    } catch (err) {
-      this.logger.error('changeConsumerPreferredLayers()', {
-        peer: this,
-        consumer,
-        error: err,
-      });
-      throw new Error('Error change preferred layer');
-    }
-  }
-
-  private async pauseRemoteConsumer(consumerId: string): Promise<SocketResponse> {
-    return this.engine.network.socket.request(MEDIASOUP_EVENTS.pauseConsumer, { consumerId });
-  }
-
-  private async resumeRemoteConsumer(consumerId: string): Promise<SocketResponse> {
-    return this.engine.network.socket.request(MEDIASOUP_EVENTS.resumeConsumer, { consumerId });
-  }
-
   private handleNewProducer(producerData: ProducerData): void {
     if (this.producers.get(producerData.id)) {
       return;
@@ -335,13 +191,13 @@ class Peer {
     });
 
     this.observer.on(MEDIASOUP_EVENTS.closeConsumer, (consumerId) => {
-      const consumer = this.consumers.get(consumerId);
+      const consumer = this.getConsumerById(consumerId);
       if (!consumer) {
         return;
       }
 
       consumer.close();
-      this.consumers.delete(consumerId);
+      this.tracks.delete(consumer.appData.producerData.label);
       this.observer.safeEmit(PEER_EVENTS.trackEnd, {
         producerId: consumer.producerId,
         kind: consumer.kind,
@@ -351,7 +207,7 @@ class Peer {
     });
 
     this.observer.on(MEDIASOUP_EVENTS.pauseConsumer, (consumerId) => {
-      const consumer = this.consumers.get(consumerId);
+      const consumer = this.getConsumerById(consumerId);
       if (!consumer) {
         return;
       }
@@ -361,7 +217,7 @@ class Peer {
     });
 
     this.observer.on(MEDIASOUP_EVENTS.resumeConsumer, (consumerId) => {
-      const consumer = this.consumers.get(consumerId);
+      const consumer = this.getConsumerById(consumerId);
       if (!consumer) {
         return;
       }
@@ -376,7 +232,7 @@ class Peer {
         return;
       }
 
-      const consumer = this.consumers.get(payload.consumerId);
+      const consumer = this.getConsumerById(payload.consumerId);
       if (!consumer) {
         return;
       }
@@ -426,6 +282,12 @@ class Peer {
         return;
       }
 
+      const track = this.engine.media.getAllTracks().find((t) => t.getProducer()?.id === payload.producerId);
+      if (!track) {
+        return;
+      }
+
+      const maxSpatialLayer = track.getProducer()?.maxSpatialLayer;
       const { scores } = payload;
       if (scores.length === 0) {
         return;
@@ -433,13 +295,18 @@ class Peer {
 
       let { score } = scores[0];
 
-      if (scores.length > 1) {
+      if (scores.length > 1 && maxSpatialLayer !== 0) {
+        // don't count disabled layers if they were disabled recently
+        if (maxSpatialLayer) {
+          scores.splice(maxSpatialLayer + 1);
+        }
+
         // video with multiple encodings has personal score for each encoding
         const scoresSum = scores
           .map(({ score: currentScore }) => currentScore)
           .reduce((acc, curr) => acc + curr, 0);
 
-        score = Math.floor(scoresSum / payload.scores.length);
+        score = Math.floor(scoresSum / scores.length);
       }
 
       this.outgoingConnectionQuality = Peer.getConnectionQualityByScore(score);
@@ -447,21 +314,30 @@ class Peer {
     });
 
     this.observer.on(MEDIASOUP_EVENTS.consumerChangePreferredLayers, (payload: ChangePreferredLayersPayload) => {
-      const consumer = this.consumers.get(payload.consumerId);
-      if (consumer instanceof VideoConsumer) {
+      const consumer = this.getConsumerById(payload.consumerId);
+      if (consumer?.isVideo) {
         consumer.setCurrentTemporalLayer(payload.temporalLayer);
         consumer.setCurrentSpatialLayer(payload.spatialLayer);
+        if (payload.spatialLayer === consumer.requestedSpatialLayer) {
+          consumer.setRequestedSpatialLayer(undefined);
+        }
+
+        this.logger.debug('consumerChangePreferredLayers()', { consumer, ...payload });
       }
     });
   }
 
   private emitConnectionQuality(): void {
-    // pick connection (incoming or outgoing) with lower quality
-    const connectionQuality = Math.min(this.outgoingConnectionQuality, this.incomingConnectionQuality);
+    // after the initialization a new layer, or new producer, scores of producers and consumers drop for a short time
+    setTimeout(() => {
+      // pick connection (incoming or outgoing) with lower quality
+      const connectionQuality = Math.min(this.outgoingConnectionQuality, this.incomingConnectionQuality);
+      this.observer.safeEmit(PEER_EVENTS.connectionQualityChanged, {
+        connectionQuality,
+      });
 
-    this.observer.safeEmit(PEER_EVENTS.connectionQualityChanged, {
-      connectionQuality,
-    });
+      this.logger.debug('emitConnectionQuality()', { peer: this, connectionQuality });
+    }, 1000);
   }
 
   static getConnectionQualityByScore(score: number): ConnectionQuality {
@@ -476,13 +352,23 @@ class Peer {
     return ConnectionQuality.GOOD;
   }
 
-  public getAllConsumers(): Consumer[] {
-    return Array.from(this.consumers.values());
+  private getAllConsumers(): PeerConsumer[] {
+    return Array.from(this.tracks.values())
+      .map((track) => track.consumer)
+      .filter((item): item is PeerConsumer => !!item);
   }
 
   public getAllProducers(): PeerProducer[] {
     return Array.from(this.producers.values());
   }
+
+  public getConsumerByProducerId(producerId: string): PeerConsumer | undefined {
+    return this.getAllConsumers().find((consumer) => consumer.producerId === producerId);
+  }
+
+  public getConsumerById(id: string): PeerConsumer | undefined {
+    return this.getAllConsumers().find((consumer) => consumer.id === id);
+  }
 }
 
 export default Peer;

package/src/engine/PeerConsumer.ts

@@ -0,0 +1,144 @@
+import { parseScalabilityMode } from 'mediasoup-client';
+import { MediaKind, RtpEncodingParameters } from 'mediasoup-client/lib/RtpParameters';
+import { Consumer as MediasoupConsumer } from 'mediasoup-client/lib/types';
+import { ConsumerData, SpatialLayerParams } from '../types/common';
+import Logger from './Logger';
+
+class PeerConsumer {
+  public score = 10;
+
+  public producerScore = 10;
+
+  public spatialLayers = 0;
+
+  public temporalLayers = 0;
+
+  public currentSpatialLayer = 0;
+
+  public currentTemporalLayer = 0;
+
+  public requestedSpatialLayer?: number;
+
+  public currentMaxSpatialLayer: number;
+
+  public availableSpatialLayers: SpatialLayerParams[] = [];
+
+  public currentSpatialLayerParams?: SpatialLayerParams;
+
+  readonly id: string;
+
+  readonly kind: MediaKind;
+
+  readonly producerId: string;
+
+  readonly appData: ConsumerData;
+
+  private readonly consumer: MediasoupConsumer;
+
+  private readonly logger: Logger;
+
+  constructor(consumer: MediasoupConsumer) {
+    this.id = consumer.id;
+    this.kind = consumer.kind as MediaKind;
+    this.producerId = consumer.producerId;
+    this.appData = consumer.appData as ConsumerData;
+    this.consumer = consumer;
+    this.logger = new Logger('PeerConsumer');
+    this.currentMaxSpatialLayer = this.appData.producerData.maxSpatialLayer;
+    this.parseScalabilityMode();
+    this.setSpatialLayersParams();
+    this.setCurrentSpatialLayerParams();
+  }
+
+  get paused() {
+    return this.consumer.paused;
+  }
+
+  pause() {
+    return this.consumer.pause();
+  }
+
+  resume() {
+    return this.consumer.resume();
+  }
+
+  close() {
+    return this.consumer.close();
+  }
+
+  get isVideo(): boolean {
+    return this.kind === 'video';
+  }
+
+  get isAudio(): boolean {
+    return this.kind === 'audio';
+  }
+
+  parseScalabilityMode(): void {
+    if (this.consumer.rtpParameters.encodings) {
+      const { scalabilityMode } = this.consumer.rtpParameters.encodings[0];
+      const { spatialLayers, temporalLayers } = parseScalabilityMode(scalabilityMode);
+      this.spatialLayers = spatialLayers;
+      this.temporalLayers = temporalLayers;
+      this.logger.debug('parseScalabilityMode()', {
+        scalabilityMode,
+        spatialLayers,
+        temporalLayers,
+      });
+    }
+  }
+
+  setSpatialLayersParams(): void {
+    const { encodings } = this.appData.producerData;
+    this.availableSpatialLayers = encodings.map((encoding) => this.parseSpatialLayerParams(encoding));
+  }
+
+  setCurrentSpatialLayerParams(): void {
+    const { encodings } = this.appData.producerData;
+    this.currentSpatialLayerParams = this.parseSpatialLayerParams(encodings[this.currentSpatialLayer]);
+    this.logger.debug('setCurrentSpatialLayerParams()', { currentSpatialLayerParams: this.currentSpatialLayerParams });
+  }
+
+  parseSpatialLayerParams(encoding: RtpEncodingParameters): SpatialLayerParams {
+    const { trackTransformParams: { width, height } } = this.appData.producerData;
+    const coefficient = encoding.scaleResolutionDownBy || 1;
+    if (width && height) {
+      return {
+        width: width / coefficient,
+        height: height / coefficient,
+        ...encoding,
+      };
+    }
+
+    return encoding;
+  }
+
+  setCurrentSpatialLayer(currentSpatialLayer: number): void {
+    if (!this.isVideo) {
+      throw new Error('setCurrentSpatialLayer applies to video consumers only');
+    }
+
+    this.currentSpatialLayer = currentSpatialLayer;
+    this.logger.debug('setCurrentSpatialLayer()', { currentSpatialLayer });
+    this.setCurrentSpatialLayerParams();
+  }
+
+  setCurrentTemporalLayer(currentTemporalLayer: number): void {
+    if (!this.isVideo) {
+      throw new Error('setCurrentTemporalLayer applies to video consumers only');
+    }
+
+    this.currentTemporalLayer = currentTemporalLayer;
+    this.logger.debug('setCurrentTemporalLayer()', { currentTemporalLayer });
+  }
+
+  setRequestedSpatialLayer(spatialLayer: number | undefined): void {
+    this.requestedSpatialLayer = spatialLayer;
+  }
+
+  setCurrentMaxSpatialLayer(spatialLayer: number): void {
+    this.currentMaxSpatialLayer = spatialLayer;
+  }
+}
+
+export default PeerConsumer;

package/src/engine/PeerProducer.ts

@@ -1,5 +1,6 @@
 import { MediaKind } from 'mediasoup-client/lib/RtpParameters';
-import { ProducerData, TrackLabel } from '../types/common';
+import { RtpEncodingParameters } from 'mediasoup-client/lib/types';
+import { ProducerData, TrackLabel, TrackTransformParams } from '../types/common';
 
 class PeerProducer {
   readonly id: string;
@@ -10,6 +11,12 @@ class PeerProducer {
 
   readonly label: TrackLabel;
 
+  readonly encodings: RtpEncodingParameters[];
+
+  readonly trackTransformParams: TrackTransformParams;
+
+  readonly maxSpatialLayer: number;
+
   public score = 10;
 
   constructor(params: ProducerData) {
@@ -21,6 +28,9 @@ class PeerProducer {
     this.kind = kind;
     this.peerId = peerId;
     this.label = label;
+    this.encodings = params.encodings;
+    this.trackTransformParams = params.trackTransformParams;
+    this.maxSpatialLayer = params.maxSpatialLayer;
   }
 }