@livedigital/client 2.1.1 → 2.2.0

package/dist/types/common.d.ts

@@ -15,11 +15,18 @@ export declare type ProduceParams = {
     rtpParameters: RtpParameters;
     appData: Record<string, unknown>;
 };
+export declare type TrackTransformParams = {
+    width?: number;
+    height?: number;
+};
 export declare type ProducerData = {
     id: string;
     kind: MediaKind;
     peerId: string;
     label: TrackLabel;
+    encodings: RtpEncodingParameters[];
+    trackTransformParams: TrackTransformParams;
+    maxSpatialLayer: number;
 };
 export declare type ConsumerData = {
     producerData: ProducerData;
@@ -191,4 +198,17 @@ export declare type CreateVideoTrackParams = {
         };
     };
     encoderConfig: VideoEncoderConfig;
+    transformParams: TransformParams;
+};
+export declare type SpatialLayerParams = RtpEncodingParameters & TransformParams;
+export declare type ProducerRequestMaxSpatialLayer = {
+    producerId: string;
+    spatialLayer: number;
+};
+export declare type ProducerSetMaxSpatialLayer = ProducerRequestMaxSpatialLayer & {
+    peerId: string;
+};
+export declare type TransformParams = {
+    width?: number;
+    height?: number;
 };

package/package.json

@@ -2,7 +2,7 @@
   "name": "@livedigital/client",
   "author": "vlprojects",
   "license": "MIT",
-  "version": "2.1.1",
+  "version": "2.2.0",
   "private": false,
   "bugs": {
     "url": "https://github.com/vlprojects/livedigital-sdk/issues"

package/src/constants/events.ts

@@ -36,6 +36,8 @@ export const MEDIASOUP_EVENTS = {
   pauseProducer: 'producer.pause',
   resumeProducer: 'producer.resume',
   producerScoreChanged: 'producer.scoreChanged',
+  producerSetMaxSpatialLayer: 'producer.setMaxSpatialLayer',
+  producerRequestMaxSpatialLayer: 'producer.requestMaxSpatialLayer',
   createConsumer: 'consumer.create',
   closeConsumer: 'consumer.close',
   pauseConsumer: 'consumer.pause',

package/src/constants/simulcastEncodings.ts

@@ -1,10 +1,24 @@
 export const WEBCAM_SIMULCAST_ENCODINGS = [
-  { scaleResolutionDownBy: 5, maxBitrate: 200000 },
-  { scaleResolutionDownBy: 4, maxBitrate: 400000 },
-  { scaleResolutionDownBy: 2, maxBitrate: 1500000 },
+  {
+    scaleResolutionDownBy: 4, // 320x180 0.15mbps 12fps by default
+    maxBitrate: 150000,
+    maxFramerate: 12,
+  },
+  {
+    scaleResolutionDownBy: 2, // 640x360 0.35mbps 18fps by default
+    maxBitrate: 350000,
+    maxFramerate: 18,
+  },
+  {
+    scaleResolutionDownBy: 1, // 1280x720 0.9mbps 24fps by default
+    maxBitrate: 900000,
+    maxFramerate: 24,
+  },
 ];
 
 export const SCREEN_SHARING_SIMULCAST_ENCODINGS = [
-  { dtx: true, maxBitrate: 1500000 },
-  { dtx: true, maxBitrate: 6000000 },
+  {
+    maxBitrate: 1000000, // 1920x1080 1mbps 30fps by default
+    dtx: true,
+  },
 ];

package/src/engine/Peer.ts

@@ -1,13 +1,9 @@
 import { ConsumerOptions } from 'mediasoup-client/lib/Consumer';
 import {
   ConsumerScoreChangedPayload,
-  PreferredLayersParams,
   ProducerData,
   ProducerScoreChangedPayload,
-  SetConsumerPriorityParams,
-  SocketResponse,
   ConnectionQuality,
-  StartTrackPayload,
   EndTrackPayload,
   PayloadOfPublishedMedia,
   PayloadOfUnpublishedMedia,
@@ -20,6 +16,7 @@ import { MEDIASOUP_EVENTS, PEER_EVENTS } from '../constants/events';
 import Logger from './Logger';
 import PeerProducer from './PeerProducer';
 import PeerConsumer from './PeerConsumer';
+import PeerTrack from './media/tracks/PeerTrack';
 
 interface PeerConstructor {
   id: string;
@@ -59,7 +56,7 @@ class Peer {
 
   private producers: Map<string, PeerProducer> = new Map();
 
-  private consumers: Map<string, PeerConsumer> = new Map();
+  public readonly tracks: Map<string, PeerTrack> = new Map();
 
   private readonly engine: Engine;
 
@@ -127,105 +124,6 @@ class Peer {
     await this.createConsumer(producer);
   }
 
-  public async setMinResolution(consumerId: string): Promise<void> {
-    const consumer = this.consumers.get(consumerId);
-    if (!consumer?.isVideo || consumer.spatialLayers === 0) {
-      return;
-    }
-
-    await this.changeConsumerPreferredLayers(consumer, { spatialLayer: 0 });
-  }
-
-  public async setMediumResolution(consumerId: string): Promise<void> {
-    const consumer = this.consumers.get(consumerId);
-    if (!consumer?.isVideo || consumer.spatialLayers === 0) {
-      return;
-    }
-
-    const mediumLayer = consumer.spatialLayers > 2
-      ? consumer.spatialLayers - 2
-      : consumer.spatialLayers;
-    await this.changeConsumerPreferredLayers(consumer, { spatialLayer: mediumLayer });
-  }
-
-  public async setMaxResolution(consumerId: string): Promise<void> {
-    const consumer = this.consumers.get(consumerId);
-    if (!consumer?.isVideo || consumer.spatialLayers === 0) {
-      return;
-    }
-
-    await this.changeConsumerPreferredLayers(consumer, {
-      spatialLayer: consumer.spatialLayers - 1,
-    });
-  }
-
-  public async pause(consumerId: string): Promise<void> {
-    const consumer = this.consumers.get(consumerId);
-    if (!consumer) {
-      this.logger.warn('pause()', { message: 'This peer has no consumer to pause', peer: this, consumerId });
-      return;
-    }
-
-    if (consumer.paused) {
-      this.logger.warn('pause()', { message: 'Already paused', peer: this, consumerId });
-      return;
-    }
-
-    try {
-      await this.pauseRemoteConsumer(consumer.id);
-      consumer.pause();
-      this.logger.debug('pause()', { peer: this, consumerId });
-    } catch (err) {
-      this.logger.error('pause()', err);
-    }
-  }
-
-  public async resume(consumerId: string): Promise<void> {
-    const consumer = this.consumers.get(consumerId);
-    if (!consumer) {
-      this.logger.warn('resume()', { message: 'This peer has no consumer to resume', peer: this, consumerId });
-      return;
-    }
-
-    if (!consumer.paused) {
-      this.logger.warn('resume()', { message: 'Already playing', peer: this, consumerId });
-      return;
-    }
-
-    try {
-      await this.resumeRemoteConsumer(consumer.id);
-      consumer.resume();
-      this.logger.debug('resume()', { peer: this });
-    } catch (err) {
-      this.logger.error('resume()', err);
-    }
-  }
-
-  public async setPriority(consumerId: string, priority: number): Promise<void> {
-    const consumer = this.consumers.get(consumerId);
-    if (!consumer) {
-      this.logger.warn('setPriority()', {
-        message: 'This peer has no consumer to change priority',
-        peer: this,
-        consumerId,
-      });
-
-      return;
-    }
-
-    await this.setConsumerPriority({ consumerId, priority });
-  }
-
-  private async setConsumerPriority(params: SetConsumerPriorityParams): Promise<void> {
-    try {
-      await this.engine.network.socket.request(MEDIASOUP_EVENTS.setConsumerPriority, params);
-      this.logger.debug('setConsumerPriority()', { peer: this, params });
-    } catch (err) {
-      this.logger.error('setConsumerPriority()', { peer: this, params });
-      throw new Error('Can`t change stream priority');
-    }
-  }
-
   private async createConsumer(producer: PeerProducer): Promise<void> {
     const transport = this.engine.network.receiveTransport;
     if (!transport) {
@@ -249,56 +147,21 @@ class Peer {
       const consumer = await transport.consume(remoteConsumer);
       await this.engine.network.socket.request(MEDIASOUP_EVENTS.resumeConsumer, { consumerId: consumer.id });
 
-      this.consumers.set(consumer.id, new PeerConsumer(consumer));
-
       this.logger.debug(`Subscribed for ${producer.kind}`, { peer: this });
-      this.observer.safeEmit(PEER_EVENTS.trackStart, {
-        producerId: producer.id,
-        consumerId: consumer.id,
-        track: consumer.track,
+      const track = new PeerTrack({
+        mediaStreamTrack: consumer.track,
         label: producer.label,
-      } as StartTrackPayload);
+        consumer: new PeerConsumer(consumer),
+        engine: this.engine,
+      });
+      this.tracks.set(track.label, track);
+      this.observer.safeEmit(PEER_EVENTS.trackStart, track);
     } catch (error) {
       this.logger.error('createConsumer()', producer, error);
       throw new Error('Error subscribe media');
     }
   }
 
-  private async changeConsumerPreferredLayers(
-    consumer: PeerConsumer,
-    { spatialLayer, temporalLayer }: PreferredLayersParams,
-  ): Promise<void> {
-    try {
-      await this.engine.network.socket.request(MEDIASOUP_EVENTS.consumerChangePreferredLayers, {
-        consumerId: consumer.id,
-        spatialLayer,
-        temporalLayer,
-      });
-
-      this.logger.debug('changeConsumerPreferredLayers()', {
-        peer: this,
-        consumer,
-        spatialLayer,
-        temporalLayer,
-      });
-    } catch (err) {
-      this.logger.error('changeConsumerPreferredLayers()', {
-        peer: this,
-        consumer,
-        error: err,
-      });
-      throw new Error('Error change preferred layer');
-    }
-  }
-
-  private async pauseRemoteConsumer(consumerId: string): Promise<SocketResponse> {
-    return this.engine.network.socket.request(MEDIASOUP_EVENTS.pauseConsumer, { consumerId });
-  }
-
-  private async resumeRemoteConsumer(consumerId: string): Promise<SocketResponse> {
-    return this.engine.network.socket.request(MEDIASOUP_EVENTS.resumeConsumer, { consumerId });
-  }
-
   private handleNewProducer(producerData: ProducerData): void {
     if (this.producers.get(producerData.id)) {
       return;
@@ -328,13 +191,13 @@ class Peer {
     });
 
     this.observer.on(MEDIASOUP_EVENTS.closeConsumer, (consumerId) => {
-      const consumer = this.consumers.get(consumerId);
+      const consumer = this.getConsumerById(consumerId);
       if (!consumer) {
         return;
       }
 
       consumer.close();
-      this.consumers.delete(consumerId);
+      this.tracks.delete(consumer.appData.producerData.label);
       this.observer.safeEmit(PEER_EVENTS.trackEnd, {
         producerId: consumer.producerId,
         kind: consumer.kind,
@@ -344,7 +207,7 @@ class Peer {
     });
 
     this.observer.on(MEDIASOUP_EVENTS.pauseConsumer, (consumerId) => {
-      const consumer = this.consumers.get(consumerId);
+      const consumer = this.getConsumerById(consumerId);
       if (!consumer) {
         return;
       }
@@ -354,7 +217,7 @@ class Peer {
     });
 
     this.observer.on(MEDIASOUP_EVENTS.resumeConsumer, (consumerId) => {
-      const consumer = this.consumers.get(consumerId);
+      const consumer = this.getConsumerById(consumerId);
       if (!consumer) {
         return;
       }
@@ -369,7 +232,7 @@ class Peer {
         return;
       }
 
-      const consumer = this.consumers.get(payload.consumerId);
+      const consumer = this.getConsumerById(payload.consumerId);
       if (!consumer) {
         return;
       }
@@ -419,6 +282,12 @@ class Peer {
         return;
       }
 
+      const track = this.engine.media.getAllTracks().find((t) => t.getProducer()?.id === payload.producerId);
+      if (!track) {
+        return;
+      }
+
+      const maxSpatialLayer = track.getProducer()?.maxSpatialLayer;
       const { scores } = payload;
       if (scores.length === 0) {
         return;
@@ -426,13 +295,18 @@ class Peer {
 
       let { score } = scores[0];
 
-      if (scores.length > 1) {
+      if (scores.length > 1 && maxSpatialLayer !== 0) {
+        // don't count disabled layers if they were disabled recently
+        if (maxSpatialLayer) {
+          scores.splice(maxSpatialLayer + 1);
+        }
+
         // video with multiple encodings has personal score for each encoding
         const scoresSum = scores
           .map(({ score: currentScore }) => currentScore)
           .reduce((acc, curr) => acc + curr, 0);
 
-        score = Math.floor(scoresSum / payload.scores.length);
+        score = Math.floor(scoresSum / scores.length);
       }
 
       this.outgoingConnectionQuality = Peer.getConnectionQualityByScore(score);
@@ -440,21 +314,30 @@ class Peer {
     });
 
     this.observer.on(MEDIASOUP_EVENTS.consumerChangePreferredLayers, (payload: ChangePreferredLayersPayload) => {
-      const consumer = this.consumers.get(payload.consumerId);
+      const consumer = this.getConsumerById(payload.consumerId);
       if (consumer?.isVideo) {
         consumer.setCurrentTemporalLayer(payload.temporalLayer);
         consumer.setCurrentSpatialLayer(payload.spatialLayer);
+        if (payload.spatialLayer === consumer.requestedSpatialLayer) {
+          consumer.setRequestedSpatialLayer(undefined);
+        }
+
+        this.logger.debug('consumerChangePreferredLayers()', { consumer, ...payload });
       }
     });
   }
 
   private emitConnectionQuality(): void {
-    // pick connection (incoming or outgoing) with lower quality
-    const connectionQuality = Math.min(this.outgoingConnectionQuality, this.incomingConnectionQuality);
+    // after the initialization a new layer, or new producer, scores of producers and consumers drop for a short time
+    setTimeout(() => {
+      // pick connection (incoming or outgoing) with lower quality
+      const connectionQuality = Math.min(this.outgoingConnectionQuality, this.incomingConnectionQuality);
+      this.observer.safeEmit(PEER_EVENTS.connectionQualityChanged, {
+        connectionQuality,
+      });
 
-    this.observer.safeEmit(PEER_EVENTS.connectionQualityChanged, {
-      connectionQuality,
-    });
+      this.logger.debug('emitConnectionQuality()', { peer: this, connectionQuality });
+    }, 1000);
   }
 
   static getConnectionQualityByScore(score: number): ConnectionQuality {
@@ -469,13 +352,23 @@ class Peer {
     return ConnectionQuality.GOOD;
   }
 
-  public getAllConsumers(): PeerConsumer[] {
-    return Array.from(this.consumers.values());
+  private getAllConsumers(): PeerConsumer[] {
+    return Array.from(this.tracks.values())
+      .map((track) => track.consumer)
+      .filter((item): item is PeerConsumer => !!item);
   }
 
   public getAllProducers(): PeerProducer[] {
     return Array.from(this.producers.values());
   }
+
+  public getConsumerByProducerId(producerId: string): PeerConsumer | undefined {
+    return this.getAllConsumers().find((consumer) => consumer.producerId === producerId);
+  }
+
+  public getConsumerById(id: string): PeerConsumer | undefined {
+    return this.getAllConsumers().find((consumer) => consumer.id === id);
+  }
 }
 
 export default Peer;

package/src/engine/PeerConsumer.ts

@@ -1,7 +1,7 @@
 import { parseScalabilityMode } from 'mediasoup-client';
-import { MediaKind } from 'mediasoup-client/lib/RtpParameters';
+import { MediaKind, RtpEncodingParameters } from 'mediasoup-client/lib/RtpParameters';
 import { Consumer as MediasoupConsumer } from 'mediasoup-client/lib/types';
-import { ConsumerData } from '../types/common';
+import { ConsumerData, SpatialLayerParams } from '../types/common';
 import Logger from './Logger';
 
 class PeerConsumer {
@@ -17,6 +17,14 @@ class PeerConsumer {
 
   public currentTemporalLayer = 0;
 
+  public requestedSpatialLayer?: number;
+
+  public currentMaxSpatialLayer: number;
+
+  public availableSpatialLayers: SpatialLayerParams[] = [];
+
+  public currentSpatialLayerParams?: SpatialLayerParams;
+
   readonly id: string;
 
   readonly kind: MediaKind;
@@ -36,6 +44,10 @@ class PeerConsumer {
     this.appData = consumer.appData as ConsumerData;
     this.consumer = consumer;
     this.logger = new Logger('PeerConsumer');
+    this.currentMaxSpatialLayer = this.appData.producerData.maxSpatialLayer;
+    this.parseScalabilityMode();
+    this.setSpatialLayersParams();
+    this.setCurrentSpatialLayerParams();
   }
 
   get paused() {
@@ -76,23 +88,57 @@ class PeerConsumer {
     }
   }
 
+  setSpatialLayersParams(): void {
+    const { encodings } = this.appData.producerData;
+    this.availableSpatialLayers = encodings.map((encoding) => this.parseSpatialLayerParams(encoding));
+  }
+
+  setCurrentSpatialLayerParams(): void {
+    const { encodings } = this.appData.producerData;
+    this.currentSpatialLayerParams = this.parseSpatialLayerParams(encodings[this.currentSpatialLayer]);
+    this.logger.debug('setCurrentSpatialLayerParams()', { currentSpatialLayerParams: this.currentSpatialLayerParams });
+  }
+
+  parseSpatialLayerParams(encoding: RtpEncodingParameters): SpatialLayerParams {
+    const { trackTransformParams: { width, height } } = this.appData.producerData;
+    const coefficient = encoding.scaleResolutionDownBy || 1;
+    if (width && height) {
+      return {
+        width: width / coefficient,
+        height: height / coefficient,
+        ...encoding,
+      };
+    }
+
+    return encoding;
+  }
+
   setCurrentSpatialLayer(currentSpatialLayer: number): void {
-    if (this.kind !== 'video') {
+    if (!this.isVideo) {
       throw new Error('setCurrentSpatialLayer applies to video consumers only');
     }
 
     this.currentSpatialLayer = currentSpatialLayer;
     this.logger.debug('setCurrentSpatialLayer()', { currentSpatialLayer });
+    this.setCurrentSpatialLayerParams();
   }
 
   setCurrentTemporalLayer(currentTemporalLayer: number): void {
-    if (this.kind !== 'video') {
+    if (!this.isVideo) {
       throw new Error('setCurrentTemporalLayer applies to video consumers only');
     }
 
     this.currentTemporalLayer = currentTemporalLayer;
     this.logger.debug('setCurrentTemporalLayer()', { currentTemporalLayer });
   }
+
+  setRequestedSpatialLayer(spatialLayer: number | undefined): void {
+    this.requestedSpatialLayer = spatialLayer;
+  }
+
+  setCurrentMaxSpatialLayer(spatialLayer: number): void {
+    this.currentMaxSpatialLayer = spatialLayer;
+  }
 }
 
 export default PeerConsumer;

package/src/engine/PeerProducer.ts

@@ -1,5 +1,6 @@
 import { MediaKind } from 'mediasoup-client/lib/RtpParameters';
-import { ProducerData, TrackLabel } from '../types/common';
+import { RtpEncodingParameters } from 'mediasoup-client/lib/types';
+import { ProducerData, TrackLabel, TrackTransformParams } from '../types/common';
 
 class PeerProducer {
   readonly id: string;
@@ -10,6 +11,12 @@ class PeerProducer {
 
   readonly label: TrackLabel;
 
+  readonly encodings: RtpEncodingParameters[];
+
+  readonly trackTransformParams: TrackTransformParams;
+
+  readonly maxSpatialLayer: number;
+
   public score = 10;
 
   constructor(params: ProducerData) {
@@ -21,6 +28,9 @@ class PeerProducer {
     this.kind = kind;
     this.peerId = peerId;
     this.label = label;
+    this.encodings = params.encodings;
+    this.trackTransformParams = params.trackTransformParams;
+    this.maxSpatialLayer = params.maxSpatialLayer;
   }
 }
 

package/src/engine/handlers/MediaSoupEventHandler.ts

@@ -2,11 +2,13 @@ import {
   ChangePreferredLayersParams,
   ConsumerScoreChangedPayload,
   ProducerData,
-  ProducerScoreChangedPayload,
+  ProducerRequestMaxSpatialLayer,
+  ProducerScoreChangedPayload, ProducerSetMaxSpatialLayer,
 } from '../../types/common';
 import Engine from '../index';
 import { MEDIASOUP_EVENTS } from '../../constants/events';
 import Logger from '../Logger';
+import VideoTrack from '../media/tracks/VideoTrack';
 
 class MediaSoupEventHandler {
   private readonly engine: Engine;
@@ -108,6 +110,83 @@ class MediaSoupEventHandler {
 
       peer.observer.safeEmit(MEDIASOUP_EVENTS.producerScoreChanged, payload);
     });
+
+    connection.on(MEDIASOUP_EVENTS.producerRequestMaxSpatialLayer, async ({
+      producerId,
+      spatialLayer,
+    }: ProducerRequestMaxSpatialLayer) => {
+      const track = this.engine.media.getAllTracks().find((t) => t.getProducer()?.id === producerId);
+      if (!track) {
+        this.logger.warn('producerRequestMaxSpatialLayer()', { message: 'Producer not found' });
+        return;
+      }
+
+      if (!(track instanceof VideoTrack)) {
+        this.logger.warn('producerRequestMaxSpatialLayer()', { message: 'Wrong producer kind' });
+        return;
+      }
+
+      const currentMaxSpatialLayer = track.getMaxSpatialLayer();
+      if (currentMaxSpatialLayer === spatialLayer) {
+        this.logger.debug('producerRequestMaxSpatialLayer()', {
+          message: 'Skip set max spatial layer',
+          requestedSpatialLayer: spatialLayer,
+          currentMaxSpatialLayer,
+        });
+        return;
+      }
+
+      await track.setMaxSpatialLayer(spatialLayer);
+      await this.engine.network.socket.request(MEDIASOUP_EVENTS.producerSetMaxSpatialLayer, {
+        producerId,
+        maxSpatialLayer: spatialLayer,
+      });
+    });
+
+    connection.on(MEDIASOUP_EVENTS.producerSetMaxSpatialLayer, async ({
+      peerId,
+      producerId,
+      spatialLayer,
+    }: ProducerSetMaxSpatialLayer) => {
+      const peer = this.engine.peers.find((item) => item.id === peerId);
+      if (!peer || peer.isMe) {
+        return;
+      }
+
+      const consumer = peer.getConsumerByProducerId(producerId);
+      if (!consumer) {
+        this.logger.warn('producerSetMaxSpatialLayer()', { message: 'Consumer not found' });
+        return;
+      }
+
+      consumer.setCurrentMaxSpatialLayer(spatialLayer);
+      this.logger.debug('producerSetMaxSpatialLayer()', {
+        producerId,
+        consumerId: consumer.id,
+        spatialLayer,
+      });
+
+      if (consumer.requestedSpatialLayer !== spatialLayer) {
+        this.logger.debug('producerSetMaxSpatialLayer()', {
+          message: 'No need to request new preferred layer',
+          currentMaxSpatialLayer: spatialLayer,
+          requestedMaxSpatialLayer: consumer.requestedSpatialLayer,
+        });
+
+        return;
+      }
+
+      await this.engine.network.socket.request(MEDIASOUP_EVENTS.consumerChangePreferredLayers, {
+        consumerId: consumer.id,
+        spatialLayer,
+      });
+
+      this.logger.debug('requestVideoPreferredLayers()', {
+        producerId,
+        consumerId: consumer.id,
+        spatialLayer,
+      });
+    });
   }
 }