@webex/plugin-meetings 3.0.0-beta.4 → 3.0.0-beta.40

package/src/multistream/remoteMediaManager.ts

@@ -1,6 +1,7 @@
+/* eslint-disable valid-jsdoc */
 import {cloneDeep, remove} from 'lodash';
 import {EventMap} from 'typed-emitter';
-import {MediaConnection as MC} from '@webex/internal-media-core';
+import {MediaType} from '@webex/internal-media-core';
 
 import LoggerProxy from '../common/logs/logger-proxy';
 import EventsScope from '../common/events/events-scope';
@@ -30,8 +31,8 @@ export interface MemberVideoPane {
 }
 
 export interface VideoLayout {
-  screenShareVideo: {
-    size: PaneSize | null; // null if you don't want to receive any screen share video
+  screenShareVideo?: {
+    size: PaneSize;
   };
   activeSpeakerVideoPaneGroups?: ActiveSpeakerVideoPaneGroup[]; // list of active speaker video pane groups
   memberVideoPanes?: MemberVideoPane[]; // list of video panes for specific members, CSI values can be changed later via setVideoPaneCsi()
@@ -40,6 +41,7 @@ export interface VideoLayout {
 export interface Configuration {
   audio: {
     numOfActiveSpeakerStreams: number; // number of audio streams we want to receive
+    numOfScreenShareStreams: number; // 1 should be enough, because in webex only 1 person at a time can be presenting screen share
   };
   video: {
     preferLiveVideo: boolean; // applies to all pane groups with active speaker policy
@@ -47,17 +49,12 @@ export interface Configuration {
 
     layouts: {[key: LayoutId]: VideoLayout}; // a map of all available layouts, a layout can be set via setLayout() method
   };
-  screenShare: {
-    audio: boolean; // whether we ever want to receive screen share audio at all
-    video: boolean; // whether we ever want to receive screen share video at all
-  };
 }
 
 /* Predefined layouts: */
 
 // An "all equal" grid, with size up to 3 x 3 = 9:
 const AllEqualLayout: VideoLayout = {
-  screenShareVideo: {size: null},
   activeSpeakerVideoPaneGroups: [
     {
       id: 'main',
@@ -70,7 +67,6 @@ const AllEqualLayout: VideoLayout = {
 
 // A layout with just a single remote active speaker video pane:
 const SingleLayout: VideoLayout = {
-  screenShareVideo: {size: null},
   activeSpeakerVideoPaneGroups: [
     {
       id: 'main',
@@ -83,7 +79,6 @@ const SingleLayout: VideoLayout = {
 
 // A layout with 1 big pane for the highest priority active speaker and 5 small panes for other active speakers:
 const OnePlusFiveLayout: VideoLayout = {
-  screenShareVideo: {size: null},
   activeSpeakerVideoPaneGroups: [
     {
       id: 'mainBigOne',
@@ -102,7 +97,6 @@ const OnePlusFiveLayout: VideoLayout = {
 
 // A layout with 2 big panes for 2 main active speakers and a strip of 6 small panes for other active speakers:
 const TwoMainPlusSixSmallLayout: VideoLayout = {
-  screenShareVideo: {size: null},
   activeSpeakerVideoPaneGroups: [
     {
       id: 'mainGroupWith2BigPanes',
@@ -134,7 +128,6 @@ const RemoteScreenShareWithSmallThumbnailsLayout: VideoLayout = {
 
 // A staged layout with 4 pre-selected meeting participants in the main 2x2 grid and 6 small panes for other active speakers at the top:
 const Stage2x2With6ThumbnailsLayout: VideoLayout = {
-  screenShareVideo: {size: null},
   activeSpeakerVideoPaneGroups: [
     {
       id: 'thumbnails',
@@ -160,6 +153,7 @@ const Stage2x2With6ThumbnailsLayout: VideoLayout = {
 export const DefaultConfiguration: Configuration = {
   audio: {
     numOfActiveSpeakerStreams: 3,
+    numOfScreenShareStreams: 1,
   },
   video: {
     preferLiveVideo: true,
@@ -173,16 +167,12 @@ export const DefaultConfiguration: Configuration = {
       ScreenShareView: RemoteScreenShareWithSmallThumbnailsLayout,
     },
   },
-  screenShare: {
-    audio: true,
-    video: true,
-  },
 };
 
 export enum Event {
   // events for audio streams
   AudioCreated = 'AudioCreated',
-  ScreenShareAudioCreated = 'ScreenShareCreated',
+  ScreenShareAudioCreated = 'ScreenShareAudioCreated',
 
   // events for video streams
   VideoLayoutChanged = 'VideoLayoutChanged',
@@ -199,7 +189,7 @@ export interface VideoLayoutChangedEventData {
 export interface Events extends EventMap {
   // audio
   [Event.AudioCreated]: (audio: RemoteMediaGroup) => void;
-  [Event.ScreenShareAudioCreated]: (screenShareAudio: RemoteMedia) => void;
+  [Event.ScreenShareAudioCreated]: (screenShareAudio: RemoteMediaGroup) => void;
 
   // video
   [Event.VideoLayoutChanged]: (data: VideoLayoutChangedEventData) => void;
@@ -223,14 +213,18 @@ export class RemoteMediaManager extends EventsScope {
   private mediaRequestManagers: {
     audio: MediaRequestManager;
     video: MediaRequestManager;
+    screenShareAudio: MediaRequestManager;
+    screenShareVideo: MediaRequestManager;
   };
 
   private currentLayout?: VideoLayout;
 
   private slots: {
     audio: ReceiveSlot[];
-    screenShareAudio?: ReceiveSlot;
-    screenShareVideo?: ReceiveSlot;
+    screenShare: {
+      audio: ReceiveSlot[];
+      video?: ReceiveSlot;
+    };
     video: {
       unused: ReceiveSlot[];
       activeSpeaker: ReceiveSlot[];
@@ -246,6 +240,10 @@ export class RemoteMediaManager extends EventsScope {
       };
       memberPanes: {[key: PaneId]: RemoteMedia};
     };
+    screenShare: {
+      audio?: RemoteMediaGroup;
+      video?: RemoteMediaGroup;
+    };
   };
 
   private receiveSlotAllocations: {
@@ -267,6 +265,8 @@ export class RemoteMediaManager extends EventsScope {
     mediaRequestManagers: {
       audio: MediaRequestManager;
       video: MediaRequestManager;
+      screenShareAudio: MediaRequestManager;
+      screenShareVideo: MediaRequestManager;
     },
     config: Configuration = DefaultConfiguration
   ) {
@@ -281,14 +281,20 @@ export class RemoteMediaManager extends EventsScope {
         activeSpeakerGroups: {},
         memberPanes: {},
       },
+      screenShare: {
+        audio: undefined,
+        video: undefined,
+      },
     };
 
     this.checkConfigValidity();
 
     this.slots = {
       audio: [],
-      screenShareAudio: undefined,
-      screenShareVideo: undefined,
+      screenShare: {
+        audio: [],
+        video: undefined,
+      },
       video: {
         unused: [],
         activeSpeaker: [],
@@ -359,8 +365,8 @@ export class RemoteMediaManager extends EventsScope {
 
     await this.createAudioMedia();
 
-    // todo: create screen share audio remote media (SPARK-377812)
-    // todo: create screen share video receive slot (SPARK-377812)
+    await this.createScreenShareReceiveSlots();
+    this.createScreenShareAudioMedia();
 
     await this.preallocateVideoReceiveSlots();
 
@@ -373,14 +379,27 @@ export class RemoteMediaManager extends EventsScope {
    */
   public stop() {
     // invalidate all remoteMedia objects
-    this.invalidateCurrentRemoteMedia({audio: true, video: true, commit: true});
+    this.invalidateCurrentRemoteMedia({
+      audio: true,
+      video: true,
+      screenShareAudio: true,
+      screenShareVideo: true,
+      commit: true,
+    });
 
     // release all audio receive slots
     this.slots.audio.forEach((slot) => this.receiveSlotManager.releaseSlot(slot));
     this.slots.audio.length = 0;
 
-    // todo: screenshare slots... (SPARK-377812)
+    // release screen share slots
+    this.slots.screenShare.audio.forEach((slot) => this.receiveSlotManager.releaseSlot(slot));
+    this.slots.screenShare.audio.length = 0;
+    if (this.slots.screenShare.video) {
+      this.receiveSlotManager.releaseSlot(this.slots.screenShare.video);
+      this.slots.screenShare.video = undefined;
+    }
 
+    // release video slots
     this.receiveSlotAllocations = {activeSpeaker: {}, receiverSelected: {}};
 
     this.slots.video.unused.push(...this.slots.video.activeSpeaker);
@@ -432,7 +451,8 @@ export class RemoteMediaManager extends EventsScope {
     while (this.slots.video.unused.length < maxNumVideoPanesRequired) {
       // eslint-disable-next-line no-await-in-loop
       this.slots.video.unused.push(
-        await this.receiveSlotManager.allocateSlot(MC.MediaType.VideoMain)
+        // eslint-disable-next-line no-await-in-loop
+        await this.receiveSlotManager.allocateSlot(MediaType.VideoMain)
       );
     }
   }
@@ -457,6 +477,7 @@ export class RemoteMediaManager extends EventsScope {
 
     await this.updateVideoReceiveSlots();
     this.updateVideoRemoteMediaObjects();
+    this.updateScreenShareVideoRemoteMediaObject();
     this.emitVideoLayoutChangedEvent();
   }
 
@@ -476,7 +497,7 @@ export class RemoteMediaManager extends EventsScope {
     // create the audio receive slots
     for (let i = 0; i < this.config.audio.numOfActiveSpeakerStreams; i += 1) {
       // eslint-disable-next-line no-await-in-loop
-      const slot = await this.receiveSlotManager.allocateSlot(MC.MediaType.AudioMain);
+      const slot = await this.receiveSlotManager.allocateSlot(MediaType.AudioMain);
 
       this.slots.audio.push(slot);
     }
@@ -496,6 +517,50 @@ export class RemoteMediaManager extends EventsScope {
     );
   }
 
+  /**
+   * Creates receive slots required for receiving screen share audio and video
+   */
+  private async createScreenShareReceiveSlots() {
+    // audio
+    for (let i = 0; i < this.config.audio.numOfScreenShareStreams; i += 1) {
+      // eslint-disable-next-line no-await-in-loop
+      const slot = await this.receiveSlotManager.allocateSlot(MediaType.AudioSlides);
+
+      this.slots.screenShare.audio.push(slot);
+    }
+
+    // video
+    const isAnyLayoutContainingScreenShareVideo = Object.values(this.config.video.layouts).some(
+      (layout) => !!layout.screenShareVideo
+    );
+
+    if (isAnyLayoutContainingScreenShareVideo) {
+      this.slots.screenShare.video = await this.receiveSlotManager.allocateSlot(
+        MediaType.VideoSlides
+      );
+    }
+  }
+
+  /**
+   * Creates RemoteMedia objects for screen share
+   */
+  private createScreenShareAudioMedia() {
+    if (this.slots.screenShare.audio.length > 0) {
+      this.media.screenShare.audio = new RemoteMediaGroup(
+        this.mediaRequestManagers.screenShareAudio,
+        this.slots.screenShare.audio,
+        255,
+        true
+      );
+
+      this.emit(
+        {file: 'multistream/remoteMediaManager', function: 'createScreenShareAudioMedia'},
+        Event.ScreenShareAudioCreated,
+        this.media.screenShare.audio
+      );
+    }
+  }
+
   /**
    * Goes over all receiver-selected slots and keeps only the ones that are required by a given layout,
    * the rest are all moved to the "unused" list
@@ -600,7 +665,8 @@ export class RemoteMediaManager extends EventsScope {
       while (numSlotsToCreate > 0) {
         // eslint-disable-next-line no-await-in-loop
         this.slots.video.unused.push(
-          await this.receiveSlotManager.allocateSlot(MC.MediaType.VideoMain)
+          // eslint-disable-next-line no-await-in-loop
+          await this.receiveSlotManager.allocateSlot(MediaType.VideoMain)
         );
         numSlotsToCreate -= 1;
       }
@@ -631,7 +697,13 @@ export class RemoteMediaManager extends EventsScope {
    */
   private updateVideoRemoteMediaObjects() {
     // invalidate all the previous remote media objects and cancel their media requests
-    this.invalidateCurrentRemoteMedia({audio: false, video: true, commit: false});
+    this.invalidateCurrentRemoteMedia({
+      audio: false,
+      video: true,
+      screenShareAudio: false,
+      screenShareVideo: false,
+      commit: false,
+    });
 
     // create new remoteMediaGroup objects
     this.media.video.activeSpeakerGroups = {};
@@ -686,16 +758,53 @@ export class RemoteMediaManager extends EventsScope {
         );
       }
     }
-    // todo: screenshare (SPARK-377812)
 
     this.mediaRequestManagers.video.commit();
   }
 
+  /**
+   * Checks if current layout requires a screen share.
+   * If it does, it creates new RemoteMediaGroup object for screen share
+   * and sends the media requests for it.
+   * If it doesn't, it makes sure we clean up any RemoteMediaGroup objects
+   * created earlier for screen share (for previous layout).
+   */
+  private updateScreenShareVideoRemoteMediaObject() {
+    this.invalidateCurrentRemoteMedia({
+      audio: false,
+      video: false,
+      screenShareAudio: false,
+      screenShareVideo: true,
+      commit: false,
+    });
+
+    this.media.screenShare.video = undefined;
+
+    if (this.currentLayout?.screenShareVideo) {
+      // we create a group of 1, because for screen share we need to use the "active speaker" policy
+      this.media.screenShare.video = new RemoteMediaGroup(
+        this.mediaRequestManagers.screenShareVideo,
+        [this.slots.screenShare.video],
+        255,
+        false,
+        {resolution: this.currentLayout.screenShareVideo.size}
+      );
+    }
+
+    this.mediaRequestManagers.screenShareVideo.commit();
+  }
+
   /**
    * Invalidates all remote media objects belonging to currently selected layout
    */
-  private invalidateCurrentRemoteMedia(options: {audio: boolean; video: boolean; commit: boolean}) {
-    const {audio, video, commit} = options;
+  private invalidateCurrentRemoteMedia(options: {
+    audio: boolean;
+    video: boolean;
+    screenShareAudio: boolean;
+    screenShareVideo: boolean;
+    commit: boolean;
+  }) {
+    const {audio, video, screenShareAudio, screenShareVideo, commit} = options;
 
     if (audio && this.media.audio) {
       this.media.audio.stop(commit);
@@ -711,12 +820,19 @@ export class RemoteMediaManager extends EventsScope {
         this.mediaRequestManagers.video.commit();
       }
     }
+
+    if (screenShareAudio && this.media.screenShare.audio) {
+      this.media.screenShare.audio.stop(commit);
+    }
+    if (screenShareVideo && this.media.screenShare.video) {
+      this.media.screenShare.video.stop(commit);
+    }
   }
 
   /** emits Event.VideoLayoutChanged */
   private emitVideoLayoutChangedEvent() {
     // todo: at this point the receive slots might still be showing a participant from previous layout, we should
-    // wait for our media requests to be fullfilled, but there is no API for that right now (we could wait for source updates
+    // wait for our media requests to be fulfilled, but there is no API for that right now (we could wait for source updates
     // but in some cases they might never come, or would need to always make sure to use a new set of receiver slots)
     // for now it's fine to have it like this, we will re-evaluate if it needs improving after more testing
 
@@ -730,7 +846,7 @@ export class RemoteMediaManager extends EventsScope {
         layoutId: this.currentLayoutId,
         activeSpeakerVideoPanes: this.media.video.activeSpeakerGroups,
         memberVideoPanes: this.media.video.memberPanes,
-        screenShareVideo: undefined, // todo: screen share (SPARK-377812)
+        screenShareVideo: this.media.screenShare.video?.getRemoteMedia()[0],
       }
     );
   }
@@ -778,7 +894,7 @@ export class RemoteMediaManager extends EventsScope {
 
     this.currentLayout.memberVideoPanes.push(newPane);
 
-    const receiveSlot = await this.receiveSlotManager.allocateSlot(MC.MediaType.VideoMain);
+    const receiveSlot = await this.receiveSlotManager.allocateSlot(MediaType.VideoMain);
 
     this.slots.video.receiverSelected.push(receiveSlot);
 

package/src/networkQualityMonitor/{index.js → index.ts}

@@ -1,16 +1,15 @@
 import EventsScope from '../common/events/events-scope';
-import {EVENT_TRIGGERS, STATS} from '../constants';
-
+import {EVENT_TRIGGERS} from '../constants';
 
 /**
-  * Meeting - network quality event
-  * Emitted on each interval of retrieving stats Analyzer data
-  * @event network:quality
-  * @type {Object}
-  * @property {string} mediaType {video|audio}
-  * @property {number} networkQualityScore - value determined in determineUplinkNetworkQuality
-  * @memberof NetworkQualityMonitor
-  */
+ * Meeting - network quality event
+ * Emitted on each interval of retrieving stats Analyzer data
+ * @event network:quality
+ * @type {Object}
+ * @property {string} mediaType {video|audio}
+ * @property {number} networkQualityScore - value determined in determineUplinkNetworkQuality
+ * @memberof NetworkQualityMonitor
+ */
 /**
  * NetworkQualityMonitor class that will emit events based on detected quality
  *
@@ -18,6 +17,13 @@ import {EVENT_TRIGGERS, STATS} from '../constants';
  * @extends {EventsScope}
  */
 export default class NetworkQualityMonitor extends EventsScope {
+  config: any;
+  frequencyTypes: any;
+  indicatorTypes: any;
+  mediaType: any;
+  networkQualityScore: any;
+  networkQualityStatus: any;
+
   /**
    * Creates a new instance of NetworkQualityMonitor
    * @constructor
@@ -29,25 +35,21 @@ export default class NetworkQualityMonitor extends EventsScope {
    * @property {Object} networkQualityStatus - hash object based on indicatorTypes and frequencyTypes
    * @property {string} mediaType - audio|video
    */
-  constructor(config) {
+  constructor(config: any) {
     super();
     this.config = config;
     this.indicatorTypes = Object.freeze({
       PACKETLOSS: 'packetLoss',
       LATENCY: 'latency',
-      JITTER: 'jitter'
+      JITTER: 'jitter',
     });
     this.frequencyTypes = Object.freeze({
       UPLINK: 'uplink',
-      DOWNLINK: 'downlink'
+      DOWNLINK: 'downlink',
     });
     this.networkQualityScore = 1;
     this.networkQualityStatus = {
-      [this.frequencyTypes.UPLINK]: {
-        [STATS.VIDEO_CORRELATE]: {},
-        [STATS.AUDIO_CORRELATE]: {},
-        [STATS.SHARE_CORRELATE]: {}
-      }
+      [this.frequencyTypes.UPLINK]: {},
     };
     this.mediaType = null;
   }
@@ -62,12 +64,12 @@ export default class NetworkQualityMonitor extends EventsScope {
     this.emit(
       {
         file: 'networkQualityMonitor',
-        function: 'emitNetworkQuality'
+        function: 'emitNetworkQuality',
       },
       EVENT_TRIGGERS.NETWORK_QUALITY,
       {
         mediaType: this.mediaType,
-        networkQualityScore: this.networkQualityScore
+        networkQualityScore: this.networkQualityScore,
       }
     );
   }
@@ -85,7 +87,6 @@ export default class NetworkQualityMonitor extends EventsScope {
     this.mediaType = null;
   }
 
-
   /**
    * filter data to determine uplink network quality, invoked on same interval as stats analyzer remote-inbout-rtp
    * @param {Object} configObj
@@ -96,7 +97,15 @@ export default class NetworkQualityMonitor extends EventsScope {
    * @public
    * @memberof NetworkQualityMonitor
    */
-  determineUplinkNetworkQuality({mediaType, remoteRtpResults, statsAnalyzerCurrentStats}) {
+  public determineUplinkNetworkQuality({
+    mediaType,
+    remoteRtpResults,
+    statsAnalyzerCurrentStats,
+  }: {
+    mediaType: string;
+    remoteRtpResults: any;
+    statsAnalyzerCurrentStats: object;
+  }) {
     const roundTripTimeInMilliseconds = remoteRtpResults.roundTripTime * 1000;
     const jitterInMilliseconds = remoteRtpResults.jitter * 1000;
     const {currentPacketLossRatio} = statsAnalyzerCurrentStats[mediaType].send;
@@ -112,8 +121,7 @@ export default class NetworkQualityMonitor extends EventsScope {
      * @returns {boolean}
      */
     const determinePacketLoss = () => {
-      if (currentPacketLossRatio >
-        this.config.videoPacketLossRatioThreshold) {
+      if (currentPacketLossRatio > this.config.videoPacketLossRatioThreshold) {
         this.networkQualityScore = 0;
 
         return false;
@@ -157,7 +165,12 @@ export default class NetworkQualityMonitor extends EventsScope {
      * @param {(number|undefined)} value
      * @returns {(number|null)}
      */
-    const determineIfUndefined = (value) => (typeof value === 'undefined' ? null : value);
+    const determineIfUndefined = (value: number | undefined) =>
+      typeof value === 'undefined' ? null : value;
+
+    if (!this.networkQualityStatus[UPLINK][mediaType]) {
+      this.networkQualityStatus[UPLINK][mediaType] = {};
+    }
 
     /**
      * Values for some browsers specifically Safari will be undefined we explicitly set to null
@@ -167,20 +180,19 @@ export default class NetworkQualityMonitor extends EventsScope {
     // PACKET LOSS
     this.networkQualityStatus[UPLINK][mediaType][PACKETLOSS] = {
       acceptable: determinePacketLoss(),
-      value: determineIfUndefined(currentPacketLossRatio)
+      value: determineIfUndefined(currentPacketLossRatio),
     };
 
-
     // LATENCY measured in Round trip time
     this.networkQualityStatus[UPLINK][mediaType][LATENCY] = {
       acceptable: determineLatency(),
-      value: determineIfUndefined(remoteRtpResults.roundTripTime)
+      value: determineIfUndefined(remoteRtpResults.roundTripTime),
     };
 
     // JITTER
     this.networkQualityStatus[UPLINK][mediaType][JITTER] = {
       acceptable: deterMineJitter(),
-      value: determineIfUndefined(remoteRtpResults.jitter)
+      value: determineIfUndefined(remoteRtpResults.jitter),
     };
 
     this.updateNetworkQualityStatus();