@webex/plugin-meetings 3.0.0-beta.8 → 3.0.0-bnr.0

package/src/multistream/remoteMediaManager.ts

@@ -1,6 +1,7 @@
-import {cloneDeep, remove} from 'lodash';
+/* eslint-disable valid-jsdoc */
+import {cloneDeep, forEach, remove} from 'lodash';
 import {EventMap} from 'typed-emitter';
-import {MediaConnection as MC} from '@webex/internal-media-core';
+import {MediaType} from '@webex/internal-media-core';
 
 import LoggerProxy from '../common/logs/logger-proxy';
 import EventsScope from '../common/events/events-scope';
@@ -30,8 +31,8 @@ export interface MemberVideoPane {
 }
 
 export interface VideoLayout {
-  screenShareVideo: {
-    size: PaneSize | null; // null if you don't want to receive any screen share video
+  screenShareVideo?: {
+    size: PaneSize;
   };
   activeSpeakerVideoPaneGroups?: ActiveSpeakerVideoPaneGroup[]; // list of active speaker video pane groups
   memberVideoPanes?: MemberVideoPane[]; // list of video panes for specific members, CSI values can be changed later via setVideoPaneCsi()
@@ -40,6 +41,7 @@ export interface VideoLayout {
 export interface Configuration {
   audio: {
     numOfActiveSpeakerStreams: number; // number of audio streams we want to receive
+    numOfScreenShareStreams: number; // 1 should be enough, because in webex only 1 person at a time can be presenting screen share
   };
   video: {
     preferLiveVideo: boolean; // applies to all pane groups with active speaker policy
@@ -47,17 +49,12 @@ export interface Configuration {
 
     layouts: {[key: LayoutId]: VideoLayout}; // a map of all available layouts, a layout can be set via setLayout() method
   };
-  screenShare: {
-    audio: boolean; // whether we ever want to receive screen share audio at all
-    video: boolean; // whether we ever want to receive screen share video at all
-  };
 }
 
 /* Predefined layouts: */
 
 // An "all equal" grid, with size up to 3 x 3 = 9:
 const AllEqualLayout: VideoLayout = {
-  screenShareVideo: {size: null},
   activeSpeakerVideoPaneGroups: [
     {
       id: 'main',
@@ -70,7 +67,6 @@ const AllEqualLayout: VideoLayout = {
 
 // A layout with just a single remote active speaker video pane:
 const SingleLayout: VideoLayout = {
-  screenShareVideo: {size: null},
   activeSpeakerVideoPaneGroups: [
     {
       id: 'main',
@@ -83,7 +79,6 @@ const SingleLayout: VideoLayout = {
 
 // A layout with 1 big pane for the highest priority active speaker and 5 small panes for other active speakers:
 const OnePlusFiveLayout: VideoLayout = {
-  screenShareVideo: {size: null},
   activeSpeakerVideoPaneGroups: [
     {
       id: 'mainBigOne',
@@ -102,7 +97,6 @@ const OnePlusFiveLayout: VideoLayout = {
 
 // A layout with 2 big panes for 2 main active speakers and a strip of 6 small panes for other active speakers:
 const TwoMainPlusSixSmallLayout: VideoLayout = {
-  screenShareVideo: {size: null},
   activeSpeakerVideoPaneGroups: [
     {
       id: 'mainGroupWith2BigPanes',
@@ -134,7 +128,6 @@ const RemoteScreenShareWithSmallThumbnailsLayout: VideoLayout = {
 
 // A staged layout with 4 pre-selected meeting participants in the main 2x2 grid and 6 small panes for other active speakers at the top:
 const Stage2x2With6ThumbnailsLayout: VideoLayout = {
-  screenShareVideo: {size: null},
   activeSpeakerVideoPaneGroups: [
     {
       id: 'thumbnails',
@@ -160,6 +153,7 @@ const Stage2x2With6ThumbnailsLayout: VideoLayout = {
 export const DefaultConfiguration: Configuration = {
   audio: {
     numOfActiveSpeakerStreams: 3,
+    numOfScreenShareStreams: 1,
   },
   video: {
     preferLiveVideo: true,
@@ -173,16 +167,12 @@ export const DefaultConfiguration: Configuration = {
       ScreenShareView: RemoteScreenShareWithSmallThumbnailsLayout,
     },
   },
-  screenShare: {
-    audio: true,
-    video: true,
-  },
 };
 
 export enum Event {
   // events for audio streams
   AudioCreated = 'AudioCreated',
-  ScreenShareAudioCreated = 'ScreenShareCreated',
+  ScreenShareAudioCreated = 'ScreenShareAudioCreated',
 
   // events for video streams
   VideoLayoutChanged = 'VideoLayoutChanged',
@@ -199,7 +189,7 @@ export interface VideoLayoutChangedEventData {
 export interface Events extends EventMap {
   // audio
   [Event.AudioCreated]: (audio: RemoteMediaGroup) => void;
-  [Event.ScreenShareAudioCreated]: (screenShareAudio: RemoteMedia) => void;
+  [Event.ScreenShareAudioCreated]: (screenShareAudio: RemoteMediaGroup) => void;
 
   // video
   [Event.VideoLayoutChanged]: (data: VideoLayoutChangedEventData) => void;
@@ -223,14 +213,18 @@ export class RemoteMediaManager extends EventsScope {
   private mediaRequestManagers: {
     audio: MediaRequestManager;
     video: MediaRequestManager;
+    screenShareAudio: MediaRequestManager;
+    screenShareVideo: MediaRequestManager;
   };
 
   private currentLayout?: VideoLayout;
 
   private slots: {
     audio: ReceiveSlot[];
-    screenShareAudio?: ReceiveSlot;
-    screenShareVideo?: ReceiveSlot;
+    screenShare: {
+      audio: ReceiveSlot[];
+      video?: ReceiveSlot;
+    };
     video: {
       unused: ReceiveSlot[];
       activeSpeaker: ReceiveSlot[];
@@ -246,6 +240,10 @@ export class RemoteMediaManager extends EventsScope {
       };
       memberPanes: {[key: PaneId]: RemoteMedia};
     };
+    screenShare: {
+      audio?: RemoteMediaGroup;
+      video?: RemoteMediaGroup;
+    };
   };
 
   private receiveSlotAllocations: {
@@ -267,6 +265,8 @@ export class RemoteMediaManager extends EventsScope {
     mediaRequestManagers: {
       audio: MediaRequestManager;
       video: MediaRequestManager;
+      screenShareAudio: MediaRequestManager;
+      screenShareVideo: MediaRequestManager;
     },
     config: Configuration = DefaultConfiguration
   ) {
@@ -281,14 +281,20 @@ export class RemoteMediaManager extends EventsScope {
         activeSpeakerGroups: {},
         memberPanes: {},
       },
+      screenShare: {
+        audio: undefined,
+        video: undefined,
+      },
     };
 
     this.checkConfigValidity();
 
     this.slots = {
       audio: [],
-      screenShareAudio: undefined,
-      screenShareVideo: undefined,
+      screenShare: {
+        audio: [],
+        video: undefined,
+      },
       video: {
         unused: [],
         activeSpeaker: [],
@@ -359,8 +365,8 @@ export class RemoteMediaManager extends EventsScope {
 
     await this.createAudioMedia();
 
-    // todo: create screen share audio remote media (SPARK-377812)
-    // todo: create screen share video receive slot (SPARK-377812)
+    await this.createScreenShareReceiveSlots();
+    this.createScreenShareAudioMedia();
 
     await this.preallocateVideoReceiveSlots();
 
@@ -373,14 +379,27 @@ export class RemoteMediaManager extends EventsScope {
    */
   public stop() {
     // invalidate all remoteMedia objects
-    this.invalidateCurrentRemoteMedia({audio: true, video: true, commit: true});
+    this.invalidateCurrentRemoteMedia({
+      audio: true,
+      video: true,
+      screenShareAudio: true,
+      screenShareVideo: true,
+      commit: true,
+    });
 
     // release all audio receive slots
     this.slots.audio.forEach((slot) => this.receiveSlotManager.releaseSlot(slot));
     this.slots.audio.length = 0;
 
-    // todo: screenshare slots... (SPARK-377812)
+    // release screen share slots
+    this.slots.screenShare.audio.forEach((slot) => this.receiveSlotManager.releaseSlot(slot));
+    this.slots.screenShare.audio.length = 0;
+    if (this.slots.screenShare.video) {
+      this.receiveSlotManager.releaseSlot(this.slots.screenShare.video);
+      this.slots.screenShare.video = undefined;
+    }
 
+    // release video slots
     this.receiveSlotAllocations = {activeSpeaker: {}, receiverSelected: {}};
 
     this.slots.video.unused.push(...this.slots.video.activeSpeaker);
@@ -432,7 +451,8 @@ export class RemoteMediaManager extends EventsScope {
     while (this.slots.video.unused.length < maxNumVideoPanesRequired) {
       // eslint-disable-next-line no-await-in-loop
       this.slots.video.unused.push(
-        await this.receiveSlotManager.allocateSlot(MC.MediaType.VideoMain)
+        // eslint-disable-next-line no-await-in-loop
+        await this.receiveSlotManager.allocateSlot(MediaType.VideoMain)
       );
     }
   }
@@ -457,6 +477,7 @@ export class RemoteMediaManager extends EventsScope {
 
     await this.updateVideoReceiveSlots();
     this.updateVideoRemoteMediaObjects();
+    this.updateScreenShareVideoRemoteMediaObject();
     this.emitVideoLayoutChangedEvent();
   }
 
@@ -476,7 +497,7 @@ export class RemoteMediaManager extends EventsScope {
     // create the audio receive slots
     for (let i = 0; i < this.config.audio.numOfActiveSpeakerStreams; i += 1) {
       // eslint-disable-next-line no-await-in-loop
-      const slot = await this.receiveSlotManager.allocateSlot(MC.MediaType.AudioMain);
+      const slot = await this.receiveSlotManager.allocateSlot(MediaType.AudioMain);
 
       this.slots.audio.push(slot);
     }
@@ -496,6 +517,50 @@ export class RemoteMediaManager extends EventsScope {
     );
   }
 
+  /**
+   * Creates receive slots required for receiving screen share audio and video
+   */
+  private async createScreenShareReceiveSlots() {
+    // audio
+    for (let i = 0; i < this.config.audio.numOfScreenShareStreams; i += 1) {
+      // eslint-disable-next-line no-await-in-loop
+      const slot = await this.receiveSlotManager.allocateSlot(MediaType.AudioSlides);
+
+      this.slots.screenShare.audio.push(slot);
+    }
+
+    // video
+    const isAnyLayoutContainingScreenShareVideo = Object.values(this.config.video.layouts).some(
+      (layout) => !!layout.screenShareVideo
+    );
+
+    if (isAnyLayoutContainingScreenShareVideo) {
+      this.slots.screenShare.video = await this.receiveSlotManager.allocateSlot(
+        MediaType.VideoSlides
+      );
+    }
+  }
+
+  /**
+   * Creates RemoteMedia objects for screen share
+   */
+  private createScreenShareAudioMedia() {
+    if (this.slots.screenShare.audio.length > 0) {
+      this.media.screenShare.audio = new RemoteMediaGroup(
+        this.mediaRequestManagers.screenShareAudio,
+        this.slots.screenShare.audio,
+        255,
+        true
+      );
+
+      this.emit(
+        {file: 'multistream/remoteMediaManager', function: 'createScreenShareAudioMedia'},
+        Event.ScreenShareAudioCreated,
+        this.media.screenShare.audio
+      );
+    }
+  }
+
   /**
    * Goes over all receiver-selected slots and keeps only the ones that are required by a given layout,
    * the rest are all moved to the "unused" list
@@ -536,25 +601,37 @@ export class RemoteMediaManager extends EventsScope {
   }
 
   /**
-   * Allocates receive slots to all video panes in the current selected layout
+   * Allocates receive slots to all active speaker video panes
+   * in the current selected layout.
+   *
+   * Allocation tries to keep the same order of the slots between the previous
+   * layout and the new one. Sorting helps making sure that highest priority slots
+   * go in the same order in the new layout.
    */
-  private allocateSlotsToVideoPaneGroups() {
-    this.receiveSlotAllocations = {activeSpeaker: {}, receiverSelected: {}};
-
-    this.currentLayout?.activeSpeakerVideoPaneGroups?.forEach((group) => {
-      this.receiveSlotAllocations.activeSpeaker[group.id] = {slots: []};
-
-      for (let paneIndex = 0; paneIndex < group.numPanes; paneIndex += 1) {
-        // allocate a slot from the "unused" list
-        const freeSlot = this.slots.video.unused.pop();
-
-        if (freeSlot) {
-          this.slots.video.activeSpeaker.push(freeSlot);
-          this.receiveSlotAllocations.activeSpeaker[group.id].slots.push(freeSlot);
+  private allocateSlotsToActiveSpeakerPaneGroups() {
+    this.currentLayout?.activeSpeakerVideoPaneGroups
+      // sorting in descending order based on group priority
+      ?.sort((a, b) => (a.priority < b.priority ? 1 : -1))
+      ?.forEach((group) => {
+        this.receiveSlotAllocations.activeSpeaker[group.id] = {slots: []};
+
+        for (let paneIndex = 0; paneIndex < group.numPanes; paneIndex += 1) {
+          // allocate a slot from the "unused" list, by grabbing in same order (shift) as previous layout
+          const freeSlot = this.slots.video.unused.shift();
+
+          if (freeSlot) {
+            this.slots.video.activeSpeaker.push(freeSlot);
+            this.receiveSlotAllocations.activeSpeaker[group.id].slots.push(freeSlot);
+          }
         }
-      }
-    });
+      });
+  }
 
+  /**
+   * Allocates receive slots to all receiver selected video panes
+   * in the current selected layout
+   */
+  private allocateSlotsToReceiverSelectedVideoPaneGroups() {
     this.currentLayout?.memberVideoPanes?.forEach((memberPane) => {
       // check if there is existing slot for this csi
       const existingSlot = this.slots.video.receiverSelected.find(
@@ -581,44 +658,81 @@ export class RemoteMediaManager extends EventsScope {
   }
 
   /**
-   * Makes sure we have the right number of receive slots created for the current layout
-   * and allocates them to the right video panes / pane groups
-   *
-   * @returns {Promise}
+   * Ensures that we have enough slots for the current layout.
    */
-  private async updateVideoReceiveSlots() {
+  private async refillRequiredSlotsIfNeeded() {
     const requiredNumSlots = this.getRequiredNumVideoSlotsForLayout(this.currentLayout);
     const totalNumSlots =
       this.slots.video.unused.length +
       this.slots.video.activeSpeaker.length +
       this.slots.video.receiverSelected.length;
 
-    // ensure we have enough total slots for current layout
     if (totalNumSlots < requiredNumSlots) {
       let numSlotsToCreate = requiredNumSlots - totalNumSlots;
 
       while (numSlotsToCreate > 0) {
         // eslint-disable-next-line no-await-in-loop
         this.slots.video.unused.push(
-          await this.receiveSlotManager.allocateSlot(MC.MediaType.VideoMain)
+          // eslint-disable-next-line no-await-in-loop
+          await this.receiveSlotManager.allocateSlot(MediaType.VideoMain)
         );
         numSlotsToCreate -= 1;
       }
     }
+  }
 
-    // move all no longer needed receiver-selected slots to "unused"
-    this.trimReceiverSelectedSlots();
-
-    // move all active speaker slots to "unused"
+  /**
+   * Move all active speaker slots to "unused"
+   */
+  private trimActiveSpeakerSlots() {
     this.slots.video.unused.push(...this.slots.video.activeSpeaker);
     this.slots.video.activeSpeaker.length = 0;
+  }
 
-    // allocate the slots to the right panes / pane groups
-    this.allocateSlotsToVideoPaneGroups();
+  /**
+   * Logs the state of the receive slots
+   */
+  private logReceieveSlots() {
+    let logMessage = '';
+    forEach(this.receiveSlotAllocations.activeSpeaker, (group, groupName) => {
+      logMessage += `group: ${groupName}\n${group.slots.map((slot) => slot.logString).join(' ')}`;
+    });
+
+    logMessage += '\nreceiverSelected:\n';
+    forEach(this.receiveSlotAllocations.receiverSelected, (slot, key) => {
+      logMessage += ` ${key}: ${slot.logString}\n`;
+    });
 
     LoggerProxy.logger.log(
-      `RemoteMediaManager#updateVideoReceiveSlots --> receive slots updated: unused=${this.slots.video.unused.length}, activeSpeaker=${this.slots.video.activeSpeaker.length}, receiverSelected=${this.slots.video.receiverSelected.length}`
+      `RemoteMediaManager#updateVideoReceiveSlots --> receive slots updated: unused=${this.slots.video.unused.length}, activeSpeaker=${this.slots.video.activeSpeaker.length}, receiverSelected=${this.slots.video.receiverSelected.length}\n${logMessage}`
     );
+  }
+
+  /**
+   * Makes sure we have the right number of receive slots created for the current layout
+   * and allocates them to the right video panes / pane groups
+   *
+   * @returns {Promise}
+   */
+  private async updateVideoReceiveSlots() {
+    // move all active speaker slots to "unused"
+    this.trimActiveSpeakerSlots();
+
+    // move all no longer needed receiver-selected slots to "unused"
+    this.trimReceiverSelectedSlots();
+
+    // ensure we have enough total slots for current layout
+    await this.refillRequiredSlotsIfNeeded();
+
+    // allocate the slots to the right panes / pane groups
+    // reset allocations
+    this.receiveSlotAllocations = {activeSpeaker: {}, receiverSelected: {}};
+    // allocate active speaker
+    this.allocateSlotsToActiveSpeakerPaneGroups();
+    // allocate receiver selected
+    this.allocateSlotsToReceiverSelectedVideoPaneGroups();
+
+    this.logReceieveSlots();
 
     // If this is the initial layout, there may be some "unused" slots left because of the preallocation
     // done in this.preallocateVideoReceiveSlots(), so release them now
@@ -631,7 +745,13 @@ export class RemoteMediaManager extends EventsScope {
    */
   private updateVideoRemoteMediaObjects() {
     // invalidate all the previous remote media objects and cancel their media requests
-    this.invalidateCurrentRemoteMedia({audio: false, video: true, commit: false});
+    this.invalidateCurrentRemoteMedia({
+      audio: false,
+      video: true,
+      screenShareAudio: false,
+      screenShareVideo: false,
+      commit: false,
+    });
 
     // create new remoteMediaGroup objects
     this.media.video.activeSpeakerGroups = {};
@@ -686,16 +806,53 @@ export class RemoteMediaManager extends EventsScope {
         );
       }
     }
-    // todo: screenshare (SPARK-377812)
 
     this.mediaRequestManagers.video.commit();
   }
 
+  /**
+   * Checks if current layout requires a screen share.
+   * If it does, it creates new RemoteMediaGroup object for screen share
+   * and sends the media requests for it.
+   * If it doesn't, it makes sure we clean up any RemoteMediaGroup objects
+   * created earlier for screen share (for previous layout).
+   */
+  private updateScreenShareVideoRemoteMediaObject() {
+    this.invalidateCurrentRemoteMedia({
+      audio: false,
+      video: false,
+      screenShareAudio: false,
+      screenShareVideo: true,
+      commit: false,
+    });
+
+    this.media.screenShare.video = undefined;
+
+    if (this.currentLayout?.screenShareVideo) {
+      // we create a group of 1, because for screen share we need to use the "active speaker" policy
+      this.media.screenShare.video = new RemoteMediaGroup(
+        this.mediaRequestManagers.screenShareVideo,
+        [this.slots.screenShare.video],
+        255,
+        false,
+        {resolution: this.currentLayout.screenShareVideo.size}
+      );
+    }
+
+    this.mediaRequestManagers.screenShareVideo.commit();
+  }
+
   /**
    * Invalidates all remote media objects belonging to currently selected layout
    */
-  private invalidateCurrentRemoteMedia(options: {audio: boolean; video: boolean; commit: boolean}) {
-    const {audio, video, commit} = options;
+  private invalidateCurrentRemoteMedia(options: {
+    audio: boolean;
+    video: boolean;
+    screenShareAudio: boolean;
+    screenShareVideo: boolean;
+    commit: boolean;
+  }) {
+    const {audio, video, screenShareAudio, screenShareVideo, commit} = options;
 
     if (audio && this.media.audio) {
       this.media.audio.stop(commit);
@@ -711,12 +868,19 @@ export class RemoteMediaManager extends EventsScope {
         this.mediaRequestManagers.video.commit();
       }
     }
+
+    if (screenShareAudio && this.media.screenShare.audio) {
+      this.media.screenShare.audio.stop(commit);
+    }
+    if (screenShareVideo && this.media.screenShare.video) {
+      this.media.screenShare.video.stop(commit);
+    }
   }
 
   /** emits Event.VideoLayoutChanged */
   private emitVideoLayoutChangedEvent() {
     // todo: at this point the receive slots might still be showing a participant from previous layout, we should
-    // wait for our media requests to be fullfilled, but there is no API for that right now (we could wait for source updates
+    // wait for our media requests to be fulfilled, but there is no API for that right now (we could wait for source updates
     // but in some cases they might never come, or would need to always make sure to use a new set of receiver slots)
     // for now it's fine to have it like this, we will re-evaluate if it needs improving after more testing
 
@@ -730,7 +894,7 @@ export class RemoteMediaManager extends EventsScope {
         layoutId: this.currentLayoutId,
         activeSpeakerVideoPanes: this.media.video.activeSpeakerGroups,
         memberVideoPanes: this.media.video.memberPanes,
-        screenShareVideo: undefined, // todo: screen share (SPARK-377812)
+        screenShareVideo: this.media.screenShare.video?.getRemoteMedia()[0],
       }
     );
   }
@@ -778,7 +942,7 @@ export class RemoteMediaManager extends EventsScope {
 
     this.currentLayout.memberVideoPanes.push(newPane);
 
-    const receiveSlot = await this.receiveSlotManager.allocateSlot(MC.MediaType.VideoMain);
+    const receiveSlot = await this.receiveSlotManager.allocateSlot(MediaType.VideoMain);
 
     this.slots.video.receiverSelected.push(receiveSlot);