@webex/plugin-meetings 3.0.0-beta.2 → 3.0.0-beta.21

package/src/multistream/receiveSlotManager.ts

@@ -1,5 +1,6 @@
+/* eslint-disable valid-jsdoc */
 /* eslint-disable import/prefer-default-export */
-import {MediaConnection as MC} from '@webex/internal-media-core';
+import {MediaType} from '@webex/internal-media-core';
 
 import LoggerProxy from '../common/logs/logger-proxy';
 import Meeting from '../meeting';
@@ -11,9 +12,9 @@ import {CSI, ReceiveSlot} from './receiveSlot';
  * so this manager has a pool in order to re-use the slots that were released earlier.
  */
 export class ReceiveSlotManager {
-  private allocatedSlots: {[key in MC.MediaType]: ReceiveSlot[]};
+  private allocatedSlots: {[key in MediaType]: ReceiveSlot[]};
 
-  private freeSlots: {[key in MC.MediaType]: ReceiveSlot[]};
+  private freeSlots: {[key in MediaType]: ReceiveSlot[]};
 
   private meeting: Meeting;
 
@@ -23,16 +24,16 @@ export class ReceiveSlotManager {
    */
   constructor(meeting) {
     this.allocatedSlots = {
-      [MC.MediaType.AudioMain]: [],
-      [MC.MediaType.VideoMain]: [],
-      [MC.MediaType.AudioSlides]: [],
-      [MC.MediaType.VideoSlides]: [],
+      [MediaType.AudioMain]: [],
+      [MediaType.VideoMain]: [],
+      [MediaType.AudioSlides]: [],
+      [MediaType.VideoSlides]: [],
     };
     this.freeSlots = {
-      [MC.MediaType.AudioMain]: [],
-      [MC.MediaType.VideoMain]: [],
-      [MC.MediaType.AudioSlides]: [],
-      [MC.MediaType.VideoSlides]: [],
+      [MediaType.AudioMain]: [],
+      [MediaType.VideoMain]: [],
+      [MediaType.AudioSlides]: [],
+      [MediaType.VideoSlides]: [],
     };
     this.meeting = meeting;
   }
@@ -40,10 +41,10 @@ export class ReceiveSlotManager {
   /**
    * Creates a new receive slot or returns one from the existing pool of free slots
    *
-   * @param {MC.MediaType} mediaType
+   * @param {MediaType} mediaType
    * @returns {Promise<ReceiveSlot>}
    */
-  async allocateSlot(mediaType: MC.MediaType): Promise<ReceiveSlot> {
+  async allocateSlot(mediaType: MediaType): Promise<ReceiveSlot> {
     if (!this.meeting?.mediaProperties?.webrtcMediaConnection) {
       return Promise.reject(new Error('Webrtc media connection is missing'));
     }
@@ -66,6 +67,7 @@ export class ReceiveSlotManager {
     const receiveSlot = new ReceiveSlot(
       mediaType,
       wcmeReceiveSlot,
+      // @ts-ignore
       (csi: CSI) => this.meeting.members.findMemberByCsi(csi)?.id
     );
 
@@ -100,16 +102,16 @@ export class ReceiveSlotManager {
    */
   reset() {
     this.allocatedSlots = {
-      [MC.MediaType.AudioMain]: [],
-      [MC.MediaType.VideoMain]: [],
-      [MC.MediaType.AudioSlides]: [],
-      [MC.MediaType.VideoSlides]: [],
+      [MediaType.AudioMain]: [],
+      [MediaType.VideoMain]: [],
+      [MediaType.AudioSlides]: [],
+      [MediaType.VideoSlides]: [],
     };
     this.freeSlots = {
-      [MC.MediaType.AudioMain]: [],
-      [MC.MediaType.VideoMain]: [],
-      [MC.MediaType.AudioSlides]: [],
-      [MC.MediaType.VideoSlides]: [],
+      [MediaType.AudioMain]: [],
+      [MediaType.VideoMain]: [],
+      [MediaType.AudioSlides]: [],
+      [MediaType.VideoSlides]: [],
     };
   }
 

package/src/multistream/remoteMedia.ts

@@ -1,4 +1,5 @@
 /* eslint-disable valid-jsdoc */
+import {MediaType, SourceState} from '@webex/internal-media-core';
 import LoggerProxy from '../common/logs/logger-proxy';
 import EventsScope from '../common/events/events-scope';
 
@@ -7,6 +8,7 @@ import {CSI, ReceiveSlot, ReceiveSlotEvents} from './receiveSlot';
 
 export const RemoteMediaEvents = {
   SourceUpdate: ReceiveSlotEvents.SourceUpdate,
+  Stopped: 'stopped',
 };
 
 export type RemoteVideoResolution =
@@ -55,7 +57,7 @@ export function getMaxFs(paneSize: RemoteVideoResolution): number {
 }
 
 type Options = {
-  resolution?: RemoteVideoResolution; // applies only to groups of type MC.MediaType.VideoMain and MC.MediaType.VideoSlides
+  resolution?: RemoteVideoResolution; // applies only to groups of type MediaType.VideoMain and MediaType.VideoSlides
 };
 
 export type RemoteMediaId = string;
@@ -108,10 +110,18 @@ export class RemoteMedia extends EventsScope {
    * @param {boolean} commit - whether to commit the cancellation of the media request
    * @internal
    */
-  public stop(commit: boolean = true) {
+  public stop(commit = true) {
     this.cancelMediaRequest(commit);
     this.receiveSlot?.removeAllListeners();
     this.receiveSlot = undefined;
+    this.emit(
+      {
+        file: 'multistream/remoteMedia',
+        function: 'stop',
+      },
+      RemoteMediaEvents.Stopped,
+      {}
+    );
   }
 
   /**
@@ -164,7 +174,7 @@ export class RemoteMedia extends EventsScope {
   private setupEventListeners() {
     if (this.receiveSlot) {
       const scope = {
-        file: 'meeting/remoteMedia',
+        file: 'multistream/remoteMedia',
         function: 'setupEventListeners',
       };
 
@@ -177,7 +187,7 @@ export class RemoteMedia extends EventsScope {
   /**
    * Getter for mediaType
    */
-  public get mediaType() {
+  public get mediaType(): MediaType {
     return this.receiveSlot?.mediaType;
   }
 
@@ -198,7 +208,7 @@ export class RemoteMedia extends EventsScope {
   /**
    * Getter for source state
    */
-  public get sourceState() {
+  public get sourceState(): SourceState {
     return this.receiveSlot?.sourceState;
   }
 

package/src/multistream/remoteMediaGroup.ts

@@ -8,8 +8,8 @@ import {MediaRequestId, MediaRequestManager} from './mediaRequestManager';
 import {CSI, ReceiveSlot} from './receiveSlot';
 
 type Options = {
-  resolution?: RemoteVideoResolution; // applies only to groups of type MC.MediaType.VideoMain and MC.MediaType.VideoSlides
-  preferLiveVideo?: boolean; // applies only to groups of type MC.MediaType.VideoMain and MC.MediaType.VideoSlides
+  resolution?: RemoteVideoResolution; // applies only to groups of type MediaType.VideoMain and MediaType.VideoSlides
+  preferLiveVideo?: boolean; // applies only to groups of type MediaType.VideoMain and MediaType.VideoSlides
 };
 
 export class RemoteMediaGroup {
@@ -62,6 +62,7 @@ export class RemoteMediaGroup {
       // return a shallow copy so that the client cannot modify this.pinnedRemoteMedia array
       return [...this.pinnedRemoteMedia];
     }
+
     return [...this.unpinnedRemoteMedia, ...this.pinnedRemoteMedia];
   }
 
@@ -189,7 +190,7 @@ export class RemoteMediaGroup {
    * @param{boolean} commit whether to commit the cancellation of media requests
    * @internal
    */
-  public stop(commit: boolean = true) {
+  public stop(commit = true) {
     this.unpinnedRemoteMedia.forEach((remoteMedia) => remoteMedia.stop(false));
     this.pinnedRemoteMedia.forEach((remoteMedia) => remoteMedia.stop(false));
     this.cancelActiveSpeakerMediaRequest(false);

package/src/multistream/remoteMediaManager.ts

@@ -1,6 +1,7 @@
+/* eslint-disable valid-jsdoc */
 import {cloneDeep, remove} from 'lodash';
 import {EventMap} from 'typed-emitter';
-import {MediaConnection as MC} from '@webex/internal-media-core';
+import {MediaType} from '@webex/internal-media-core';
 
 import LoggerProxy from '../common/logs/logger-proxy';
 import EventsScope from '../common/events/events-scope';
@@ -30,8 +31,8 @@ export interface MemberVideoPane {
 }
 
 export interface VideoLayout {
-  screenShareVideo: {
-    size: PaneSize | null; // null if you don't want to receive any screen share video
+  screenShareVideo?: {
+    size: PaneSize;
   };
   activeSpeakerVideoPaneGroups?: ActiveSpeakerVideoPaneGroup[]; // list of active speaker video pane groups
   memberVideoPanes?: MemberVideoPane[]; // list of video panes for specific members, CSI values can be changed later via setVideoPaneCsi()
@@ -40,6 +41,7 @@ export interface VideoLayout {
 export interface Configuration {
   audio: {
     numOfActiveSpeakerStreams: number; // number of audio streams we want to receive
+    numOfScreenShareStreams: number; // 1 should be enough, because in webex only 1 person at a time can be presenting screen share
   };
   video: {
     preferLiveVideo: boolean; // applies to all pane groups with active speaker policy
@@ -47,17 +49,12 @@ export interface Configuration {
 
     layouts: {[key: LayoutId]: VideoLayout}; // a map of all available layouts, a layout can be set via setLayout() method
   };
-  screenShare: {
-    audio: boolean; // whether we ever want to receive screen share audio at all
-    video: boolean; // whether we ever want to receive screen share video at all
-  };
 }
 
 /* Predefined layouts: */
 
 // An "all equal" grid, with size up to 3 x 3 = 9:
 const AllEqualLayout: VideoLayout = {
-  screenShareVideo: {size: null},
   activeSpeakerVideoPaneGroups: [
     {
       id: 'main',
@@ -70,7 +67,6 @@ const AllEqualLayout: VideoLayout = {
 
 // A layout with just a single remote active speaker video pane:
 const SingleLayout: VideoLayout = {
-  screenShareVideo: {size: null},
   activeSpeakerVideoPaneGroups: [
     {
       id: 'main',
@@ -83,7 +79,6 @@ const SingleLayout: VideoLayout = {
 
 // A layout with 1 big pane for the highest priority active speaker and 5 small panes for other active speakers:
 const OnePlusFiveLayout: VideoLayout = {
-  screenShareVideo: {size: null},
   activeSpeakerVideoPaneGroups: [
     {
       id: 'mainBigOne',
@@ -102,7 +97,6 @@ const OnePlusFiveLayout: VideoLayout = {
 
 // A layout with 2 big panes for 2 main active speakers and a strip of 6 small panes for other active speakers:
 const TwoMainPlusSixSmallLayout: VideoLayout = {
-  screenShareVideo: {size: null},
   activeSpeakerVideoPaneGroups: [
     {
       id: 'mainGroupWith2BigPanes',
@@ -121,7 +115,7 @@ const TwoMainPlusSixSmallLayout: VideoLayout = {
 
 // A strip of 8 small video panes (thumbnails) displayed at the top of a remote screenshare:
 const RemoteScreenShareWithSmallThumbnailsLayout: VideoLayout = {
-  screenShareVideo: {size: 'best'},
+  // screenShareVideo: {size: 'best'}, // todo: SPARK-393485: uncomment this once backend supports screen sharing
   activeSpeakerVideoPaneGroups: [
     {
       id: 'thumbnails',
@@ -134,7 +128,6 @@ const RemoteScreenShareWithSmallThumbnailsLayout: VideoLayout = {
 
 // A staged layout with 4 pre-selected meeting participants in the main 2x2 grid and 6 small panes for other active speakers at the top:
 const Stage2x2With6ThumbnailsLayout: VideoLayout = {
-  screenShareVideo: {size: null},
   activeSpeakerVideoPaneGroups: [
     {
       id: 'thumbnails',
@@ -160,6 +153,7 @@ const Stage2x2With6ThumbnailsLayout: VideoLayout = {
 export const DefaultConfiguration: Configuration = {
   audio: {
     numOfActiveSpeakerStreams: 3,
+    numOfScreenShareStreams: 0, // todo: SPARK-393485: change to 1 once backend supports screen sharing
   },
   video: {
     preferLiveVideo: true,
@@ -173,16 +167,12 @@ export const DefaultConfiguration: Configuration = {
       ScreenShareView: RemoteScreenShareWithSmallThumbnailsLayout,
     },
   },
-  screenShare: {
-    audio: true,
-    video: true,
-  },
 };
 
 export enum Event {
   // events for audio streams
   AudioCreated = 'AudioCreated',
-  ScreenShareAudioCreated = 'ScreenShareCreated',
+  ScreenShareAudioCreated = 'ScreenShareAudioCreated',
 
   // events for video streams
   VideoLayoutChanged = 'VideoLayoutChanged',
@@ -199,7 +189,7 @@ export interface VideoLayoutChangedEventData {
 export interface Events extends EventMap {
   // audio
   [Event.AudioCreated]: (audio: RemoteMediaGroup) => void;
-  [Event.ScreenShareAudioCreated]: (screenShareAudio: RemoteMedia) => void;
+  [Event.ScreenShareAudioCreated]: (screenShareAudio: RemoteMediaGroup) => void;
 
   // video
   [Event.VideoLayoutChanged]: (data: VideoLayoutChangedEventData) => void;
@@ -223,14 +213,18 @@ export class RemoteMediaManager extends EventsScope {
   private mediaRequestManagers: {
     audio: MediaRequestManager;
     video: MediaRequestManager;
+    screenShareAudio: MediaRequestManager;
+    screenShareVideo: MediaRequestManager;
   };
 
   private currentLayout?: VideoLayout;
 
   private slots: {
     audio: ReceiveSlot[];
-    screenShareAudio?: ReceiveSlot;
-    screenShareVideo?: ReceiveSlot;
+    screenShare: {
+      audio: ReceiveSlot[];
+      video?: ReceiveSlot;
+    };
     video: {
       unused: ReceiveSlot[];
       activeSpeaker: ReceiveSlot[];
@@ -246,6 +240,10 @@ export class RemoteMediaManager extends EventsScope {
       };
       memberPanes: {[key: PaneId]: RemoteMedia};
     };
+    screenShare: {
+      audio?: RemoteMediaGroup;
+      video?: RemoteMediaGroup;
+    };
   };
 
   private receiveSlotAllocations: {
@@ -267,6 +265,8 @@ export class RemoteMediaManager extends EventsScope {
     mediaRequestManagers: {
       audio: MediaRequestManager;
       video: MediaRequestManager;
+      screenShareAudio: MediaRequestManager;
+      screenShareVideo: MediaRequestManager;
     },
     config: Configuration = DefaultConfiguration
   ) {
@@ -281,14 +281,20 @@ export class RemoteMediaManager extends EventsScope {
         activeSpeakerGroups: {},
         memberPanes: {},
       },
+      screenShare: {
+        audio: undefined,
+        video: undefined,
+      },
     };
 
     this.checkConfigValidity();
 
     this.slots = {
       audio: [],
-      screenShareAudio: undefined,
-      screenShareVideo: undefined,
+      screenShare: {
+        audio: [],
+        video: undefined,
+      },
       video: {
         unused: [],
         activeSpeaker: [],
@@ -359,8 +365,8 @@ export class RemoteMediaManager extends EventsScope {
 
     await this.createAudioMedia();
 
-    // todo: create screen share audio remote media (SPARK-377812)
-    // todo: create screen share video receive slot (SPARK-377812)
+    await this.createScreenShareReceiveSlots();
+    this.createScreenShareAudioMedia();
 
     await this.preallocateVideoReceiveSlots();
 
@@ -373,14 +379,27 @@ export class RemoteMediaManager extends EventsScope {
    */
   public stop() {
     // invalidate all remoteMedia objects
-    this.invalidateCurrentRemoteMedia({audio: true, video: true, commit: true});
+    this.invalidateCurrentRemoteMedia({
+      audio: true,
+      video: true,
+      screenShareAudio: true,
+      screenShareVideo: true,
+      commit: true,
+    });
 
     // release all audio receive slots
     this.slots.audio.forEach((slot) => this.receiveSlotManager.releaseSlot(slot));
     this.slots.audio.length = 0;
 
-    // todo: screenshare slots... (SPARK-377812)
+    // release screen share slots
+    this.slots.screenShare.audio.forEach((slot) => this.receiveSlotManager.releaseSlot(slot));
+    this.slots.screenShare.audio.length = 0;
+    if (this.slots.screenShare.video) {
+      this.receiveSlotManager.releaseSlot(this.slots.screenShare.video);
+      this.slots.screenShare.video = undefined;
+    }
 
+    // release video slots
     this.receiveSlotAllocations = {activeSpeaker: {}, receiverSelected: {}};
 
     this.slots.video.unused.push(...this.slots.video.activeSpeaker);
@@ -432,7 +451,8 @@ export class RemoteMediaManager extends EventsScope {
     while (this.slots.video.unused.length < maxNumVideoPanesRequired) {
       // eslint-disable-next-line no-await-in-loop
       this.slots.video.unused.push(
-        await this.receiveSlotManager.allocateSlot(MC.MediaType.VideoMain)
+        // eslint-disable-next-line no-await-in-loop
+        await this.receiveSlotManager.allocateSlot(MediaType.VideoMain)
       );
     }
   }
@@ -457,6 +477,7 @@ export class RemoteMediaManager extends EventsScope {
 
     await this.updateVideoReceiveSlots();
     this.updateVideoRemoteMediaObjects();
+    this.updateScreenShareVideoRemoteMediaObject();
     this.emitVideoLayoutChangedEvent();
   }
 
@@ -476,7 +497,7 @@ export class RemoteMediaManager extends EventsScope {
     // create the audio receive slots
     for (let i = 0; i < this.config.audio.numOfActiveSpeakerStreams; i += 1) {
       // eslint-disable-next-line no-await-in-loop
-      const slot = await this.receiveSlotManager.allocateSlot(MC.MediaType.AudioMain);
+      const slot = await this.receiveSlotManager.allocateSlot(MediaType.AudioMain);
 
       this.slots.audio.push(slot);
     }
@@ -496,6 +517,50 @@ export class RemoteMediaManager extends EventsScope {
     );
   }
 
+  /**
+   * Creates receive slots required for receiving screen share audio and video
+   */
+  private async createScreenShareReceiveSlots() {
+    // audio
+    for (let i = 0; i < this.config.audio.numOfScreenShareStreams; i += 1) {
+      // eslint-disable-next-line no-await-in-loop
+      const slot = await this.receiveSlotManager.allocateSlot(MediaType.AudioSlides);
+
+      this.slots.screenShare.audio.push(slot);
+    }
+
+    // video
+    const isAnyLayoutContainingScreenShareVideo = Object.values(this.config.video.layouts).some(
+      (layout) => !!layout.screenShareVideo
+    );
+
+    if (isAnyLayoutContainingScreenShareVideo) {
+      this.slots.screenShare.video = await this.receiveSlotManager.allocateSlot(
+        MediaType.VideoSlides
+      );
+    }
+  }
+
+  /**
+   * Creates RemoteMedia objects for screen share
+   */
+  private createScreenShareAudioMedia() {
+    if (this.slots.screenShare.audio.length > 0) {
+      this.media.screenShare.audio = new RemoteMediaGroup(
+        this.mediaRequestManagers.screenShareAudio,
+        this.slots.screenShare.audio,
+        255,
+        true
+      );
+
+      this.emit(
+        {file: 'multistream/remoteMediaManager', function: 'createScreenShareAudioMedia'},
+        Event.ScreenShareAudioCreated,
+        this.media.screenShare.audio
+      );
+    }
+  }
+
   /**
    * Goes over all receiver-selected slots and keeps only the ones that are required by a given layout,
    * the rest are all moved to the "unused" list
@@ -600,7 +665,8 @@ export class RemoteMediaManager extends EventsScope {
       while (numSlotsToCreate > 0) {
         // eslint-disable-next-line no-await-in-loop
         this.slots.video.unused.push(
-          await this.receiveSlotManager.allocateSlot(MC.MediaType.VideoMain)
+          // eslint-disable-next-line no-await-in-loop
+          await this.receiveSlotManager.allocateSlot(MediaType.VideoMain)
         );
         numSlotsToCreate -= 1;
       }
@@ -631,7 +697,13 @@ export class RemoteMediaManager extends EventsScope {
    */
   private updateVideoRemoteMediaObjects() {
     // invalidate all the previous remote media objects and cancel their media requests
-    this.invalidateCurrentRemoteMedia({audio: false, video: true, commit: false});
+    this.invalidateCurrentRemoteMedia({
+      audio: false,
+      video: true,
+      screenShareAudio: false,
+      screenShareVideo: false,
+      commit: false,
+    });
 
     // create new remoteMediaGroup objects
     this.media.video.activeSpeakerGroups = {};
@@ -686,16 +758,53 @@ export class RemoteMediaManager extends EventsScope {
         );
       }
     }
-    // todo: screenshare (SPARK-377812)
 
     this.mediaRequestManagers.video.commit();
   }
 
+  /**
+   * Checks if current layout requires a screen share.
+   * If it does, it creates new RemoteMediaGroup object for screen share
+   * and sends the media requests for it.
+   * If it doesn't, it makes sure we clean up any RemoteMediaGroup objects
+   * created earlier for screen share (for previous layout).
+   */
+  private updateScreenShareVideoRemoteMediaObject() {
+    this.invalidateCurrentRemoteMedia({
+      audio: false,
+      video: false,
+      screenShareAudio: false,
+      screenShareVideo: true,
+      commit: false,
+    });
+
+    this.media.screenShare.video = undefined;
+
+    if (this.currentLayout?.screenShareVideo) {
+      // we create a group of 1, because for screen share we need to use the "active speaker" policy
+      this.media.screenShare.video = new RemoteMediaGroup(
+        this.mediaRequestManagers.screenShareVideo,
+        [this.slots.screenShare.video],
+        255,
+        false,
+        {resolution: this.currentLayout.screenShareVideo.size}
+      );
+    }
+
+    this.mediaRequestManagers.screenShareVideo.commit();
+  }
+
   /**
    * Invalidates all remote media objects belonging to currently selected layout
    */
-  private invalidateCurrentRemoteMedia(options: {audio: boolean; video: boolean; commit: boolean}) {
-    const {audio, video, commit} = options;
+  private invalidateCurrentRemoteMedia(options: {
+    audio: boolean;
+    video: boolean;
+    screenShareAudio: boolean;
+    screenShareVideo: boolean;
+    commit: boolean;
+  }) {
+    const {audio, video, screenShareAudio, screenShareVideo, commit} = options;
 
     if (audio && this.media.audio) {
       this.media.audio.stop(commit);
@@ -711,12 +820,19 @@ export class RemoteMediaManager extends EventsScope {
         this.mediaRequestManagers.video.commit();
       }
     }
+
+    if (screenShareAudio && this.media.screenShare.audio) {
+      this.media.screenShare.audio.stop(commit);
+    }
+    if (screenShareVideo && this.media.screenShare.video) {
+      this.media.screenShare.video.stop(commit);
+    }
   }
 
   /** emits Event.VideoLayoutChanged */
   private emitVideoLayoutChangedEvent() {
     // todo: at this point the receive slots might still be showing a participant from previous layout, we should
-    // wait for our media requests to be fullfilled, but there is no API for that right now (we could wait for source updates
+    // wait for our media requests to be fulfilled, but there is no API for that right now (we could wait for source updates
     // but in some cases they might never come, or would need to always make sure to use a new set of receiver slots)
     // for now it's fine to have it like this, we will re-evaluate if it needs improving after more testing
 
@@ -730,7 +846,7 @@ export class RemoteMediaManager extends EventsScope {
         layoutId: this.currentLayoutId,
         activeSpeakerVideoPanes: this.media.video.activeSpeakerGroups,
         memberVideoPanes: this.media.video.memberPanes,
-        screenShareVideo: undefined, // todo: screen share (SPARK-377812)
+        screenShareVideo: this.media.screenShare.video?.getRemoteMedia()[0],
       }
     );
   }
@@ -778,7 +894,7 @@ export class RemoteMediaManager extends EventsScope {
 
     this.currentLayout.memberVideoPanes.push(newPane);
 
-    const receiveSlot = await this.receiveSlotManager.allocateSlot(MC.MediaType.VideoMain);
+    const receiveSlot = await this.receiveSlotManager.allocateSlot(MediaType.VideoMain);
 
     this.slots.video.receiverSelected.push(receiveSlot);