@webex/plugin-meetings 3.0.0-beta.39 → 3.0.0-beta.391

package/src/multistream/remoteMediaGroup.ts

@@ -1,6 +1,8 @@
 /* eslint-disable valid-jsdoc */
 /* eslint-disable require-jsdoc */
 /* eslint-disable import/prefer-default-export */
+import {forEach} from 'lodash';
+import {NamedMediaGroup} from '@webex/json-multistream';
 import LoggerProxy from '../common/logs/logger-proxy';
 
 import {getMaxFs, RemoteMedia, RemoteVideoResolution} from './remoteMedia';
@@ -10,6 +12,7 @@ import {CSI, ReceiveSlot} from './receiveSlot';
 type Options = {
   resolution?: RemoteVideoResolution; // applies only to groups of type MediaType.VideoMain and MediaType.VideoSlides
   preferLiveVideo?: boolean; // applies only to groups of type MediaType.VideoMain and MediaType.VideoSlides
+  namedMediaGroup?: NamedMediaGroup; // applies only to named media groups for audio
 };
 
 export class RemoteMediaGroup {
@@ -66,6 +69,53 @@ export class RemoteMediaGroup {
     return [...this.unpinnedRemoteMedia, ...this.pinnedRemoteMedia];
   }
 
+  /**
+   * Sets CSIs for multiple RemoteMedia instances belonging to this RemoteMediaGroup.
+   * For each entry in the remoteMediaCsis array:
+   * - if csi is specified, the RemoteMedia instance is pinned to that CSI
+   * - if csi is undefined, the RemoteMedia instance is unpinned
+   * @internal
+   */
+  public setActiveSpeakerCsis(
+    remoteMediaCsis: {remoteMedia: RemoteMedia; csi?: number}[],
+    commit = true
+  ): void {
+    forEach(remoteMediaCsis, ({csi, remoteMedia}) => {
+      if (csi) {
+        if (!(this.pinnedRemoteMedia.indexOf(remoteMedia) >= 0)) {
+          const unpinId = this.unpinnedRemoteMedia.indexOf(remoteMedia);
+          if (unpinId >= 0) {
+            this.unpinnedRemoteMedia.splice(unpinId, 1);
+            this.pinnedRemoteMedia.push(remoteMedia);
+          } else {
+            throw new Error(
+              `failed to pin a remote media object ${remoteMedia.id}, because it is not found in this remote media group`
+            );
+          }
+        }
+        remoteMedia.sendMediaRequest(csi, false);
+      } else {
+        if (!(this.unpinnedRemoteMedia.indexOf(remoteMedia) >= 0)) {
+          const pinId = this.pinnedRemoteMedia.indexOf(remoteMedia);
+          if (pinId >= 0) {
+            this.pinnedRemoteMedia.splice(pinId, 1);
+            this.unpinnedRemoteMedia.push(remoteMedia);
+          } else {
+            throw new Error(
+              `failed to unpin a remote media object ${remoteMedia.id}, because it is not found in this remote media group`
+            );
+          }
+        }
+        remoteMedia.cancelMediaRequest(false);
+      }
+    });
+    this.cancelActiveSpeakerMediaRequest(false);
+    this.sendActiveSpeakerMediaRequest(false);
+    if (commit) {
+      this.mediaRequestManager.commit();
+    }
+  }
+
   /**
    * Pins a specific remote media instance to a specfic CSI, so the media will
    * no longer come from active speaker, but from that CSI.
@@ -151,6 +201,17 @@ export class RemoteMediaGroup {
     throw new Error(`remote media object ${remoteMedia.id} not found in the group`);
   }
 
+  /**
+   * setPreferLiveVideo - sets preferLiveVideo to true/false
+   * @internal
+   */
+  public setPreferLiveVideo(preferLiveVideo: boolean, commit: boolean) {
+    if (this.options.preferLiveVideo !== preferLiveVideo) {
+      this.options.preferLiveVideo = preferLiveVideo;
+      this.sendActiveSpeakerMediaRequest(commit);
+    }
+  }
+
   private sendActiveSpeakerMediaRequest(commit: boolean) {
     this.cancelActiveSpeakerMediaRequest(false);
 
@@ -162,6 +223,9 @@ export class RemoteMediaGroup {
           crossPriorityDuplication: false,
           crossPolicyDuplication: false,
           preferLiveVideo: !!this.options?.preferLiveVideo,
+          namedMediaGroups: this.options.namedMediaGroup?.value
+            ? [this.options?.namedMediaGroup]
+            : undefined,
         },
         receiveSlots: this.unpinnedRemoteMedia.map((remoteMedia) =>
           remoteMedia.getUnderlyingReceiveSlot()
@@ -182,6 +246,20 @@ export class RemoteMediaGroup {
     }
   }
 
+  /**
+   * setNamedMediaGroup - sets named media group type and value
+   * @internal
+   */
+  public setNamedMediaGroup(namedMediaGroup: NamedMediaGroup, commit: boolean) {
+    if (
+      this.options.namedMediaGroup.value !== namedMediaGroup.value ||
+      this.options.namedMediaGroup.type !== namedMediaGroup.type
+    ) {
+      this.options.namedMediaGroup = namedMediaGroup;
+      this.sendActiveSpeakerMediaRequest(commit);
+    }
+  }
+
   /**
    * Invalidates the remote media group by clearing the references to the receive slots
    * used by all remote media from that group and cancelling all media requests.

package/src/multistream/remoteMediaManager.ts

@@ -1,7 +1,8 @@
 /* eslint-disable valid-jsdoc */
-import {cloneDeep, remove} from 'lodash';
+import {cloneDeep, forEach, remove} from 'lodash';
 import {EventMap} from 'typed-emitter';
 import {MediaType} from '@webex/internal-media-core';
+import {NamedMediaGroup} from '@webex/json-multistream';
 
 import LoggerProxy from '../common/logs/logger-proxy';
 import EventsScope from '../common/events/events-scope';
@@ -11,6 +12,7 @@ import {ReceiveSlot, CSI} from './receiveSlot';
 import {ReceiveSlotManager} from './receiveSlotManager';
 import {RemoteMediaGroup} from './remoteMediaGroup';
 import {MediaRequestManager} from './mediaRequestManager';
+import {NAMED_MEDIA_GROUP_TYPE_AUDIO} from '../constants';
 
 export type PaneSize = RemoteVideoResolution;
 export type LayoutId = string;
@@ -49,6 +51,7 @@ export interface Configuration {
 
     layouts: {[key: LayoutId]: VideoLayout}; // a map of all available layouts, a layout can be set via setLayout() method
   };
+  namedMediaGroup?: NamedMediaGroup;
 }
 
 /* Predefined layouts: */
@@ -96,6 +99,7 @@ const OnePlusFiveLayout: VideoLayout = {
 };
 
 // A layout with 2 big panes for 2 main active speakers and a strip of 6 small panes for other active speakers:
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
 const TwoMainPlusSixSmallLayout: VideoLayout = {
   activeSpeakerVideoPaneGroups: [
     {
@@ -172,6 +176,7 @@ export const DefaultConfiguration: Configuration = {
 export enum Event {
   // events for audio streams
   AudioCreated = 'AudioCreated',
+  InterpretationAudioCreated = 'InterpretationAudioCreated',
   ScreenShareAudioCreated = 'ScreenShareAudioCreated',
 
   // events for video streams
@@ -220,7 +225,10 @@ export class RemoteMediaManager extends EventsScope {
   private currentLayout?: VideoLayout;
 
   private slots: {
-    audio: ReceiveSlot[];
+    audio: {
+      main: ReceiveSlot[];
+      si: ReceiveSlot;
+    };
     screenShare: {
       audio: ReceiveSlot[];
       video?: ReceiveSlot;
@@ -233,7 +241,10 @@ export class RemoteMediaManager extends EventsScope {
   };
 
   private media: {
-    audio?: RemoteMediaGroup;
+    audio: {
+      main?: RemoteMediaGroup;
+      si?: RemoteMediaGroup;
+    };
     video: {
       activeSpeakerGroups: {
         [key: PaneGroupId]: RemoteMediaGroup;
@@ -276,7 +287,10 @@ export class RemoteMediaManager extends EventsScope {
     this.receiveSlotManager = receiveSlotManager;
     this.mediaRequestManagers = mediaRequestManagers;
     this.media = {
-      audio: undefined,
+      audio: {
+        main: undefined,
+        si: undefined,
+      },
       video: {
         activeSpeakerGroups: {},
         memberPanes: {},
@@ -290,7 +304,10 @@ export class RemoteMediaManager extends EventsScope {
     this.checkConfigValidity();
 
     this.slots = {
-      audio: [],
+      audio: {
+        main: [],
+        si: undefined,
+      },
       screenShare: {
         audio: [],
         video: undefined,
@@ -388,8 +405,11 @@ export class RemoteMediaManager extends EventsScope {
     });
 
     // release all audio receive slots
-    this.slots.audio.forEach((slot) => this.receiveSlotManager.releaseSlot(slot));
-    this.slots.audio.length = 0;
+    this.slots.audio.main.forEach((slot) => this.receiveSlotManager.releaseSlot(slot));
+    this.slots.audio.main.length = 0;
+    if (this.slots.audio.si) {
+      this.receiveSlotManager.releaseSlot(this.slots.audio.si);
+    }
 
     // release screen share slots
     this.slots.screenShare.audio.forEach((slot) => this.receiveSlotManager.releaseSlot(slot));
@@ -472,6 +492,8 @@ export class RemoteMediaManager extends EventsScope {
     if (!this.started) {
       throw new Error('setLayout() called before start()');
     }
+    LoggerProxy.logger.log(`RemoteMediaManager#setLayout --> new layout selected: ${layoutId}`);
+
     this.currentLayoutId = layoutId;
     this.currentLayout = cloneDeep(this.config.video.layouts[this.currentLayoutId]);
 
@@ -490,22 +512,86 @@ export class RemoteMediaManager extends EventsScope {
     return this.currentLayoutId;
   }
 
+  /**
+   * sets the preferLiveVideo
+   */
+  public setPreferLiveVideo(preferLiveVideo: boolean) {
+    LoggerProxy.logger.log(
+      `RemoteMediaManager#setPreferLiveVideo --> setPreferLiveVideo is called to set preferLiveVideo to ${preferLiveVideo}`
+    );
+    this.config.video.preferLiveVideo = preferLiveVideo;
+    Object.values(this.media.video.activeSpeakerGroups).forEach((activeSpeakerGroup) => {
+      activeSpeakerGroup.setPreferLiveVideo(preferLiveVideo, false);
+    });
+    this.mediaRequestManagers.video.commit();
+  }
+
+  /**
+   * Sets CSIs for multiple RemoteMedia instances belonging to RemoteMediaGroup.
+   * For each entry in the remoteMediaCsis array:
+   * - if csi is specified, the RemoteMedia instance is pinned to that CSI
+   * - if csi is undefined, the RemoteMedia instance is unpinned
+   */
+  public setActiveSpeakerCsis(remoteMediaCsis: {remoteMedia: RemoteMedia; csi?: number}[]) {
+    Object.values(this.media.video.activeSpeakerGroups).forEach((remoteMediaGroup) => {
+      const groupRemoteMediaCsis = remoteMediaCsis.filter(({remoteMedia}) =>
+        remoteMediaGroup.includes(remoteMedia)
+      );
+      if (groupRemoteMediaCsis.length > 0) {
+        remoteMediaGroup.setActiveSpeakerCsis(groupRemoteMediaCsis, false);
+      }
+    });
+    this.mediaRequestManagers.video.commit();
+  }
+
+  /**
+   * Sets which named media group need receiving
+   * @param {MediaType} mediaType of the stream
+   * @param {number} languageCode of the stream. If the languageId is 0, the named media group request will be canceled,
+   * and only receive the main audio stream.
+   * @returns {void}
+   */
+  public async setReceiveNamedMediaGroup(mediaType: MediaType, languageId: number) {
+    if (mediaType !== MediaType.AudioMain) {
+      throw new Error(`cannot set receive named media group which media type is ${mediaType}`);
+    }
+
+    const value = languageId;
+    if (value === this.config.namedMediaGroup?.value) {
+      return;
+    }
+
+    this.config.namedMediaGroup = {
+      type: NAMED_MEDIA_GROUP_TYPE_AUDIO,
+      value,
+    };
+
+    if (!this.media.audio.si) {
+      await this.createInterpretationAudioMedia(true);
+    } else {
+      this.media.audio.si.setNamedMediaGroup(this.config.namedMediaGroup, true);
+    }
+  }
+
   /**
    * Creates the audio slots
    */
   private async createAudioMedia() {
-    // create the audio receive slots
+    // create si audio request
+    await this.createInterpretationAudioMedia(false);
+
+    // create main audio receive slots
     for (let i = 0; i < this.config.audio.numOfActiveSpeakerStreams; i += 1) {
       // eslint-disable-next-line no-await-in-loop
       const slot = await this.receiveSlotManager.allocateSlot(MediaType.AudioMain);
 
-      this.slots.audio.push(slot);
+      this.slots.audio.main.push(slot);
     }
 
-    // create a remote media group
-    this.media.audio = new RemoteMediaGroup(
+    // create a remote media group for main audio
+    this.media.audio.main = new RemoteMediaGroup(
       this.mediaRequestManagers.audio,
-      this.slots.audio,
+      this.slots.audio.main,
       255,
       true
     );
@@ -513,10 +599,40 @@ export class RemoteMediaManager extends EventsScope {
     this.emit(
       {file: 'multistream/remoteMediaManager', function: 'createAudioMedia'},
       Event.AudioCreated,
-      this.media.audio
+      this.media.audio.main
     );
   }
 
+  /**
+   * Creates the audio slots for named media
+   */
+  private async createInterpretationAudioMedia(commitRequest: boolean) {
+    // create slot for interpretation language audio
+    if (
+      this.config.namedMediaGroup?.type === NAMED_MEDIA_GROUP_TYPE_AUDIO &&
+      this.config.namedMediaGroup?.value
+    ) {
+      this.slots.audio.si = await this.receiveSlotManager.allocateSlot(MediaType.AudioMain);
+
+      // create a remote media group for si audio
+      this.media.audio.si = new RemoteMediaGroup(
+        this.mediaRequestManagers.audio,
+        [this.slots.audio.si],
+        255,
+        commitRequest,
+        {
+          namedMediaGroup: this.config.namedMediaGroup,
+        }
+      );
+
+      this.emit(
+        {file: 'multistream/remoteMediaManager', function: 'createInterpretationAudioMedia'},
+        Event.InterpretationAudioCreated,
+        this.media.audio.si
+      );
+    }
+  }
+
   /**
    * Creates receive slots required for receiving screen share audio and video
    */
@@ -601,25 +717,37 @@ export class RemoteMediaManager extends EventsScope {
   }
 
   /**
-   * Allocates receive slots to all video panes in the current selected layout
+   * Allocates receive slots to all active speaker video panes
+   * in the current selected layout.
+   *
+   * Allocation tries to keep the same order of the slots between the previous
+   * layout and the new one. Sorting helps making sure that highest priority slots
+   * go in the same order in the new layout.
    */
-  private allocateSlotsToVideoPaneGroups() {
-    this.receiveSlotAllocations = {activeSpeaker: {}, receiverSelected: {}};
-
-    this.currentLayout?.activeSpeakerVideoPaneGroups?.forEach((group) => {
-      this.receiveSlotAllocations.activeSpeaker[group.id] = {slots: []};
-
-      for (let paneIndex = 0; paneIndex < group.numPanes; paneIndex += 1) {
-        // allocate a slot from the "unused" list
-        const freeSlot = this.slots.video.unused.pop();
-
-        if (freeSlot) {
-          this.slots.video.activeSpeaker.push(freeSlot);
-          this.receiveSlotAllocations.activeSpeaker[group.id].slots.push(freeSlot);
+  private allocateSlotsToActiveSpeakerPaneGroups() {
+    this.currentLayout?.activeSpeakerVideoPaneGroups
+      // sorting in descending order based on group priority
+      ?.sort((a, b) => (a.priority < b.priority ? 1 : -1))
+      ?.forEach((group) => {
+        this.receiveSlotAllocations.activeSpeaker[group.id] = {slots: []};
+
+        for (let paneIndex = 0; paneIndex < group.numPanes; paneIndex += 1) {
+          // allocate a slot from the "unused" list, by grabbing in same order (shift) as previous layout
+          const freeSlot = this.slots.video.unused.shift();
+
+          if (freeSlot) {
+            this.slots.video.activeSpeaker.push(freeSlot);
+            this.receiveSlotAllocations.activeSpeaker[group.id].slots.push(freeSlot);
+          }
         }
-      }
-    });
+      });
+  }
 
+  /**
+   * Allocates receive slots to all receiver selected video panes
+   * in the current selected layout
+   */
+  private allocateSlotsToReceiverSelectedVideoPaneGroups() {
     this.currentLayout?.memberVideoPanes?.forEach((memberPane) => {
       // check if there is existing slot for this csi
       const existingSlot = this.slots.video.receiverSelected.find(
@@ -646,19 +774,15 @@ export class RemoteMediaManager extends EventsScope {
   }
 
   /**
-   * Makes sure we have the right number of receive slots created for the current layout
-   * and allocates them to the right video panes / pane groups
-   *
-   * @returns {Promise}
+   * Ensures that we have enough slots for the current layout.
    */
-  private async updateVideoReceiveSlots() {
+  private async refillRequiredSlotsIfNeeded() {
     const requiredNumSlots = this.getRequiredNumVideoSlotsForLayout(this.currentLayout);
     const totalNumSlots =
       this.slots.video.unused.length +
       this.slots.video.activeSpeaker.length +
       this.slots.video.receiverSelected.length;
 
-    // ensure we have enough total slots for current layout
     if (totalNumSlots < requiredNumSlots) {
       let numSlotsToCreate = requiredNumSlots - totalNumSlots;
 
@@ -671,20 +795,95 @@ export class RemoteMediaManager extends EventsScope {
         numSlotsToCreate -= 1;
       }
     }
+  }
 
-    // move all no longer needed receiver-selected slots to "unused"
-    this.trimReceiverSelectedSlots();
-
-    // move all active speaker slots to "unused"
+  /**
+   * Move all active speaker slots to "unused"
+   */
+  private trimActiveSpeakerSlots() {
     this.slots.video.unused.push(...this.slots.video.activeSpeaker);
     this.slots.video.activeSpeaker.length = 0;
+  }
 
-    // allocate the slots to the right panes / pane groups
-    this.allocateSlotsToVideoPaneGroups();
+  /**
+   * Logs the state of the receive slots
+   */
+  private logMainVideoReceiveSlots() {
+    let logMessage = '';
+    forEach(this.receiveSlotAllocations.activeSpeaker, (group, groupName) => {
+      logMessage += `\ngroup: ${groupName}\n${group.slots
+        .map((slot) => slot.logString)
+        .join(', ')}`;
+    });
+
+    logMessage += '\nreceiverSelected:\n';
+    forEach(this.receiveSlotAllocations.receiverSelected, (slot, key) => {
+      logMessage += ` ${key}: ${slot.logString}\n`;
+    });
+
+    LoggerProxy.logger.log(
+      `RemoteMediaManager#logMainVideoReceiveSlots --> MAIN VIDEO receive slots: unused=${this.slots.video.unused.length}, activeSpeaker=${this.slots.video.activeSpeaker.length}, receiverSelected=${this.slots.video.receiverSelected.length}${logMessage}`
+    );
+  }
+
+  /** logs main audio slots */
+  private logMainAudioReceiveSlots() {
+    LoggerProxy.logger.log(
+      `RemoteMediaManager#logMainAudioReceiveSlots --> MAIN AUDIO receive slots: ${this.slots.audio.main
+        .map((slot) => slot.logString)
+        .join(', ')}`
+    );
+  }
+
+  /** logs slides video slots */
+  private logSlidesVideoReceiveSlots() {
+    LoggerProxy.logger.log(
+      `RemoteMediaManager#logSlidesVideoReceiveSlots --> SLIDES VIDEO receive slot: ${this.slots.screenShare.video?.logString}`
+    );
+  }
 
+  /** logs slides audio slots */
+  private logSlidesAudioReceiveSlots() {
     LoggerProxy.logger.log(
-      `RemoteMediaManager#updateVideoReceiveSlots --> receive slots updated: unused=${this.slots.video.unused.length}, activeSpeaker=${this.slots.video.activeSpeaker.length}, receiverSelected=${this.slots.video.receiverSelected.length}`
+      `RemoteMediaManager#logSlidesAudioReceiveSlots --> SLIDES AUDIO receive slots: ${this.slots.screenShare.audio
+        .map((slot) => slot.logString)
+        .join(', ')}`
     );
+  }
+
+  /** Logs all current receive slots */
+  public logAllReceiveSlots() {
+    this.logMainVideoReceiveSlots();
+    this.logMainAudioReceiveSlots();
+    this.logSlidesVideoReceiveSlots();
+    this.logSlidesAudioReceiveSlots();
+  }
+
+  /**
+   * Makes sure we have the right number of receive slots created for the current layout
+   * and allocates them to the right video panes / pane groups
+   *
+   * @returns {Promise}
+   */
+  private async updateVideoReceiveSlots() {
+    // move all active speaker slots to "unused"
+    this.trimActiveSpeakerSlots();
+
+    // move all no longer needed receiver-selected slots to "unused"
+    this.trimReceiverSelectedSlots();
+
+    // ensure we have enough total slots for current layout
+    await this.refillRequiredSlotsIfNeeded();
+
+    // allocate the slots to the right panes / pane groups
+    // reset allocations
+    this.receiveSlotAllocations = {activeSpeaker: {}, receiverSelected: {}};
+    // allocate active speaker
+    this.allocateSlotsToActiveSpeakerPaneGroups();
+    // allocate receiver selected
+    this.allocateSlotsToReceiverSelectedVideoPaneGroups();
+
+    this.logMainVideoReceiveSlots();
 
     // If this is the initial layout, there may be some "unused" slots left because of the preallocation
     // done in this.preallocateVideoReceiveSlots(), so release them now
@@ -806,8 +1005,13 @@ export class RemoteMediaManager extends EventsScope {
   }) {
     const {audio, video, screenShareAudio, screenShareVideo, commit} = options;
 
-    if (audio && this.media.audio) {
-      this.media.audio.stop(commit);
+    if (audio) {
+      if (this.media.audio.main) {
+        this.media.audio.main.stop(commit);
+      }
+      if (this.media.audio.si) {
+        this.media.audio.si.stop(commit);
+      }
     }
     if (video) {
       Object.values(this.media.video.activeSpeakerGroups).forEach((remoteMediaGroup) => {