@webex/plugin-meetings 3.0.0-beta.17 → 3.0.0-beta.171

package/src/multistream/mediaRequestManager.ts

@@ -1,9 +1,20 @@
 /* eslint-disable require-jsdoc */
-import {MediaConnection as MC} from '@webex/internal-media-core';
+import {
+  StreamRequest,
+  Policy,
+  ActiveSpeakerInfo,
+  ReceiverSelectedInfo,
+  CodecInfo as WcmeCodecInfo,
+  H264Codec,
+  getRecommendedMaxBitrateForFrameSize,
+  RecommendedOpusBitrates,
+} from '@webex/internal-media-core';
+import {cloneDeepWith, debounce, isEmpty} from 'lodash';
 
 import LoggerProxy from '../common/logs/logger-proxy';
 
-import {ReceiveSlot, ReceiveSlotId} from './receiveSlot';
+import {ReceiveSlot, ReceiveSlotEvents} from './receiveSlot';
+import {getMaxFs} from './remoteMedia';
 
 export interface ActiveSpeakerPolicyInfo {
   policy: 'active-speaker';
@@ -35,6 +46,8 @@ export interface MediaRequest {
   policyInfo: PolicyInfo;
   receiveSlots: Array<ReceiveSlot>;
   codecInfo?: CodecInfo;
+  preferredMaxFs?: number;
+  handleMaxFs?: ({maxFs}: {maxFs: number}) => void;
 }
 
 export type MediaRequestId = string;
@@ -47,91 +60,326 @@ const CODEC_DEFAULTS = {
   },
 };
 
-type SendMediaRequestsCallback = (mediaRequests: MC.MediaRequest[]) => void;
+const DEBOUNCED_SOURCE_UPDATE_TIME = 1000;
+
+type DegradationPreferences = {
+  maxMacroblocksLimit: number;
+};
+
+type SendMediaRequestsCallback = (streamRequests: StreamRequest[]) => void;
+type Kind = 'audio' | 'video';
+
+type Options = {
+  degradationPreferences: DegradationPreferences;
+  kind: Kind;
+  trimRequestsToNumOfSources: boolean; // if enabled, AS speaker requests will be trimmed based on the calls to setNumCurrentSources()
+};
+
+type ClientRequestsMap = {[key: MediaRequestId]: MediaRequest};
 
 export class MediaRequestManager {
   private sendMediaRequestsCallback: SendMediaRequestsCallback;
 
-  private counter;
+  private kind: Kind;
+
+  private counter: number;
+
+  private clientRequests: ClientRequestsMap;
 
-  private clientRequests: {[key: MediaRequestId]: MediaRequest};
+  private degradationPreferences: DegradationPreferences;
 
-  private slotsActiveInLastMediaRequest: {[key: ReceiveSlotId]: ReceiveSlot};
+  private sourceUpdateListener: () => void;
 
-  constructor(sendMediaRequestsCallback: SendMediaRequestsCallback) {
+  private debouncedSourceUpdateListener: () => void;
+
+  private previousStreamRequests: Array<StreamRequest> = [];
+
+  private trimRequestsToNumOfSources: boolean;
+  private numTotalSources: number;
+  private numLiveSources: number;
+
+  constructor(sendMediaRequestsCallback: SendMediaRequestsCallback, options: Options) {
     this.sendMediaRequestsCallback = sendMediaRequestsCallback;
     this.counter = 0;
+    this.numLiveSources = 0;
+    this.numTotalSources = 0;
     this.clientRequests = {};
-    this.slotsActiveInLastMediaRequest = {};
+    this.degradationPreferences = options.degradationPreferences;
+    this.kind = options.kind;
+    this.trimRequestsToNumOfSources = options.trimRequestsToNumOfSources;
+    this.sourceUpdateListener = this.commit.bind(this);
+    this.debouncedSourceUpdateListener = debounce(
+      this.sourceUpdateListener,
+      DEBOUNCED_SOURCE_UPDATE_TIME
+    );
   }
 
-  private resetInactiveReceiveSlots() {
-    const activeSlots: {[key: ReceiveSlotId]: ReceiveSlot} = {};
-
-    // create a map of all currently used slot ids
-    Object.values(this.clientRequests).forEach((request) =>
-      request.receiveSlots.forEach((slot) => {
-        activeSlots[slot.id] = slot;
-      })
-    );
+  public setDegradationPreferences(degradationPreferences: DegradationPreferences) {
+    this.degradationPreferences = degradationPreferences;
+    this.sendRequests(); // re-send requests after preferences are set
+  }
 
-    // when we stop using some receive slots and they are not included in the new media request,
-    // we will never get a 'no source' notification for them, so we reset their state,
-    // so that the client doesn't try to display their video anymore
-    for (const [slotId, slot] of Object.entries(this.slotsActiveInLastMediaRequest)) {
-      if (!(slotId in activeSlots)) {
-        LoggerProxy.logger.info(
-          `multistream:mediaRequestManager --> resetting sourceState to "no source" for slot ${slot.id}`
+  private getDegradedClientRequests(clientRequests: ClientRequestsMap) {
+    const maxFsLimits = [
+      getMaxFs('best'),
+      getMaxFs('large'),
+      getMaxFs('medium'),
+      getMaxFs('small'),
+      getMaxFs('very small'),
+      getMaxFs('thumbnail'),
+    ];
+
+    // reduce max-fs until total macroblocks is below limit
+    for (let i = 0; i < maxFsLimits.length; i += 1) {
+      let totalMacroblocksRequested = 0;
+      Object.values(clientRequests).forEach((mr) => {
+        if (mr.codecInfo) {
+          mr.codecInfo.maxFs = Math.min(
+            mr.preferredMaxFs || CODEC_DEFAULTS.h264.maxFs,
+            mr.codecInfo.maxFs || CODEC_DEFAULTS.h264.maxFs,
+            maxFsLimits[i]
+          );
+          // we only consider sources with "live" state
+          const slotsWithLiveSource = mr.receiveSlots.filter((rs) => rs.sourceState === 'live');
+          totalMacroblocksRequested += mr.codecInfo.maxFs * slotsWithLiveSource.length;
+        }
+      });
+      if (totalMacroblocksRequested <= this.degradationPreferences.maxMacroblocksLimit) {
+        if (i !== 0) {
+          LoggerProxy.logger.warn(
+            `multistream:mediaRequestManager --> too many streams with high max-fs, frame size will be limited to ${maxFsLimits[i]}`
+          );
+        }
+        break;
+      } else if (i === maxFsLimits.length - 1) {
+        LoggerProxy.logger.warn(
+          `multistream:mediaRequestManager --> even with frame size limited to ${maxFsLimits[i]} you are still requesting too many streams, consider reducing the number of requests`
         );
-        slot.resetSourceState();
       }
     }
+  }
 
-    this.slotsActiveInLastMediaRequest = activeSlots;
+  /**
+   * Returns true if two stream requests are the same, false otherwise.
+   *
+   * @param {StreamRequest} streamRequestA - Stream request A for comparison.
+   * @param {StreamRequest} streamRequestB - Stream request B for comparison.
+   * @returns {boolean} - Whether they are equal.
+   */
+  // eslint-disable-next-line class-methods-use-this
+  public isEqual(streamRequestA: StreamRequest, streamRequestB: StreamRequest) {
+    return (
+      JSON.stringify(streamRequestA._toJmpStreamRequest()) ===
+      JSON.stringify(streamRequestB._toJmpStreamRequest())
+    );
   }
 
-  private sendRequests() {
-    const wcmeMediaRequests: MC.MediaRequest[] = [];
+  /**
+   * Compares new stream requests to previous ones and determines
+   * if they are the same.
+   *
+   * @param {StreamRequest[]} newRequests - Array with new requests.
+   * @returns {boolean} - True if they are equal, false otherwise.
+   */
+  private checkIsNewRequestsEqualToPrev(newRequests: StreamRequest[]) {
+    return (
+      !isEmpty(this.previousStreamRequests) &&
+      this.previousStreamRequests.length === newRequests.length &&
+      this.previousStreamRequests.every((req, idx) => this.isEqual(req, newRequests[idx]))
+    );
+  }
+
+  /**
+   * Returns the maxPayloadBitsPerSecond per Stream
+   *
+   * If MediaRequestManager kind is "audio", a constant bitrate will be returned.
+   * If MediaRequestManager kind is "video", the bitrate will be calculated based
+   * on maxFs (default h264 maxFs as fallback if maxFs is not defined)
+   *
+   * @param {MediaRequest} mediaRequest  - mediaRequest to take data from
+   * @returns {number} maxPayloadBitsPerSecond
+   */
+  private getMaxPayloadBitsPerSecond(mediaRequest: MediaRequest): number {
+    if (this.kind === 'audio') {
+      // return mono_music bitrate default if the kind of mediarequest manager is audio:
+      return RecommendedOpusBitrates.FB_MONO_MUSIC;
+    }
 
-    // todo: check how many streams we're asking for and what resolution and introduce some limits (spark-377701)
-    const maxPayloadBitsPerSecond = 10 * 1000 * 1000;
+    return getRecommendedMaxBitrateForFrameSize(
+      mediaRequest.codecInfo.maxFs || CODEC_DEFAULTS.h264.maxFs
+    );
+  }
+
+  /**
+   * Returns the max Macro Blocks per second (maxMbps) per H264 Stream
+   *
+   * The maxMbps will be calculated based on maxFs and maxFps
+   * (default h264 maxFps as fallback if maxFps is not defined)
+   *
+   * @param {MediaRequest} mediaRequest  - mediaRequest to take data from
+   * @returns {number} maxMbps
+   */
+  // eslint-disable-next-line class-methods-use-this
+  private getH264MaxMbps(mediaRequest: MediaRequest): number {
+    // fallback for maxFps (not needed for maxFs, since there is a fallback already in getDegradedClientRequests)
+    const maxFps = mediaRequest.codecInfo.maxFps || CODEC_DEFAULTS.h264.maxFps;
+
+    // divided by 100 since maxFps is 3000 (for 30 frames per seconds)
+    return (mediaRequest.codecInfo.maxFs * maxFps) / 100;
+  }
+
+  /**
+   * Clears the previous stream requests.
+   *
+   * @returns {void}
+   */
+  public clearPreviousRequests(): void {
+    this.previousStreamRequests = [];
+  }
+
+  /** Modifies the passed in clientRequests and makes sure that in total they don't ask
+   *  for more streams than there are available.
+   *
+   * @param {Object} clientRequests
+   * @returns {void}
+   */
+  private trimRequests(clientRequests: ClientRequestsMap) {
+    const preferLiveVideo = this.getPreferLiveVideo();
+
+    if (!this.trimRequestsToNumOfSources) {
+      return;
+    }
+
+    // preferLiveVideo being undefined means that there are no active-speaker requests so we don't need to do any trimming
+    if (preferLiveVideo === undefined) {
+      return;
+    }
+
+    let numStreamsAvailable = preferLiveVideo ? this.numLiveSources : this.numTotalSources;
+
+    Object.values(clientRequests)
+      .sort((a, b) => {
+        // we have to count how many streams we're asking for
+        // and should not ask for more than numStreamsAvailable in total,
+        // so we might need to trim active-speaker requests and first ones to trim should be
+        // the ones with lowest priority
+
+        // receiver-selected requests have priority over active-speakers
+        if (a.policyInfo.policy === 'receiver-selected') {
+          return -1;
+        }
+        if (b.policyInfo.policy === 'receiver-selected') {
+          return 1;
+        }
+
+        // and active-speakers are sorted by descending priority
+        return b.policyInfo.priority - a.policyInfo.priority;
+      })
+      .forEach((request) => {
+        // we only trim active-speaker requests
+        if (request.policyInfo.policy === 'active-speaker') {
+          const trimmedCount = Math.min(numStreamsAvailable, request.receiveSlots.length);
+
+          request.receiveSlots.length = trimmedCount;
+
+          numStreamsAvailable -= trimmedCount;
+        } else {
+          numStreamsAvailable -= request.receiveSlots.length;
+        }
+
+        if (numStreamsAvailable < 0) {
+          numStreamsAvailable = 0;
+        }
+      });
+  }
+
+  private getPreferLiveVideo(): boolean | undefined {
+    let preferLiveVideo;
 
-    // map all the client media requests to wcme media requests
     Object.values(this.clientRequests).forEach((mr) => {
-      wcmeMediaRequests.push(
-        new MC.MediaRequest(
-          mr.policyInfo.policy === 'active-speaker'
-            ? MC.Policy.ActiveSpeaker
-            : MC.Policy.ReceiverSelected,
-          mr.policyInfo.policy === 'active-speaker'
-            ? new MC.ActiveSpeakerInfo(
-                mr.policyInfo.priority,
-                mr.policyInfo.crossPriorityDuplication,
-                mr.policyInfo.crossPolicyDuplication,
-                mr.policyInfo.preferLiveVideo
-              )
-            : new MC.ReceiverSelectedInfo(mr.policyInfo.csi),
-          mr.receiveSlots.map((receiveSlot) => receiveSlot.wcmeReceiveSlot),
-          maxPayloadBitsPerSecond,
-          mr.codecInfo && [
-            new MC.CodecInfo(
-              0x80,
-              new MC.H264Codec(
-                mr.codecInfo.maxFs || CODEC_DEFAULTS.h264.maxFs,
-                mr.codecInfo.maxFps || CODEC_DEFAULTS.h264.maxFps,
-                mr.codecInfo.maxMbps || CODEC_DEFAULTS.h264.maxMbps,
-                mr.codecInfo.maxWidth,
-                mr.codecInfo.maxHeight
-              )
-            ),
-          ]
-        )
-      );
+      if (mr.policyInfo.policy === 'active-speaker') {
+        // take the value from first encountered active speaker request
+        if (preferLiveVideo === undefined) {
+          preferLiveVideo = mr.policyInfo.preferLiveVideo;
+        }
+
+        if (mr.policyInfo.preferLiveVideo !== preferLiveVideo) {
+          throw new Error(
+            'a mix of active-speaker groups with different values for preferLiveVideo is not supported'
+          );
+        }
+      }
     });
 
-    this.sendMediaRequestsCallback(wcmeMediaRequests);
+    return preferLiveVideo;
+  }
+
+  private cloneClientRequests(): ClientRequestsMap {
+    // we clone the client requests but without cloning the ReceiveSlots that they reference
+    return cloneDeepWith(this.clientRequests, (value, key) => {
+      if (key === 'receiveSlots') {
+        return [...value];
+      }
+
+      return undefined;
+    });
+  }
+
+  private sendRequests() {
+    const streamRequests: StreamRequest[] = [];
+
+    // clone the requests so that any modifications we do to them don't affect the original ones
+    const clientRequests = this.cloneClientRequests();
+
+    this.trimRequests(clientRequests);
+    this.getDegradedClientRequests(clientRequests);
+
+    // map all the client media requests to wcme stream requests
+    Object.values(clientRequests).forEach((mr) => {
+      if (mr.receiveSlots.length > 0) {
+        streamRequests.push(
+          new StreamRequest(
+            mr.policyInfo.policy === 'active-speaker'
+              ? Policy.ActiveSpeaker
+              : Policy.ReceiverSelected,
+            mr.policyInfo.policy === 'active-speaker'
+              ? new ActiveSpeakerInfo(
+                  mr.policyInfo.priority,
+                  mr.policyInfo.crossPriorityDuplication,
+                  mr.policyInfo.crossPolicyDuplication,
+                  mr.policyInfo.preferLiveVideo
+                )
+              : new ReceiverSelectedInfo(mr.policyInfo.csi),
+            mr.receiveSlots.map((receiveSlot) => receiveSlot.wcmeReceiveSlot),
+            this.getMaxPayloadBitsPerSecond(mr),
+            mr.codecInfo && [
+              new WcmeCodecInfo(
+                0x80,
+                new H264Codec(
+                  mr.codecInfo.maxFs,
+                  mr.codecInfo.maxFps || CODEC_DEFAULTS.h264.maxFps,
+                  this.getH264MaxMbps(mr),
+                  mr.codecInfo.maxWidth,
+                  mr.codecInfo.maxHeight
+                )
+              ),
+            ]
+          )
+        );
+      }
+    });
 
-    this.resetInactiveReceiveSlots();
+    //! IMPORTANT: this is only a temporary fix. This will soon be done in the jmp layer (@webex/json-multistream)
+    // https://jira-eng-gpk2.cisco.com/jira/browse/WEBEX-326713
+    if (!this.checkIsNewRequestsEqualToPrev(streamRequests)) {
+      this.sendMediaRequestsCallback(streamRequests);
+      this.previousStreamRequests = streamRequests;
+      LoggerProxy.logger.info(`multistream:sendRequests --> media requests sent. `);
+    } else {
+      LoggerProxy.logger.info(
+        `multistream:sendRequests --> detected duplicate WCME requests, skipping them... `
+      );
+    }
   }
 
   public addRequest(mediaRequest: MediaRequest, commit = true): MediaRequestId {
@@ -140,6 +388,17 @@ export class MediaRequestManager {
 
     this.clientRequests[newId] = mediaRequest;
 
+    const eventHandler = ({maxFs}) => {
+      mediaRequest.preferredMaxFs = maxFs;
+      this.debouncedSourceUpdateListener();
+    };
+    mediaRequest.handleMaxFs = eventHandler;
+
+    mediaRequest.receiveSlots.forEach((rs) => {
+      rs.on(ReceiveSlotEvents.SourceUpdate, this.sourceUpdateListener);
+      rs.on(ReceiveSlotEvents.MaxFsUpdate, mediaRequest.handleMaxFs);
+    });
+
     if (commit) {
       this.commit();
     }
@@ -148,6 +407,13 @@ export class MediaRequestManager {
   }
 
   public cancelRequest(requestId: MediaRequestId, commit = true) {
+    const mediaRequest = this.clientRequests[requestId];
+
+    mediaRequest?.receiveSlots.forEach((rs) => {
+      rs.off(ReceiveSlotEvents.SourceUpdate, this.sourceUpdateListener);
+      rs.off(ReceiveSlotEvents.MaxFsUpdate, mediaRequest.handleMaxFs);
+    });
+
     delete this.clientRequests[requestId];
 
     if (commit) {
@@ -161,6 +427,14 @@ export class MediaRequestManager {
 
   public reset() {
     this.clientRequests = {};
-    this.slotsActiveInLastMediaRequest = {};
+    this.numTotalSources = 0;
+    this.numLiveSources = 0;
+  }
+
+  public setNumCurrentSources(numTotalSources: number, numLiveSources: number) {
+    this.numTotalSources = numTotalSources;
+    this.numLiveSources = numLiveSources;
+
+    this.sendRequests();
   }
 }

package/src/multistream/receiveSlot.ts

@@ -1,14 +1,20 @@
 /* eslint-disable valid-jsdoc */
-import {MediaConnection as MC} from '@webex/internal-media-core';
+import {
+  MediaType,
+  ReceiveSlot as WcmeReceiveSlot,
+  ReceiveSlotEvents as WcmeReceiveSlotEvents,
+  StreamState,
+} from '@webex/internal-media-core';
 
 import LoggerProxy from '../common/logs/logger-proxy';
 import EventsScope from '../common/events/events-scope';
 
 export const ReceiveSlotEvents = {
   SourceUpdate: 'sourceUpdate',
+  MaxFsUpdate: 'maxFsUpdate',
 };
 
-export type SourceState = MC.SourceState;
+export type {StreamState} from '@webex/internal-media-core';
 export type CSI = number;
 export type MemberId = string;
 export type ReceiveSlotId = string;
@@ -22,31 +28,31 @@ export type FindMemberIdCallback = (csi: CSI) => MemberId | undefined;
  * for example some participant's main video or audio
  */
 export class ReceiveSlot extends EventsScope {
-  private readonly mcReceiveSlot: MC.ReceiveSlot;
+  private readonly mcReceiveSlot: WcmeReceiveSlot;
 
   private readonly findMemberIdCallback: FindMemberIdCallback;
 
   public readonly id: ReceiveSlotId;
 
-  public readonly mediaType: MC.MediaType;
+  public readonly mediaType: MediaType;
 
   #memberId?: MemberId;
 
   #csi?: CSI;
 
-  #sourceState: MC.SourceState;
+  #sourceState: StreamState;
 
   /**
    * constructor - don't use it directly, you should always use meeting.receiveSlotManager.allocateSlot()
    * to create any receive slots
    *
-   * @param {MC.MediaType} mediaType
-   * @param {MC.ReceiveSlot} mcReceiveSlot
+   * @param {MediaType} mediaType
+   * @param {ReceiveSlot} mcReceiveSlot
    * @param {FindMemberIdCallback} findMemberIdCallback callback for finding memberId for given CSI
    */
   constructor(
-    mediaType: MC.MediaType,
-    mcReceiveSlot: MC.ReceiveSlot,
+    mediaType: MediaType,
+    mcReceiveSlot: WcmeReceiveSlot,
     findMemberIdCallback: FindMemberIdCallback
   ) {
     super();
@@ -76,6 +82,25 @@ export class ReceiveSlot extends EventsScope {
     return this.#csi;
   }
 
+  /**
+   * Set the max frame size for this slot
+   * @param newFs frame size
+   */
+  public setMaxFs(newFs) {
+    // emit event for media request manager to listen to
+
+    this.emit(
+      {
+        file: 'meeting/receiveSlot',
+        function: 'findMemberId',
+      },
+      ReceiveSlotEvents.MaxFsUpdate,
+      {
+        maxFs: newFs,
+      }
+    );
+  }
+
   /**
    * Getter for sourceState
    */
@@ -86,15 +111,15 @@ export class ReceiveSlot extends EventsScope {
   /**
    * registers event handlers with the underlying ReceiveSlot
    */
-  setupEventListeners() {
+  private setupEventListeners() {
     const scope = {
       file: 'meeting/receiveSlot',
       function: 'setupEventListeners',
     };
 
     this.mcReceiveSlot.on(
-      MC.ReceiveSlotEvents.SourceUpdate,
-      (state: MC.SourceState, csi?: number) => {
+      WcmeReceiveSlotEvents.SourceUpdate,
+      (state: StreamState, csi?: number) => {
         LoggerProxy.logger.log(
           `ReceiveSlot#setupEventListeners --> got source update on receive slot ${this.id}, mediaType=${this.mediaType}, csi=${csi}, state=${state}`
         );
@@ -111,6 +136,36 @@ export class ReceiveSlot extends EventsScope {
     );
   }
 
+  /** Tries to find the member id for this receive slot if it hasn't got one */
+  public findMemberId() {
+    if (this.#memberId === undefined && this.#csi) {
+      this.#memberId = this.findMemberIdCallback(this.#csi);
+
+      if (this.#memberId) {
+        // if we found the memberId, simulate source update so that the client app knows that something's changed
+        this.emit(
+          {
+            file: 'meeting/receiveSlot',
+            function: 'findMemberId',
+          },
+          ReceiveSlotEvents.SourceUpdate,
+          {
+            state: this.#sourceState,
+            csi: this.#csi,
+            memberId: this.#memberId,
+          }
+        );
+      }
+    }
+  }
+
+  /**
+   * @returns {string} a log message used to identify the receive slot
+   */
+  public get logString() {
+    return `ReceiveSlot - ${this.id}: ${JSON.stringify(this.mcReceiveSlot.id)}`;
+  }
+
   /**
    * The MediaStream object associated with this slot.
    *
@@ -123,20 +178,7 @@ export class ReceiveSlot extends EventsScope {
   /**
    * The underlying WCME receive slot
    */
-  get wcmeReceiveSlot(): MC.ReceiveSlot {
+  get wcmeReceiveSlot(): WcmeReceiveSlot {
     return this.mcReceiveSlot;
   }
-
-  /**
-   * Resets the source state to the default 'no source' value.
-   * This function should be called on receive slots that are
-   * no longer part of a media request. It's needed because WCME
-   * does not send any more events on such slots, so the sourceState
-   * value would not represent the truth anymore.
-   */
-  public resetSourceState() {
-    this.#sourceState = 'no source';
-    this.#csi = undefined;
-    this.#memberId = undefined;
-  }
 }