@webex/plugin-meetings 3.12.0-next.57 → 3.12.0-next.59

package/src/meeting/index.ts

@@ -22,7 +22,6 @@ import {
   MediaConnectionEventNames,
   MediaContent,
   MediaType,
-  MediaCodecMimeType,
   RemoteTrackType,
   RoapMessage,
   StatsAnalyzer,
@@ -31,7 +30,7 @@ import {
   NetworkQualityMonitor,
   StatsMonitor,
   StatsMonitorEventNames,
-  InboundAudioIssueSubTypes,
+  MediaCodecMimeType,
 } from '@webex/internal-media-core';
 
 import {DataChannelTokenType} from '@webex/internal-plugin-llm';
@@ -969,6 +968,7 @@ export default class Meeting extends StatelessWebexPlugin {
       },
       (csi: CSI) => (this.members.findMemberByCsi(csi) as any)?.id
     );
+
     /**
      * Object containing helper classes for managing media requests for audio/video/screenshare (for multistream media connections)
      * All multistream media requests sent out for this meeting have to go through them.
@@ -988,6 +988,7 @@ export default class Meeting extends StatelessWebexPlugin {
             mediaRequests
           );
         },
+        this.getIngressPayloadTypeCallback.bind(this),
         {
           // @ts-ignore - config coming from registerPlugin
           degradationPreferences: this.config.degradationPreferences,
@@ -1009,6 +1010,7 @@ export default class Meeting extends StatelessWebexPlugin {
             mediaRequests
           );
         },
+        this.getIngressPayloadTypeCallback.bind(this),
         {
           // @ts-ignore - config coming from registerPlugin
           degradationPreferences: this.config.degradationPreferences,
@@ -1030,6 +1032,7 @@ export default class Meeting extends StatelessWebexPlugin {
             mediaRequests
           );
         },
+        this.getIngressPayloadTypeCallback.bind(this),
         {
           // @ts-ignore - config coming from registerPlugin
           degradationPreferences: this.config.degradationPreferences,
@@ -1051,11 +1054,14 @@ export default class Meeting extends StatelessWebexPlugin {
             mediaRequests
           );
         },
+        this.getIngressPayloadTypeCallback.bind(this),
         {
           // @ts-ignore - config coming from registerPlugin
           degradationPreferences: this.config.degradationPreferences,
           kind: 'video',
           trimRequestsToNumOfSources: false,
+          // @ts-ignore - config coming from registerPlugin
+          enableAv1: this.config.enableAv1SlidesSupport,
         }
       ),
     };
@@ -1716,6 +1722,37 @@ export default class Meeting extends StatelessWebexPlugin {
     this.mediaServerIp = undefined;
   }
 
+  /**
+   * Get the ingress payload type for a given media type and codec mime type
+   * @param {MediaType} mediaType - The media type
+   * @param {MediaCodecMimeType} codecMimeType - The codec mime type
+   * @returns {number | undefined} - The ingress payload type
+   * @private
+   * @memberof Meeting
+   */
+  private getIngressPayloadTypeCallback(
+    mediaType: MediaType,
+    codecMimeType: MediaCodecMimeType
+  ): number | undefined {
+    if (this.isMultistream) {
+      try {
+        return this.mediaProperties.webrtcMediaConnection.getIngressPayloadType(
+          mediaType,
+          codecMimeType
+        );
+      } catch (error) {
+        LoggerProxy.logger.info(
+          `Meeting:index#mediaRequestManager --> failed to get ingress payload type for mediaType=${mediaType}, codecMimeType=${codecMimeType}`,
+          error
+        );
+
+        return undefined;
+      }
+    }
+
+    return undefined;
+  }
+
   /**
    * Temporary func to return webex object,
    * in order to access internal plugin metrics
@@ -7768,6 +7805,8 @@ export default class Meeting extends StatelessWebexPlugin {
         disableAudioMainDtx: this.config.experimental.disableAudioMainDtx,
         // @ts-ignore - config coming from registerPlugin
         enableAudioTwcc: this.config.enableAudioTwccForMultistream,
+        // @ts-ignore - config coming from registerPlugin
+        enableAv1SlidesSupport: this.config.enableAv1SlidesSupport,
         stopIceGatheringAfterFirstRelayCandidate:
           // @ts-ignore - config coming from registerPlugin
           this.config.stopIceGatheringAfterFirstRelayCandidate,

package/src/meetings/index.ts

@@ -935,6 +935,27 @@ export default class Meetings extends WebexPlugin {
     }
   }
 
+  /**
+   * API to toggle AV1 codec support for video slides in multistream,
+   * needs to be called before webex.meetings.joinWithMedia()
+   *
+   * @param {Boolean} newValue
+   * @private
+   * @memberof Meetings
+   * @returns {undefined}
+   */
+  private _toggleEnableAv1SlidesSupport(newValue: boolean) {
+    if (typeof newValue !== 'boolean') {
+      return;
+    }
+
+    // @ts-ignore
+    if (this.config.enableAv1SlidesSupport !== newValue) {
+      // @ts-ignore
+      this.config.enableAv1SlidesSupport = newValue;
+    }
+  }
+
   /**
    * API to toggle stopping ICE Candidates Gathering after first relay candidate,
    * needs to be called before webex.meetings.joinWithMedia()

package/src/multistream/codec/constants.ts

@@ -0,0 +1,58 @@
+import {AV1EncodingParams, SupportedResolution} from '@webex/internal-media-core';
+
+export const AV1_CODEC_PARAMETERS: Record<SupportedResolution, AV1EncodingParams> = {
+  '90p': {
+    levelIdx: 0,
+    tier: 0,
+    maxWidth: 160,
+    maxHeight: 90,
+    maxPicSize: 160 * 90,
+    maxDecodeRate: 5_529_600,
+  },
+  '180p': {
+    levelIdx: 0,
+    tier: 0,
+    maxWidth: 320,
+    maxHeight: 180,
+    maxPicSize: 320 * 180,
+    maxDecodeRate: 5_529_600,
+  },
+  '360p': {
+    levelIdx: 1,
+    tier: 0,
+    maxWidth: 640,
+    maxHeight: 360,
+    maxPicSize: 640 * 360,
+    maxDecodeRate: 10_454_400,
+  },
+  '540p': {
+    levelIdx: 4,
+    tier: 0,
+    maxWidth: 960,
+    maxHeight: 540,
+    maxPicSize: 960 * 540,
+    maxDecodeRate: 24_969_600,
+  },
+  '720p': {
+    levelIdx: 5,
+    tier: 0,
+    maxWidth: 1280,
+    maxHeight: 720,
+    maxPicSize: 1280 * 720,
+    maxDecodeRate: 39_938_400,
+  },
+  '1080p': {
+    levelIdx: 8,
+    tier: 0,
+    maxWidth: 1920,
+    maxHeight: 1080,
+    maxPicSize: 1920 * 1080,
+    maxDecodeRate: 77_856_768,
+  },
+};
+
+export const H264_CODEC_PARAMETERS = {
+  maxFs: 8192,
+  maxFps: 3000,
+  maxMbps: 245760,
+};

package/src/multistream/mediaRequestManager.ts

@@ -9,6 +9,11 @@ import {
   getRecommendedMaxBitrateForFrameSize,
   RecommendedOpusBitrates,
   NamedMediaGroup,
+  AV1Codec,
+  SupportedResolution,
+  AV1EncodingParams,
+  MediaType,
+  MediaCodecMimeType,
 } from '@webex/internal-media-core';
 import {cloneDeepWith, debounce} from 'lodash';
 
@@ -16,6 +21,7 @@ import LoggerProxy from '../common/logs/logger-proxy';
 
 import {ReceiveSlot, ReceiveSlotEvents} from './receiveSlot';
 import {MAX_FS_VALUES} from './remoteMedia';
+import {AV1_CODEC_PARAMETERS, H264_CODEC_PARAMETERS} from './codec/constants';
 
 export interface ActiveSpeakerPolicyInfo {
   policy: 'active-speaker';
@@ -54,34 +60,49 @@ export interface MediaRequest {
 
 export type MediaRequestId = string;
 
-const CODEC_DEFAULTS = {
-  h264: {
-    maxFs: 8192,
-    maxFps: 3000,
-    maxMbps: 245760,
-  },
-};
-
 const DEBOUNCED_SOURCE_UPDATE_TIME = 1000;
 
+const RESOLUTION_BUCKETS: Array<[SupportedResolution, number]> = [
+  ['90p', MAX_FS_VALUES['90p']],
+  ['180p', MAX_FS_VALUES['180p']],
+  ['360p', MAX_FS_VALUES['360p']],
+  ['540p', MAX_FS_VALUES['540p']],
+  ['720p', MAX_FS_VALUES['720p']],
+];
+
 type DegradationPreferences = {
   maxMacroblocksLimit: number;
 };
 
 type SendMediaRequestsCallback = (streamRequests: StreamRequest[]) => void;
+type GetIngressPayloadTypeCallback = (
+  mediaType: MediaType,
+  codecMimeType: MediaCodecMimeType
+) => number | undefined;
 type Kind = 'audio' | 'video';
 
-type Options = {
+type AudioMediaRequestManagerOptions = {
   degradationPreferences: DegradationPreferences;
-  kind: Kind;
+  kind: 'audio';
   trimRequestsToNumOfSources: boolean; // if enabled, AS speaker requests will be trimmed based on the calls to setNumCurrentSources()
 };
 
+type VideoMediaRequestManagerOptions = {
+  degradationPreferences: DegradationPreferences;
+  kind: 'video';
+  trimRequestsToNumOfSources: boolean;
+  enableAv1?: boolean;
+};
+
+type Options = AudioMediaRequestManagerOptions | VideoMediaRequestManagerOptions;
+
 type ClientRequestsMap = {[key: MediaRequestId]: MediaRequest};
 
 export class MediaRequestManager {
   private sendMediaRequestsCallback: SendMediaRequestsCallback;
 
+  private getIngressPayloadTypeCallback: GetIngressPayloadTypeCallback;
+
   private kind: Kind;
 
   private counter: number;
@@ -95,11 +116,17 @@ export class MediaRequestManager {
   private debouncedSourceUpdateListener: () => void;
 
   private trimRequestsToNumOfSources: boolean;
+  private enableAv1: boolean;
   private numTotalSources: number;
   private numLiveSources: number;
 
-  constructor(sendMediaRequestsCallback: SendMediaRequestsCallback, options: Options) {
+  constructor(
+    sendMediaRequestsCallback: SendMediaRequestsCallback,
+    getIngressPayloadTypeCallback: GetIngressPayloadTypeCallback,
+    options: Options
+  ) {
     this.sendMediaRequestsCallback = sendMediaRequestsCallback;
+    this.getIngressPayloadTypeCallback = getIngressPayloadTypeCallback;
     this.counter = 0;
     this.numLiveSources = 0;
     this.numTotalSources = 0;
@@ -107,6 +134,7 @@ export class MediaRequestManager {
     this.degradationPreferences = options.degradationPreferences;
     this.kind = options.kind;
     this.trimRequestsToNumOfSources = options.trimRequestsToNumOfSources;
+    this.enableAv1 = options.kind === 'video' && !!options.enableAv1;
     this.sourceUpdateListener = this.commit.bind(this);
     this.debouncedSourceUpdateListener = debounce(
       this.sourceUpdateListener,
@@ -135,8 +163,8 @@ export class MediaRequestManager {
       Object.values(clientRequests).forEach((mr) => {
         if (mr.codecInfo) {
           mr.codecInfo.maxFs = Math.min(
-            mr.preferredMaxFs || CODEC_DEFAULTS.h264.maxFs,
-            mr.codecInfo.maxFs || CODEC_DEFAULTS.h264.maxFs,
+            mr.preferredMaxFs || H264_CODEC_PARAMETERS.maxFs,
+            mr.codecInfo.maxFs || H264_CODEC_PARAMETERS.maxFs,
             maxFsLimits[i]
           );
           // we only consider sources with "live" state
@@ -176,7 +204,7 @@ export class MediaRequestManager {
     }
 
     return getRecommendedMaxBitrateForFrameSize(
-      mediaRequest.codecInfo.maxFs || CODEC_DEFAULTS.h264.maxFs
+      mediaRequest.codecInfo.maxFs || H264_CODEC_PARAMETERS.maxFs
     );
   }
 
@@ -192,12 +220,80 @@ export class MediaRequestManager {
   // eslint-disable-next-line class-methods-use-this
   private getH264MaxMbps(mediaRequest: MediaRequest): number {
     // fallback for maxFps (not needed for maxFs, since there is a fallback already in getDegradedClientRequests)
-    const maxFps = mediaRequest.codecInfo.maxFps || CODEC_DEFAULTS.h264.maxFps;
+    const maxFps = mediaRequest.codecInfo.maxFps || H264_CODEC_PARAMETERS.maxFps;
 
     // divided by 100 since maxFps is 3000 (for 30 frames per seconds)
     return (mediaRequest.codecInfo.maxFs * maxFps) / 100;
   }
 
+  /**
+   * Returns the AV1 encoding parameters for a media request
+   * @param mediaRequest - The media request to get the AV1 encoding parameters for
+   * @returns {AV1EncodingParams} The AV1 encoding parameters
+   */
+  // eslint-disable-next-line class-methods-use-this
+  private getAv1EncodingParams(mediaRequest: MediaRequest): AV1EncodingParams {
+    const frameSize = mediaRequest.codecInfo.maxFs || H264_CODEC_PARAMETERS.maxFs;
+    const resolution = RESOLUTION_BUCKETS.find(([, maxFs]) => frameSize <= maxFs)?.[0] ?? '1080p';
+
+    return AV1_CODEC_PARAMETERS[resolution];
+  }
+
+  private buildH264CodecInfo(mr: MediaRequest): WcmeCodecInfo | undefined {
+    if (!mr.codecInfo) {
+      return undefined;
+    }
+
+    const h264PayloadType = this.getIngressPayloadTypeCallback(
+      mr.receiveSlots[0].mediaType,
+      MediaCodecMimeType.H264
+    );
+
+    if (h264PayloadType === undefined) {
+      return undefined;
+    }
+
+    return WcmeCodecInfo.fromH264(
+      h264PayloadType,
+      new H264Codec(
+        mr.codecInfo.maxFs,
+        mr.codecInfo.maxFps || H264_CODEC_PARAMETERS.maxFps,
+        this.getH264MaxMbps(mr),
+        mr.codecInfo.maxWidth,
+        mr.codecInfo.maxHeight
+      )
+    );
+  }
+
+  private buildAv1CodecInfo(mr: MediaRequest): WcmeCodecInfo | undefined {
+    if (!this.enableAv1 || !mr.codecInfo) {
+      return undefined;
+    }
+
+    const av1PayloadType = this.getIngressPayloadTypeCallback(
+      mr.receiveSlots[0].mediaType,
+      MediaCodecMimeType.AV1
+    );
+
+    if (av1PayloadType === undefined) {
+      return undefined;
+    }
+
+    const av1EncodingParams = this.getAv1EncodingParams(mr);
+
+    return WcmeCodecInfo.fromAv1(
+      av1PayloadType,
+      new AV1Codec(
+        av1EncodingParams.levelIdx,
+        av1EncodingParams.tier,
+        mr.codecInfo.maxWidth || av1EncodingParams.maxWidth,
+        mr.codecInfo.maxHeight || av1EncodingParams.maxHeight,
+        av1EncodingParams.maxPicSize,
+        av1EncodingParams.maxDecodeRate
+      )
+    );
+  }
+
   /** Modifies the passed in clientRequests and makes sure that in total they don't ask
    *  for more streams than there are available.
    *
@@ -298,6 +394,12 @@ export class MediaRequestManager {
     // map all the client media requests to wcme stream requests
     Object.values(clientRequests).forEach((mr) => {
       if (mr.receiveSlots.length > 0) {
+        const codecInfos: WcmeCodecInfo[] = mr.codecInfo
+          ? [this.buildH264CodecInfo(mr), this.buildAv1CodecInfo(mr)].filter(
+              (info): info is WcmeCodecInfo => info !== undefined
+            )
+          : [];
+
         streamRequests.push(
           new StreamRequest(
             mr.policyInfo.policy === 'active-speaker'
@@ -314,25 +416,14 @@ export class MediaRequestManager {
               : new ReceiverSelectedInfo(mr.policyInfo.csi),
             mr.receiveSlots.map((receiveSlot) => receiveSlot.wcmeReceiveSlot),
             this.getMaxPayloadBitsPerSecond(mr),
-            mr.codecInfo && [
-              WcmeCodecInfo.fromH264(
-                0x80,
-                new H264Codec(
-                  mr.codecInfo.maxFs,
-                  mr.codecInfo.maxFps || CODEC_DEFAULTS.h264.maxFps,
-                  this.getH264MaxMbps(mr),
-                  mr.codecInfo.maxWidth,
-                  mr.codecInfo.maxHeight
-                )
-              ),
-            ]
+            codecInfos
           )
         );
       }
     });
 
     this.sendMediaRequestsCallback(streamRequests);
-    LoggerProxy.logger.info(`multistream:sendRequests --> media requests sent. `);
+    LoggerProxy.logger.info(`multistream:sendRequests --> media requests sent.`);
   }
 
   public addRequest(mediaRequest: MediaRequest, commit = true): MediaRequestId {

package/test/unit/spec/breakouts/index.ts

@@ -1849,6 +1849,53 @@ describe('plugin-meetings', () => {
       });
     });
 
+    describe('#removeFromBreakout', () => {
+      it('should make a POST request with correct body and return the result', async () => {
+        breakouts.request = sinon.stub().returns(Promise.resolve('REQUEST_RETURN_VALUE'));
+        breakouts.set('url', 'url');
+        breakouts.set('mainGroupId', 'mainGroupId');
+        breakouts.set('mainSessionId', 'mainSessionId');
+
+        const participants = ['participant1', 'participant2'];
+        const result = await breakouts.removeFromBreakout(participants);
+
+        assert.calledOnceWithExactly(breakouts.request, {
+          method: 'POST',
+          uri: 'url/move',
+          body: {
+            groups: [
+              {
+                id: 'mainGroupId',
+                sessions: [
+                  {
+                    id: 'mainSessionId',
+                    participants,
+                  },
+                ],
+              },
+            ],
+          },
+        });
+        assert.equal(result, 'REQUEST_RETURN_VALUE');
+      });
+
+      it('should throw an error if mainGroupId is missing', () => {
+        breakouts.set('mainSessionId', 'mainSessionId');
+        assert.throws(
+          () => breakouts.removeFromBreakout(['participant1']),
+          'Main group ID and session ID must be available to remove participants from breakout'
+        );
+      });
+
+      it('should throw an error if mainSessionId is missing', () => {
+        breakouts.set('mainGroupId', 'mainGroupId');
+        assert.throws(
+          () => breakouts.removeFromBreakout(['participant1']),
+          'Main group ID and session ID must be available to remove participants from breakout'
+        );
+      });
+    });
+
     describe('#triggerReturnToMainEvent', () => {
       const checkTrigger = ({breakout, shouldTrigger}) => {
         breakouts.trigger = sinon.stub();

package/test/unit/spec/media/index.ts

@@ -295,6 +295,7 @@ describe('createMediaConnection', () => {
         bundlePolicy: 'max-bundle',
         disableAudioMainDtx: false,
         disableAudioTwcc: false,
+        enableAV1SlidesSupport: false,
       },
       'meeting id'
     );
@@ -322,6 +323,26 @@ describe('createMediaConnection', () => {
     assert.calledOnce(rtcMetrics.sendMetricsInQueue);
   });
 
+  it('passes enableAV1SlidesSupport: true to MultistreamRoapMediaConnection when enableAv1SlidesSupport is set', () => {
+    const multistreamRoapMediaConnectionConstructorStub = sinon
+      .stub(InternalMediaCoreModule, 'MultistreamRoapMediaConnection')
+      .returns(fakeRoapMediaConnection);
+
+    Media.createMediaConnection(true, 'some debug id', 'meeting id', {
+      enableAv1SlidesSupport: true,
+    });
+    assert.calledOnce(multistreamRoapMediaConnectionConstructorStub);
+    assert.calledWith(
+      multistreamRoapMediaConnectionConstructorStub,
+      sinon.match({
+        iceServers: [],
+        disableAudioTwcc: true,
+        enableAV1SlidesSupport: true,
+      }),
+      'meeting id'
+    );
+  });
+
   it('multistream non-firefox does not care about stopIceGatheringAfterFirstRelayCandidate', () => {
     const multistreamRoapMediaConnectionConstructorStub = sinon
       .stub(InternalMediaCoreModule, 'MultistreamRoapMediaConnection')
@@ -336,6 +357,7 @@ describe('createMediaConnection', () => {
       {
         iceServers: [],
         disableAudioTwcc: true,
+        enableAV1SlidesSupport: false,
       },
       'meeting id'
     );
@@ -359,6 +381,7 @@ describe('createMediaConnection', () => {
         doFullIce: true,
         stopIceGatheringAfterFirstRelayCandidate: true,
         disableAudioTwcc: true,
+        enableAV1SlidesSupport: false,
       },
       'meeting id'
     );
@@ -382,6 +405,7 @@ describe('createMediaConnection', () => {
         doFullIce: true,
         stopIceGatheringAfterFirstRelayCandidate: false,
         disableAudioTwcc: true,
+        enableAV1SlidesSupport: false,
       },
       'meeting id'
     );
@@ -418,6 +442,7 @@ describe('createMediaConnection', () => {
         {
           iceServers: [],
           disableAudioTwcc: true,
+          enableAV1SlidesSupport: false,
         },
         'meeting id'
       );
@@ -448,6 +473,7 @@ describe('createMediaConnection', () => {
       {
         iceServers: [],
         disableAudioTwcc: true,
+        enableAV1SlidesSupport: false,
       },
       'meeting id'
     );
@@ -477,6 +503,7 @@ describe('createMediaConnection', () => {
       {
         iceServers: [],
         disableAudioTwcc: true,
+        enableAV1SlidesSupport: false,
       },
       'meeting id'
     );
@@ -505,6 +532,7 @@ describe('createMediaConnection', () => {
       {
         iceServers: [],
         disableAudioTwcc: true,
+        enableAV1SlidesSupport: false,
       },
       'meeting id'
     );
@@ -591,6 +619,7 @@ describe('createMediaConnection', () => {
     {
       iceServers: [],
       disableAudioTwcc: true,
+      enableAV1SlidesSupport: false,
       enableInboundAudioLevelMonitoring: true,
     }
   );
@@ -602,6 +631,7 @@ describe('createMediaConnection', () => {
     {
       iceServers: [],
       disableAudioTwcc: true,
+      enableAV1SlidesSupport: false,
       enableInboundAudioLevelMonitoring: true,
     }
   );
@@ -613,6 +643,7 @@ describe('createMediaConnection', () => {
     {
       iceServers: [],
       disableAudioTwcc: true,
+      enableAV1SlidesSupport: false,
       doFullIce: true,
       stopIceGatheringAfterFirstRelayCandidate: undefined,
     }