@webex/plugin-meetings 3.12.0-next.58 → 3.12.0-next.59

package/src/meetings/index.ts

@@ -935,6 +935,27 @@ export default class Meetings extends WebexPlugin {
     }
   }
 
+  /**
+   * API to toggle AV1 codec support for video slides in multistream,
+   * needs to be called before webex.meetings.joinWithMedia()
+   *
+   * @param {Boolean} newValue
+   * @private
+   * @memberof Meetings
+   * @returns {undefined}
+   */
+  private _toggleEnableAv1SlidesSupport(newValue: boolean) {
+    if (typeof newValue !== 'boolean') {
+      return;
+    }
+
+    // @ts-ignore
+    if (this.config.enableAv1SlidesSupport !== newValue) {
+      // @ts-ignore
+      this.config.enableAv1SlidesSupport = newValue;
+    }
+  }
+
   /**
    * API to toggle stopping ICE Candidates Gathering after first relay candidate,
    * needs to be called before webex.meetings.joinWithMedia()

package/src/multistream/codec/constants.ts

@@ -0,0 +1,58 @@
+import {AV1EncodingParams, SupportedResolution} from '@webex/internal-media-core';
+
+export const AV1_CODEC_PARAMETERS: Record<SupportedResolution, AV1EncodingParams> = {
+  '90p': {
+    levelIdx: 0,
+    tier: 0,
+    maxWidth: 160,
+    maxHeight: 90,
+    maxPicSize: 160 * 90,
+    maxDecodeRate: 5_529_600,
+  },
+  '180p': {
+    levelIdx: 0,
+    tier: 0,
+    maxWidth: 320,
+    maxHeight: 180,
+    maxPicSize: 320 * 180,
+    maxDecodeRate: 5_529_600,
+  },
+  '360p': {
+    levelIdx: 1,
+    tier: 0,
+    maxWidth: 640,
+    maxHeight: 360,
+    maxPicSize: 640 * 360,
+    maxDecodeRate: 10_454_400,
+  },
+  '540p': {
+    levelIdx: 4,
+    tier: 0,
+    maxWidth: 960,
+    maxHeight: 540,
+    maxPicSize: 960 * 540,
+    maxDecodeRate: 24_969_600,
+  },
+  '720p': {
+    levelIdx: 5,
+    tier: 0,
+    maxWidth: 1280,
+    maxHeight: 720,
+    maxPicSize: 1280 * 720,
+    maxDecodeRate: 39_938_400,
+  },
+  '1080p': {
+    levelIdx: 8,
+    tier: 0,
+    maxWidth: 1920,
+    maxHeight: 1080,
+    maxPicSize: 1920 * 1080,
+    maxDecodeRate: 77_856_768,
+  },
+};
+
+export const H264_CODEC_PARAMETERS = {
+  maxFs: 8192,
+  maxFps: 3000,
+  maxMbps: 245760,
+};

package/src/multistream/mediaRequestManager.ts

@@ -9,6 +9,11 @@ import {
   getRecommendedMaxBitrateForFrameSize,
   RecommendedOpusBitrates,
   NamedMediaGroup,
+  AV1Codec,
+  SupportedResolution,
+  AV1EncodingParams,
+  MediaType,
+  MediaCodecMimeType,
 } from '@webex/internal-media-core';
 import {cloneDeepWith, debounce} from 'lodash';
 
@@ -16,6 +21,7 @@ import LoggerProxy from '../common/logs/logger-proxy';
 
 import {ReceiveSlot, ReceiveSlotEvents} from './receiveSlot';
 import {MAX_FS_VALUES} from './remoteMedia';
+import {AV1_CODEC_PARAMETERS, H264_CODEC_PARAMETERS} from './codec/constants';
 
 export interface ActiveSpeakerPolicyInfo {
   policy: 'active-speaker';
@@ -54,34 +60,49 @@ export interface MediaRequest {
 
 export type MediaRequestId = string;
 
-const CODEC_DEFAULTS = {
-  h264: {
-    maxFs: 8192,
-    maxFps: 3000,
-    maxMbps: 245760,
-  },
-};
-
 const DEBOUNCED_SOURCE_UPDATE_TIME = 1000;
 
+const RESOLUTION_BUCKETS: Array<[SupportedResolution, number]> = [
+  ['90p', MAX_FS_VALUES['90p']],
+  ['180p', MAX_FS_VALUES['180p']],
+  ['360p', MAX_FS_VALUES['360p']],
+  ['540p', MAX_FS_VALUES['540p']],
+  ['720p', MAX_FS_VALUES['720p']],
+];
+
 type DegradationPreferences = {
   maxMacroblocksLimit: number;
 };
 
 type SendMediaRequestsCallback = (streamRequests: StreamRequest[]) => void;
+type GetIngressPayloadTypeCallback = (
+  mediaType: MediaType,
+  codecMimeType: MediaCodecMimeType
+) => number | undefined;
 type Kind = 'audio' | 'video';
 
-type Options = {
+type AudioMediaRequestManagerOptions = {
   degradationPreferences: DegradationPreferences;
-  kind: Kind;
+  kind: 'audio';
   trimRequestsToNumOfSources: boolean; // if enabled, AS speaker requests will be trimmed based on the calls to setNumCurrentSources()
 };
 
+type VideoMediaRequestManagerOptions = {
+  degradationPreferences: DegradationPreferences;
+  kind: 'video';
+  trimRequestsToNumOfSources: boolean;
+  enableAv1?: boolean;
+};
+
+type Options = AudioMediaRequestManagerOptions | VideoMediaRequestManagerOptions;
+
 type ClientRequestsMap = {[key: MediaRequestId]: MediaRequest};
 
 export class MediaRequestManager {
   private sendMediaRequestsCallback: SendMediaRequestsCallback;
 
+  private getIngressPayloadTypeCallback: GetIngressPayloadTypeCallback;
+
   private kind: Kind;
 
   private counter: number;
@@ -95,11 +116,17 @@ export class MediaRequestManager {
   private debouncedSourceUpdateListener: () => void;
 
   private trimRequestsToNumOfSources: boolean;
+  private enableAv1: boolean;
   private numTotalSources: number;
   private numLiveSources: number;
 
-  constructor(sendMediaRequestsCallback: SendMediaRequestsCallback, options: Options) {
+  constructor(
+    sendMediaRequestsCallback: SendMediaRequestsCallback,
+    getIngressPayloadTypeCallback: GetIngressPayloadTypeCallback,
+    options: Options
+  ) {
     this.sendMediaRequestsCallback = sendMediaRequestsCallback;
+    this.getIngressPayloadTypeCallback = getIngressPayloadTypeCallback;
     this.counter = 0;
     this.numLiveSources = 0;
     this.numTotalSources = 0;
@@ -107,6 +134,7 @@ export class MediaRequestManager {
     this.degradationPreferences = options.degradationPreferences;
     this.kind = options.kind;
     this.trimRequestsToNumOfSources = options.trimRequestsToNumOfSources;
+    this.enableAv1 = options.kind === 'video' && !!options.enableAv1;
     this.sourceUpdateListener = this.commit.bind(this);
     this.debouncedSourceUpdateListener = debounce(
       this.sourceUpdateListener,
@@ -135,8 +163,8 @@ export class MediaRequestManager {
       Object.values(clientRequests).forEach((mr) => {
         if (mr.codecInfo) {
           mr.codecInfo.maxFs = Math.min(
-            mr.preferredMaxFs || CODEC_DEFAULTS.h264.maxFs,
-            mr.codecInfo.maxFs || CODEC_DEFAULTS.h264.maxFs,
+            mr.preferredMaxFs || H264_CODEC_PARAMETERS.maxFs,
+            mr.codecInfo.maxFs || H264_CODEC_PARAMETERS.maxFs,
             maxFsLimits[i]
           );
           // we only consider sources with "live" state
@@ -176,7 +204,7 @@ export class MediaRequestManager {
     }
 
     return getRecommendedMaxBitrateForFrameSize(
-      mediaRequest.codecInfo.maxFs || CODEC_DEFAULTS.h264.maxFs
+      mediaRequest.codecInfo.maxFs || H264_CODEC_PARAMETERS.maxFs
     );
   }
 
@@ -192,12 +220,80 @@ export class MediaRequestManager {
   // eslint-disable-next-line class-methods-use-this
   private getH264MaxMbps(mediaRequest: MediaRequest): number {
     // fallback for maxFps (not needed for maxFs, since there is a fallback already in getDegradedClientRequests)
-    const maxFps = mediaRequest.codecInfo.maxFps || CODEC_DEFAULTS.h264.maxFps;
+    const maxFps = mediaRequest.codecInfo.maxFps || H264_CODEC_PARAMETERS.maxFps;
 
     // divided by 100 since maxFps is 3000 (for 30 frames per seconds)
     return (mediaRequest.codecInfo.maxFs * maxFps) / 100;
   }
 
+  /**
+   * Returns the AV1 encoding parameters for a media request
+   * @param mediaRequest - The media request to get the AV1 encoding parameters for
+   * @returns {AV1EncodingParams} The AV1 encoding parameters
+   */
+  // eslint-disable-next-line class-methods-use-this
+  private getAv1EncodingParams(mediaRequest: MediaRequest): AV1EncodingParams {
+    const frameSize = mediaRequest.codecInfo.maxFs || H264_CODEC_PARAMETERS.maxFs;
+    const resolution = RESOLUTION_BUCKETS.find(([, maxFs]) => frameSize <= maxFs)?.[0] ?? '1080p';
+
+    return AV1_CODEC_PARAMETERS[resolution];
+  }
+
+  private buildH264CodecInfo(mr: MediaRequest): WcmeCodecInfo | undefined {
+    if (!mr.codecInfo) {
+      return undefined;
+    }
+
+    const h264PayloadType = this.getIngressPayloadTypeCallback(
+      mr.receiveSlots[0].mediaType,
+      MediaCodecMimeType.H264
+    );
+
+    if (h264PayloadType === undefined) {
+      return undefined;
+    }
+
+    return WcmeCodecInfo.fromH264(
+      h264PayloadType,
+      new H264Codec(
+        mr.codecInfo.maxFs,
+        mr.codecInfo.maxFps || H264_CODEC_PARAMETERS.maxFps,
+        this.getH264MaxMbps(mr),
+        mr.codecInfo.maxWidth,
+        mr.codecInfo.maxHeight
+      )
+    );
+  }
+
+  private buildAv1CodecInfo(mr: MediaRequest): WcmeCodecInfo | undefined {
+    if (!this.enableAv1 || !mr.codecInfo) {
+      return undefined;
+    }
+
+    const av1PayloadType = this.getIngressPayloadTypeCallback(
+      mr.receiveSlots[0].mediaType,
+      MediaCodecMimeType.AV1
+    );
+
+    if (av1PayloadType === undefined) {
+      return undefined;
+    }
+
+    const av1EncodingParams = this.getAv1EncodingParams(mr);
+
+    return WcmeCodecInfo.fromAv1(
+      av1PayloadType,
+      new AV1Codec(
+        av1EncodingParams.levelIdx,
+        av1EncodingParams.tier,
+        mr.codecInfo.maxWidth || av1EncodingParams.maxWidth,
+        mr.codecInfo.maxHeight || av1EncodingParams.maxHeight,
+        av1EncodingParams.maxPicSize,
+        av1EncodingParams.maxDecodeRate
+      )
+    );
+  }
+
   /** Modifies the passed in clientRequests and makes sure that in total they don't ask
    *  for more streams than there are available.
    *
@@ -298,6 +394,12 @@ export class MediaRequestManager {
     // map all the client media requests to wcme stream requests
     Object.values(clientRequests).forEach((mr) => {
       if (mr.receiveSlots.length > 0) {
+        const codecInfos: WcmeCodecInfo[] = mr.codecInfo
+          ? [this.buildH264CodecInfo(mr), this.buildAv1CodecInfo(mr)].filter(
+              (info): info is WcmeCodecInfo => info !== undefined
+            )
+          : [];
+
         streamRequests.push(
           new StreamRequest(
             mr.policyInfo.policy === 'active-speaker'
@@ -314,25 +416,14 @@ export class MediaRequestManager {
               : new ReceiverSelectedInfo(mr.policyInfo.csi),
             mr.receiveSlots.map((receiveSlot) => receiveSlot.wcmeReceiveSlot),
             this.getMaxPayloadBitsPerSecond(mr),
-            mr.codecInfo && [
-              WcmeCodecInfo.fromH264(
-                0x80,
-                new H264Codec(
-                  mr.codecInfo.maxFs,
-                  mr.codecInfo.maxFps || CODEC_DEFAULTS.h264.maxFps,
-                  this.getH264MaxMbps(mr),
-                  mr.codecInfo.maxWidth,
-                  mr.codecInfo.maxHeight
-                )
-              ),
-            ]
+            codecInfos
           )
         );
       }
     });
 
     this.sendMediaRequestsCallback(streamRequests);
-    LoggerProxy.logger.info(`multistream:sendRequests --> media requests sent. `);
+    LoggerProxy.logger.info(`multistream:sendRequests --> media requests sent.`);
   }
 
   public addRequest(mediaRequest: MediaRequest, commit = true): MediaRequestId {

package/test/unit/spec/media/index.ts

@@ -295,6 +295,7 @@ describe('createMediaConnection', () => {
         bundlePolicy: 'max-bundle',
         disableAudioMainDtx: false,
         disableAudioTwcc: false,
+        enableAV1SlidesSupport: false,
       },
       'meeting id'
     );
@@ -322,6 +323,26 @@ describe('createMediaConnection', () => {
     assert.calledOnce(rtcMetrics.sendMetricsInQueue);
   });
 
+  it('passes enableAV1SlidesSupport: true to MultistreamRoapMediaConnection when enableAv1SlidesSupport is set', () => {
+    const multistreamRoapMediaConnectionConstructorStub = sinon
+      .stub(InternalMediaCoreModule, 'MultistreamRoapMediaConnection')
+      .returns(fakeRoapMediaConnection);
+
+    Media.createMediaConnection(true, 'some debug id', 'meeting id', {
+      enableAv1SlidesSupport: true,
+    });
+    assert.calledOnce(multistreamRoapMediaConnectionConstructorStub);
+    assert.calledWith(
+      multistreamRoapMediaConnectionConstructorStub,
+      sinon.match({
+        iceServers: [],
+        disableAudioTwcc: true,
+        enableAV1SlidesSupport: true,
+      }),
+      'meeting id'
+    );
+  });
+
   it('multistream non-firefox does not care about stopIceGatheringAfterFirstRelayCandidate', () => {
     const multistreamRoapMediaConnectionConstructorStub = sinon
       .stub(InternalMediaCoreModule, 'MultistreamRoapMediaConnection')
@@ -336,6 +357,7 @@ describe('createMediaConnection', () => {
       {
         iceServers: [],
         disableAudioTwcc: true,
+        enableAV1SlidesSupport: false,
       },
       'meeting id'
     );
@@ -359,6 +381,7 @@ describe('createMediaConnection', () => {
         doFullIce: true,
         stopIceGatheringAfterFirstRelayCandidate: true,
         disableAudioTwcc: true,
+        enableAV1SlidesSupport: false,
       },
       'meeting id'
     );
@@ -382,6 +405,7 @@ describe('createMediaConnection', () => {
         doFullIce: true,
         stopIceGatheringAfterFirstRelayCandidate: false,
         disableAudioTwcc: true,
+        enableAV1SlidesSupport: false,
       },
       'meeting id'
     );
@@ -418,6 +442,7 @@ describe('createMediaConnection', () => {
         {
           iceServers: [],
           disableAudioTwcc: true,
+          enableAV1SlidesSupport: false,
         },
         'meeting id'
       );
@@ -448,6 +473,7 @@ describe('createMediaConnection', () => {
       {
         iceServers: [],
         disableAudioTwcc: true,
+        enableAV1SlidesSupport: false,
       },
       'meeting id'
     );
@@ -477,6 +503,7 @@ describe('createMediaConnection', () => {
       {
         iceServers: [],
         disableAudioTwcc: true,
+        enableAV1SlidesSupport: false,
       },
       'meeting id'
     );
@@ -505,6 +532,7 @@ describe('createMediaConnection', () => {
       {
         iceServers: [],
         disableAudioTwcc: true,
+        enableAV1SlidesSupport: false,
       },
       'meeting id'
     );
@@ -591,6 +619,7 @@ describe('createMediaConnection', () => {
     {
       iceServers: [],
       disableAudioTwcc: true,
+      enableAV1SlidesSupport: false,
       enableInboundAudioLevelMonitoring: true,
     }
   );
@@ -602,6 +631,7 @@ describe('createMediaConnection', () => {
     {
       iceServers: [],
       disableAudioTwcc: true,
+      enableAV1SlidesSupport: false,
       enableInboundAudioLevelMonitoring: true,
     }
   );
@@ -613,6 +643,7 @@ describe('createMediaConnection', () => {
     {
       iceServers: [],
       disableAudioTwcc: true,
+      enableAV1SlidesSupport: false,
       doFullIce: true,
       stopIceGatheringAfterFirstRelayCandidate: undefined,
     }

package/test/unit/spec/meeting/index.js

@@ -420,6 +420,160 @@ describe('plugin-meetings', () => {
           assert.instanceOf(meeting.mediaRequestManagers.screenShareVideo, MediaRequestManager);
         });
 
+        it('getIngressPayloadType on webrtcMediaConnection is invoked for H264 when sending multistream video requests', () => {
+          const getIngressPayloadType = sinon.stub().returns(97);
+
+          meeting.isMultistream = true;
+          meeting.mediaProperties.webrtcMediaConnection = {
+            getIngressPayloadType,
+            requestMedia: sinon.stub(),
+          };
+
+          const fakeReceiveSlot = {
+            on: sinon.stub(),
+            off: sinon.stub(),
+            sourceState: 'live',
+            mediaType: MediaType.VideoMain,
+            wcmeReceiveSlot: {id: 'fake-wcme-slot'},
+          };
+
+          meeting.mediaRequestManagers.video.addRequest(
+            {
+              policyInfo: {
+                policy: 'receiver-selected',
+                csi: 42,
+              },
+              receiveSlots: [fakeReceiveSlot],
+              codecInfo: {
+                codec: 'h264',
+                maxFs: 3600,
+              },
+            },
+            true
+          );
+
+          assert.calledOnceWithExactly(
+            getIngressPayloadType,
+            MediaType.VideoMain,
+            MediaCodecMimeType.H264
+          );
+        });
+
+        it('getIngressPayloadType on webrtcMediaConnection is invoked for H264 and AV1 for slides video when AV1 slides support is enabled', () => {
+          const localWebex = new MockWebex({
+            children: {
+              meetings: Meetings,
+              credentials: Credentials,
+              support: Support,
+              llm: LLM,
+              mercury: Mercury,
+            },
+            config: {
+              credentials: {
+                client_id: 'mock-client-id',
+              },
+              meetings: {
+                reconnection: {
+                  enabled: false,
+                },
+                mediaSettings: {},
+                metrics: {},
+                stats: {},
+                experimental: {enableUnifiedMeetings: true},
+                degradationPreferences: {maxMacroblocksLimit: 8192},
+                enableAv1SlidesSupport: true,
+              },
+              metrics: {
+                type: ['behavioral'],
+              },
+            },
+          });
+
+          localWebex.internal.newMetrics.callDiagnosticMetrics.clearErrorCache = sinon.stub();
+          localWebex.internal.newMetrics.callDiagnosticMetrics.clearEventLimitsForCorrelationId =
+            sinon.stub();
+          localWebex.internal.support.submitLogs = sinon.stub().returns(Promise.resolve());
+          localWebex.internal.services = {get: sinon.stub().returns('locus-url')};
+          localWebex.credentials.getOrgId = sinon.stub().returns('fake-org-id');
+          localWebex.internal.metrics.submitClientMetrics = sinon.stub().returns(Promise.resolve());
+          localWebex.meetings.uploadLogs = sinon.stub().returns(Promise.resolve());
+          localWebex.meetings.reachability = {
+            isAnyPublicClusterReachable: sinon.stub().resolves(true),
+            getReachabilityResults: sinon.stub().resolves(undefined),
+            getReachabilityMetrics: sinon.stub().resolves({}),
+            stopReachability: sinon.stub(),
+            isSubnetReachable: sinon.stub().returns(true),
+          };
+          localWebex.internal.llm.isDataChannelTokenEnabled = sinon.stub().resolves(false);
+          localWebex.internal.llm.on = sinon.stub();
+          localWebex.internal.voicea.announce = sinon.stub();
+          localWebex.internal.newMetrics.callDiagnosticLatencies = new CallDiagnosticLatencies(
+            {},
+            {parent: localWebex}
+          );
+
+          Metrics.initialSetup(localWebex);
+
+          const localMeeting = new Meeting(
+            {
+              userId: uuid1,
+              resource: uuid2,
+              deviceUrl: uuid3,
+              locus: {url: url1},
+              destination: testDestination,
+              destinationType: DESTINATION_TYPE.MEETING_ID,
+              correlationId,
+              selfId: uuid1,
+            },
+            {
+              parent: localWebex,
+            }
+          );
+
+          const getIngressPayloadType = sinon.stub().callsFake((_mediaType, codecMimeType) => {
+            if (codecMimeType === MediaCodecMimeType.H264) {
+              return 97;
+            }
+            if (codecMimeType === MediaCodecMimeType.AV1) {
+              return 98;
+            }
+
+            return undefined;
+          });
+
+          localMeeting.isMultistream = true;
+          localMeeting.mediaProperties.webrtcMediaConnection = {
+            getIngressPayloadType,
+            requestMedia: sinon.stub(),
+          };
+
+          const fakeReceiveSlot = {
+            on: sinon.stub(),
+            off: sinon.stub(),
+            sourceState: 'live',
+            mediaType: MediaType.VideoSlides,
+            wcmeReceiveSlot: {id: 'fake-wcme-slides-slot'},
+          };
+
+          localMeeting.mediaRequestManagers.screenShareVideo.addRequest(
+            {
+              policyInfo: {
+                policy: 'receiver-selected',
+                csi: 42,
+              },
+              receiveSlots: [fakeReceiveSlot],
+              codecInfo: {
+                codec: 'h264',
+                maxFs: 3600,
+              },
+            },
+            true
+          );
+
+          assert.calledWith(getIngressPayloadType, MediaType.VideoSlides, MediaCodecMimeType.H264);
+          assert.calledWith(getIngressPayloadType, MediaType.VideoSlides, MediaCodecMimeType.AV1);
+        });
+
         it('uses meeting id as correlation id if not provided in constructor', () => {
           const newMeeting = new Meeting(
             {

package/test/unit/spec/meetings/index.js

@@ -426,6 +426,33 @@ describe('plugin-meetings', () => {
       });
     });
 
+    describe('#_toggleEnableAv1SlidesSupport', () => {
+      it('should have _toggleEnableAv1SlidesSupport', () => {
+        assert.equal(typeof webex.meetings._toggleEnableAv1SlidesSupport, 'function');
+      });
+
+      describe('success', () => {
+        it('should update meetings config to enable AV1 slides support', () => {
+          webex.meetings._toggleEnableAv1SlidesSupport(true);
+          assert.equal(webex.meetings.config.enableAv1SlidesSupport, true);
+
+          webex.meetings._toggleEnableAv1SlidesSupport(false);
+          assert.equal(webex.meetings.config.enableAv1SlidesSupport, false);
+        });
+
+        it('should not update config when called with a non-boolean value', () => {
+          webex.meetings._toggleEnableAv1SlidesSupport(true);
+          assert.equal(webex.meetings.config.enableAv1SlidesSupport, true);
+
+          webex.meetings._toggleEnableAv1SlidesSupport('invalid');
+          assert.equal(webex.meetings.config.enableAv1SlidesSupport, true);
+
+          webex.meetings._toggleEnableAv1SlidesSupport(undefined);
+          assert.equal(webex.meetings.config.enableAv1SlidesSupport, true);
+        });
+      });
+    });
+
     describe('#_toggleStopIceGatheringAfterFirstRelayCandidate', () => {
       it('should have _toggleStopIceGatheringAfterFirstRelayCandidate', () => {
         assert.equal(