@webex/plugin-meetings 3.0.0-beta.1 → 3.0.0-beta.2

package/src/meeting/request.js

@@ -499,6 +499,7 @@ export default class MeetingRequest extends StatelessWebexPlugin {
    * @param {String} options.deviceUrl Url of a device
    * @param {String} options.resourceId Populated if you are paired to a device
    * @param {String} options.localMedias local sdps
+   * @param {Boolean} options.preferTranscoding false for multistream (Homer), true for transcoded media (Edonus)
    * @returns {Promise}
    */
   remoteAudioVideoToggle(options) {

package/src/meeting/util.js

@@ -58,7 +58,8 @@ MeetingUtil.remoteUpdateAudioVideo = (audioMuted, videoMuted, meeting) => {
     selfId: meeting.selfId,
     localMedias,
     deviceUrl: meeting.deviceUrl,
-    correlationId: meeting.correlationId
+    correlationId: meeting.correlationId,
+    preferTranscoding: !meeting.isMultistream,
   }).then((response) => {
     Metrics.postEvent({event: eventType.MEDIA_RESPONSE, meeting});
 
@@ -95,7 +96,7 @@ MeetingUtil.joinMeeting = (meeting, options) => {
       moderator: options.moderator,
       pin: options.pin,
       moveToResource: options.moveToResource,
-      preferTranscoding: options.preferTranscoding,
+      preferTranscoding: !meeting.isMultistream,
       asResourceOccupant: options.asResourceOccupant
     })
     .then((res) => {
@@ -130,7 +131,6 @@ MeetingUtil.cleanUp = (meeting) => {
       meeting.unsetPeerConnections();
       meeting.reconnectionManager.cleanUp();
     })
-    .then(() => meeting.roap.stop(meeting.correlationId, meeting.roapSeq))
     .then(() => meeting.stopKeepAlive());
 };
 
@@ -279,49 +279,6 @@ MeetingUtil.joinMeetingOptions = (meeting, options = {}) => {
     });
 };
 
-MeetingUtil.updateTransceiver = (options, meetingOptions) => {
-  const {
-    type,
-    sendTrack,
-    receiveTrack,
-    track,
-    transceiver,
-    peerConnection,
-    previousMediaDirection
-  } = options;
-
-  if ((sendTrack !== undefined && sendTrack !== previousMediaDirection.sendTrack) ||
-  (receiveTrack !== undefined && receiveTrack !== previousMediaDirection.receiveTrack)) {
-    return Media.updateTransceiver({
-      meetingId: meetingOptions.meeting.id,
-      remoteQualityLevel: meetingOptions.mediaProperties.remoteQualityLevel,
-      enableRtx: meetingOptions.meeting.config.enableRtx,
-      enableExtmap: meetingOptions.meeting.config.enableExtmap
-    }, peerConnection, transceiver,
-    {
-      track,
-      type,
-      receiveTrack,
-      sendTrack
-    })
-      .then(() => meetingOptions.meeting.roap
-        .sendRoapMediaRequest({
-          sdp: meetingOptions.mediaProperties.peerConnection.sdp,
-          roapSeq: meetingOptions.meeting.roapSeq,
-          meeting: meetingOptions.meeting // or can pass meeting ID
-        }))
-      .catch((e) => {
-        LoggerProxy.logger.error(`Meeting:util#updateTransceiver --> Error updating the ${type} streams with error: ${e}`);
-      });
-  } if (track) {
-    transceiver.sender.replaceTrack(track);
-
-    return Promise.resolve();
-  }
-
-  return Promise.reject(new ParameterError('update Failed: please pass valid parameter'));
-};
-
 MeetingUtil.validateOptions = (options) => {
   const {
     sendVideo, sendAudio, sendShare, localStream, localShare

package/src/meetings/index.js

@@ -2,6 +2,7 @@
 import '@webex/internal-plugin-mercury';
 import '@webex/internal-plugin-conversation';
 import {WebexPlugin} from '@webex/webex-core';
+import {MediaConnection as MC} from '@webex/internal-media-core';
 
 import 'webrtc-adapter';
 
@@ -51,7 +52,33 @@ import CaptchaError from '../common/errors/captcha-error';
 import MeetingCollection from './collection';
 import MeetingsUtil from './util';
 
+let mediaLogger;
 
+class MediaLogger {
+  info(...args) {
+    LoggerProxy.logger.info(...args);
+  }
+
+  log(...args) {
+    LoggerProxy.logger.log(...args);
+  }
+
+  error(...args) {
+    LoggerProxy.logger.error(...args);
+  }
+
+  warn(...args) {
+    LoggerProxy.logger.warn(...args);
+  }
+
+  trace(...args) {
+    LoggerProxy.logger.trace(...args);
+  }
+
+  debug(...args) {
+    LoggerProxy.logger.debug(...args);
+  }
+}
 /**
    * Meetings Ready Event
    * Emitted when the meetings instance on webex is ready
@@ -374,6 +401,9 @@ export default class Meetings extends WebexPlugin {
       LoggerConfig.set(this.config.logging);
       LoggerProxy.set(this.webex.logger);
 
+      mediaLogger = new MediaLogger();
+      MC.setLogger(mediaLogger);
+
       /**
        * The MeetingInfo object to interact with server
        * @instance
@@ -805,7 +835,6 @@ export default class Meetings extends WebexPlugin {
         userId: this.webex.internal.device.userId,
         deviceUrl: this.webex.internal.device.url,
         orgId: this.webex.internal.device.orgId,
-        roapSeq: 0,
         locus: type === _LOCUS_ID_ ? destination : null, // pass the locus object if present
         meetingInfoProvider: this.meetingInfo,
         destination,

package/src/meetings/util.js

@@ -6,7 +6,8 @@ import {
   _CREATED_,
   LOCUSEVENT,
   CORRELATION_ID,
-  EVENT_TRIGGERS
+  EVENT_TRIGGERS,
+  ROAP
 } from '../constants';
 import LoggerProxy from '../common/logs/logger-proxy';
 import Trigger from '../common/events/trigger-proxy';
@@ -41,7 +42,27 @@ MeetingsUtil.handleRoapMercury = (envelope, meetingCollection) => {
     const meeting = meetingCollection.getByKey(CORRELATION_ID, data.correlationId);
 
     if (meeting) {
-      meeting.roap.roapEvent(data);
+      const {
+        seq, messageType, tieBreaker, errorType, errorCause
+      } = data.message;
+
+      if (messageType === ROAP.ROAP_TYPES.TURN_DISCOVERY_RESPONSE) {
+        // turn discovery is not part of normal roap protocol and so we are not handling it
+        // through the usual roap state machine
+        meeting.roap.turnDiscovery.handleTurnDiscoveryResponse(data.message);
+      }
+      else {
+        const roapMessage = {
+          seq,
+          messageType,
+          sdp: data.message.sdps?.length > 0 ? data.message.sdps[0] : undefined,
+          tieBreaker,
+          errorType,
+          errorCause
+        };
+
+        meeting.mediaProperties.webrtcMediaConnection.roapMessageReceived(roapMessage);
+      }
     }
   }
 };

package/src/members/index.js

@@ -148,6 +148,18 @@ export default class Members extends StatelessWebexPlugin {
      * @memberof Members
     */
     this.recordingId = null;
+
+    /**
+     * reference to a ReceiveSlotManager instance (for multistream)
+     * @private
+     */
+    this.receiveSlotManager = attrs.receiveSlotManager;
+
+    /**
+     * reference to a MediaRequestManager instance that manages main video requests (for multistream)
+     * @private
+     */
+    this.mediaRequestManagers = attrs.mediaRequestManagers;
   }
 
   /**
@@ -833,4 +845,40 @@ export default class Members extends StatelessWebexPlugin {
 
     return Promise.reject(new Error('Members:index#sendDialPadKey --> cannot send DTMF, meeting does not have a connection to the "locus" call control service.'));
   }
+
+  /** Finds a member that has any device with a csi matching provided value
+   *
+   * @param {number} csi
+   * @returns {Member}
+   */
+  findMemberByCsi(csi) {
+    return Object.values(this.membersCollection.getAll())
+      .find((member) => (
+        member.participant?.devices?.find((device) => (
+          device.csis?.find((memberCsi) => memberCsi === csi)))));
+  }
+
+  /**
+   * Returns an array of a member's CSIs matching the mediaType and mediaContent
+   *
+   * @param {string} memberId
+   * @param {string} mediaType 'audio' or 'video'
+   * @param {string} mediaContent 'main' or 'slides'
+   * @returns {Member}
+   */
+  getCsisForMember(memberId, mediaType = 'video', mediaContent = 'main') {
+    const csis = [];
+
+    this.membersCollection.get(memberId)?.participant?.devices?.forEach((device) => {
+      if (device.mediaSessions) {
+        const deviceCsis = device.mediaSessions
+          ?.filter((mediaSession) => mediaSession.mediaType === mediaType && mediaSession.mediaContent === mediaContent)
+          .map((mediaSession) => mediaSession.csi);
+
+        csis.push(...deviceCsis);
+      }
+    });
+
+    return csis;
+  }
 }

package/src/multistream/mediaRequestManager.ts

@@ -0,0 +1,164 @@
+/* eslint-disable require-jsdoc */
+import {MediaConnection as MC} from '@webex/internal-media-core';
+
+import LoggerProxy from '../common/logs/logger-proxy';
+
+import {ReceiveSlot, ReceiveSlotId} from './receiveSlot';
+
+export interface ActiveSpeakerPolicyInfo {
+  policy: 'active-speaker';
+  priority: number;
+  crossPriorityDuplication: boolean;
+  crossPolicyDuplication: boolean;
+  preferLiveVideo: boolean;
+}
+
+export interface ReceiverSelectedPolicyInfo {
+  policy: 'receiver-selected';
+  csi: number;
+}
+
+export type PolicyInfo = ActiveSpeakerPolicyInfo | ReceiverSelectedPolicyInfo;
+
+export interface H264CodecInfo {
+  codec: 'h264';
+  maxFs?: number;
+  maxFps?: number;
+  maxMbps?: number;
+  maxWidth?: number;
+  maxHeight?: number;
+}
+
+export type CodecInfo = H264CodecInfo; // we'll add AV1 here in the future when it's available
+
+export interface MediaRequest {
+  policyInfo: PolicyInfo;
+  receiveSlots: Array<ReceiveSlot>;
+  codecInfo?: CodecInfo;
+}
+
+export type MediaRequestId = string;
+
+const CODEC_DEFAULTS = {
+  h264: {
+    maxFs: 8192,
+    maxFps: 3000,
+    maxMbps: 245760,
+  },
+};
+
+type SendMediaRequestsCallback = (mediaRequests: MC.MediaRequest[]) => void;
+
+export class MediaRequestManager {
+  private sendMediaRequestsCallback: SendMediaRequestsCallback;
+
+  private counter;
+
+  private clientRequests: {[key: MediaRequestId]: MediaRequest};
+
+  private slotsActiveInLastMediaRequest: {[key: ReceiveSlotId]: ReceiveSlot};
+
+  constructor(sendMediaRequestsCallback: SendMediaRequestsCallback) {
+    this.sendMediaRequestsCallback = sendMediaRequestsCallback;
+    this.counter = 0;
+    this.clientRequests = {};
+    this.slotsActiveInLastMediaRequest = {};
+  }
+
+  private resetInactiveReceiveSlots() {
+    const activeSlots: {[key: ReceiveSlotId]: ReceiveSlot} = {};
+
+    // create a map of all currently used slot ids
+    Object.values(this.clientRequests).forEach((request) =>
+      request.receiveSlots.forEach((slot) => {
+        activeSlots[slot.id] = slot;
+      })
+    );
+
+    // when we stop using some receive slots and they are not included in the new media request,
+    // we will never get a 'no source' notification for them, so we reset their state,
+    // so that the client doesn't try to display their video anymore
+    for (const [slotId, slot] of Object.entries(this.slotsActiveInLastMediaRequest)) {
+      if (!(slotId in activeSlots)) {
+        LoggerProxy.logger.info(
+          `multistream:mediaRequestManager --> resetting sourceState to "no source" for slot ${slot.id}`
+        );
+        slot.resetSourceState();
+      }
+    }
+
+    this.slotsActiveInLastMediaRequest = activeSlots;
+  }
+
+  private sendRequests() {
+    const wcmeMediaRequests: MC.MediaRequest[] = [];
+
+    // todo: check how many streams we're asking for and what resolution and introduce some limits (spark-377701)
+
+    // map all the client media requests to wcme media requests
+    Object.values(this.clientRequests).forEach((mr) => {
+      wcmeMediaRequests.push(
+        new MC.MediaRequest(
+          mr.policyInfo.policy === 'active-speaker'
+            ? MC.Policy.ActiveSpeaker
+            : MC.Policy.ReceiverSelected,
+          mr.policyInfo.policy === 'active-speaker'
+            ? new MC.ActiveSpeakerInfo(
+                mr.policyInfo.priority,
+                mr.policyInfo.crossPriorityDuplication,
+                mr.policyInfo.crossPolicyDuplication,
+                mr.policyInfo.preferLiveVideo
+              )
+            : new MC.ReceiverSelectedInfo(mr.policyInfo.csi),
+          mr.receiveSlots.map((receiveSlot) => receiveSlot.wcmeReceiveSlot),
+          mr.codecInfo && [
+            new MC.CodecInfo(
+              0x80,
+              new MC.H264Codec(
+                mr.codecInfo.maxFs || CODEC_DEFAULTS.h264.maxFs,
+                mr.codecInfo.maxFps || CODEC_DEFAULTS.h264.maxFps,
+                mr.codecInfo.maxMbps || CODEC_DEFAULTS.h264.maxMbps,
+                mr.codecInfo.maxWidth,
+                mr.codecInfo.maxHeight
+              )
+            ),
+          ]
+        )
+      );
+    });
+
+    this.sendMediaRequestsCallback(wcmeMediaRequests);
+
+    this.resetInactiveReceiveSlots();
+  }
+
+  public addRequest(mediaRequest: MediaRequest, commit = true): MediaRequestId {
+    // eslint-disable-next-line no-plusplus
+    const newId = `${this.counter++}`;
+
+    this.clientRequests[newId] = mediaRequest;
+
+    if (commit) {
+      this.commit();
+    }
+
+    return newId;
+  }
+
+  public cancelRequest(requestId: MediaRequestId, commit = true) {
+    delete this.clientRequests[requestId];
+
+    if (commit) {
+      this.commit();
+    }
+  }
+
+  public commit() {
+    return this.sendRequests();
+  }
+
+  public reset() {
+    this.clientRequests = {};
+    this.slotsActiveInLastMediaRequest = {};
+  }
+}

package/src/multistream/multistreamMedia.ts

@@ -0,0 +1,92 @@
+/* eslint-disable import/prefer-default-export */
+
+import {AUDIO, VIDEO} from '../constants';
+import createMuteState from '../meeting/muteState';
+import Meeting from '../meeting';
+
+/**
+ * Class wrapping all the multistream specific APIs that need to be exposed at meeting level.
+ *
+ */
+export class MultistreamMedia {
+  private meeting: Meeting;
+
+  /**
+   * Constructor
+   * @param {Meeting} meeting
+   */
+  constructor(meeting: Meeting) {
+    this.meeting = meeting;
+  }
+
+  /**
+   * throws if we don't have a media connection created
+   */
+  private checkMediaConnection() {
+    if (this.meeting?.mediaProperties?.webrtcMediaConnection) {
+      return;
+    }
+    throw new Error('Webrtc media connection is missing');
+  }
+
+  /**
+   * Publishes a local track in the meeting
+   *
+   * @param {MediaStreamTrack} track
+   * @returns {Promise}
+   */
+  publishTrack(track: MediaStreamTrack): Promise<void> {
+    this.checkMediaConnection();
+
+    /* todo: for now we don't support screen share (waiting for server to support bundling, see SPARK-377812)
+       for sharing:
+          we have to call meeting.stopFloorRequest() before unpublishing a track
+          we have to call meeting.share() after publishing a track
+    */
+
+    // todo: depending on how muting is done with Local tracks, this code here might need to change...
+
+    if (track.kind === 'audio') {
+      this.meeting.setLocalAudioTrack(track);
+      this.meeting.mediaProperties.mediaDirection.sendAudio = true;
+
+      // audio state could be undefined if you have not sent audio before
+      this.meeting.audio =
+        this.meeting.audio ||
+        createMuteState(AUDIO, this.meeting, this.meeting.mediaProperties.mediaDirection);
+    } else if (track.kind === 'video') {
+      this.meeting.setLocalVideoTrack(track);
+      this.meeting.mediaProperties.mediaDirection.sendVideo = true;
+
+      // video state could be undefined if you have not sent video before
+      this.meeting.video =
+        this.meeting.video ||
+        createMuteState(VIDEO, this.meeting, this.meeting.mediaProperties.mediaDirection);
+    }
+
+    return this.meeting.mediaProperties.webrtcMediaConnection.publishTrack(track);
+  }
+
+  /**
+   * Unpublishes a local track in the meeting
+   *
+   * @param {MediaStreamTrack} track
+   * @returns {Promise}
+   */
+  unpublishTrack(track: MediaStreamTrack): Promise<void> {
+    // todo: see todos in publishTrack() - they all apply here too:
+    // screen sharing - SPARK-377812
+    // muting etc
+
+    if (track.kind === 'audio') {
+      this.meeting.setLocalVideoTrack(null);
+      this.meeting.mediaProperties.mediaDirection.sendAudio = false;
+    } else if (track.kind === 'video') {
+      this.meeting.setLocalAudioTrack(null);
+      this.meeting.mediaProperties.mediaDirection.sendVideo = false;
+    }
+    this.checkMediaConnection();
+
+    return this.meeting.mediaProperties.webrtcMediaConnection.unpublishTrack(track);
+  }
+}

package/src/multistream/receiveSlot.ts

@@ -0,0 +1,141 @@
+import {MediaConnection as MC} from '@webex/internal-media-core';
+
+import LoggerProxy from '../common/logs/logger-proxy';
+import EventsScope from '../common/events/events-scope';
+
+export const ReceiveSlotEvents = {
+  SourceUpdate: 'sourceUpdate',
+};
+
+export type SourceState = MC.SourceState;
+export type CSI = number;
+export type MemberId = string;
+export type ReceiveSlotId = string;
+
+let receiveSlotCounter = 0;
+
+export type FindMemberIdCallback = (csi: CSI) => MemberId | undefined;
+
+/**
+ * Class representing a receive slot. A single receive slot is able to receive a single track
+ * for example some participant's main video or audio
+ */
+export class ReceiveSlot extends EventsScope {
+  private readonly mcReceiveSlot: MC.ReceiveSlot;
+
+  private readonly findMemberIdCallback: FindMemberIdCallback;
+
+  public readonly id: ReceiveSlotId;
+
+  public readonly mediaType: MC.MediaType;
+
+  #memberId?: MemberId;
+
+  #csi?: CSI;
+
+  #sourceState: MC.SourceState;
+
+  /**
+   * constructor - don't use it directly, you should always use meeting.receiveSlotManager.allocateSlot()
+   * to create any receive slots
+   *
+   * @param {MC.MediaType} mediaType
+   * @param {MC.ReceiveSlot} mcReceiveSlot
+   * @param {FindMemberIdCallback} findMemberIdCallback callback for finding memberId for given CSI
+   */
+  constructor(
+    mediaType: MC.MediaType,
+    mcReceiveSlot: MC.ReceiveSlot,
+    findMemberIdCallback: FindMemberIdCallback
+  ) {
+    super();
+
+    receiveSlotCounter += 1;
+
+    this.findMemberIdCallback = findMemberIdCallback;
+    this.mediaType = mediaType;
+    this.mcReceiveSlot = mcReceiveSlot;
+    this.#sourceState = 'no source';
+    this.id = `r${receiveSlotCounter}`;
+
+    this.setupEventListeners();
+  }
+
+  /**
+   * Getter for memberId
+   */
+  public get memberId() {
+    return this.#memberId;
+  }
+
+  /**
+   * Getter for csi
+   */
+  public get csi() {
+    return this.#csi;
+  }
+
+  /**
+   * Getter for sourceState
+   */
+  public get sourceState() {
+    return this.#sourceState;
+  }
+
+  /**
+   * registers event handlers with the underlying ReceiveSlot
+   */
+  setupEventListeners() {
+    const scope = {
+      file: 'meeting/receiveSlot',
+      function: 'setupEventListeners',
+    };
+
+    this.mcReceiveSlot.on(
+      MC.ReceiveSlotEvents.SourceUpdate,
+      (state: MC.SourceState, csi?: number) => {
+        LoggerProxy.logger.log(
+          `ReceiveSlot#setupEventListeners --> got source update on receive slot ${this.id}, mediaType=${this.mediaType}, csi=${csi}, state=${state}`
+        );
+        this.#memberId = csi ? this.findMemberIdCallback(csi) : undefined;
+        this.#csi = csi;
+        this.#sourceState = state;
+
+        this.emit(scope, ReceiveSlotEvents.SourceUpdate, {
+          state: this.#sourceState,
+          csi: this.#csi,
+          memberId: this.#memberId,
+        });
+      }
+    );
+  }
+
+  /**
+   * The MediaStream object associated with this slot.
+   *
+   * @returns {MediaStream} The MediaStreamTrack.
+   */
+  get stream(): MediaStream {
+    return this.mcReceiveSlot.stream;
+  }
+
+  /**
+   * The underlying WCME receive slot
+   */
+  get wcmeReceiveSlot(): MC.ReceiveSlot {
+    return this.mcReceiveSlot;
+  }
+
+  /**
+   * Resets the source state to the default 'no source' value.
+   * This function should be called on receive slots that are
+   * no longer part of a media request. It's needed because WCME
+   * does not send any more events on such slots, so the sourceState
+   * value would not represent the truth anymore.
+   */
+  public resetSourceState() {
+    this.#sourceState = 'no source';
+    this.#csi = undefined;
+    this.#memberId = undefined;
+  }
+}