@webex/plugin-meetings 3.0.0-beta.39 → 3.0.0-beta.391

package/src/locus-info/selfUtils.ts

@@ -32,9 +32,11 @@ SelfUtils.parse = (self: any, deviceId: string) => {
     const pstnDevices = self.devices.filter((device) => PSTN_DEVICE_TYPE === device.deviceType);
 
     return {
+      remoteVideoMuted: SelfUtils.getRemoteVideoMuted(self),
       remoteMuted: SelfUtils.getRemoteMuted(self),
       unmuteAllowed: SelfUtils.getUnmuteAllowed(self),
       localAudioUnmuteRequested: SelfUtils.getLocalAudioUnmuteRequested(self),
+      localAudioUnmuteRequestedTimeStamp: SelfUtils.getLocalAudioUnmuteRequestedTimeStamp(self),
       localAudioUnmuteRequired: SelfUtils.getLocalAudioUnmuteRequired(self),
       lastModified: SelfUtils.getLastModified(self),
       modifiedBy: SelfUtils.getModifiedBy(self),
@@ -61,14 +63,18 @@ SelfUtils.parse = (self: any, deviceId: string) => {
       layout: SelfUtils.getLayout(self),
       canNotViewTheParticipantList: SelfUtils.canNotViewTheParticipantList(self),
       isSharingBlocked: SelfUtils.isSharingBlocked(self),
-      breakoutSessions: SelfUtils.getBreakouts(self),
+      breakoutSessions: SelfUtils.getBreakoutSessions(self),
+      breakout: SelfUtils.getBreakout(self),
+      interpretation: SelfUtils.getInterpretation(self),
     };
   }
 
   return null;
 };
 
-SelfUtils.getBreakouts = (self) => self?.controls?.breakout?.sessions;
+SelfUtils.getBreakoutSessions = (self) => self?.controls?.breakout?.sessions;
+SelfUtils.getBreakout = (self) => self?.controls?.breakout;
+SelfUtils.getInterpretation = (self) => self?.controls?.interpretation;
 
 SelfUtils.getLayout = (self) =>
   Array.isArray(self?.controls?.layouts) ? self.controls.layouts[0].type : undefined;
@@ -93,6 +99,7 @@ SelfUtils.getSelves = (oldSelf, newSelf, deviceId) => {
 
   updates.isUserUnadmitted = SelfUtils.isUserUnadmitted(current);
   updates.isUserAdmitted = SelfUtils.isUserAdmitted(previous, current);
+  updates.isVideoMutedByOthersChanged = SelfUtils.videoMutedByOthersChanged(previous, current);
   updates.isMutedByOthersChanged = SelfUtils.mutedByOthersChanged(previous, current);
   updates.localAudioUnmuteRequestedByServer = SelfUtils.localAudioUnmuteRequestedByServer(
     previous,
@@ -103,6 +110,7 @@ SelfUtils.getSelves = (oldSelf, newSelf, deviceId) => {
     current
   );
   updates.moderatorChanged = SelfUtils.moderatorChanged(previous, current);
+  updates.isRolesChanged = SelfUtils.isRolesChanged(previous, current);
   updates.isMediaInactiveOrReleased = SelfUtils.wasMediaInactiveOrReleased(previous, current);
   updates.isUserObserving = SelfUtils.isDeviceObserving(previous, current);
   updates.layoutChanged = SelfUtils.layoutChanged(previous, current);
@@ -119,6 +127,7 @@ SelfUtils.getSelves = (oldSelf, newSelf, deviceId) => {
     previous?.canNotViewTheParticipantList !== current.canNotViewTheParticipantList;
   updates.isSharingBlockedChanged = previous?.isSharingBlocked !== current.isSharingBlocked;
   updates.breakoutsChanged = SelfUtils.breakoutsChanged(previous, current);
+  updates.interpretationChanged = SelfUtils.interpretationChanged(previous, current);
 
   return {
     previous,
@@ -139,13 +148,16 @@ SelfUtils.isJoined = (self: any) => self?.state === _JOINED_;
  *
  * @param {Self} previous - Previous self state
  * @param {Self} current - Current self state [per event]
- * @returns {boolean} - If the MEeting Layout Controls Layout has changed.
+ * @returns {boolean} - If the Meeting Layout Controls Layout has changed.
  */
 SelfUtils.layoutChanged = (previous: any, current: any) =>
   current?.layout && previous?.layout !== current?.layout;
 
 SelfUtils.breakoutsChanged = (previous, current) =>
-  !isEqual(previous?.breakoutSessions, current?.breakoutSessions);
+  !isEqual(previous?.breakoutSessions, current?.breakoutSessions) && !!current?.breakout;
+
+SelfUtils.interpretationChanged = (previous, current) =>
+  !isEqual(previous?.interpretation, current?.interpretation) && !!current?.interpretation;
 
 SelfUtils.isMediaInactive = (previous, current) => {
   if (
@@ -236,6 +248,19 @@ SelfUtils.getSelfIdentity = (self: any) => {
   return self.person.id;
 };
 
+/**
+ * get the "remote video mute" property from the self object
+ * @param {Object} self
+ * @returns {Boolean}
+ */
+SelfUtils.getRemoteVideoMuted = (self: any) => {
+  if (!self || !self.controls || !self.controls.video) {
+    return null;
+  }
+
+  return self.controls.video.muted;
+};
+
 /**
  * get the "remote mute" property from the self object
  * @param {Object} self
@@ -251,6 +276,10 @@ SelfUtils.getRemoteMuted = (self: any) => {
 
 SelfUtils.getLocalAudioUnmuteRequested = (self) => !!self?.controls?.audio?.requestedToUnmute;
 
+// requestedToUnmute timestamp
+SelfUtils.getLocalAudioUnmuteRequestedTimeStamp = (self) =>
+  Date.parse(self?.controls?.audio?.lastModifiedRequestedToUnmute) || 0;
+
 SelfUtils.getUnmuteAllowed = (self) => {
   if (!self || !self.controls || !self.controls.audio) {
     return null;
@@ -275,6 +304,7 @@ SelfUtils.getStatus = (status) => ({
 SelfUtils.wasMediaInactiveOrReleased = (oldSelf: any = {}, changedSelf: any) =>
   oldSelf.joinedWith &&
   oldSelf.joinedWith.state === _JOINED_ &&
+  changedSelf.joinedWith &&
   changedSelf.joinedWith.state === _LEFT_ &&
   (changedSelf.joinedWith.reason === MEETING_END_REASON.INACTIVE ||
     changedSelf.joinedWith.reason === MEETING_END_REASON.MEDIA_RELEASED);
@@ -319,6 +349,20 @@ SelfUtils.moderatorChanged = (oldSelf, changedSelf) => {
   return oldSelf.moderator !== changedSelf.moderator;
 };
 
+/**
+ * determine whether the roles of self is changed or not
+ * @param {Object} oldSelf
+ * @param {Object} changedSelf
+ * @returns {Boolean}
+ */
+SelfUtils.isRolesChanged = (oldSelf, changedSelf) => {
+  if (!changedSelf) {
+    // no new self means no change
+    return false;
+  }
+
+  return !isEqual(oldSelf?.roles, changedSelf?.roles);
+};
 /**
  * @param {Object} oldSelf
  * @param {Object} changedSelf
@@ -351,6 +395,25 @@ SelfUtils.isUserAdmitted = (oldSelf: object, changedSelf: object) => {
   return SelfUtils.isLocusUserUnadmitted(oldSelf) && SelfUtils.isLocusUserAdmitted(changedSelf);
 };
 
+SelfUtils.videoMutedByOthersChanged = (oldSelf, changedSelf) => {
+  if (!changedSelf) {
+    throw new ParameterError(
+      'New self must be defined to determine if self was video muted by others.'
+    );
+  }
+
+  if (!oldSelf || oldSelf.remoteVideoMuted === null) {
+    if (changedSelf.remoteVideoMuted) {
+      return true; // this happens when host disables "Allow start video"
+    }
+
+    // we don't want to be sending the 'meeting:self:videoUnmutedByOthers' notification on meeting join
+    return false;
+  }
+
+  return oldSelf.remoteVideoMuted !== changedSelf.remoteVideoMuted;
+};
+
 SelfUtils.mutedByOthersChanged = (oldSelf, changedSelf) => {
   if (!changedSelf) {
     throw new ParameterError('New self must be defined to determine if self was muted by others.');
@@ -379,7 +442,10 @@ SelfUtils.localAudioUnmuteRequestedByServer = (oldSelf: any = {}, changedSelf: a
     );
   }
 
-  return changedSelf.localAudioUnmuteRequested && !oldSelf.localAudioUnmuteRequested;
+  return (
+    changedSelf.localAudioUnmuteRequested &&
+    changedSelf.localAudioUnmuteRequestedTimeStamp > oldSelf.localAudioUnmuteRequestedTimeStamp
+  );
 };
 
 SelfUtils.localAudioUnmuteRequiredByServer = (oldSelf: any = {}, changedSelf: any) => {
@@ -432,4 +498,14 @@ SelfUtils.getMediaStatus = (mediaSessions = []) => {
   return mediaStatus;
 };
 
+SelfUtils.getReplacedBreakoutMoveId = (self: any, deviceId: string) => {
+  if (self && Array.isArray(self.devices)) {
+    const joinedDevice = self.devices.find((device) => deviceId === device.url);
+    if (Array.isArray(joinedDevice?.replaces)) {
+      return joinedDevice.replaces[0]?.breakoutMoveId;
+    }
+  }
+
+  return null;
+};
 export default SelfUtils;

package/src/media/index.ts

@@ -4,21 +4,28 @@
 /* globals navigator */
 
 import {
-  LocalCameraTrack,
-  LocalDisplayTrack,
-  LocalMicrophoneTrack,
   RoapMediaConnection,
   MultistreamRoapMediaConnection,
+  type MultistreamConnectionConfig,
 } from '@webex/internal-media-core';
+import {
+  LocalStream,
+  LocalCameraStream,
+  LocalDisplayStream,
+  LocalSystemAudioStream,
+  LocalMicrophoneStream,
+} from '@webex/media-helpers';
 import LoggerProxy from '../common/logs/logger-proxy';
-import {AUDIO_INPUT, VIDEO_INPUT, MEDIA_TRACK_CONSTRAINT} from '../constants';
+import {MEDIA_TRACK_CONSTRAINT} from '../constants';
 import Config from '../config';
 import StaticConfig from '../common/config';
-import MediaError from '../common/errors/media';
 import BrowserDetection from '../common/browser-detection';
+import RtcMetrics from '../rtcMetrics';
 
 const {isBrowser} = BrowserDetection();
 
+export type BundlePolicy = MultistreamConnectionConfig['bundlePolicy'];
+
 /**
  * MediaDirection
  * @typedef {Object} MediaDirection
@@ -49,29 +56,6 @@ const {isBrowser} = BrowserDetection();
  */
 const Media: any = {};
 
-/**
- * format the media array for send
- * @param {String} mediaId
- * @param {Boolean} audioMuted
- * @param {Boolean} videoMuted
- * @returns {Array} medias
- */
-Media.generateLocalMedias = (mediaId: string, audioMuted: boolean, videoMuted: boolean) => {
-  if (mediaId) {
-    return [
-      {
-        localSdp: JSON.stringify({
-          audioMuted,
-          videoMuted,
-        }),
-        mediaId,
-      },
-    ];
-  }
-
-  return [];
-};
-
 /**
  * make a browser call to get the media
  * @param {SendOptions} options
@@ -99,33 +83,60 @@ Media.getLocalMedia = (options: any, config: object) => {
   return Promise.resolve(undefined);
 };
 
+Media.getDirection = (forceSendRecv: boolean, receive: boolean, send: boolean) => {
+  if (!receive && !send) {
+    return 'inactive';
+  }
+
+  if (forceSendRecv || (receive && send)) {
+    return 'sendrecv';
+  }
+
+  if (receive) {
+    return 'recvonly';
+  }
+
+  return 'sendonly';
+};
+
 /**
  * creates a webrtc media connection with provided tracks and mediaDirection configuration
  *
  * @param {boolean} isMultistream
  * @param {string} debugId string useful for debugging (will appear in media connection logs)
+ * @param {object} webex main `webex` object.
+ * @param {string} meetingId id for the meeting using this connection
+ * @param {string} correlationId id used in requests to correlate to this session
  * @param {Object} options
- * @param {Object} [options.mediaProperties] only applicable to non-multistream connections, contains mediaDirection and local tracks:
- *                                 audioTrack, videoTrack and shareTrack
+ * @param {Object} [options.mediaProperties] contains mediaDirection and local tracks:
+ *                                 audioTrack, videoTrack, shareVideoTrack, and shareAudioTrack
  * @param {string} [options.remoteQualityLevel] LOW|MEDIUM|HIGH applicable only to non-multistream connections
  * @param {boolean} [options.enableRtx] applicable only to non-multistream connections
  * @param {boolean} [options.enableExtmap] applicable only to non-multistream connections
  * @param {Object} [options.turnServerInfo]
+ * @param {BundlePolicy} [options.bundlePolicy]
  * @returns {RoapMediaConnection | MultistreamRoapMediaConnection}
  */
 Media.createMediaConnection = (
   isMultistream: boolean,
   debugId: string,
+  webex: object,
+  meetingId: string,
+  correlationId: string,
   options: {
-    mediaProperties?: {
+    mediaProperties: {
       mediaDirection?: {
         receiveAudio: boolean;
         receiveVideo: boolean;
         receiveShare: boolean;
+        sendAudio: boolean;
+        sendVideo: boolean;
+        sendShare: boolean;
       };
-      audioTrack?: LocalMicrophoneTrack;
-      videoTrack?: LocalCameraTrack;
-      shareTrack?: LocalDisplayTrack;
+      audioStream?: LocalMicrophoneStream;
+      videoStream?: LocalCameraStream;
+      shareVideoStream?: LocalDisplayStream;
+      shareAudioStream?: LocalSystemAudioStream;
     };
     remoteQualityLevel?: 'LOW' | 'MEDIUM' | 'HIGH';
     enableRtx?: boolean;
@@ -135,13 +146,23 @@ Media.createMediaConnection = (
       username: string;
       password: string;
     };
+    bundlePolicy?: BundlePolicy;
   }
 ) => {
-  const {mediaProperties, remoteQualityLevel, enableRtx, enableExtmap, turnServerInfo} = options;
+  const {
+    mediaProperties,
+    remoteQualityLevel,
+    enableRtx,
+    enableExtmap,
+    turnServerInfo,
+    bundlePolicy,
+  } = options;
 
   const iceServers = [];
 
-  if (turnServerInfo) {
+  // we might not have any TURN server if TURN discovery failed or wasn't done or
+  // we might get an empty TURN url if we land on a video mesh node
+  if (turnServerInfo?.url) {
     iceServers.push({
       urls: turnServerInfo.url,
       username: turnServerInfo.username || '',
@@ -150,11 +171,23 @@ Media.createMediaConnection = (
   }
 
   if (isMultistream) {
+    const config: MultistreamConnectionConfig = {
+      iceServers,
+    };
+
+    if (bundlePolicy) {
+      config.bundlePolicy = bundlePolicy;
+    }
+
+    const rtcMetrics = new RtcMetrics(webex, meetingId, correlationId);
+
     return new MultistreamRoapMediaConnection(
-      {
-        iceServers,
-      },
-      debugId
+      config,
+      meetingId,
+      /* the rtc metrics objects callbacks */
+      (data) => rtcMetrics.addMetrics(data),
+      () => rtcMetrics.closeMetrics(),
+      () => rtcMetrics.sendMetricsInQueue()
     );
   }
 
@@ -162,7 +195,8 @@ Media.createMediaConnection = (
     throw new Error('mediaProperties have to be provided for non-multistream media connections');
   }
 
-  const {mediaDirection, audioTrack, videoTrack, shareTrack} = mediaProperties;
+  const {mediaDirection, audioStream, videoStream, shareVideoStream, shareAudioStream} =
+    mediaProperties;
 
   return new RoapMediaConnection(
     {
@@ -183,17 +217,23 @@ Media.createMediaConnection = (
       },
     },
     {
-      send: {
-        audio: audioTrack?.underlyingTrack,
-        video: videoTrack?.underlyingTrack,
-        screenShareVideo: shareTrack?.underlyingTrack,
-      },
-      receive: {
-        audio: mediaDirection.receiveAudio,
-        video: mediaDirection.receiveVideo,
-        screenShareVideo: mediaDirection.receiveShare,
-        remoteQualityLevel,
+      // TODO: RoapMediaConnection is not ready to use stream classes yet, so we pass the raw MediaStreamTrack for now SPARK-460530
+      localTracks: {
+        audio: audioStream?.outputStream?.getTracks()[0],
+        video: videoStream?.outputStream?.getTracks()[0],
+        screenShareVideo: shareVideoStream?.outputStream?.getTracks()[0],
+        screenShareAudio: shareAudioStream?.outputStream?.getTracks()[0], // TODO: add type for screenShareAudio in internal-media-core SPARK-446923
+      } as unknown,
+      direction: {
+        audio: Media.getDirection(true, mediaDirection.receiveAudio, mediaDirection.sendAudio),
+        video: Media.getDirection(true, mediaDirection.receiveVideo, mediaDirection.sendVideo),
+        screenShareVideo: Media.getDirection(
+          false,
+          mediaDirection.receiveShare,
+          mediaDirection.sendShare
+        ),
       },
+      remoteQualityLevel,
     },
     debugId
   );
@@ -338,52 +378,6 @@ Media.getMedia = (audio: any | boolean, video: any | boolean, config: any) => {
   });
 };
 
-/**
- * Checks if the machine has at least one audio or video device (Dont use this for screen share)
- * @param {object} [options]
- * {
- *    sendAudio: true/false,
- *    sendVideo: true/false
- * }
- * @returns {Object} {
- *    sendAudio: true/false,
- *    sendVideo: true/false
- *}
- */
-Media.getSupportedDevice = ({sendAudio, sendVideo}: {sendAudio: boolean; sendVideo: boolean}) =>
-  Promise.resolve().then(() => {
-    if (!navigator.mediaDevices || navigator.mediaDevices.enumerateDevices === undefined) {
-      return {
-        sendAudio: false,
-        sendVideo: false,
-      };
-    }
-
-    return navigator.mediaDevices.enumerateDevices().then((devices) => {
-      const supported = {
-        audio: devices.filter((device) => device.kind === AUDIO_INPUT).length > 0,
-        video: devices.filter((device) => device.kind === VIDEO_INPUT).length > 0,
-      };
-
-      return {
-        sendAudio: supported.audio && sendAudio,
-        sendVideo: supported.video && sendVideo,
-      };
-    });
-  });
-
-/**
- * proxy to browser navigator.mediaDevices.enumerateDevices()
- * @returns {Promise}
- */
-Media.getDevices = () => {
-  if (navigator && navigator.mediaDevices && navigator.mediaDevices.enumerateDevices) {
-    return navigator.mediaDevices.enumerateDevices();
-  }
-
-  return Promise.reject(new MediaError('enumerateDevices not supported.'));
-};
-
 /**
  *
  * Toggle a specific stream
@@ -394,23 +388,21 @@ Media.toggleStream = () => {};
 
 /**
  * Stop input stream
- * @param {MediaTrack} track A media stream
+ * @param {LocalStream} stream A local stream
  * @returns {null}
  */
-Media.stopTracks = (track: any) => {
-  if (!track) {
+Media.stopStream = (stream: LocalStream) => {
+  if (!stream) {
     return Promise.resolve();
   }
 
   return Promise.resolve().then(() => {
-    if (track && track.stop) {
-      try {
-        track.stop();
-      } catch (e) {
-        LoggerProxy.logger.error(
-          `Media:index#stopTracks --> Unable to stop the track with state ${track.readyState}, error: ${e}`
-        );
-      }
+    try {
+      stream.stop();
+    } catch (e) {
+      LoggerProxy.logger.error(
+        `Media:index#stopStream --> Unable to stop the stream with ready state => ${stream.readyState}, error: ${e}`
+      );
     }
   });
 };