@webex/plugin-meetings 3.0.0-stream-classes.5 → 3.0.0-test.1

package/src/multistream/remoteMediaManager.ts

@@ -2,6 +2,7 @@
 import {cloneDeep, forEach, remove} from 'lodash';
 import {EventMap} from 'typed-emitter';
 import {MediaType} from '@webex/internal-media-core';
+import {NamedMediaGroup} from '@webex/json-multistream';
 
 import LoggerProxy from '../common/logs/logger-proxy';
 import EventsScope from '../common/events/events-scope';
@@ -11,6 +12,7 @@ import {ReceiveSlot, CSI} from './receiveSlot';
 import {ReceiveSlotManager} from './receiveSlotManager';
 import {RemoteMediaGroup} from './remoteMediaGroup';
 import {MediaRequestManager} from './mediaRequestManager';
+import {NAMED_MEDIA_GROUP_TYPE_AUDIO} from '../constants';
 
 export type PaneSize = RemoteVideoResolution;
 export type LayoutId = string;
@@ -49,6 +51,7 @@ export interface Configuration {
 
     layouts: {[key: LayoutId]: VideoLayout}; // a map of all available layouts, a layout can be set via setLayout() method
   };
+  namedMediaGroup?: NamedMediaGroup;
 }
 
 /* Predefined layouts: */
@@ -173,6 +176,7 @@ export const DefaultConfiguration: Configuration = {
 export enum Event {
   // events for audio streams
   AudioCreated = 'AudioCreated',
+  InterpretationAudioCreated = 'InterpretationAudioCreated',
   ScreenShareAudioCreated = 'ScreenShareAudioCreated',
 
   // events for video streams
@@ -221,7 +225,10 @@ export class RemoteMediaManager extends EventsScope {
   private currentLayout?: VideoLayout;
 
   private slots: {
-    audio: ReceiveSlot[];
+    audio: {
+      main: ReceiveSlot[];
+      si: ReceiveSlot;
+    };
     screenShare: {
       audio: ReceiveSlot[];
       video?: ReceiveSlot;
@@ -234,7 +241,10 @@ export class RemoteMediaManager extends EventsScope {
   };
 
   private media: {
-    audio?: RemoteMediaGroup;
+    audio: {
+      main?: RemoteMediaGroup;
+      si?: RemoteMediaGroup;
+    };
     video: {
       activeSpeakerGroups: {
         [key: PaneGroupId]: RemoteMediaGroup;
@@ -277,7 +287,10 @@ export class RemoteMediaManager extends EventsScope {
     this.receiveSlotManager = receiveSlotManager;
     this.mediaRequestManagers = mediaRequestManagers;
     this.media = {
-      audio: undefined,
+      audio: {
+        main: undefined,
+        si: undefined,
+      },
       video: {
         activeSpeakerGroups: {},
         memberPanes: {},
@@ -291,7 +304,10 @@ export class RemoteMediaManager extends EventsScope {
     this.checkConfigValidity();
 
     this.slots = {
-      audio: [],
+      audio: {
+        main: [],
+        si: undefined,
+      },
       screenShare: {
         audio: [],
         video: undefined,
@@ -389,8 +405,11 @@ export class RemoteMediaManager extends EventsScope {
     });
 
     // release all audio receive slots
-    this.slots.audio.forEach((slot) => this.receiveSlotManager.releaseSlot(slot));
-    this.slots.audio.length = 0;
+    this.slots.audio.main.forEach((slot) => this.receiveSlotManager.releaseSlot(slot));
+    this.slots.audio.main.length = 0;
+    if (this.slots.audio.si) {
+      this.receiveSlotManager.releaseSlot(this.slots.audio.si);
+    }
 
     // release screen share slots
     this.slots.screenShare.audio.forEach((slot) => this.receiveSlotManager.releaseSlot(slot));
@@ -525,22 +544,54 @@ export class RemoteMediaManager extends EventsScope {
     this.mediaRequestManagers.video.commit();
   }
 
+  /**
+   * Sets which named media group need receiving
+   * @param {MediaType} mediaType of the stream
+   * @param {number} languageCode of the stream. If the languageId is 0, the named media group request will be canceled,
+   * and only receive the main audio stream.
+   * @returns {void}
+   */
+  public async setReceiveNamedMediaGroup(mediaType: MediaType, languageId: number) {
+    if (mediaType !== MediaType.AudioMain) {
+      throw new Error(`cannot set receive named media group which media type is ${mediaType}`);
+    }
+
+    const value = languageId;
+    if (value === this.config.namedMediaGroup?.value) {
+      return;
+    }
+
+    this.config.namedMediaGroup = {
+      type: NAMED_MEDIA_GROUP_TYPE_AUDIO,
+      value,
+    };
+
+    if (!this.media.audio.si) {
+      await this.createInterpretationAudioMedia(true);
+    } else {
+      this.media.audio.si.setNamedMediaGroup(this.config.namedMediaGroup, true);
+    }
+  }
+
   /**
    * Creates the audio slots
    */
   private async createAudioMedia() {
-    // create the audio receive slots
+    // create si audio request
+    await this.createInterpretationAudioMedia(false);
+
+    // create main audio receive slots
     for (let i = 0; i < this.config.audio.numOfActiveSpeakerStreams; i += 1) {
       // eslint-disable-next-line no-await-in-loop
       const slot = await this.receiveSlotManager.allocateSlot(MediaType.AudioMain);
 
-      this.slots.audio.push(slot);
+      this.slots.audio.main.push(slot);
     }
 
-    // create a remote media group
-    this.media.audio = new RemoteMediaGroup(
+    // create a remote media group for main audio
+    this.media.audio.main = new RemoteMediaGroup(
       this.mediaRequestManagers.audio,
-      this.slots.audio,
+      this.slots.audio.main,
       255,
       true
     );
@@ -548,10 +599,40 @@ export class RemoteMediaManager extends EventsScope {
     this.emit(
       {file: 'multistream/remoteMediaManager', function: 'createAudioMedia'},
       Event.AudioCreated,
-      this.media.audio
+      this.media.audio.main
     );
   }
 
+  /**
+   * Creates the audio slots for named media
+   */
+  private async createInterpretationAudioMedia(commitRequest: boolean) {
+    // create slot for interpretation language audio
+    if (
+      this.config.namedMediaGroup?.type === NAMED_MEDIA_GROUP_TYPE_AUDIO &&
+      this.config.namedMediaGroup?.value
+    ) {
+      this.slots.audio.si = await this.receiveSlotManager.allocateSlot(MediaType.AudioMain);
+
+      // create a remote media group for si audio
+      this.media.audio.si = new RemoteMediaGroup(
+        this.mediaRequestManagers.audio,
+        [this.slots.audio.si],
+        255,
+        commitRequest,
+        {
+          namedMediaGroup: this.config.namedMediaGroup,
+        }
+      );
+
+      this.emit(
+        {file: 'multistream/remoteMediaManager', function: 'createInterpretationAudioMedia'},
+        Event.InterpretationAudioCreated,
+        this.media.audio.si
+      );
+    }
+  }
+
   /**
    * Creates receive slots required for receiving screen share audio and video
    */
@@ -748,7 +829,7 @@ export class RemoteMediaManager extends EventsScope {
   /** logs main audio slots */
   private logMainAudioReceiveSlots() {
     LoggerProxy.logger.log(
-      `RemoteMediaManager#logMainAudioReceiveSlots --> MAIN AUDIO receive slots: ${this.slots.audio
+      `RemoteMediaManager#logMainAudioReceiveSlots --> MAIN AUDIO receive slots: ${this.slots.audio.main
         .map((slot) => slot.logString)
         .join(', ')}`
     );
@@ -924,8 +1005,13 @@ export class RemoteMediaManager extends EventsScope {
   }) {
     const {audio, video, screenShareAudio, screenShareVideo, commit} = options;
 
-    if (audio && this.media.audio) {
-      this.media.audio.stop(commit);
+    if (audio) {
+      if (this.media.audio.main) {
+        this.media.audio.main.stop(commit);
+      }
+      if (this.media.audio.si) {
+        this.media.audio.si.stop(commit);
+      }
     }
     if (video) {
       Object.values(this.media.video.activeSpeakerGroups).forEach((remoteMediaGroup) => {

package/src/multistream/sendSlotManager.ts

@@ -5,6 +5,8 @@ import {
   MultistreamRoapMediaConnection,
 } from '@webex/internal-media-core';
 
+import {NamedMediaGroup} from '@webex/json-multistream';
+
 export default class SendSlotManager {
   private readonly slots: Map<MediaType, SendSlot> = new Map();
   private readonly LoggerProxy: any;
@@ -55,6 +57,33 @@ export default class SendSlotManager {
     return slot;
   }
 
+  /**
+   * Allow users to specify 'namedMediaGroups' to indicate which named media group its audio should be sent to.
+   * @param {MediaType} mediaType MediaType of the sendSlot to which the audio stream needs to be send to the media group
+   * @param {[]}namedMediaGroups - Allow users to specify 'namedMediaGroups'.If the value of 'namedMediaGroups' is zero,
+   * named media group will be canceled and the audio stream will be sent to the floor.
+   * @returns {void}
+   */
+  public setNamedMediaGroups(mediaType: MediaType, namedMediaGroups: NamedMediaGroup[]) {
+    if (mediaType !== MediaType.AudioMain) {
+      throw new Error(
+        `sendSlotManager cannot set named media group which media type is ${mediaType}`
+      );
+    }
+
+    const slot = this.slots.get(mediaType);
+
+    if (!slot) {
+      throw new Error(`Slot for ${mediaType} does not exist`);
+    }
+
+    slot.setNamedMediaGroups(namedMediaGroups);
+
+    this.LoggerProxy.logger.info(
+      `SendSlotsManager->setNamedMediaGroups#set named media group ${namedMediaGroups}`
+    );
+  }
+
   /**
    * This method publishes the given stream to the sendSlot for the given mediaType
    * @param {MediaType} mediaType MediaType of the sendSlot to which a stream needs to be published (AUDIO_MAIN/VIDEO_MAIN/AUDIO_SLIDES/VIDEO_SLIDES)

package/src/reachability/clusterReachability.ts

@@ -0,0 +1,320 @@
+import {Defer} from '@webex/common';
+
+import LoggerProxy from '../common/logs/logger-proxy';
+import {ClusterNode} from './request';
+import {convertStunUrlToTurn} from './util';
+
+import {ICE_GATHERING_STATE, CONNECTION_STATE} from '../constants';
+
+const DEFAULT_TIMEOUT = 3000;
+const VIDEO_MESH_TIMEOUT = 1000;
+
+// result for a specific transport protocol (like udp or tcp)
+export type TransportResult = {
+  result: 'reachable' | 'unreachable' | 'untested';
+  latencyInMilliseconds?: number; // amount of time it took to get the first ICE candidate
+  clientMediaIPs?: string[];
+};
+
+// reachability result for a specific media cluster
+export type ClusterReachabilityResult = {
+  udp: TransportResult;
+  tcp: TransportResult;
+  xtls: TransportResult;
+};
+
+/**
+ * A class that handles reachability checks for a single cluster.
+ */
+export class ClusterReachability {
+  private numUdpUrls: number;
+  private numTcpUrls: number;
+  private result: ClusterReachabilityResult;
+  private pc?: RTCPeerConnection;
+  private defer: Defer; // this defer is resolved once reachability checks for this cluster are completed
+  private startTimestamp: number;
+  public readonly isVideoMesh: boolean;
+  public readonly name;
+
+  /**
+   * Constructor for ClusterReachability
+   * @param {string} name cluster name
+   * @param {ClusterNode} clusterInfo information about the media cluster
+   */
+  constructor(name: string, clusterInfo: ClusterNode) {
+    this.name = name;
+    this.isVideoMesh = clusterInfo.isVideoMesh;
+    this.numUdpUrls = clusterInfo.udp.length;
+    this.numTcpUrls = clusterInfo.tcp.length;
+
+    this.pc = this.createPeerConnection(clusterInfo);
+
+    this.defer = new Defer();
+    this.result = {
+      udp: {
+        result: 'untested',
+      },
+      tcp: {
+        result: 'untested',
+      },
+      xtls: {
+        result: 'untested',
+      },
+    };
+  }
+
+  /**
+   * Gets total elapsed time, can be called only after start() is called
+   * @returns {Number} Milliseconds
+   */
+  private getElapsedTime() {
+    return Math.round(performance.now() - this.startTimestamp);
+  }
+
+  /**
+   * Generate peerConnection config settings
+   * @param {ClusterNode} cluster
+   * @returns {RTCConfiguration} peerConnectionConfig
+   */
+  private buildPeerConnectionConfig(cluster: ClusterNode): RTCConfiguration {
+    const udpIceServers = cluster.udp.map((url) => ({
+      username: '',
+      credential: '',
+      urls: [url],
+    }));
+
+    // STUN servers are contacted only using UDP, so in order to test TCP reachability
+    // we pretend that Linus is a TURN server, because we can explicitly say "transport=tcp" in TURN urls.
+    // We then check for relay candidates to know if TURN-TCP worked (see registerIceCandidateListener()).
+    const tcpIceServers = cluster.tcp.map((urlString: string) => {
+      return {
+        username: 'webexturnreachuser',
+        credential: 'webexturnreachpwd',
+        urls: [convertStunUrlToTurn(urlString, 'tcp')],
+      };
+    });
+
+    return {
+      iceServers: [...udpIceServers, ...tcpIceServers],
+      iceCandidatePoolSize: 0,
+      iceTransportPolicy: 'all',
+    };
+  }
+
+  /**
+   * Creates an RTCPeerConnection
+   * @param {ClusterNode} clusterInfo information about the media cluster
+   * @returns {RTCPeerConnection} peerConnection
+   */
+  private createPeerConnection(clusterInfo: ClusterNode) {
+    try {
+      const config = this.buildPeerConnectionConfig(clusterInfo);
+
+      const peerConnection = new RTCPeerConnection(config);
+
+      return peerConnection;
+    } catch (peerConnectionError) {
+      LoggerProxy.logger.warn(
+        `Reachability:index#createPeerConnection --> Error creating peerConnection:`,
+        peerConnectionError
+      );
+
+      return undefined;
+    }
+  }
+
+  /**
+   * @returns {ClusterReachabilityResult} reachability result for this cluster
+   */
+  getResult() {
+    return this.result;
+  }
+
+  /**
+   * Closes the peerConnection
+   *
+   * @returns {void}
+   */
+  private closePeerConnection() {
+    if (this.pc) {
+      this.pc.onicecandidate = null;
+      this.pc.onicegatheringstatechange = null;
+      this.pc.close();
+    }
+  }
+
+  /**
+   * Resolves the defer, indicating that reachability checks for this cluster are completed
+   *
+   * @returns {void}
+   */
+  private finishReachabilityCheck() {
+    this.defer.resolve();
+  }
+
+  /**
+   * Adds public IP (client media IPs)
+   * @param {string} protocol
+   * @param {string} publicIP
+   * @returns {void}
+   */
+  private addPublicIP(protocol: 'udp' | 'tcp', publicIP?: string | null) {
+    const result = this.result[protocol];
+
+    if (publicIP) {
+      if (result.clientMediaIPs) {
+        if (!result.clientMediaIPs.includes(publicIP)) {
+          result.clientMediaIPs.push(publicIP);
+        }
+      } else {
+        result.clientMediaIPs = [publicIP];
+      }
+    }
+  }
+
+  /**
+   * Registers a listener for the iceGatheringStateChange event
+   *
+   * @returns {void}
+   */
+  private registerIceGatheringStateChangeListener() {
+    this.pc.onicegatheringstatechange = () => {
+      const {COMPLETE} = ICE_GATHERING_STATE;
+
+      if (this.pc.iceConnectionState === COMPLETE) {
+        this.closePeerConnection();
+        this.finishReachabilityCheck();
+      }
+    };
+  }
+
+  /**
+   * Checks if we have the results for all the protocols (UDP and TCP)
+   *
+   * @returns {boolean} true if we have all results, false otherwise
+   */
+  private haveWeGotAllResults(): boolean {
+    return ['udp', 'tcp'].every(
+      (protocol) =>
+        this.result[protocol].result === 'reachable' || this.result[protocol].result === 'untested'
+    );
+  }
+
+  /**
+   * Stores the latency in the result for the given protocol and marks it as reachable
+   *
+   * @param {string} protocol
+   * @param {number} latency
+   * @returns {void}
+   */
+  private storeLatencyResult(protocol: 'udp' | 'tcp', latency: number) {
+    const result = this.result[protocol];
+
+    if (result.latencyInMilliseconds === undefined) {
+      LoggerProxy.logger.log(
+        // @ts-ignore
+        `Reachability:index#storeLatencyResult --> Successfully reached ${this.name} over ${protocol}: ${latency}ms`
+      );
+      result.latencyInMilliseconds = latency;
+      result.result = 'reachable';
+    }
+  }
+
+  /**
+   * Registers a listener for the icecandidate event
+   *
+   * @returns {void}
+   */
+  private registerIceCandidateListener() {
+    this.pc.onicecandidate = (e) => {
+      const CANDIDATE_TYPES = {
+        SERVER_REFLEXIVE: 'srflx',
+        RELAY: 'relay',
+      };
+
+      if (e.candidate) {
+        if (e.candidate.type === CANDIDATE_TYPES.SERVER_REFLEXIVE) {
+          this.storeLatencyResult('udp', this.getElapsedTime());
+          this.addPublicIP('udp', e.candidate.address);
+        }
+
+        if (e.candidate.type === CANDIDATE_TYPES.RELAY) {
+          this.storeLatencyResult('tcp', this.getElapsedTime());
+          // we don't add public IP for TCP, because in the case of relay candidates
+          // e.candidate.address is the TURN server address, not the client's public IP
+        }
+
+        if (this.haveWeGotAllResults()) {
+          this.closePeerConnection();
+          this.finishReachabilityCheck();
+        }
+      }
+    };
+  }
+
+  /**
+   * Starts the process of doing UDP and TCP reachability checks on the media cluster.
+   * XTLS reachability checking is not supported.
+   *
+   * @returns {Promise}
+   */
+  async start(): Promise<ClusterReachabilityResult> {
+    if (!this.pc) {
+      LoggerProxy.logger.warn(
+        `Reachability:ClusterReachability#start --> Error: peerConnection is undefined`
+      );
+
+      return this.result;
+    }
+
+    // Initialize this.result as saying that nothing is reachable.
+    // It will get updated as we go along and successfully gather ICE candidates.
+    this.result.udp = {
+      result: this.numUdpUrls > 0 ? 'unreachable' : 'untested',
+    };
+    this.result.tcp = {
+      result: this.numTcpUrls > 0 ? 'unreachable' : 'untested',
+    };
+
+    try {
+      const offer = await this.pc.createOffer({offerToReceiveAudio: true});
+
+      this.startTimestamp = performance.now();
+
+      // not awaiting the next call on purpose, because we're not sending the offer anywhere and there won't be any answer
+      // we just need to make this call to trigger the ICE gathering process
+      this.pc.setLocalDescription(offer);
+
+      await this.gatherIceCandidates();
+    } catch (error) {
+      LoggerProxy.logger.warn(`Reachability:ClusterReachability#start --> Error: `, error);
+    }
+
+    return this.result;
+  }
+
+  /**
+   * Starts the process of gathering ICE candidates
+   *
+   * @returns {Promise} promise that's resolved once reachability checks for this cluster are completed or timeout is reached
+   */
+  private gatherIceCandidates() {
+    const timeout = this.isVideoMesh ? VIDEO_MESH_TIMEOUT : DEFAULT_TIMEOUT;
+
+    this.registerIceGatheringStateChangeListener();
+    this.registerIceCandidateListener();
+
+    // Set maximum timeout
+    setTimeout(() => {
+      const {CLOSED} = CONNECTION_STATE;
+
+      // Close any open peerConnections
+      if (this.pc.connectionState !== CLOSED) {
+        this.closePeerConnection();
+        this.finishReachabilityCheck();
+      }
+    }, timeout);
+
+    return this.defer.promise;
+  }
+}