@webex/plugin-meetings 2.60.1-next.2 → 2.60.1-next.4

package/src/meeting/index.ts

@@ -33,6 +33,12 @@ import {
   RemoteStream,
 } from '@webex/media-helpers';
 
+import {
+  EVENT_TRIGGERS as VOICEAEVENTS,
+  TURN_ON_CAPTION_STATUS,
+} from '@webex/internal-plugin-voicea';
+import {processNewCaptions, processHighlightCreated} from './voicea-meeting';
+
 import {
   MeetingNotActiveError,
   UserInLobbyError,
@@ -40,6 +46,7 @@ import {
   UserNotJoinedError,
   AddMediaFailed,
 } from '../common/errors/webex-errors';
+
 import {StatsAnalyzer, EVENTS as StatsAnalyzerEvents} from '../statsAnalyzer';
 import NetworkQualityMonitor from '../networkQualityMonitor';
 import LoggerProxy from '../common/logs/logger-proxy';
@@ -59,7 +66,6 @@ import MeetingsUtil from '../meetings/util';
 import RecordingUtil from '../recording-controller/util';
 import ControlsOptionsUtil from '../controls-options-manager/util';
 import MediaUtil from '../media/util';
-import Transcription from '../transcription';
 import {Reactions, SkinTones} from '../reactions/reactions';
 import PasswordError from '../common/errors/password-error';
 import CaptchaError from '../common/errors/captcha-error';
@@ -97,7 +103,6 @@ import {
   SHARE_STATUS,
   SHARE_STOPPED_REASON,
   VIDEO,
-  HTTP_VERBS,
   SELF_ROLES,
   INTERPRETATION,
   SELF_POLICY,
@@ -105,6 +110,7 @@ import {
   MEETING_PERMISSION_TOKEN_REFRESH_REASON,
   ROAP_OFFER_ANSWER_EXCHANGE_TIMEOUT,
   RECONNECTION,
+  LANGUAGE_ENGLISH,
 } from '../constants';
 import BEHAVIORAL_METRICS from '../metrics/constants';
 import ParameterError from '../common/errors/parameter';
@@ -158,6 +164,37 @@ const logRequest = (request: any, {logText = ''}) => {
     });
 };
 
+export type CaptionData = {
+  id: string;
+  isFinal: boolean;
+  translations: Array<string>;
+  text: string;
+  currentCaptionLanguage: string;
+  timestamp: string;
+  speaker: string;
+};
+
+export type Transcription = {
+  languageOptions: {
+    captionLanguages?: string; // list of supported caption languages from backend
+    maxLanguages?: number;
+    spokenLanguages?: Array<string>; // list of supported spoken languages from backend
+    currentCaptionLanguage?: string; // current caption language - default is english
+    requestedCaptionLanguage?: string; // requested caption language
+    currentSpokenLanguage?: string; // current spoken language - default is english
+  };
+  status: string;
+  isListening: boolean;
+  commandText: string;
+  captions: Array<CaptionData>;
+  highlights: Array<any>;
+  showCaptionBox: boolean;
+  transcribingRequestStatus: string;
+  isCaptioning: boolean;
+  speakerProxy: Map<string, any>;
+  interimCaptions: Map<string, CaptionData>;
+};
+
 export type LocalStreams = {
   microphone?: LocalMicrophoneStream;
   camera?: LocalCameraStream;
@@ -549,7 +586,6 @@ export default class Meeting extends StatelessWebexPlugin {
   screenShareFloorState: ScreenShareFloorStatus;
   statsAnalyzer: StatsAnalyzer;
   transcription: Transcription;
-  receiveTranscription: boolean;
   updateMediaConnections: (mediaConnections: any[]) => void;
   userDisplayHints: any;
   endCallInitJoinReq: any;
@@ -581,6 +617,49 @@ export default class Meeting extends StatelessWebexPlugin {
   allowMediaInLobby: boolean;
   turnDiscoverySkippedReason: string;
   turnServerUsed: boolean;
+  areVoiceaEventsSetup = false;
+  voiceaListenerCallbacks: object = {
+    [VOICEAEVENTS.VOICEA_ANNOUNCEMENT]: (payload: Transcription['languageOptions']) => {
+      this.transcription.languageOptions = payload;
+      Trigger.trigger(
+        this,
+        {
+          file: 'meeting/index',
+          function: 'setUpVoiceaListeners',
+        },
+        EVENT_TRIGGERS.MEETING_STARTED_RECEIVING_TRANSCRIPTION,
+        payload
+      );
+    },
+    [VOICEAEVENTS.CAPTIONS_TURNED_ON]: () => {
+      this.transcription.status = TURN_ON_CAPTION_STATUS.ENABLED;
+    },
+    [VOICEAEVENTS.EVA_COMMAND]: (payload) => {
+      const {data} = payload;
+
+      this.transcription.isListening = !!data.isListening;
+      this.transcription.commandText = data.text ?? '';
+    },
+    [VOICEAEVENTS.NEW_CAPTION]: (data) => {
+      processNewCaptions({data, meeting: this});
+      Trigger.trigger(
+        this,
+        {
+          file: 'meeting/index',
+          function: 'setUpVoiceaListeners',
+        },
+        EVENT_TRIGGERS.MEETING_CAPTION_RECEIVED,
+        {
+          captions: this.transcription.captions,
+          interimCaptions: this.transcription.interimCaptions,
+        }
+      );
+    },
+    [VOICEAEVENTS.HIGHLIGHT_CREATED]: (data) => {
+      processHighlightCreated({data, meeting: this});
+    },
+  };
+
   private retriedWithTurnServer: boolean;
   private sendSlotManager: SendSlotManager = new SendSlotManager(LoggerProxy);
   private deferSDPAnswer?: Defer; // used for waiting for a response
@@ -1179,7 +1258,18 @@ export default class Meeting extends StatelessWebexPlugin {
      * @private
      * @memberof Meeting
      */
-    this.transcription = undefined;
+    this.transcription = {
+      captions: [],
+      highlights: [],
+      isListening: false,
+      commandText: '',
+      languageOptions: {},
+      showCaptionBox: false,
+      transcribingRequestStatus: 'INACTIVE',
+      isCaptioning: false,
+      interimCaptions: {} as Map<string, CaptionData>,
+      speakerProxy: {} as Map<string, any>,
+    } as Transcription;
 
     /**
      * Password status. If it's PASSWORD_STATUS.REQUIRED then verifyPassword() needs to be called
@@ -1840,6 +1930,7 @@ export default class Meeting extends StatelessWebexPlugin {
    * @memberof Meeting
    */
   private setUpInterpretationListener() {
+    // TODO: check if its getting used or not
     this.simultaneousInterpretation.on(INTERPRETATION.EVENTS.SUPPORT_LANGUAGES_UPDATE, () => {
       Trigger.trigger(
         this,
@@ -1850,7 +1941,7 @@ export default class Meeting extends StatelessWebexPlugin {
         EVENT_TRIGGERS.MEETING_INTERPRETATION_SUPPORT_LANGUAGES_UPDATE
       );
     });
-
+    // TODO: check if its getting used or not
     this.simultaneousInterpretation.on(
       INTERPRETATION.EVENTS.HANDOFF_REQUESTS_ARRIVED,
       (payload) => {
@@ -1867,6 +1958,44 @@ export default class Meeting extends StatelessWebexPlugin {
     );
   }
 
+  /**
+   * Set up the listeners for captions
+   * @returns {undefined}
+   * @private
+   * @memberof Meeting
+   */
+  private setUpVoiceaListeners() {
+    // @ts-ignore
+    this.webex.internal.voicea.on(
+      VOICEAEVENTS.VOICEA_ANNOUNCEMENT,
+      this.voiceaListenerCallbacks[VOICEAEVENTS.VOICEA_ANNOUNCEMENT]
+    );
+
+    // @ts-ignore
+    this.webex.internal.voicea.on(
+      VOICEAEVENTS.CAPTIONS_TURNED_ON,
+      this.voiceaListenerCallbacks[VOICEAEVENTS.CAPTIONS_TURNED_ON]
+    );
+
+    // @ts-ignore
+    this.webex.internal.voicea.on(
+      VOICEAEVENTS.EVA_COMMAND,
+      this.voiceaListenerCallbacks[VOICEAEVENTS.EVA_COMMAND]
+    );
+
+    // @ts-ignore
+    this.webex.internal.voicea.on(
+      VOICEAEVENTS.NEW_CAPTION,
+      this.voiceaListenerCallbacks[VOICEAEVENTS.NEW_CAPTION]
+    );
+
+    // @ts-ignore
+    this.webex.internal.voicea.on(
+      VOICEAEVENTS.HIGHLIGHT_CREATED,
+      this.voiceaListenerCallbacks[VOICEAEVENTS.HIGHLIGHT_CREATED]
+    );
+  }
+
   /**
    * Set up the locus info listener for meetings disconnected due to inactivity
    * @returns {undefined}
@@ -2186,19 +2315,22 @@ export default class Meeting extends StatelessWebexPlugin {
     this.locusInfo.on(
       LOCUSINFO.EVENTS.CONTROLS_MEETING_TRANSCRIBE_UPDATED,
       ({caption, transcribing}) => {
-        // @ts-ignore - config coming from registerPlugin
-        if (transcribing && !this.transcription && this.config.receiveTranscription) {
-          this.startTranscription();
-        } else if (!transcribing && this.transcription) {
-          Trigger.trigger(
-            this,
-            {
-              file: 'meeting/index',
-              function: 'setupLocusControlsListener',
-            },
-            EVENT_TRIGGERS.MEETING_STOPPED_RECEIVING_TRANSCRIPTION,
-            {caption, transcribing}
-          );
+        // user need to be joined to start the llm and receive transcription
+        if (this.isJoined()) {
+          // @ts-ignore - config coming from registerPlugin
+          if (transcribing && !this.transcription) {
+            this.startTranscription();
+          } else if (!transcribing && this.transcription) {
+            Trigger.trigger(
+              this,
+              {
+                file: 'meeting/index',
+                function: 'setupLocusControlsListener',
+              },
+              EVENT_TRIGGERS.MEETING_STOPPED_RECEIVING_TRANSCRIPTION,
+              {caption, transcribing}
+            );
+          }
         }
       }
     );
@@ -2852,15 +2984,6 @@ export default class Meeting extends StatelessWebexPlugin {
     });
     this.locusInfo.on(LOCUSINFO.EVENTS.SELF_ADMITTED_GUEST, async (payload) => {
       this.stopKeepAlive();
-      // @ts-ignore
-      if (!this.transcription && (this.config.receiveTranscription || this.receiveTranscription)) {
-        if (this.isTranscriptionSupported()) {
-          await this.startTranscription();
-          LoggerProxy.logger.info(
-            'Meeting:index#setUpLocusInfoSelfListener --> enabled to receive transcription for guest user!'
-          );
-        }
-      }
 
       if (payload) {
         Trigger.trigger(
@@ -4419,7 +4542,7 @@ export default class Meeting extends StatelessWebexPlugin {
     }
 
     LoggerProxy.logger.error(
-      'Meeting:index#isTranscriptionSupported --> Webex Assistant is not supported'
+      'Meeting:index#isTranscriptionSupported --> Webex Assistant is not enabled/supported'
     );
 
     return false;
@@ -4440,109 +4563,136 @@ export default class Meeting extends StatelessWebexPlugin {
   }
 
   /**
-   * Monitor the Low-Latency Mercury (LLM) web socket connection on `onError` and `onClose` states
-   * @private
-   * @returns {void}
+   * sets Caption language for the meeting
+   * @param {string} language
+   * @returns {Promise}
    */
-  private monitorTranscriptionSocketConnection() {
-    this.transcription.onCloseSocket((event) => {
-      LoggerProxy.logger.info(
-        `Meeting:index#onCloseSocket -->
-        unable to continue receiving transcription;
-        low-latency mercury web socket connection is closed now.
-        ${event}`
-      );
-
-      this.triggerStopReceivingTranscriptionEvent();
-    });
+  public setCaptionLanguage(language: string) {
+    return new Promise((resolve, reject) => {
+      if (!this.isTranscriptionSupported()) {
+        LoggerProxy.logger.error(
+          'Meeting:index#setCaptionLanguage --> Webex Assistant is not enabled/supported'
+        );
 
-    this.transcription.onErrorSocket((event) => {
-      LoggerProxy.logger.error(
-        `Meeting:index#onErrorSocket -->
-         unable to continue receiving transcription;
-         low-latency mercury web socket connection error had occured.
-        ${event}`
-      );
+        reject(new Error('Webex Assistant is not enabled/supported'));
+      }
 
-      this.triggerStopReceivingTranscriptionEvent();
+      try {
+        const voiceaListenerCaptionUpdate = (payload) => {
+          // @ts-ignore
+          this.webex.internal.voicea.off(
+            VOICEAEVENTS.CAPTION_LANGUAGE_UPDATE,
+            voiceaListenerCaptionUpdate
+          );
+          const {statusCode} = payload;
+
+          if (statusCode === 200) {
+            const currentCaptionLanguage =
+              this.transcription.languageOptions.requestedCaptionLanguage ?? LANGUAGE_ENGLISH;
+            this.transcription.languageOptions = {
+              ...this.transcription.languageOptions,
+              currentCaptionLanguage,
+            };
+            resolve(currentCaptionLanguage);
+          } else {
+            reject(payload);
+          }
+        };
+        // @ts-ignore
+        this.webex.internal.voicea.on(
+          VOICEAEVENTS.CAPTION_LANGUAGE_UPDATE,
+          voiceaListenerCaptionUpdate
+        );
+        // @ts-ignore
+        this.webex.internal.voicea.requestLanguage(language);
+      } catch (error) {
+        LoggerProxy.logger.error(`Meeting:index#setCaptionLanguage --> ${error}`);
 
-      Metrics.sendBehavioralMetric(BEHAVIORAL_METRICS.RECEIVE_TRANSCRIPTION_FAILURE, {
-        correlation_id: this.correlationId,
-        reason: 'unexpected error: transcription LLM web socket connection error had occured.',
-        event,
-      });
+        reject(error);
+      }
     });
   }
 
   /**
-   * Request for a WebSocket Url, open and monitor the WebSocket connection
-   * @private
-   * @returns {Promise<void>} a promise to open the WebSocket connection
+   * sets Spoken language for the meeting
+   * @param {string} language
+   * @returns {Promise}
    */
-  private async startTranscription() {
-    LoggerProxy.logger.info(
-      `Meeting:index#startTranscription -->
-      Attempting to generate a web socket url.`
-    );
+  public setSpokenLanguage(language: string) {
+    return new Promise((resolve, reject) => {
+      if (!this.isTranscriptionSupported()) {
+        LoggerProxy.logger.error(
+          'Meeting:index#setCaptionLanguage --> Webex Assistant is not enabled/supported'
+        );
 
-    try {
-      const {datachannelUrl} = this.locusInfo.info;
-      // @ts-ignore - fix type
-      const {
-        body: {webSocketUrl},
-        // @ts-ignore
-      } = await this.request({
-        method: HTTP_VERBS.POST,
-        uri: datachannelUrl,
-        body: {deviceUrl: this.deviceUrl},
-      });
+        reject(new Error('Webex Assistant is not enabled/supported'));
+      }
 
-      LoggerProxy.logger.info(
-        `Meeting:index#startTranscription -->
-        Generated web socket url succesfully.`
-      );
+      try {
+        const voiceaListenerLanguageUpdate = (payload) => {
+          // @ts-ignore
+          this.webex.internal.voicea.off(
+            VOICEAEVENTS.SPOKEN_LANGUAGE_UPDATE,
+            voiceaListenerLanguageUpdate
+          );
+          const {languageCode} = payload;
 
-      this.transcription = new Transcription(
-        webSocketUrl,
-        // @ts-ignore - fix type
-        this.webex.sessionId,
-        this.members
-      );
+          if (languageCode) {
+            this.transcription.languageOptions = {
+              ...this.transcription.languageOptions,
+              currentSpokenLanguage: languageCode,
+            };
+            resolve(languageCode);
+          } else {
+            reject(payload);
+          }
+        };
+        // @ts-ignore
+        this.webex.internal.voicea.on(
+          VOICEAEVENTS.SPOKEN_LANGUAGE_UPDATE,
+          voiceaListenerLanguageUpdate
+        );
+        // @ts-ignore
+        this.webex.internal.voicea.setSpokenLanguage(language);
+      } catch (error) {
+        LoggerProxy.logger.error(`Meeting:index#setSpokenLanguage --> ${error}`);
 
+        reject(error);
+      }
+    });
+  }
+
+  /**
+   * This method will enable the transcription for the current meeting if the meeting has enabled/supports Webex Assistant
+   * @param {Object} options object with spokenlanguage setting
+   * @public
+   * @returns {Promise<void>} a promise to open the WebSocket connection
+   */
+  public async startTranscription(options?: {spokenLanguage?: string}) {
+    if (this.isJoined()) {
       LoggerProxy.logger.info(
-        `Meeting:index#startTranscription -->
-        opened LLM web socket connection successfully.`
+        'Meeting:index#startTranscription --> Attempting to enable transcription!'
       );
 
-      if (!this.inMeetingActions.isClosedCaptionActive) {
-        LoggerProxy.logger.error(
-          `Meeting:index#receiveTranscription --> Transcription cannot be started until a licensed user enables it`
-        );
+      try {
+        if (!this.areVoiceaEventsSetup) {
+          this.setUpVoiceaListeners();
+          this.areVoiceaEventsSetup = true;
+        }
+        // @ts-ignore
+        await this.webex.internal.voicea.toggleTranscribing(true, options?.spokenLanguage);
+      } catch (error) {
+        LoggerProxy.logger.error(`Meeting:index#startTranscription --> ${error}`);
+        Metrics.sendBehavioralMetric(BEHAVIORAL_METRICS.RECEIVE_TRANSCRIPTION_FAILURE, {
+          correlation_id: this.correlationId,
+          reason: error.message,
+          stack: error.stack,
+        });
       }
-
-      // retrieve and pass the payload
-      this.transcription.subscribe((payload) => {
-        Trigger.trigger(
-          this,
-          {
-            file: 'meeting/index',
-            function: 'join',
-          },
-          EVENT_TRIGGERS.MEETING_STARTED_RECEIVING_TRANSCRIPTION,
-          payload
-        );
-      });
-
-      this.monitorTranscriptionSocketConnection();
-      // @ts-ignore - fix type
-      this.transcription.connect(this.webex.credentials.supertoken.access_token);
-    } catch (error) {
-      LoggerProxy.logger.error(`Meeting:index#startTranscription --> ${error}`);
-      Metrics.sendBehavioralMetric(BEHAVIORAL_METRICS.RECEIVE_TRANSCRIPTION_FAILURE, {
-        correlation_id: this.correlationId,
-        reason: error.message,
-        stack: error.stack,
-      });
+    } else {
+      LoggerProxy.logger.error(
+        `Meeting:index#startTranscription --> meeting joined : ${this.isJoined()}`
+      );
     }
   }
 
@@ -4585,13 +4735,42 @@ export default class Meeting extends StatelessWebexPlugin {
   };
 
   /**
-   * stop recieving Transcription by closing
-   * the web socket connection properly
+   * This method stops receiving transcription for the current meeting
    * @returns {void}
    */
-  stopReceivingTranscription() {
+  stopTranscription() {
     if (this.transcription) {
-      this.transcription.closeSocket();
+      // @ts-ignore
+      this.webex.internal.voicea.off(
+        VOICEAEVENTS.VOICEA_ANNOUNCEMENT,
+        this.voiceaListenerCallbacks[VOICEAEVENTS.VOICEA_ANNOUNCEMENT]
+      );
+
+      // @ts-ignore
+      this.webex.internal.voicea.off(
+        VOICEAEVENTS.CAPTIONS_TURNED_ON,
+        this.voiceaListenerCallbacks[VOICEAEVENTS.CAPTIONS_TURNED_ON]
+      );
+
+      // @ts-ignore
+      this.webex.internal.voicea.off(
+        VOICEAEVENTS.EVA_COMMAND,
+        this.voiceaListenerCallbacks[VOICEAEVENTS.EVA_COMMAND]
+      );
+
+      // @ts-ignore
+      this.webex.internal.voicea.off(
+        VOICEAEVENTS.NEW_CAPTION,
+        this.voiceaListenerCallbacks[VOICEAEVENTS.NEW_CAPTION]
+      );
+
+      // @ts-ignore
+      this.webex.internal.voicea.off(
+        VOICEAEVENTS.HIGHLIGHT_CREATED,
+        this.voiceaListenerCallbacks[VOICEAEVENTS.HIGHLIGHT_CREATED]
+      );
+      this.areVoiceaEventsSetup = false;
+      this.triggerStopReceivingTranscriptionEvent();
     }
   }
 
@@ -4604,7 +4783,7 @@ export default class Meeting extends StatelessWebexPlugin {
   private triggerStopReceivingTranscriptionEvent() {
     LoggerProxy.logger.info(`
       Meeting:index#stopReceivingTranscription -->
-      closed transcription LLM web socket connection successfully.`);
+      closed voicea event listeners successfully.`);
 
     Trigger.trigger(
       this,
@@ -4792,7 +4971,6 @@ export default class Meeting extends StatelessWebexPlugin {
         joinSuccess(join);
 
         this.deferJoin = undefined;
-        this.receiveTranscription = !!options.receiveTranscription;
 
         return join;
       })
@@ -4847,39 +5025,6 @@ export default class Meeting extends StatelessWebexPlugin {
           });
         }
 
-        return join;
-      })
-      .then((join) => {
-        if (isBrowser) {
-          // @ts-ignore - config coming from registerPlugin
-          if (this.config.receiveTranscription || options.receiveTranscription) {
-            if (this.isTranscriptionSupported()) {
-              LoggerProxy.logger.info(
-                'Meeting:index#join --> Attempting to enabled to receive transcription!'
-              );
-              this.startTranscription().catch((error) => {
-                LoggerProxy.logger.error(
-                  'Meeting:index#join --> Receive Transcription Failed',
-                  error
-                );
-
-                Metrics.sendBehavioralMetric(
-                  BEHAVIORAL_METRICS.RECEIVE_TRANSCRIPTION_AFTER_JOIN_FAILURE,
-                  {
-                    correlation_id: this.correlationId,
-                    reason: error?.message,
-                    stack: error.stack,
-                  }
-                );
-              });
-            }
-          }
-        } else {
-          LoggerProxy.logger.error(
-            'Meeting:index#join --> Receving transcription is not supported on this platform'
-          );
-        }
-
         return join;
       });
   }
@@ -7618,8 +7763,7 @@ export default class Meeting extends StatelessWebexPlugin {
     this.queuedMediaUpdates = [];
 
     if (this.transcription) {
-      this.transcription.closeSocket();
-      this.triggerStopReceivingTranscriptionEvent();
+      this.stopTranscription();
       this.transcription = undefined;
     }
   };