@webex/internal-plugin-voicea 2.23.0

package/src/constants.ts

@@ -0,0 +1,31 @@
+export const EVENT_TRIGGERS = {
+  VOICEA_ANNOUNCEMENT: 'voicea:announcement',
+  CAPTION_LANGUAGE_UPDATE: 'voicea:captionLanguageUpdate',
+  SPOKEN_LANGUAGE_UPDATE: 'voicea:spokenLanguageUpdate',
+  CAPTIONS_TURNED_ON: 'voicea:captionOn',
+  TRANSCRIBING_ON: 'voicea:transcribingOn',
+  TRANSCRIBING_OFF: 'voicea:transcribingOff',
+
+  NEW_CAPTION: 'voicea:newCaption',
+  EVA_COMMAND: 'voicea:wxa',
+  HIGHLIGHT_CREATED: 'voicea:highlightCreated'
+};
+
+export const VOICEA_RELAY_TYPES = {
+  ANNOUNCEMENT: 'voicea.annc',
+  CLIENT_ANNOUNCEMENT: 'client.annc',
+  TRANSLATION_REQUEST: 'voicea.transl.req',
+  TRANSLATION_RESPONSE: 'voicea.transl.rsp',
+  TRANSCRIPTION: 'voicea.transcription'
+};
+
+export const TRANSCRIPTION_TYPE = {
+  UNKNOWN: 'unknown',
+  EVA_WAKE: 'eva_wake',
+  EVA_THANKS: 'eva_thanks',
+  EVA_CANCEL: 'eva_cancel',
+  HIGHLIGHT_CREATED: 'highlight_created',
+  TRANSCRIPT_INTERIM_RESULTS: 'transcript_interim_results',
+  TRANSCRIPT_FINAL_RESULT: 'transcript_final_result'
+
+};

package/src/index.js

@@ -0,0 +1,6 @@
+import * as WebexCore from '@webex/webex-core';
+import VoiceaChannel from '@webex/internal-plugin-voicea/src/voicea';
+
+WebexCore.registerInternalPlugin('voicea', VoiceaChannel, {});
+
+export {default} from '@webex/internal-plugin-voicea/src/voicea';

package/src/utils.ts

@@ -0,0 +1,8 @@
+// eslint-disable-next-line import/prefer-default-export
+export const millisToMinutesAndSeconds = (millis: number) => {
+  const minutes = Math.floor(millis / 60000);
+  const seconds = ((millis % 60000) / 1000).toFixed(0);
+
+  return `${minutes}:${+(seconds) < 10 ? '0' : ''}${seconds}`;
+};
+

package/src/voicea.ts

@@ -0,0 +1,306 @@
+import uuid from 'uuid';
+import LLMChannel from '@webex/internal-plugin-llm';
+import config from '@webex/webex-core/src/config';
+import Trigger from '@webex/plugin-meetings/src/common/events/trigger-proxy';
+
+import {EVENT_TRIGGERS, VOICEA_RELAY_TYPES, TRANSCRIPTION_TYPE} from './constants';
+// eslint-disable-next-line no-unused-vars
+import {AnnouncementPayload, CaptionLanguageResponse, TranscriptionResponse, IVoiceaChannel} from './voicea.types';
+import {millisToMinutesAndSeconds} from './utils';
+/**
+ * VoiceaChannel to hold single instance of LLM
+ */
+export class VoiceaChannel extends LLMChannel implements IVoiceaChannel {
+  private seqNum: number;
+
+  private hasVoiceaJoined: boolean;
+
+  private areCaptionsEnabled: boolean;
+
+  private isTranscribingEnabled:boolean;
+
+  private vmcDeviceId: string;
+
+  /**
+   * Initializes Voicea plugin
+   * @param {any} args
+   */
+  constructor(...args) {
+    super(...args);
+    this.seqNum = 1;
+    this.hasVoiceaJoined = false;
+    this.areCaptionsEnabled = false;
+    this.isTranscribingEnabled = false;
+    this.vmcDeviceId = undefined;
+
+    this.on('event:relay.event', (e) => {
+      this.seqNum = e.sequenceNumber + 1;
+      switch (e.data.relayType) {
+        case VOICEA_RELAY_TYPES.ANNOUNCEMENT:
+          this.vmcDeviceId = e.headers.from;
+          this.hasVoiceaJoined = true;
+          this.processAnnouncementMessage(e.voiceaPayload);
+          break;
+        case VOICEA_RELAY_TYPES.TRANSLATION_RESPONSE:
+          this.processCaptionLanguageResponse(e.voiceaPayload);
+          break;
+        case VOICEA_RELAY_TYPES.TRANSCRIPTION:
+          this.processTranscription(e.voiceaPayload);
+          break;
+        default:
+          break;
+      }
+    });
+  }
+
+  /**
+   * Process Transcript and send alert
+   * @param {TranscriptionResponse} voiceaPayload
+   * @returns {void}
+   */
+  private processTranscription = (voiceaPayload: TranscriptionResponse): void => {
+    switch (voiceaPayload.type) {
+      case TRANSCRIPTION_TYPE.TRANSCRIPT_INTERIM_RESULTS:
+        Trigger.trigger(this, {
+          file: 'voicea',
+          function: 'processTranscription'
+        },
+        EVENT_TRIGGERS.NEW_CAPTION, {
+          isFinal: false,
+          transcriptId: voiceaPayload.transcript_id,
+          translations: voiceaPayload.translations,
+          transcript: {
+            csis: voiceaPayload.csis,
+            text: voiceaPayload.data
+          },
+        });
+        break;
+
+      case TRANSCRIPTION_TYPE.TRANSCRIPT_FINAL_RESULT:
+        Trigger.trigger(this, {
+          file: 'voicea',
+          function: 'processTranscription'
+        },
+        EVENT_TRIGGERS.NEW_CAPTION, {
+          isFinal: true,
+          transcriptId: voiceaPayload.transcript_id,
+          translations: voiceaPayload.translations,
+          transcript: {
+            csis: voiceaPayload.csis,
+            text: voiceaPayload.data
+          },
+          timestamp: millisToMinutesAndSeconds(voiceaPayload.transcript.end_millis - voiceaPayload.transcript.start_millis)
+        });
+        break;
+
+      case TRANSCRIPTION_TYPE.HIGHLIGHT_CREATED:
+        Trigger.trigger(this, {
+          file: 'voicea',
+          function: 'processTranscription'
+        },
+        EVENT_TRIGGERS.HIGHLIGHT_CREATED, {
+          csis: voiceaPayload.highlight.csis,
+          highlightId: voiceaPayload.highlight.highlight_id,
+          text: voiceaPayload.highlight.transcript,
+          highlightLabel: voiceaPayload.highlight.highlight_label,
+          highlightSource: voiceaPayload.highlight.highlight_source,
+          timestamp: millisToMinutesAndSeconds(voiceaPayload.highlight.end_millis - voiceaPayload.highlight.start_millis)
+        });
+        break;
+
+      case TRANSCRIPTION_TYPE.EVA_THANKS:
+        Trigger.trigger(this, {
+          file: 'voicea',
+          function: 'processTranscription'
+        },
+        EVENT_TRIGGERS.EVA_COMMAND, {
+          isListening: false, text: voiceaPayload.command_response
+        });
+        break;
+
+      case TRANSCRIPTION_TYPE.EVA_WAKE:
+      case TRANSCRIPTION_TYPE.EVA_CANCEL:
+        Trigger.trigger(this, {
+          file: 'voicea',
+          function: 'processTranscription'
+        },
+        EVENT_TRIGGERS.EVA_COMMAND, {
+          isListening: voiceaPayload.type === TRANSCRIPTION_TYPE.EVA_WAKE
+        });
+        break;
+
+      default:
+        break;
+    }
+  };
+
+  /**
+   * Processes Caption Language Response
+   * @param {CaptionLanguageResponse} voiceaPayload
+   * @returns {void}
+   */
+  private processCaptionLanguageResponse = (voiceaPayload: CaptionLanguageResponse):void => {
+    if (voiceaPayload.statusCode === 200) {
+      Trigger.trigger(this, {
+        file: 'voicea',
+        function: 'processCaptionLanguageResponse'
+      },
+      EVENT_TRIGGERS.CAPTION_LANGUAGE_UPDATE, {statusCode: 200});
+    }
+    else {
+      Trigger.trigger(this, {
+        file: 'voicea',
+        function: 'processCaptionLanguageResponse'
+      },
+      EVENT_TRIGGERS.CAPTION_LANGUAGE_UPDATE, {statusCode: voiceaPayload.errorCode, errorMessage: voiceaPayload.message});
+    }
+  };
+
+  /**
+ * processes voicea announcement response and triggers event
+ * @param {Object} voiceaPayload
+ * @returns {void}
+ */
+  private processAnnouncementMessage = (voiceaPayload: AnnouncementPayload):void => {
+    const voiceaLanguageOptions = {
+      captionLanguages: voiceaPayload?.translation?.allowed_languages ?? [],
+      maxLanguages: voiceaPayload?.translation?.max_languages ?? 0,
+      spokenLanguages: voiceaPayload?.ASR?.spoken_languages ?? [],
+    };
+
+    Trigger.trigger(
+      this,
+      {
+        file: 'voicea',
+        function: 'processAnnouncementMessage'
+      },
+      EVENT_TRIGGERS.VOICEA_ANNOUNCEMENT,
+      voiceaLanguageOptions
+    );
+  };
+
+
+  /**
+   * Sends Announcement to add voicea to the meeting
+   * @returns {void}
+   */
+  private sendAnnouncement = ():void => {
+    if (this.hasVoiceaJoined) return;
+    this.socket.send({
+      id: `${this.seqNum}`,
+      type: 'publishRequest',
+      recipients: {
+        route: this.getBinding(),
+      },
+      headers: {},
+      data: {
+        clientPayload: {
+          version: 'v2',
+        },
+        eventType: 'relay.event',
+        relayType: VOICEA_RELAY_TYPES.CLIENT_ANNOUNCEMENT,
+      },
+      trackingId: `${config.trackingIdPrefix}_${uuid.v4().toString()}`
+    });
+    this.seqNum += 1;
+  };
+
+  /**
+   * Set Spoken Language for the meeting
+   * @param {string} languageCode
+   * @returns {Promise}
+   */
+  public setSpokenLanguage = (languageCode: string):Promise<void> => this.request({
+    method: 'PUT',
+    url: `${this.locusUrl}/controls/`,
+    body: {
+      languageCode
+    }
+  }).then(() => {
+    Trigger.trigger(this, {
+      file: 'voicea',
+      function: 'setSpokenLanguage'
+    },
+    EVENT_TRIGGERS.SPOKEN_LANGUAGE_UPDATE,
+    {languageCode});
+  });
+
+  /**
+   * Request Language translation
+   * @param {string} languageCode
+   * @returns {void}
+   */
+  public requestLanguage = (languageCode: string): void => {
+    this.socket.send({
+      id: `${this.seqNum}`,
+      type: 'publishRequest',
+      recipients: {
+        route: this.getBinding(),
+      },
+      headers: {
+        to: this.vmcDeviceId,
+      },
+      data: {
+        clientPayload: {
+          translationLanguage: languageCode,
+          id: uuid.v4(),
+        },
+        eventType: 'relay.event',
+        relayType: VOICEA_RELAY_TYPES.TRANSLATION_REQUEST,
+      },
+      trackingId: `${config.trackingIdPrefix}_${uuid.v4().toString()}`
+    });
+    this.seqNum += 1;
+  };
+
+  /**
+   * Turn on Captions
+   * @returns {Promise}
+   */
+  public turnOnCaptions = async (): undefined | Promise<void> => {
+    if (this.hasVoiceaJoined && this.areCaptionsEnabled) return undefined;
+
+    return this.request({
+      method: 'PUT',
+      url: `${this.locusUrl}/controls/`,
+      body: {
+        transcribe: {caption: true}
+      }
+    }).then(() => {
+      Trigger.trigger(this, {
+        file: 'voicea',
+        function: 'turnOnCaptions'
+      },
+      EVENT_TRIGGERS.CAPTIONS_TURNED_ON);
+      this.areCaptionsEnabled = true;
+      this.sendAnnouncement();
+    });
+  };
+
+  /**
+   * Toggle transcribing for highlights
+   * @param {bool} activate if true transcribing is turned on
+   * @returns {Promise}
+   */
+  public toggleTranscribing = async (activate:boolean):undefined|Promise<void> => {
+    if (this.isTranscribingEnabled === activate) return undefined;
+
+    return this.request({
+      method: 'PUT',
+      url: `${this.locusUrl}/controls/`,
+      body: {
+        transcribe: {transcribing: activate}
+      }
+    }).then(() => {
+      Trigger.trigger(this, {
+        file: 'voicea',
+        function: 'toggleTranscribing'
+      },
+      activate ? EVENT_TRIGGERS.TRANSCRIBING_ON : EVENT_TRIGGERS.TRANSCRIBING_OFF);
+      this.isTranscribingEnabled = activate;
+      if (activate && !this.areCaptionsEnabled && !this.hasVoiceaJoined) this.turnOnCaptions();
+    });
+  };
+}
+
+export default VoiceaChannel;

package/src/voicea.types.ts

@@ -0,0 +1,81 @@
+/* eslint-disable camelcase */
+/* eslint-disable no-undef */
+
+/**
+ * Type for payload
+ */
+interface AnnouncementPayload {
+  translation:{
+    max_languages: number;
+    allowed_languages: string[]
+  };
+
+  ASR: {
+    spoken_languages: string[];
+  };
+}
+
+// eslint-disable-next-line no-undef
+type TRANSCRIPTION_TYPES = 'unknown'|
+'eva_wake' |
+'eva_thanks' |
+'eva_cancel' |
+'highlight_created'|
+'transcript_interim_results'|
+'transcript_final_result';
+
+/**
+ * Class for an Transcription Object
+ */
+interface Transcription {
+  start_millis:number;
+  end_millis:number;
+  text:string;
+  transcript_language_code: string;
+  translations:{[x:string]:string};
+  csis:number[];
+}
+
+/**
+ * Highlights
+ */
+ interface Highlight {
+  highlight_id: string;
+  transcript: string;
+  highlight_label: string;
+  highlight_source: string;
+  start_millis:number;
+  end_millis:number;
+  csis:number[];
+}
+/**
+ * Type for Transcription message
+ */
+interface TranscriptionResponse {
+  type: TRANSCRIPTION_TYPES;
+  transcript_id: string;
+  translations?: {[x:string]:string};
+  transcripts?: Transcription[];
+  transcript?: Transcription;
+  highlight?: Highlight;
+  csis: number[];
+  data:string;
+  command_response: string;
+}
+/**
+ * Type for CaptionLanguageResponse
+ */
+interface CaptionLanguageResponse {
+  requestId: string;
+  statusCode: number;
+  errorCode:number;
+  message:string;
+}
+
+interface IVoiceaChannel {
+  setSpokenLanguage: (languageCode: string) => Promise<void>;
+  requestLanguage: (languageCode: string) => void;
+  turnOnCaptions: () => undefined | Promise<void>;
+  toggleTranscribing: (activate:boolean) => undefined|Promise<void>;
+}
+export {AnnouncementPayload, CaptionLanguageResponse, TranscriptionResponse, Transcription, Highlight, IVoiceaChannel};

package/test/unit/spec/utils.js

@@ -0,0 +1,12 @@
+import {assert} from '@webex/test-helper-chai';
+import {millisToMinutesAndSeconds} from '@webex/internal-plugin-voicea/src/utils';
+
+describe('Voicea utils', () => {
+  describe('#millisToMinutesAndSeconds()', () => {
+    it('returns the correct timestamp', () => {
+      const result = millisToMinutesAndSeconds(2000);
+
+      assert.equal(result, '0:02');
+    });
+  });
+});