agora-appbuilder-core 3.0.13 → 3.0.28-beta-9

package/template/src/subComponents/caption/proto/ptoto.js

@@ -0,0 +1,91 @@
+/*eslint-disable block-scoped-var, id-length, no-control-regex, no-magic-numbers, no-prototype-builtins, no-redeclare, no-shadow, no-var, sort-vars*/
+"use strict";
+
+var $protobuf = require("protobufjs/light");
+
+var $root = ($protobuf.roots["default"] || ($protobuf.roots["default"] = new $protobuf.Root()))
+    .addJSON({
+        agora: {
+            nested: {
+                audio2text: {
+                    options: {
+                        java_package: "io.agora.rtc.audio2text",
+                        java_outer_classname: "Audio2TextProtobuffer"
+                    },
+                    nested: {
+                        Text: {
+                            fields: {
+                                vendor: {
+                                    type: "int32",
+                                    id: 1
+                                },
+                                version: {
+                                    type: "int32",
+                                    id: 2
+                                },
+                                seqnum: {
+                                    type: "int32",
+                                    id: 3
+                                },
+                                uid: {
+                                    type: "uint32",
+                                    id: 4
+                                },
+                                flag: {
+                                    type: "int32",
+                                    id: 5
+                                },
+                                time: {
+                                    type: "int64",
+                                    id: 6
+                                },
+                                lang: {
+                                    type: "int32",
+                                    id: 7
+                                },
+                                starttime: {
+                                    type: "int32",
+                                    id: 8
+                                },
+                                offtime: {
+                                    type: "int32",
+                                    id: 9
+                                },
+                                words: {
+                                    rule: "repeated",
+                                    type: "Word",
+                                    id: 10
+                                }
+                            }
+                        },
+                        Word: {
+                            fields: {
+                                text: {
+                                    type: "string",
+                                    id: 1
+                                },
+                                startMs: {
+                                    type: "int32",
+                                    id: 2
+                                },
+                                durationMs: {
+                                    type: "int32",
+                                    id: 3
+                                },
+                                isFinal: {
+                                    type: "bool",
+                                    id: 4
+                                },
+                                confidence: {
+                                    type: "double",
+                                    id: 5
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    });
+
+module.exports = $root;

package/template/src/subComponents/caption/proto/test.proto

@@ -0,0 +1,23 @@
+syntax = "proto3";
+package agora.audio2text;
+option java_package = "io.agora.rtc.audio2text";
+option java_outer_classname = "Audio2TextProtobuffer";
+message Text {
+ int32 vendor = 1;
+ int32 version = 2;
+ int32 seqnum = 3;
+ uint32 uid = 4;
+ int32 flag = 5;
+ int64 time = 6;
+ int32 lang = 7;
+ int32 starttime = 8;
+ int32 offtime = 9;
+ repeated Word words = 10;
+}
+message Word {
+ string text = 1;
+ int32 start_ms = 2;
+ int32 duration_ms = 3;
+ bool is_final = 4;
+ double confidence = 5;
+}

package/template/src/subComponents/caption/useCaption.tsx

@@ -0,0 +1,123 @@
+import {createHook} from 'customization-implementation';
+import React from 'react';
+import {LanguageType} from './utils';
+
+export type TranscriptItem = {
+  uid: string;
+  time: number;
+  text: string;
+};
+
+type CaptionObj = {
+  [key: string]: {
+    text: string;
+    lastUpdated: number;
+  };
+};
+
+export const CaptionContext = React.createContext<{
+  // for caption btn state
+  isCaptionON: boolean;
+  setIsCaptionON: React.Dispatch<React.SetStateAction<boolean>>;
+
+  // for error state
+  isSTTError: boolean;
+  setIsSTTError: React.Dispatch<React.SetStateAction<boolean>>;
+
+  // to check if stt is active in the call
+  isSTTActive: boolean;
+  setIsSTTActive: React.Dispatch<React.SetStateAction<boolean>>;
+
+  // holds the language selection for stt
+  language: LanguageType[];
+  setLanguage: React.Dispatch<React.SetStateAction<LanguageType[]>>;
+
+  // holds meeting transcript
+  meetingTranscript: TranscriptItem[];
+  setMeetingTranscript: React.Dispatch<React.SetStateAction<TranscriptItem[]>>;
+
+  // holds status of stt language change process
+  isLangChangeInProgress: boolean;
+  setIsLangChangeInProgress: React.Dispatch<React.SetStateAction<boolean>>;
+
+  // holds live captions
+  captionObj: CaptionObj;
+  setCaptionObj: React.Dispatch<React.SetStateAction<CaptionObj>>;
+
+  // holds status of StreamMessageCallback listener added (caption/transcript)
+  isSTTListenerAdded: boolean;
+  setIsSTTListenerAdded: React.Dispatch<React.SetStateAction<boolean>>;
+
+  activeSpeakerRef: React.MutableRefObject<string>;
+  prevSpeakerRef: React.MutableRefObject<string>;
+}>({
+  isCaptionON: false,
+  setIsCaptionON: () => {},
+  isSTTError: false,
+  setIsSTTError: () => {},
+  isSTTActive: false,
+  setIsSTTActive: () => {},
+  language: ['en-US'],
+  setLanguage: () => {},
+  meetingTranscript: [],
+  setMeetingTranscript: () => {},
+  isLangChangeInProgress: false,
+  setIsLangChangeInProgress: () => {},
+  captionObj: {},
+  setCaptionObj: () => {},
+  isSTTListenerAdded: false,
+  setIsSTTListenerAdded: () => {},
+  activeSpeakerRef: {current: ''},
+  prevSpeakerRef: {current: ''},
+});
+
+const CaptionProvider = ({children}) => {
+  const [isSTTError, setIsSTTError] = React.useState<boolean>(false);
+  const [isCaptionON, setIsCaptionON] = React.useState<boolean>(false);
+  const [isSTTActive, setIsSTTActive] = React.useState<boolean>(false);
+  const [language, setLanguage] = React.useState<[LanguageType]>(['']);
+  const [isLangChangeInProgress, setIsLangChangeInProgress] =
+    React.useState<boolean>(false);
+  const [meetingTranscript, setMeetingTranscript] = React.useState<
+    TranscriptItem[]
+  >([]);
+  const [captionObj, setCaptionObj] = React.useState<CaptionObj>({});
+  const [isSTTListenerAdded, setIsSTTListenerAdded] =
+    React.useState<boolean>(false);
+  const [activeSpeakerUID, setActiveSpeakerUID] = React.useState<string>('');
+  const [prevActiveSpeakerUID, setPrevActiveSpeakerUID] =
+    React.useState<string>('');
+
+  const activeSpeakerRef = React.useRef('');
+  const prevSpeakerRef = React.useRef('');
+
+  return (
+    <CaptionContext.Provider
+      value={{
+        isCaptionON,
+        setIsCaptionON,
+        isSTTError,
+        setIsSTTError,
+        isSTTActive,
+        setIsSTTActive,
+        language,
+        setLanguage,
+        meetingTranscript,
+        setMeetingTranscript,
+        isLangChangeInProgress,
+        setIsLangChangeInProgress,
+        captionObj,
+        setCaptionObj,
+        isSTTListenerAdded,
+        setIsSTTListenerAdded,
+        activeSpeakerRef,
+        prevSpeakerRef,
+      }}>
+      {children}
+    </CaptionContext.Provider>
+  );
+};
+
+const useCaption = createHook(CaptionContext);
+
+export {CaptionProvider, useCaption};

package/template/src/subComponents/caption/useCaptionWidth.ts

@@ -0,0 +1,27 @@
+import React from 'react';
+import {useWindowDimensions} from 'react-native';
+import {useCaption} from './useCaption';
+import {useSidePanel, SidePanelType} from 'customization-api';
+import {CAPTION_CONTAINER_HEIGHT} from '../../../src/components/CommonStyles';
+
+interface CaptionWidthReturnType {
+  isCaptionNotFullWidth: boolean;
+  transcriptHeight: string;
+}
+
+const useCaptionWidth = (): CaptionWidthReturnType => {
+  const {isCaptionON} = useCaption();
+  const {width: windowWidth, height: windowHeight} = useWindowDimensions();
+  const {sidePanel} = useSidePanel();
+
+  const isCaptionNotFullWidth =
+    isCaptionON && sidePanel !== SidePanelType.None && windowWidth > 1200;
+
+  const transcriptHeight = isCaptionNotFullWidth
+    ? `calc(100% + ${CAPTION_CONTAINER_HEIGHT}px + 4px)` //10px is gap b/w <Video> & <Caption/>
+    : '';
+
+  return {isCaptionNotFullWidth, transcriptHeight};
+};
+
+export default useCaptionWidth;

package/template/src/subComponents/caption/useSTTAPI.tsx

@@ -0,0 +1,275 @@
+import React, {useContext} from 'react';
+import StorageContext from '../../components/StorageContext';
+import {useRoomInfo} from '../../components/room-info/useRoomInfo';
+import {useCaption} from './useCaption';
+import events, {PersistanceLevel} from '../../rtm-events-api';
+import {EventNames} from '../../rtm-events';
+import {getLanguageLabel, LanguageType} from './utils';
+import useGetName from '../../utils/useGetName';
+import {capitalizeFirstLetter} from '../../utils/common';
+import {PropsContext, useLocalUid} from '../../../agora-rn-uikit';
+import {logger, LogSource} from '../../logger/AppBuilderLogger';
+import getUniqueID from '../../utils/getUniqueID';
+
+interface IuseSTTAPI {
+  start: (lang: LanguageType[]) => Promise<{message: string} | null>;
+  stop: () => Promise<void>;
+  restart: (lang: LanguageType[]) => Promise<void>;
+  isAuthorizedSTTUser: () => boolean;
+  isAuthorizedTranscriptUser: () => boolean;
+}
+
+const useSTTAPI = (): IuseSTTAPI => {
+  const {store} = React.useContext(StorageContext);
+  const {
+    data: {roomId, isHost},
+  } = useRoomInfo();
+  const {
+    language,
+    isSTTActive,
+    setIsSTTActive,
+    setIsLangChangeInProgress,
+    setLanguage,
+    setMeetingTranscript,
+    setIsSTTError,
+  } = useCaption();
+
+  const currentLangRef = React.useRef<LanguageType[]>([]);
+  const STT_API_URL = `${$config.BACKEND_ENDPOINT}/v1/stt`;
+  const username = useGetName();
+  const localUid = useLocalUid();
+  const {rtcProps} = useContext(PropsContext);
+
+  React.useEffect(() => {
+    currentLangRef.current = language;
+  }, [language]);
+
+  const apiCall = async (method: string, lang: LanguageType[] = []) => {
+    const requestId = getUniqueID();
+    const startReqTs = Date.now();
+    logger.log(
+      LogSource.NetworkRest,
+      'stt',
+      `Trying to ${method} stt for lang ${lang}`,
+      {
+        method,
+        lang,
+        requestId,
+        startReqTs,
+      },
+    );
+    try {
+      const response = await fetch(`${STT_API_URL}/${method}`, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          authorization: store.token ? `Bearer ${store.token}` : '',
+          'X-Request-Id': requestId,
+          'X-Session-Id': logger.getSessionId(),
+        },
+        body: JSON.stringify({
+          passphrase: roomId?.host || '',
+          lang: lang,
+          dataStream_uid: 111111, // bot ID
+          encryption_mode: $config.ENCRYPTION_ENABLED
+            ? rtcProps.encryption.mode
+            : null,
+        }),
+      });
+      const res = await response.json();
+      const endReqTs = Date.now();
+      const latency = endReqTs - startReqTs;
+      logger.log(
+        LogSource.NetworkRest,
+        'stt',
+        `STT API Success - Called ${method} on stt with lang ${lang}`,
+        {
+          responseData: res,
+          requestId,
+          startReqTs,
+          endReqTs,
+          latency,
+        },
+      );
+      return res;
+    } catch (error) {
+      const endReqTs = Date.now();
+      const latency = endReqTs - startReqTs;
+      logger.error(
+        LogSource.NetworkRest,
+        'stt',
+        `STT API Failure - Called ${method} on stt with lang ${lang}`,
+        error,
+        {
+          requestId,
+          startReqTs,
+          endReqTs,
+          latency,
+        },
+      );
+    }
+  };
+
+  const startWithDelay = (lang: LanguageType[]): Promise<string> =>
+    new Promise(resolve => {
+      setTimeout(async () => {
+        const res = await start(lang);
+        resolve(res);
+      }, 1000); // Delay of 1 seconds (1000 milliseconds) to allow existing stt service to fully stop
+    });
+
+  const start = async (lang: LanguageType[]) => {
+    try {
+      setIsLangChangeInProgress(true);
+      const res = await apiCall('start', lang);
+      // null means stt startred successfully
+      const isSTTAlreadyActive =
+        res?.error?.message
+          ?.toLowerCase()
+          .indexOf('current status is started') !== -1 || false;
+
+      if (res?.error?.message) {
+        setIsSTTError(true);
+        logger.error(
+          LogSource.NetworkRest,
+          'stt',
+          `start stt for lang ${lang} failed`,
+          res?.error,
+        );
+      } else {
+        logger.log(
+          LogSource.NetworkRest,
+          'stt',
+          `start stt for lang ${lang} succesfull`,
+          res,
+        );
+        setIsSTTError(false);
+      }
+      if (res === null || isSTTAlreadyActive) {
+        // once STT is active in the channel , notify others so that they dont' trigger start again
+        events.send(
+          EventNames.STT_ACTIVE,
+          JSON.stringify({active: true}),
+          PersistanceLevel.Sender,
+        );
+        setIsSTTActive(true);
+        logger.debug(
+          LogSource.NetworkRest,
+          'stt',
+          `stt lang update from: ${language} to ${lang}`,
+        );
+        // inform about the language set for stt
+        events.send(
+          EventNames.STT_LANGUAGE,
+          JSON.stringify({
+            username: capitalizeFirstLetter(username),
+            uid: localUid,
+            prevLang: language,
+            newLang: lang,
+          }),
+          PersistanceLevel.Sender,
+        );
+        setLanguage(lang);
+
+        // updaing transcript for self
+        const actionText =
+          language.indexOf('') !== -1
+            ? `has set the spoken language to  "${getLanguageLabel(lang)}" `
+            : `changed the spoken language from "${getLanguageLabel(
+                language,
+              )}" to "${getLanguageLabel(lang)}" `;
+        //const msg = `${capitalizeFirstLetter(username)} ${actionText} `;
+        setMeetingTranscript(prev => {
+          return [
+            ...prev,
+            {
+              name: 'langUpdate',
+              time: new Date().getTime(),
+              uid: `langUpdate-${localUid}`,
+              text: actionText,
+            },
+          ];
+        });
+      }
+      return res;
+    } catch (errorMsg) {
+      logger.error(
+        LogSource.NetworkRest,
+        'stt',
+        'There was error in start stt',
+        errorMsg,
+      );
+      throw errorMsg;
+    } finally {
+      setIsLangChangeInProgress(false);
+    }
+  };
+
+  const stop = async () => {
+    try {
+      const res = await apiCall('stop');
+      // once STT is non-active in the channel , notify others so that they dont' trigger start again
+      // events.send(
+      //   EventNames.STT_ACTIVE,
+      //   JSON.stringify({active: false}),
+      //   PersistanceLevel.Session,
+      // );
+      setIsSTTActive(false);
+      if (res?.error?.message) {
+        setIsSTTError(true);
+      } else {
+        logger.log(LogSource.NetworkRest, 'stt', 'stop stt succesfull', res);
+        setIsSTTError(false);
+      }
+      return res;
+    } catch (error) {
+      logger.error(
+        LogSource.NetworkRest,
+        'stt',
+        'There was error in stop stt',
+        error,
+      );
+      throw error;
+    }
+  };
+  const restart = async (lang: LanguageType[]) => {
+    try {
+      setIsLangChangeInProgress(true);
+      await stop();
+      await startWithDelay(lang);
+      return Promise.resolve();
+    } catch (error) {
+      logger.error(
+        LogSource.NetworkRest,
+        'stt',
+        'There was error error in re-starting STT',
+        error,
+      );
+      return Promise.reject(error);
+    } finally {
+      setIsLangChangeInProgress(false);
+    }
+  };
+
+  // attendee can view option if any host has started STT
+  const isAuthorizedSTTUser = () =>
+    $config.ENABLE_STT &&
+    $config.ENABLE_CAPTION &&
+    (isHost || (!isHost && isSTTActive));
+
+  const isAuthorizedTranscriptUser = () =>
+    $config.ENABLE_STT &&
+    $config.ENABLE_CAPTION &&
+    $config.ENABLE_MEETING_TRANSCRIPT &&
+    (isHost || (!isHost && isSTTActive));
+
+  return {
+    start,
+    stop,
+    restart,
+    isAuthorizedSTTUser,
+    isAuthorizedTranscriptUser,
+  };
+};
+
+export default useSTTAPI;