agora-appbuilder-core 4.0.0 → 4.0.1

package/template/src/subComponents/caption/TranscriptIcon.tsx

@@ -0,0 +1,123 @@
+import {StyleSheet, View} from 'react-native';
+import React from 'react';
+import {SidePanelType, useSidePanel} from 'customization-api';
+import IconButton, {IconButtonProps} from '../../atoms/IconButton';
+import LanguageSelectorPopup from './LanguageSelectorPopup';
+import {useCaption} from './useCaption';
+import useSTTAPI from './useSTTAPI';
+import {useString} from '../../utils/useString';
+import {toolbarItemTranscriptText} from '../../language/default-labels/videoCallScreenLabels';
+
+interface TranscriptIconProps {
+  plainIconHoverEffect?: boolean;
+  showToolTip?: boolean;
+  showLabel?: boolean;
+  disabled?: boolean;
+  isOnActionSheet?: boolean;
+  isMobileView?: boolean;
+}
+
+const TranscriptIcon = (props: TranscriptIconProps) => {
+  const {setSidePanel, sidePanel} = useSidePanel();
+  const {
+    showToolTip = false,
+    showLabel = $config.ICON_TEXT,
+    disabled = false,
+    isOnActionSheet = false,
+    isMobileView = false,
+  } = props;
+
+  const {start, restart, isAuthorizedSTTUser} = useSTTAPI();
+  const {isSTTActive, language: prevLang, isSTTError} = useCaption();
+  const isDisabled = !isAuthorizedSTTUser();
+  const [isLanguagePopupOpen, setLanguagePopup] =
+    React.useState<boolean>(false);
+  const isFirstTimePopupOpen = React.useRef(false);
+
+  const isTranscriptON = sidePanel === SidePanelType.Transcript;
+  const onPress = () => {
+    if (isSTTError) {
+      setSidePanel(
+        isTranscriptON ? SidePanelType.None : SidePanelType.Transcript,
+      );
+      return;
+    }
+    if (isSTTActive) {
+      setSidePanel(
+        isTranscriptON ? SidePanelType.None : SidePanelType.Transcript,
+      );
+    } else {
+      isFirstTimePopupOpen.current = true;
+      setLanguagePopup(true);
+    }
+  };
+
+  const label = useString<boolean>(toolbarItemTranscriptText);
+  const iconButtonProps: IconButtonProps = {
+    onPress,
+    iconProps: {
+      name: 'transcript',
+      iconBackgroundColor: isTranscriptON
+        ? $config.PRIMARY_ACTION_BRAND_COLOR
+        : '',
+      tintColor: isDisabled
+        ? $config.SEMANTIC_NEUTRAL
+        : isTranscriptON
+        ? $config.PRIMARY_ACTION_TEXT_COLOR
+        : $config.SECONDARY_ACTION_COLOR,
+    },
+    disabled: isDisabled,
+    btnTextProps: {
+      text: showLabel
+        ? isOnActionSheet
+          ? label(isTranscriptON)?.replace(' ', '\n')
+          : label(isTranscriptON)
+        : '',
+      textColor: $config.FONT_COLOR,
+      numberOfLines: 2,
+    },
+  };
+  iconButtonProps.isOnActionSheet = isOnActionSheet;
+  if (!isOnActionSheet) {
+    iconButtonProps.toolTipMessage = label(isTranscriptON);
+  }
+
+  const onConfirm = async (langChanged, language) => {
+    setLanguagePopup(false);
+
+    isFirstTimePopupOpen.current = false;
+    const method = isTranscriptON ? 'stop' : 'start';
+    if (method === 'stop') return; // not closing the stt service as it will stop for whole channel
+    if (method === 'start' && isSTTActive === true) return; // not triggering the start service if STT Service already started by anyone else in the channel
+    if (!isTranscriptON) {
+      setSidePanel(SidePanelType.Transcript);
+    } else {
+      setSidePanel(SidePanelType.None);
+    }
+    try {
+      const res = await start(language);
+      if (res?.message.includes('STARTED')) {
+        // channel is already started now restart
+        await restart(language);
+      }
+    } catch (error) {
+      console.log('eror in starting stt', error);
+    }
+  };
+
+  return (
+    <View>
+      <IconButton {...iconButtonProps} />
+      <LanguageSelectorPopup
+        modalVisible={isLanguagePopupOpen}
+        setModalVisible={setLanguagePopup}
+        onConfirm={onConfirm}
+        isFirstTimePopupOpen={isFirstTimePopupOpen.current}
+      />
+    </View>
+  );
+};
+
+export default TranscriptIcon;
+
+const styles = StyleSheet.create({});

package/template/src/subComponents/caption/TranscriptText.tsx

@@ -0,0 +1,98 @@
+import {Text, View, StyleSheet} from 'react-native';
+import React from 'react';
+
+import ThemeConfig from '../../../src/theme';
+import hexadecimalTransparency from '../../../src/utils/hexadecimalTransparency';
+import {formatTime} from './utils';
+
+interface TranscriptTextProps {
+  user: string;
+  time: number;
+  value: string;
+  searchQuery?: string;
+}
+
+export const TranscriptText = ({
+  user,
+  time,
+  value,
+  searchQuery = '',
+}: TranscriptTextProps) => {
+  const t = time ? formatTime(Number(time)) : '';
+  const regex = searchQuery ? new RegExp(`(${searchQuery})`, 'gi') : ' ';
+  const parts = value.split(regex);
+
+  return (
+    <View key={user} style={styles.transcriptTextContainer}>
+      <View style={styles.nameTimeContainer}>
+        <Text
+          numberOfLines={1}
+          textBreakStrategy="simple"
+          ellipsizeMode="tail"
+          style={[styles.transcriptName]}>
+          {user}
+        </Text>
+        <Text style={styles.timestamp}>{t}</Text>
+      </View>
+
+      <View>
+        <Text style={[styles.transciptText]}>
+          {/* If substring matches search query then highlight it */}
+          {parts.map((part, index) =>
+            part.toLowerCase() === searchQuery.toLowerCase() &&
+            searchQuery !== '' ? (
+              <Text key={index} style={styles.highlightedText}>
+                {searchQuery ? part : part + ' '}
+              </Text>
+            ) : (
+              <Text key={index}>{searchQuery ? part : part + ' '}</Text>
+            ),
+          )}
+        </Text>
+      </View>
+    </View>
+  );
+};
+
+const styles = StyleSheet.create({
+  nameTimeContainer: {
+    flexDirection: 'row',
+    alignItems: 'baseline',
+    marginBottom: 4,
+    justifyContent: 'flex-start',
+  },
+
+  transcriptTextContainer: {
+    alignItems: 'flex-start',
+    marginBottom: 20,
+  },
+
+  transciptText: {
+    fontSize: 16,
+    lineHeight: 22,
+    fontFamily: ThemeConfig.FontFamily.sansPro,
+    fontWeight: '400',
+    color: $config.FONT_COLOR + hexadecimalTransparency['70%'],
+  },
+
+  transcriptName: {
+    fontSize: 16,
+    lineHeight: 19,
+    fontWeight: '600',
+    fontFamily: ThemeConfig.FontFamily.sansPro,
+    color: $config.FONT_COLOR,
+    maxWidth: 160,
+  },
+
+  timestamp: {
+    fontSize: 12,
+    lineHeight: 16,
+    marginLeft: 8,
+    color: $config.FONT_COLOR + hexadecimalTransparency['40%'],
+    fontFamily: ThemeConfig.FontFamily.sansPro,
+    fontWeight: '400',
+  },
+  highlightedText: {
+    backgroundColor: $config.SEMANTIC_NEUTRAL,
+  },
+});

package/template/src/subComponents/caption/index.ts

@@ -0,0 +1,3 @@
+import CaptionButton from './CaptionButton';
+
+export {CaptionButton};

package/template/src/subComponents/caption/proto/ptoto.js

@@ -0,0 +1,91 @@
+/*eslint-disable block-scoped-var, id-length, no-control-regex, no-magic-numbers, no-prototype-builtins, no-redeclare, no-shadow, no-var, sort-vars*/
+"use strict";
+
+var $protobuf = require("protobufjs/light");
+
+var $root = ($protobuf.roots["default"] || ($protobuf.roots["default"] = new $protobuf.Root()))
+    .addJSON({
+        agora: {
+            nested: {
+                audio2text: {
+                    options: {
+                        java_package: "io.agora.rtc.audio2text",
+                        java_outer_classname: "Audio2TextProtobuffer"
+                    },
+                    nested: {
+                        Text: {
+                            fields: {
+                                vendor: {
+                                    type: "int32",
+                                    id: 1
+                                },
+                                version: {
+                                    type: "int32",
+                                    id: 2
+                                },
+                                seqnum: {
+                                    type: "int32",
+                                    id: 3
+                                },
+                                uid: {
+                                    type: "uint32",
+                                    id: 4
+                                },
+                                flag: {
+                                    type: "int32",
+                                    id: 5
+                                },
+                                time: {
+                                    type: "int64",
+                                    id: 6
+                                },
+                                lang: {
+                                    type: "int32",
+                                    id: 7
+                                },
+                                starttime: {
+                                    type: "int32",
+                                    id: 8
+                                },
+                                offtime: {
+                                    type: "int32",
+                                    id: 9
+                                },
+                                words: {
+                                    rule: "repeated",
+                                    type: "Word",
+                                    id: 10
+                                }
+                            }
+                        },
+                        Word: {
+                            fields: {
+                                text: {
+                                    type: "string",
+                                    id: 1
+                                },
+                                startMs: {
+                                    type: "int32",
+                                    id: 2
+                                },
+                                durationMs: {
+                                    type: "int32",
+                                    id: 3
+                                },
+                                isFinal: {
+                                    type: "bool",
+                                    id: 4
+                                },
+                                confidence: {
+                                    type: "double",
+                                    id: 5
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    });
+
+module.exports = $root;

package/template/src/subComponents/caption/proto/test.proto

@@ -0,0 +1,23 @@
+syntax = "proto3";
+package agora.audio2text;
+option java_package = "io.agora.rtc.audio2text";
+option java_outer_classname = "Audio2TextProtobuffer";
+message Text {
+ int32 vendor = 1;
+ int32 version = 2;
+ int32 seqnum = 3;
+ uint32 uid = 4;
+ int32 flag = 5;
+ int64 time = 6;
+ int32 lang = 7;
+ int32 starttime = 8;
+ int32 offtime = 9;
+ repeated Word words = 10;
+}
+message Word {
+ string text = 1;
+ int32 start_ms = 2;
+ int32 duration_ms = 3;
+ bool is_final = 4;
+ double confidence = 5;
+}

package/template/src/subComponents/caption/useCaption.tsx

@@ -0,0 +1,123 @@
+import {createHook} from 'customization-implementation';
+import React from 'react';
+import {LanguageType} from './utils';
+
+export type TranscriptItem = {
+  uid: string;
+  time: number;
+  text: string;
+};
+
+type CaptionObj = {
+  [key: string]: {
+    text: string;
+    lastUpdated: number;
+  };
+};
+
+export const CaptionContext = React.createContext<{
+  // for caption btn state
+  isCaptionON: boolean;
+  setIsCaptionON: React.Dispatch<React.SetStateAction<boolean>>;
+
+  // for error state
+  isSTTError: boolean;
+  setIsSTTError: React.Dispatch<React.SetStateAction<boolean>>;
+
+  // to check if stt is active in the call
+  isSTTActive: boolean;
+  setIsSTTActive: React.Dispatch<React.SetStateAction<boolean>>;
+
+  // holds the language selection for stt
+  language: LanguageType[];
+  setLanguage: React.Dispatch<React.SetStateAction<LanguageType[]>>;
+
+  // holds meeting transcript
+  meetingTranscript: TranscriptItem[];
+  setMeetingTranscript: React.Dispatch<React.SetStateAction<TranscriptItem[]>>;
+
+  // holds status of stt language change process
+  isLangChangeInProgress: boolean;
+  setIsLangChangeInProgress: React.Dispatch<React.SetStateAction<boolean>>;
+
+  // holds live captions
+  captionObj: CaptionObj;
+  setCaptionObj: React.Dispatch<React.SetStateAction<CaptionObj>>;
+
+  // holds status of StreamMessageCallback listener added (caption/transcript)
+  isSTTListenerAdded: boolean;
+  setIsSTTListenerAdded: React.Dispatch<React.SetStateAction<boolean>>;
+
+  activeSpeakerRef: React.MutableRefObject<string>;
+  prevSpeakerRef: React.MutableRefObject<string>;
+}>({
+  isCaptionON: false,
+  setIsCaptionON: () => {},
+  isSTTError: false,
+  setIsSTTError: () => {},
+  isSTTActive: false,
+  setIsSTTActive: () => {},
+  language: ['en-US'],
+  setLanguage: () => {},
+  meetingTranscript: [],
+  setMeetingTranscript: () => {},
+  isLangChangeInProgress: false,
+  setIsLangChangeInProgress: () => {},
+  captionObj: {},
+  setCaptionObj: () => {},
+  isSTTListenerAdded: false,
+  setIsSTTListenerAdded: () => {},
+  activeSpeakerRef: {current: ''},
+  prevSpeakerRef: {current: ''},
+});
+
+const CaptionProvider = ({children}) => {
+  const [isSTTError, setIsSTTError] = React.useState<boolean>(false);
+  const [isCaptionON, setIsCaptionON] = React.useState<boolean>(false);
+  const [isSTTActive, setIsSTTActive] = React.useState<boolean>(false);
+  const [language, setLanguage] = React.useState<[LanguageType]>(['']);
+  const [isLangChangeInProgress, setIsLangChangeInProgress] =
+    React.useState<boolean>(false);
+  const [meetingTranscript, setMeetingTranscript] = React.useState<
+    TranscriptItem[]
+  >([]);
+  const [captionObj, setCaptionObj] = React.useState<CaptionObj>({});
+  const [isSTTListenerAdded, setIsSTTListenerAdded] =
+    React.useState<boolean>(false);
+  const [activeSpeakerUID, setActiveSpeakerUID] = React.useState<string>('');
+  const [prevActiveSpeakerUID, setPrevActiveSpeakerUID] =
+    React.useState<string>('');
+
+  const activeSpeakerRef = React.useRef('');
+  const prevSpeakerRef = React.useRef('');
+
+  return (
+    <CaptionContext.Provider
+      value={{
+        isCaptionON,
+        setIsCaptionON,
+        isSTTError,
+        setIsSTTError,
+        isSTTActive,
+        setIsSTTActive,
+        language,
+        setLanguage,
+        meetingTranscript,
+        setMeetingTranscript,
+        isLangChangeInProgress,
+        setIsLangChangeInProgress,
+        captionObj,
+        setCaptionObj,
+        isSTTListenerAdded,
+        setIsSTTListenerAdded,
+        activeSpeakerRef,
+        prevSpeakerRef,
+      }}>
+      {children}
+    </CaptionContext.Provider>
+  );
+};
+
+const useCaption = createHook(CaptionContext);
+
+export {CaptionProvider, useCaption};

package/template/src/subComponents/caption/useCaptionWidth.ts

@@ -0,0 +1,27 @@
+import React from 'react';
+import {useWindowDimensions} from 'react-native';
+import {useCaption} from './useCaption';
+import {useSidePanel, SidePanelType} from 'customization-api';
+import {CAPTION_CONTAINER_HEIGHT} from '../../../src/components/CommonStyles';
+
+interface CaptionWidthReturnType {
+  isCaptionNotFullWidth: boolean;
+  transcriptHeight: string;
+}
+
+const useCaptionWidth = (): CaptionWidthReturnType => {
+  const {isCaptionON} = useCaption();
+  const {width: windowWidth, height: windowHeight} = useWindowDimensions();
+  const {sidePanel} = useSidePanel();
+
+  const isCaptionNotFullWidth =
+    isCaptionON && sidePanel !== SidePanelType.None && windowWidth > 1200;
+
+  const transcriptHeight = isCaptionNotFullWidth
+    ? `calc(100% + ${CAPTION_CONTAINER_HEIGHT}px + 4px)` //10px is gap b/w <Video> & <Caption/>
+    : '';
+
+  return {isCaptionNotFullWidth, transcriptHeight};
+};
+
+export default useCaptionWidth;

package/template/src/subComponents/caption/useSTTAPI.tsx

@@ -0,0 +1,179 @@
+import React, {useContext} from 'react';
+import StorageContext from '../../components/StorageContext';
+import {useRoomInfo} from '../../components/room-info/useRoomInfo';
+import {useCaption} from './useCaption';
+import events, {PersistanceLevel} from '../../rtm-events-api';
+import {EventNames} from '../../rtm-events';
+import {getLanguageLabel, LanguageType} from './utils';
+import useGetName from '../../utils/useGetName';
+import {capitalizeFirstLetter} from '../../utils/common';
+import {PropsContext, useLocalUid} from '../../../agora-rn-uikit';
+
+interface IuseSTTAPI {
+  start: (lang: LanguageType[]) => Promise<{message: string} | null>;
+  stop: () => Promise<void>;
+  restart: (lang: LanguageType[]) => Promise<void>;
+  isAuthorizedSTTUser: () => boolean;
+}
+
+const useSTTAPI = (): IuseSTTAPI => {
+  const {store} = React.useContext(StorageContext);
+  const {
+    data: {roomId, isHost},
+  } = useRoomInfo();
+  const {
+    language,
+    isSTTActive,
+    setIsSTTActive,
+    setIsLangChangeInProgress,
+    setLanguage,
+    setMeetingTranscript,
+    setIsSTTError,
+  } = useCaption();
+
+  const currentLangRef = React.useRef<LanguageType[]>([]);
+  const STT_API_URL = `${$config.BACKEND_ENDPOINT}/v1/stt`;
+  const username = useGetName();
+  const localUid = useLocalUid();
+  const {rtcProps} = useContext(PropsContext);
+
+  React.useEffect(() => {
+    currentLangRef.current = language;
+  }, [language]);
+
+  const apiCall = async (method: string, lang: LanguageType[] = []) => {
+    const response = await fetch(`${STT_API_URL}/${method}`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        authorization: store.token ? `Bearer ${store.token}` : '',
+      },
+      body: JSON.stringify({
+        passphrase: roomId?.host || '',
+        lang: lang,
+        dataStream_uid: 111111, // bot ID
+        encryption_mode: $config.ENCRYPTION_ENABLED
+          ? rtcProps.encryption.mode
+          : null,
+      }),
+    });
+    const res = await response.json();
+    return res;
+  };
+
+  const startWithDelay = (lang: LanguageType[]): Promise<string> =>
+    new Promise(resolve => {
+      setTimeout(async () => {
+        const res = await start(lang);
+        resolve(res);
+      }, 1000); // Delay of 1 seconds (1000 milliseconds) to allow existing stt service to fully stop
+    });
+
+  const start = async (lang: LanguageType[]) => {
+    try {
+      setIsLangChangeInProgress(true);
+      const res = await apiCall('start', lang);
+      // null means stt startred successfully
+      const isSTTAlreadyActive =
+        res?.error?.message
+          ?.toLowerCase()
+          .indexOf('current status is started') !== -1 || false;
+
+      if (res?.error?.message) {
+        setIsSTTError(true);
+      } else {
+        setIsSTTError(false);
+      }
+      if (res === null || isSTTAlreadyActive) {
+        // once STT is active in the channel , notify others so that they dont' trigger start again
+        events.send(
+          EventNames.STT_ACTIVE,
+          JSON.stringify({active: true}),
+          PersistanceLevel.Sender,
+        );
+        setIsSTTActive(true);
+
+        console.log(`stt lang update from: ${language} to ${lang}`);
+        // inform about the language set for stt
+        events.send(
+          EventNames.STT_LANGUAGE,
+          JSON.stringify({
+            username: capitalizeFirstLetter(username),
+            uid: localUid,
+            prevLang: language,
+            newLang: lang,
+          }),
+          PersistanceLevel.Sender,
+        );
+        setLanguage(lang);
+
+        // updaing transcript for self
+        const actionText =
+          language.indexOf('') !== -1
+            ? `has set the spoken language to  "${getLanguageLabel(lang)}" `
+            : `changed the spoken language from "${getLanguageLabel(
+                language,
+              )}" to "${getLanguageLabel(lang)}" `;
+        //const msg = `${capitalizeFirstLetter(username)} ${actionText} `;
+        setMeetingTranscript(prev => {
+          return [
+            ...prev,
+            {
+              name: 'langUpdate',
+              time: new Date().getTime(),
+              uid: `langUpdate-${localUid}`,
+              text: actionText,
+            },
+          ];
+        });
+      }
+      return res;
+    } catch (errorMsg) {
+      throw errorMsg;
+    } finally {
+      setIsLangChangeInProgress(false);
+    }
+  };
+
+  const stop = async () => {
+    try {
+      const res = await apiCall('stop');
+      // once STT is non-active in the channel , notify others so that they dont' trigger start again
+      // events.send(
+      //   EventNames.STT_ACTIVE,
+      //   JSON.stringify({active: false}),
+      //   PersistanceLevel.Session,
+      // );
+      setIsSTTActive(false);
+      if (res?.error?.message) {
+        setIsSTTError(true);
+      } else {
+        setIsSTTError(false);
+      }
+      return res;
+    } catch (error) {
+      throw error;
+    }
+  };
+  const restart = async (lang: LanguageType[]) => {
+    try {
+      setIsLangChangeInProgress(true);
+      await stop();
+      await startWithDelay(lang);
+      return Promise.resolve();
+    } catch (error) {
+      console.log('error in re-starting STT', error);
+      return Promise.reject(error);
+    } finally {
+      setIsLangChangeInProgress(false);
+    }
+  };
+
+  // attendee can view option if any host has started STT
+  const isAuthorizedSTTUser = () =>
+    $config.ENABLE_STT && (isHost || (!isHost && isSTTActive));
+
+  return {start, stop, restart, isAuthorizedSTTUser};
+};
+
+export default useSTTAPI;