agora-appbuilder-core 4.0.19 → 4.0.20-beta.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agora-appbuilder-core",
-  "version": "4.0.19",
+  "version": "4.0.20-beta.1",
   "description": "React Native template for RTE app builder",
   "main": "index.js",
   "files": [

package/template/customization-api/sub-components.ts

@@ -72,3 +72,6 @@ export type {
   ToolbarCustomItem,
   ToolbarPresetProps,
 } from '../src/atoms/ToolbarPreset';
+export {default as TranscriptPanel} from '../src/subComponents/caption/Transcript';
+export type {TranscriptProps} from '../src/subComponents/caption/Transcript';
+export {default as CaptionPanel} from '../src/subComponents/caption/CaptionContainer';

package/template/customization-api/typeDefinition.ts

@@ -68,6 +68,8 @@ export interface VideoCallInterface extends BeforeAndAfterInterface {
   //settingsPanel?: React.ComponentType;
   participantsPanel?: React.ComponentType;
   chat?: ChatCmpInterface;
+  captionPanel?: React.ComponentType;
+  transcriptPanel?: React.ComponentType;
   customLayout?: (layouts: LayoutItem[]) => LayoutItem[];
   wrapper?: React.ComponentType;
   invitePopup?: {

package/template/customization-api/utils.ts

@@ -32,3 +32,6 @@ export {isWeb, isIOS, isAndroid, isDesktop} from '../src/utils/common';
 export {default as isMobileOrTablet} from '../src/utils/isMobileOrTablet';
 export {useLocalUid} from '../agora-rn-uikit';
 export {default as useLocalAudio} from '../src/utils/useLocalAudio';
+export {default as useLocalVideo} from '../src/utils/useLocalVideo';
+export type {LanguageType} from '../src/subComponents/caption/utils';
+export {default as useSpeechToText} from '../src/utils/useSpeechToText';

package/template/defaultConfig.js

@@ -61,6 +61,8 @@ const DefaultConfig = {
   ENABLE_IDP_AUTH: false,
   ENABLE_TOKEN_AUTH: false,
   ENABLE_STT: false,
+  ENABLE_CAPTION: true,
+  ENABLE_MEETING_TRANSCRIPT: true,
   ENABLE_NOISE_CANCELLATION: true,
   ENABLE_VIRTUAL_BACKGROUND: true,
   ENABLE_WHITEBOARD: false,
@@ -69,7 +71,7 @@ const DefaultConfig = {
   WHITEBOARD_APPIDENTIFIER: '',
   WHITEBOARD_REGION: '',
   ENABLE_NOISE_CANCELLATION_BY_DEFAULT: false,
-  CHAT_ORG_NAME:'',
+  CHAT_ORG_NAME: '',
   CHAT_APP_NAME: '',
   CHAT_URL: '',
 };

package/template/global.d.ts

@@ -104,6 +104,8 @@ interface ConfigInterface {
   ENABLE_IDP_AUTH: boolean;
   PROJECT_ID: string;
   ENABLE_STT: boolean;
+  ENABLE_CAPTION: boolean;
+  ENABLE_MEETING_TRANSCRIPT: boolean;
   ENABLE_NOISE_CANCELLATION: boolean;
   ENABLE_VIRTUAL_BACKGROUND: boolean;
   ENABLE_WHITEBOARD: boolean;

package/template/src/components/Controls.tsx

@@ -465,12 +465,16 @@ const MoreButton = () => {
 
   // host can see stt options and attendee can view only when stt is enabled by a host in the channel
 
-  if ($config.ENABLE_STT) {
+  if ($config.ENABLE_STT && $config.ENABLE_CAPTION) {
     actionMenuitems.push({
       icon: `${isCaptionON ? 'captions-off' : 'captions'}`,
       iconColor: $config.SECONDARY_ACTION_COLOR,
       textColor: $config.FONT_COLOR,
-      disabled: !($config.ENABLE_STT && (isHost || (!isHost && isSTTActive))),
+      disabled: !(
+        $config.ENABLE_STT &&
+        $config.ENABLE_CAPTION &&
+        (isHost || (!isHost && isSTTActive))
+      ),
       title: captionLabel(isCaptionON),
       callback: () => {
         setActionMenuVisible(false);
@@ -489,31 +493,38 @@ const MoreButton = () => {
       },
     });
 
-    actionMenuitems.push({
-      icon: 'transcript',
-      iconColor: $config.SECONDARY_ACTION_COLOR,
-      textColor: $config.FONT_COLOR,
-      disabled: !($config.ENABLE_STT && (isHost || (!isHost && isSTTActive))),
-      title: transcriptLabel(isTranscriptON),
-      callback: () => {
-        setActionMenuVisible(false);
-        STT_clicked.current = !isTranscriptON ? 'transcript' : null;
-        if (isSTTError) {
-          !isTranscriptON
-            ? setSidePanel(SidePanelType.Transcript)
-            : setSidePanel(SidePanelType.None);
-          return;
-        }
-        if (isSTTActive) {
-          !isTranscriptON
-            ? setSidePanel(SidePanelType.Transcript)
-            : setSidePanel(SidePanelType.None);
-        } else {
-          isFirstTimePopupOpen.current = true;
-          setLanguagePopup(true);
-        }
-      },
-    });
+    if ($config.ENABLE_MEETING_TRANSCRIPT) {
+      actionMenuitems.push({
+        icon: 'transcript',
+        iconColor: $config.SECONDARY_ACTION_COLOR,
+        textColor: $config.FONT_COLOR,
+        disabled: !(
+          $config.ENABLE_STT &&
+          $config.ENABLE_CAPTION &&
+          $config.ENABLE_MEETING_TRANSCRIPT &&
+          (isHost || (!isHost && isSTTActive))
+        ),
+        title: transcriptLabel(isTranscriptON),
+        callback: () => {
+          setActionMenuVisible(false);
+          STT_clicked.current = !isTranscriptON ? 'transcript' : null;
+          if (isSTTError) {
+            !isTranscriptON
+              ? setSidePanel(SidePanelType.Transcript)
+              : setSidePanel(SidePanelType.None);
+            return;
+          }
+          if (isSTTActive) {
+            !isTranscriptON
+              ? setSidePanel(SidePanelType.Transcript)
+              : setSidePanel(SidePanelType.None);
+          } else {
+            isFirstTimePopupOpen.current = true;
+            setLanguagePopup(true);
+          }
+        },
+      });
+    }
   }
 
   // view recordings
@@ -880,7 +891,9 @@ export const MoreButtonToolbarItem = () => {
   }, [isHost]);
 
   return width < BREAKPOINTS.md ||
-    ($config.ENABLE_STT && (isHost || (!isHost && isSTTActive))) ||
+    ($config.ENABLE_STT &&
+      $config.ENABLE_CAPTION &&
+      (isHost || (!isHost && isSTTActive))) ||
     $config.ENABLE_NOISE_CANCELLATION ||
     ($config.ENABLE_VIRTUAL_BACKGROUND && !$config.AUDIO_ROOM) ||
     (isHost && $config.ENABLE_WHITEBOARD && isWebInternal()) ? (

package/template/src/pages/video-call/ActionSheetContent.tsx

@@ -235,6 +235,9 @@ const TranscriptIconBtn = (props: TranscriptIconProps) => {
     $config.ENABLE_STT &&
     (isHost || (!isHost && isSTTActive))
   );
+  if (!$config.ENABLE_MEETING_TRANSCRIPT) {
+    return null;
+  }
   return (
     <ToolbarItem>
       <TranscriptIcon isOnActionSheet={true} showLabel={true} />

package/template/src/pages/video-call/VideoCallScreen.tsx

@@ -29,12 +29,15 @@ import {useRoomInfo} from '../../components/room-info/useRoomInfo';
 import {
   ToolbarCustomItem,
   controlMessageEnum,
+  useCaption,
   useUserName,
 } from 'customization-api';
 import events, {PersistanceLevel} from '../../rtm-events-api';
 import VideoCallMobileView from './VideoCallMobileView';
 import CaptionContainer from '../../subComponents/caption/CaptionContainer';
-import Transcript from '../../subComponents/caption/Transcript';
+import Transcript, {
+  TranscriptProps,
+} from '../../subComponents/caption/Transcript';
 import Spacer from '../../atoms/Spacer';
 import Leftbar, {LeftbarProps} from '../../components/Leftbar';
 import Rightbar, {RightbarProps} from '../../components/Rightbar';
@@ -50,11 +53,14 @@ const VideoCallScreen = () => {
   const {
     data: {meetingTitle, isHost},
   } = useRoomInfo();
+  const {isCaptionON} = useCaption();
   const {
     ChatComponent,
     VideocallComponent,
     BottombarComponent,
     ParticipantsComponent,
+    TranscriptComponent,
+    CaptionComponent,
     SettingsComponent,
     TopbarComponent,
     VideocallBeforeView,
@@ -73,6 +79,8 @@ const VideoCallScreen = () => {
       ChatComponent: React.ComponentType<ChatProps>;
       BottombarComponent: React.ComponentType<ControlsProps>;
       ParticipantsComponent: React.ComponentType;
+      TranscriptComponent: React.ComponentType<TranscriptProps>;
+      CaptionComponent: React.ComponentType;
       SettingsComponent: React.ComponentType;
       TopbarComponent: React.ComponentType<NavbarProps>;
       VideocallBeforeView: React.ComponentType;
@@ -88,6 +96,8 @@ const VideoCallScreen = () => {
       TopbarComponent: Navbar,
       ChatComponent: Chat,
       ParticipantsComponent: ParticipantsView,
+      TranscriptComponent: Transcript,
+      CaptionComponent: CaptionContainer,
       SettingsComponent: SettingsView,
       VideocallAfterView: React.Fragment,
       VideocallBeforeView: React.Fragment,
@@ -206,6 +216,23 @@ const VideoCallScreen = () => {
           data?.components?.videoCall.participantsPanel;
       }
 
+      if (
+        data?.components?.videoCall.transcriptPanel &&
+        typeof data?.components?.videoCall.transcriptPanel !== 'object' &&
+        isValidReactComponent(data?.components?.videoCall.transcriptPanel)
+      ) {
+        components.TranscriptComponent =
+          data?.components?.videoCall.transcriptPanel;
+      }
+
+      if (
+        data?.components?.videoCall.captionPanel &&
+        typeof data?.components?.videoCall.captionPanel !== 'object' &&
+        isValidReactComponent(data?.components?.videoCall.captionPanel)
+      ) {
+        components.CaptionComponent = data?.components?.videoCall.captionPanel;
+      }
+
       //todo hari - need to remove wrapper
       if (
         data?.components?.videoCall.wrapper &&
@@ -313,7 +340,15 @@ const VideoCallScreen = () => {
               ) : (
                 <></>
               )}
-              {sidePanel === SidePanelType.Transcript ? <Transcript /> : <></>}
+              {sidePanel === SidePanelType.Transcript ? (
+                $config.ENABLE_MEETING_TRANSCRIPT ? (
+                  <TranscriptComponent />
+                ) : (
+                  <></>
+                )
+              ) : (
+                <></>
+              )}
               {sidePanel === SidePanelType.VirtualBackground ? (
                 <VBPanel />
               ) : (
@@ -331,7 +366,7 @@ const VideoCallScreen = () => {
                   />
                 ) : (
                   <>
-                    <CaptionContainer />
+                    {isCaptionON ? <CaptionComponent /> : <></>}
                     <Spacer size={10} />
                     <View
                       style={

package/template/src/subComponents/EndcallPopup.tsx

@@ -57,7 +57,11 @@ const EndcallPopup = (props: EndcallPopupProps) => {
   const leaveMeetingPopupActionButton = useString(leavePopupPrimaryBtnText)();
   const cancelLabel = useString(cancelText)();
   const {isSTTActive} = useCaption();
-  const isTranscriptAvailable = $config.ENABLE_STT && isSTTActive;
+  const isTranscriptAvailable =
+    $config.ENABLE_STT &&
+    $config.ENABLE_CAPTION &&
+    $config.ENABLE_MEETING_TRANSCRIPT &&
+    isSTTActive;
 
   const stayBtnLabel = cancelLabel;
   const leaveBtnLabel = leaveMeetingPopupActionButton;

package/template/src/subComponents/caption/Transcript.tsx

@@ -41,7 +41,7 @@ import {
   sttTranscriptPanelViewLatestText,
 } from '../../../src/language/default-labels/videoCallScreenLabels';
 
-interface TranscriptProps {
+export interface TranscriptProps {
   showHeader?: boolean;
 }
 

package/template/src/subComponents/caption/TranscriptIcon.tsx

@@ -27,9 +27,9 @@ const TranscriptIcon = (props: TranscriptIconProps) => {
     isMobileView = false,
   } = props;
 
-  const {start, restart, isAuthorizedSTTUser} = useSTTAPI();
+  const {start, restart, isAuthorizedTranscriptUser} = useSTTAPI();
   const {isSTTActive, language: prevLang, isSTTError} = useCaption();
-  const isDisabled = !isAuthorizedSTTUser();
+  const isDisabled = !isAuthorizedTranscriptUser();
   const [isLanguagePopupOpen, setLanguagePopup] =
     React.useState<boolean>(false);
   const isFirstTimePopupOpen = React.useRef(false);

package/template/src/subComponents/caption/useSTTAPI.tsx

@@ -15,6 +15,7 @@ interface IuseSTTAPI {
   stop: () => Promise<void>;
   restart: (lang: LanguageType[]) => Promise<void>;
   isAuthorizedSTTUser: () => boolean;
+  isAuthorizedTranscriptUser: () => boolean;
 }
 
 const useSTTAPI = (): IuseSTTAPI => {
@@ -213,9 +214,23 @@ const useSTTAPI = (): IuseSTTAPI => {
 
   // attendee can view option if any host has started STT
   const isAuthorizedSTTUser = () =>
-    $config.ENABLE_STT && (isHost || (!isHost && isSTTActive));
+    $config.ENABLE_STT &&
+    $config.ENABLE_CAPTION &&
+    (isHost || (!isHost && isSTTActive));
 
-  return {start, stop, restart, isAuthorizedSTTUser};
+  const isAuthorizedTranscriptUser = () =>
+    $config.ENABLE_STT &&
+    $config.ENABLE_CAPTION &&
+    $config.ENABLE_MEETING_TRANSCRIPT &&
+    (isHost || (!isHost && isSTTActive));
+
+  return {
+    start,
+    stop,
+    restart,
+    isAuthorizedSTTUser,
+    isAuthorizedTranscriptUser,
+  };
 };
 
 export default useSTTAPI;

package/template/src/subComponents/recording/useRecording.tsx

@@ -139,7 +139,11 @@ const RecordingProvider = (props: RecordingProviderProps) => {
         setSidePanel(SidePanelType.Chat);
         setChatType(ChatType.Group);
       }
-      if (recordingBotUIConfig.stt && $config.ENABLE_STT) {
+      if (
+        recordingBotUIConfig.stt &&
+        $config.ENABLE_STT &&
+        $config.ENABLE_CAPTION
+      ) {
         setIsCaptionON(true);
       }
     }

package/template/src/utils/useLocalAudio.ts

@@ -12,6 +12,7 @@
 
 import {DispatchContext, RtcContext} from '../../agora-rn-uikit';
 import {useContext} from 'react';
+import {isWeb} from './common';
 
 function useLocalAudio() {
   const {dispatch} = useContext(DispatchContext);
@@ -32,7 +33,37 @@ function useLocalAudio() {
     });
   };
 
-  return {enableAudioButton, disableAudioButton};
+  const getLocalAudioStream = () => {
+    try {
+      return isWeb() ? window?.engine?.localStream?.audio : null;
+    } catch (error) {
+      throw error;
+    }
+  };
+
+  const getRemoteAudioStream = (uid: number) => {
+    try {
+      return isWeb() ? window?.engine?.remoteStreams?.get(uid)?.audio : null;
+    } catch (error) {
+      throw error;
+    }
+  };
+
+  const getLocalScreenshareAudioStream = () => {
+    try {
+      return isWeb() ? window?.engine?.screenStream?.audio : null;
+    } catch (error) {
+      throw error;
+    }
+  };
+
+  return {
+    enableAudioButton,
+    disableAudioButton,
+    getLocalAudioStream,
+    getRemoteAudioStream,
+    getLocalScreenshareAudioStream,
+  };
 }
 
 export default useLocalAudio;

package/template/src/utils/useLocalVideo.ts

@@ -0,0 +1,69 @@
+/*
+********************************************
+ Copyright © 2021 Agora Lab, Inc., all rights reserved.
+ AppBuilder and all associated components, source code, APIs, services, and documentation 
+ (the “Materials”) are owned by Agora Lab, Inc. and its licensors. The Materials may not be 
+ accessed, used, modified, or distributed for any purpose without a license from Agora Lab, Inc.  
+ Use without a license or in violation of any license terms and conditions (including use for 
+ any purpose competitive to Agora Lab, Inc.’s business) is strictly prohibited. For more 
+ information visit https://appbuilder.agora.io. 
+*********************************************
+*/
+
+import {DispatchContext, RtcContext} from '../../agora-rn-uikit';
+import {useContext} from 'react';
+import {isWeb} from './common';
+
+function useLocalVideo() {
+  const {dispatch} = useContext(DispatchContext);
+  const {RtcEngineUnsafe} = useContext(RtcContext);
+
+  const disableVideoButton = () => {
+    RtcEngineUnsafe.muteLocalVideoStream(true);
+    dispatch({
+      type: 'LocalMuteVideo',
+      value: [0, true],
+    });
+  };
+
+  const enableVideoButton = () => {
+    dispatch({
+      type: 'LocalMuteVideo',
+      value: [0, false],
+    });
+  };
+
+  const getLocalVideoStream = () => {
+    try {
+      return isWeb() ? window?.engine?.localStream?.video : null;
+    } catch (error) {
+      throw error;
+    }
+  };
+
+  const getRemoteVideoStream = (uid: number) => {
+    try {
+      return isWeb() ? window?.engine?.remoteStreams?.get(uid)?.video : null;
+    } catch (error) {
+      throw error;
+    }
+  };
+
+  const getLocalScreenshareVideoStream = () => {
+    try {
+      return isWeb() ? window?.engine?.screenStream?.video : null;
+    } catch (error) {
+      throw error;
+    }
+  };
+
+  return {
+    enableVideoButton,
+    disableVideoButton,
+    getLocalVideoStream,
+    getRemoteVideoStream,
+    getLocalScreenshareVideoStream,
+  };
+}
+
+export default useLocalVideo;

package/template/src/utils/useSpeechToText.ts

@@ -0,0 +1,102 @@
+import {useEffect, useRef} from 'react';
+import {
+  SidePanelType,
+  customEvents,
+  useContent,
+  useSTTAPI,
+  useSidePanel,
+} from 'customization-api';
+import useTranscriptDownload from '../subComponents/caption/useTranscriptDownload';
+import {useCaption} from '../subComponents/caption/useCaption';
+import {LanguageType} from '../subComponents/caption/utils';
+
+const useSpeechToText = () => {
+  const {
+    isCaptionON: isSpeechToTextOn,
+    setIsCaptionON,
+    meetingTranscript: transcriptData,
+    captionObj: captionData,
+    prevSpeakerRef,
+    activeSpeakerRef,
+  } = useCaption();
+  const {setSidePanel} = useSidePanel();
+
+  const {start, restart, stop, isAuthorizedSTTUser} = useSTTAPI();
+  const {defaultContent} = useContent();
+
+  const isAuthorizedSTTUserRef = useRef(isAuthorizedSTTUser);
+  const defaultContentRef = useRef(defaultContent);
+
+  const showTranscriptPanel = (show: boolean) => {
+    show
+      ? setSidePanel(SidePanelType.Transcript)
+      : setSidePanel(SidePanelType.None);
+  };
+
+  const showCaptionPanel = (show: boolean) => {
+    show ? setIsCaptionON(true) : setIsCaptionON(false);
+  };
+
+  useEffect(() => {
+    if (!$config.ENABLE_STT) {
+      throw new Error('Speech To Text is not enabled');
+    }
+  }, []);
+
+  useEffect(() => {
+    isAuthorizedSTTUserRef.current = isAuthorizedSTTUser;
+  }, [isAuthorizedSTTUser]);
+
+  useEffect(() => {
+    defaultContentRef.current = defaultContent;
+  }, [defaultContent]);
+
+  const {downloadTranscript} = useTranscriptDownload();
+
+  const getActiveSpeakerName = (): string => {
+    return defaultContentRef.current[activeSpeakerRef.current]?.name || '';
+  };
+
+  const getPrevSpeakerName = (): string => {
+    return defaultContentRef.current[prevSpeakerRef.current]?.name || '';
+  };
+
+  const startSpeechToText = async (language: LanguageType[]) => {
+    if (!isAuthorizedSTTUserRef.current) {
+      throw new Error('Invalid user');
+    }
+    return await start(language);
+  };
+
+  const stopSpeechToText = async () => {
+    if (!isAuthorizedSTTUserRef.current) {
+      throw new Error('Invalid user');
+    }
+    return await stop();
+  };
+
+  const changeSpeakingLanguage = async (language: LanguageType[]) => {
+    if (!isAuthorizedSTTUserRef.current) {
+      throw new Error('Invalid user');
+    }
+    return await restart(language);
+  };
+
+  return $config.ENABLE_STT
+    ? {
+        isSpeechToTextOn,
+        startSpeechToText,
+        stopSpeechToText,
+        changeSpeakingLanguage,
+        downloadTranscript,
+        getActiveSpeakerName,
+        getPrevSpeakerName,
+        showTranscriptPanel,
+        showCaptionPanel,
+        transcriptData,
+        captionData,
+      }
+    : {};
+};
+
+export default useSpeechToText;