agora-appbuilder-core 4.1.5 → 4.1.6-1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agora-appbuilder-core",
-  "version": "4.1.5",
+  "version": "4.1.6-1",
   "description": "React Native template for RTE app builder",
   "main": "index.js",
   "files": [

package/template/bridge/rtc/webNg/RtcEngine.ts

@@ -797,8 +797,11 @@ export default class RtcEngine {
         mediaType,
       );
       if (mediaType === 'audio') {
-        const {audio, ...rest} = this.remoteStreams.get(user.uid);
-        this.remoteStreams.set(user.uid, rest);
+        const data = this.remoteStreams.get(user.uid);
+        try {
+          delete data['audio'];
+        } catch (error) {}
+        this.remoteStreams.set(user.uid, {...data});
         (this.eventsMap.get('onRemoteAudioStateChanged') as callbackType)(
           {},
           user.uid,
@@ -807,8 +810,11 @@ export default class RtcEngine {
           0,
         );
       } else {
-        const {video, ...rest} = this.remoteStreams.get(user.uid);
-        this.remoteStreams.set(user.uid, rest);
+        const data = this.remoteStreams.get(user.uid);
+        try {
+          delete data['video'];
+        } catch (error) {}
+        this.remoteStreams.set(user.uid, {...data});
         (this.eventsMap.get('onRemoteVideoStateChanged') as callbackType)(
           {},
           user.uid,

package/template/defaultConfig.js

@@ -77,8 +77,8 @@ const DefaultConfig = {
   CHAT_ORG_NAME: '',
   CHAT_APP_NAME: '',
   CHAT_URL: '',
-  CLI_VERSION: '3.1.5',
-  CORE_VERSION: '4.1.5',
+  CLI_VERSION: '3.1.6-1',
+  CORE_VERSION: '4.1.6-1',
   DISABLE_LANDSCAPE_MODE: false,
   STT_AUTO_START: false,
   CLOUD_RECORDING_AUTO_START: false,

package/template/package.json

@@ -65,7 +65,7 @@
     "agora-extension-beauty-effect": "^1.0.2-beta",
     "agora-extension-virtual-background": "^1.1.3",
     "agora-react-native-rtm": "1.5.1",
-    "agora-rtc-sdk-ng": "4.23.2",
+    "agora-rtc-sdk-ng": "4.23.3",
     "agora-rtm-sdk": "1.5.1",
     "buffer": "^6.0.3",
     "electron-log": "4.3.5",

package/template/src/ai-agent/ai-interface/AIAgentInterface.tsx

@@ -4,8 +4,9 @@ import {
   useContent,
   useLocalAudio,
 } from 'customization-api';
-import {View, StyleSheet, Platform} from 'react-native';
-import hark from 'hark';
+import {View, StyleSheet, Platform, Text} from 'react-native';
+import ThemeConfig from '../../theme';
+import {isAndroid, isIOS, isMobileUA} from '../../utils/common';
 import {AgentContext} from '../components/AgentControls/AgentContext';
 
 const NotJoinedMp4 = require('./1.Not-Joined.mp4').default;
@@ -22,6 +23,17 @@ const AI_ANIMATION_VIDEO = {
   DISCONNECTING: DisconnectingMp4,
 };
 
+const cssHideVideoControls = `
+video {
+  -webkit-appearance: none; /* Remove default styling */
+}
+video::-webkit-media-controls {
+  display: none !important; /* Hide the controls */
+}
+video::-webkit-media-controls-enclosure {
+  display:none !important;
+}`;
+
 export default function AiAgentCustomView({
   connectionState,
 }: CustomAgentInterfaceProps) {
@@ -34,27 +46,27 @@ export default function AiAgentCustomView({
   const aiMediaStream = useRef(null);
 
   useEffect(() => {
-    if (activeUids?.indexOf(agentUID) !== -1 && !aiMediaStream.current) {
+    if (
+      activeUids?.indexOf(agentUID) !== -1 &&
+      !aiMediaStream?.current &&
+      !(isAndroid() || isIOS())
+    ) {
       const track = getRemoteAudioStream(agentUID);
       if (track) {
+        const hark = require('hark');
         try {
-          if (!aiMediaStream.current) {
-            console.log('debugging creating MediaStream here');
+          if (!aiMediaStream?.current) {
             aiMediaStream.current = new MediaStream([
               track.getMediaStreamTrack(),
             ]);
-          } else {
-            console.log('debugging existing MediaStream used');
           }
-          const harkai = hark(aiMediaStream.current, {
+          const harkai = hark(aiMediaStream?.current, {
             interval: 100,
           });
           harkai.on('speaking', () => {
-            console.log('debugging ai is speaking');
             setAISpeaking(true);
           });
           harkai.on('stopped_speaking', () => {
-            console.log('debugging ai is not speaking');
             setAISpeaking(false);
           });
         } catch (error) {
@@ -93,19 +105,58 @@ export default function AiAgentCustomView({
   }, [connectionState, animation, isAISpeaking]);
   return (
     <View style={styles.container}>
-      <video
-        autoPlay
-        style={{pointerEvents: 'none'}}
-        loop
-        src={AI_ANIMATION_VIDEO[animation]}
-        width="40%"
-        height="40%"
-      />
+      {isAndroid() || isIOS() ? (
+        <View style={styles.nativeTextContainer}>
+          <Text style={styles.nativeText}>AI Agent...</Text>
+        </View>
+      ) : (
+        <div style={styles.videoContainer}>
+          <style type="text/css">{cssHideVideoControls}</style>
+          <video
+            disablePictureInPicture
+            playsInline
+            id="animation-video"
+            autoPlay
+            style={{pointerEvents: 'none'}}
+            loop
+            src={AI_ANIMATION_VIDEO[animation]}
+            width={isMobileUA() ? '40%' : 'auto'}
+            height={'auto'}
+          />
+          <div style={styles.overlay} />
+        </div>
+      )}
     </View>
   );
 }
 
 const styles = StyleSheet.create({
+  videoContainer: {
+    flex: 1,
+    position: 'relative',
+    display: 'flex',
+    alignItems: 'center',
+    justifyContent: 'center',
+  },
+  overlay: {
+    position: 'absolute',
+    top: 0,
+    left: 0,
+    width: '100%',
+    height: '100%',
+    zIndex: 1,
+  },
+  nativeTextContainer: {
+    flex: 1,
+    justifyContent: 'center',
+    alignItems: 'center',
+  },
+  nativeText: {
+    fontFamily: ThemeConfig.FontFamily.sansPro,
+    fontSize: 18,
+    fontWeight: '800',
+    color: $config.FONT_COLOR,
+  },
   container: {
     flex: 1,
     backgroundColor: 'black',

package/template/src/ai-agent/components/AgentControls/AgentContext.tsx

@@ -330,6 +330,18 @@ export const AgentProvider: React.FC<{children: React.ReactNode}> = ({
               secondaryBtn: null,
               leadingIcon: null,
             });
+          } else if (agentConnectError.toString().indexOf('403') !== -1) {
+            Toast.show({
+              leadingIconName: 'alert',
+              type: 'error',
+              text1: 'Uh oh! Agent failed to connect',
+              text2:
+                "Verify if you've enabled conversational AI on agora console or if they have configured correct customerId & customer certificate for your project on appbuilder console",
+              visibilityTime: 5000,
+              primaryBtn: null,
+              secondaryBtn: null,
+              leadingIcon: null,
+            });
           } else {
             Toast.show({
               leadingIconName: 'alert',

package/template/src/ai-agent/components/ControlButtons.tsx

@@ -17,7 +17,7 @@ import TranscriptIcon from '../assets/transcript.png';
 //@ts-ignore
 import SettingsIcon from '../assets/settings.png';
 //@ts-ignore
-import DisconnectIcon from '../assets/close.png';
+import LeaveCallIcon from '../assets/leave-call.png';
 
 export const MicButton = () => {
   const {audio} = useLocalUserInfo();
@@ -117,7 +117,7 @@ export const DisconnectButton = () => {
       }}>
       <Image
         style={[styles.iconStyle, {tintColor: $config.FONT_COLOR}]}
-        source={DisconnectIcon}
+        source={LeaveCallIcon}
       />
     </TouchableOpacity>
   );

package/template/src/ai-agent/components/SelectAiAgent.tsx

@@ -35,7 +35,10 @@ const SelectAiAgent = () => {
   }, [agents]);
 
   useEffect(() => {
-    if (!agentId && data && data.length) {
+    const storedAgentValidAndActive =
+      agentId && agents.find(item => item.id === agentId)?.is_active;
+
+    if (!storedAgentValidAndActive && data && data.length) {
       //set default agent
       setAgentId(data[0]?.value);
       setStore(prevState => {

package/template/src/ai-agent/components/mobile/Bottombar.tsx

@@ -2,13 +2,20 @@ import React from 'react';
 import {ToolbarPreset} from 'customization-api';
 import {AgentControl} from '../AgentControls';
 import {CustomSettingButton, CustomTranscriptButton} from '../Bottombar';
+import {isAndroid, isIOS} from '../../../utils/common';
 
 const Bottombar = () => {
   const AI_LAYOUT = $config.AI_LAYOUT ? $config.AI_LAYOUT : 'LAYOUT_TYPE_1';
   return (
     <ToolbarPreset
       align="bottom"
-      snapPointsMinMax={AI_LAYOUT === 'LAYOUT_TYPE_1' ? [100, 100] : [0, 0]}
+      snapPointsMinMax={
+        isAndroid() || isIOS()
+          ? [100, 100]
+          : AI_LAYOUT === 'LAYOUT_TYPE_2'
+          ? [100, 100]
+          : [0, 0]
+      }
       items={{
         layout: {hide: true},
         invite: {hide: true},

package/template/src/ai-agent/components/mobile/Topbar.tsx

@@ -2,16 +2,26 @@ import React from 'react';
 import {View, StyleSheet, Text} from 'react-native';
 import {useRoomInfo} from 'customization-api';
 import ThemeConfig from '../../../theme';
+import {SettingButton} from '../ControlButtons';
+import {getAILayoutType} from '../../utils';
 
 const MobileTopbar = () => {
   const {
     data: {meetingTitle},
   } = useRoomInfo();
+
   return (
     <View style={style.rootStyle}>
       <View style={style.containerStyle}>
         <Text style={style.textStyle}>{meetingTitle}</Text>
       </View>
+      {getAILayoutType() !== 'LAYOUT_TYPE_2' ? (
+        <View>
+          <SettingButton />
+        </View>
+      ) : (
+        <></>
+      )}
     </View>
   );
 };

package/template/src/ai-agent/index.tsx

@@ -1,6 +1,6 @@
 import React from 'react';
 import {CustomizationApiInterface} from 'customization-api';
-import {isMobileUA} from '../utils/common';
+import {isIOS, isAndroid, isMobileUA} from '../utils/common';
 import Bottombar from './components/Bottombar';
 import CustomCreate from './components/CustomCreate';
 import MobileTopBar from './components/mobile/Topbar';
@@ -8,24 +8,20 @@ import MobileBottombar from './components/mobile/Bottombar';
 import CustomChatPanel from './components/CustomChatPanel';
 import CustomSettingsPanel from './components/CustomSettingsPanel';
 import {AgentProvider} from './components/AgentControls/AgentContext';
-
+import {getAILayoutType} from './utils';
 //LAYOUT_TYPE_1
-import DefaultLayout from './layout/DefaultLayout';
+import NewAnimationLayout from './layout/NewAnimation';
 //LAYOUT_TYPE_2
-import ConversationalAILayout from './layout/ConversationalAI';
+import AIWithLocalUser from './layout/AIWithLocalUser';
 //LAYOUT_TYPE_3
-import NewAnimationLayout from './layout/NewAnimation';
+import ConversationalAILayout from './layout/ConversationalAI';
 
 const DummyComponent = () => {
   return <></>;
 };
 
-const getAILayoutType = () => {
-  return $config.AI_LAYOUT ? $config.AI_LAYOUT : 'LAYOUT_TYPE_1';
-};
-
 const getTopBarComponent = () => {
-  return isMobileUA() || getAILayoutType() !== 'LAYOUT_TYPE_1'
+  return isMobileUA() || getAILayoutType() !== 'LAYOUT_TYPE_2'
     ? MobileTopBar
     : DummyComponent;
 };
@@ -33,17 +29,19 @@ const getTopBarComponent = () => {
 const getBottombarComponent = () => {
   return isMobileUA()
     ? MobileBottombar
-    : getAILayoutType() === 'LAYOUT_TYPE_1'
+    : getAILayoutType() === 'LAYOUT_TYPE_2'
     ? Bottombar
     : DummyComponent;
 };
 
 const getCustomLayoutComponent = () => {
-  return getAILayoutType() === 'LAYOUT_TYPE_3'
-    ? NewAnimationLayout
-    : getAILayoutType() === 'LAYOUT_TYPE_2'
+  return isAndroid() || isIOS()
+    ? AIWithLocalUser
+    : getAILayoutType() === 'LAYOUT_TYPE_3'
     ? ConversationalAILayout
-    : DefaultLayout;
+    : getAILayoutType() === 'LAYOUT_TYPE_2'
+    ? AIWithLocalUser
+    : NewAnimationLayout;
 };
 
 export const AI_AGENT_CUSTOMIZATION: CustomizationApiInterface = {

package/template/src/ai-agent/layout/{DefaultLayout.tsx → AIWithLocalUser.tsx}

@@ -14,7 +14,7 @@ import AudioVisualizer, {DisconnectedView} from '../components/AudioVisualizer';
 import {AgentState} from '../components/AgentControls/const';
 import {AgentContext} from '../components/AgentControls/AgentContext';
 
-export default function DefaultLayout() {
+export default function AIWithLocalUser() {
   const localUid = useLocalUid();
   const {defaultContent, activeUids} = useContent();
   const {RtcEngineUnsafe} = useRtc();

package/template/src/ai-agent/layout/ConversationalAI.tsx

@@ -7,18 +7,17 @@ import {
   Image,
   ActivityIndicator,
 } from 'react-native';
-import {useSidePanel, isAndroid, isIOS} from 'customization-api';
+import {useSidePanel} from 'customization-api';
 import ThemeConfig from '../../theme';
 import {AgentContext} from '../components/AgentControls/AgentContext';
 import {AgentState} from '../components/AgentControls/const';
 import {useIsAgentAvailable} from '../components/utils';
-import {isMobileUA} from '../../utils/common';
+import {isMobileUA, isAndroid, isIOS} from '../../utils/common';
 //@ts-ignore
 import JoinCallIcon from '../assets/join-call.png';
 import {
   DisconnectButton,
   MicButton,
-  SettingButton,
   TranscriptButton,
 } from '../components/ControlButtons';
 
@@ -39,9 +38,15 @@ export default function ConversationalAI() {
           const {Application} = require('@splinetool/runtime');
           // start the application and load the scene
           const spline = new Application(canvas);
-          spline.load(
-            'https://d1i64xs2div6cu.cloudfront.net/scene-250216.splinecode',
-          );
+          spline
+            ?.load(
+              'https://d1i64xs2div6cu.cloudfront.net/scene-250216.splinecode',
+            )
+            ?.then(() => {
+              if (isMobileUA()) {
+                spline?.setZoom(0.5);
+              }
+            });
         }
       });
     }
@@ -83,7 +88,6 @@ export default function ConversationalAI() {
           <View style={styles.controlsContainer}>
             <MicButton />
             <TranscriptButton />
-            <SettingButton />
             <DisconnectButton />
           </View>
         ) : (
@@ -94,6 +98,7 @@ export default function ConversationalAI() {
               agentConnectionState === AgentState.AGENT_DISCONNECT_REQUEST
                 ? {backgroundColor: $config.SEMANTIC_ERROR}
                 : {},
+              isLoading || !isAgentAvailable ? styles.disabledOpacity : {},
             ]}
             onPress={() => {
               toggleAgentConnection();
@@ -142,7 +147,7 @@ const styles = StyleSheet.create({
   },
   layoutRootContainer: {
     flex: 1,
-    backgroundColor: $config.VIDEO_AUDIO_TILE_COLOR,
+    backgroundColor: '#1D1D1D',
     borderRadius: 8,
   },
   callAgentBtnInnerContainer: {
@@ -158,6 +163,9 @@ const styles = StyleSheet.create({
     display: 'flex',
     flex: 1,
   },
+  disabledOpacity: {
+    opacity: 0.6,
+  },
   callAgentBtnText: {
     fontFamily: ThemeConfig.FontFamily.sansPro,
     fontSize: 18,

package/template/src/ai-agent/layout/NewAnimation.tsx

@@ -1,4 +1,4 @@
-import React, {useContext} from 'react';
+import React, {useContext, useEffect} from 'react';
 import {
   StyleSheet,
   TouchableOpacity,
@@ -15,15 +15,18 @@ import AiAgentCustomView from '../ai-interface/AIAgentInterface';
 import {
   DisconnectButton,
   MicButton,
-  SettingButton,
   TranscriptButton,
 } from '../components/ControlButtons';
+import {isMobileUA} from '../../utils/common';
+import {useSidePanel} from 'customization-api';
+
 //@ts-ignore
 import JoinCallIcon from '../assets/join-call.png';
 
 export default function NewAnimation() {
   const {agentConnectionState, toggleAgentConnection} =
     useContext(AgentContext);
+  const {setSidePanel} = useSidePanel();
 
   const isLoading =
     agentConnectionState === AgentState.REQUEST_SENT ||
@@ -36,6 +39,12 @@ export default function NewAnimation() {
     isAwaitingLeave;
   const isAgentAvailable = useIsAgentAvailable();
 
+  useEffect(() => {
+    setTimeout(() => {
+      !isMobileUA() && setSidePanel('custom-settings-panel');
+    });
+  }, []);
+
   return (
     <View style={styles.layoutRootContainer}>
       <View style={styles.container}>
@@ -46,7 +55,6 @@ export default function NewAnimation() {
           <View style={styles.controlsContainer}>
             <MicButton />
             <TranscriptButton />
-            <SettingButton />
             <DisconnectButton />
           </View>
         ) : (
@@ -57,6 +65,7 @@ export default function NewAnimation() {
               agentConnectionState === AgentState.AGENT_DISCONNECT_REQUEST
                 ? {backgroundColor: $config.SEMANTIC_ERROR}
                 : {},
+              isLoading || !isAgentAvailable ? styles.disabledOpacity : {},
             ]}
             onPress={() => {
               toggleAgentConnection();
@@ -109,7 +118,6 @@ const styles = StyleSheet.create({
   },
   layoutRootContainer: {
     flex: 1,
-    backgroundColor: $config.VIDEO_AUDIO_TILE_COLOR,
     borderRadius: 8,
   },
   callAgentBtnInnerContainer: {
@@ -125,6 +133,9 @@ const styles = StyleSheet.create({
     display: 'flex',
     flex: 1,
   },
+  disabledOpacity: {
+    opacity: 0.6,
+  },
   callAgentBtnText: {
     fontFamily: ThemeConfig.FontFamily.sansPro,
     fontSize: 18,
@@ -140,6 +151,7 @@ const styles = StyleSheet.create({
     marginRight: 'auto',
     maxWidth: 500,
     bottom: 50,
+    zIndex: 999,
   },
   container: {
     flex: 1,

package/template/src/ai-agent/utils.ts

@@ -1,5 +1,7 @@
 import {IMicrophoneAudioTrack} from 'agora-rtc-sdk-ng';
 import {useState, useEffect} from 'react';
+import {isAndroid, isIOS} from '../utils/common';
+
 //for native its require text decoder to be imported
 const TextDecoder = require('text-encoding').TextDecoder;
 export const useMultibandTrackVolume = (
@@ -131,3 +133,11 @@ export const Base64 = {
     return output;
   },
 };
+
+export const getAILayoutType = () => {
+  return isAndroid() || isIOS()
+    ? 'LAYOUT_TYPE_2'
+    : $config.AI_LAYOUT
+    ? $config.AI_LAYOUT
+    : 'LAYOUT_TYPE_1';
+};

package/template/src/utils/useSpeechToText.ts

@@ -52,7 +52,7 @@ const useSpeechToText = () => {
   useEffect(() => {
     if (!$config.ENABLE_STT) {
       //throw new Error('Speech To Text is not enabled');
-      console.error('Speech To Text is not enabled');
+      console.log('Speech To Text is not enabled');
     }
   }, []);