agora-appbuilder-core 4.1.0-beta-2 → 4.1.0-beta-4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agora-appbuilder-core",
-  "version": "4.1.0-beta-2",
+  "version": "4.1.0-beta-4",
   "description": "React Native template for RTE app builder",
   "main": "index.js",
   "files": [

package/template/customization-api/atoms.ts

@@ -7,3 +7,4 @@ export {default as PrimaryButton} from '../src/atoms/PrimaryButton';
 export {default as TertiaryButton} from '../src/atoms/TertiaryButton';
 export {default as ActionMenu} from '../src/atoms/ActionMenu';
 export {default as IconButton} from '../src/atoms/IconButton';
+export {default as Dropdown} from '../src/atoms/Dropdown';

package/template/customization-api/customize.tsx

@@ -177,6 +177,17 @@ const mergeCustomization = (
     );
   }
 
+  if (externalConfig?.components?.videoCall?.wrapper) {
+    const AiAgentVideoCallWrapper = aiAgentConfig.components.videoCall.wrapper;
+    const ExternalVideoCallWrapper =
+      externalConfig.components.videoCall.wrapper;
+    mergedData.components.videoCall.wrapper = props => (
+      <AiAgentVideoCallWrapper>
+        <ExternalVideoCallWrapper>{props.children}</ExternalVideoCallWrapper>
+      </AiAgentVideoCallWrapper>
+    );
+  }
+
   //override the i18n
   if (externalConfig?.i18n && externalConfig?.i18n?.length) {
     mergedData.i18n = externalConfig.i18n;

package/template/customization-api/sub-components.ts

@@ -91,3 +91,5 @@ export {default as Loading} from '../src/subComponents/Loading';
 export {default as UserAvatar} from '../src/atoms/UserAvatar';
 export {default as Card} from '../src/atoms/Card';
 export {default as ThemeConfig} from '../src/theme';
+export {default as SelectDevice} from '../src/subComponents/SelectDevice';
+export {EditName} from '../src/components/SettingsView';

package/template/src/ai-agent/components/AgentControls/AgentConnectionWrapper.tsx

@@ -0,0 +1,248 @@
+import {UidType, useContent, useRoomInfo, Toast} from 'customization-api';
+import React, {createContext, useContext, useEffect} from 'react';
+import {AgentContext} from './AgentContext';
+import {AgentState} from './const';
+import StorageContext from '../../../components/StorageContext';
+
+export interface AgentContextInterface {
+  toggleAgentConnection: (forceStop?: boolean) => Promise<boolean>;
+}
+
+export const AgentConnectionContext = createContext<AgentContextInterface>({
+  toggleAgentConnection: () => {
+    return Promise.resolve(false);
+  },
+});
+
+export const AgentConnectionProvider: React.FC<{children: React.ReactNode}> = ({
+  children,
+}) => {
+  const {activeUids: users} = useContent();
+  const {
+    agentUID,
+    agentConnectionState,
+    setAgentConnectionState,
+    agentId,
+    setAgentUID,
+    prompt,
+  } = useContext(AgentContext);
+  const {
+    data: {channel: channel_name, uid: localUid, agents},
+  } = useRoomInfo();
+  const {store} = useContext(StorageContext);
+
+  useEffect(() => {
+    console.log('debugging users agent contrl', {users});
+    // welcome agent
+    const aiAgentUID = users.filter(item => item === agentUID);
+
+    if (
+      aiAgentUID.length &&
+      agentConnectionState === AgentState.AWAITING_JOIN
+    ) {
+      setAgentConnectionState(AgentState.AGENT_CONNECTED);
+
+      Toast.show({
+        leadingIconName: 'tick-fill',
+        type: 'success',
+        text1: 'Say Hi!!',
+        text2: null,
+        visibilityTime: 3000,
+        primaryBtn: null,
+        secondaryBtn: null,
+        leadingIcon: null,
+      });
+    }
+    // when agent leaves, show left toast, and set agent to not connected state
+    if (
+      !aiAgentUID.length &&
+      agentConnectionState === AgentState.AWAITING_LEAVE
+    ) {
+      setAgentConnectionState(AgentState.NOT_CONNECTED);
+    }
+  }, [users, agentUID]);
+
+  const handleConnectionToggle = async (forceStop: boolean = false) => {
+    try {
+      // connect to agent when agent is in not connected state or when earlier connect failed
+      if (
+        agentConnectionState === AgentState.NOT_CONNECTED ||
+        agentConnectionState === AgentState.AGENT_REQUEST_FAILED ||
+        agentConnectionState === AgentState.AWAITING_LEAVE
+      ) {
+        try {
+          setAgentConnectionState(AgentState.REQUEST_SENT);
+          const data = await connectToAIAgent(
+            'start',
+            channel_name,
+            localUid,
+            store.token,
+            {
+              agent_id: agentId,
+              prompt: prompt,
+              voice: agents.find(a => a.id === agentId)?.config?.tts?.params
+                ?.voice_name,
+            },
+          );
+          // console.log("response X-Client-ID", newClientId, typeof newClientId)
+          // @ts-ignore
+          const {agent_uid = null} = data;
+
+          //setClientId(agent_id);
+          setAgentUID(agent_uid);
+
+          setAgentConnectionState(AgentState.AWAITING_JOIN);
+
+          Toast.show({
+            leadingIconName: 'tick-fill',
+            type: 'success',
+            text1: 'Agent requested to join',
+            text2: null,
+            visibilityTime: 3000,
+            primaryBtn: null,
+            secondaryBtn: null,
+            leadingIcon: null,
+          });
+          return Promise.resolve(true);
+        } catch (agentConnectError) {
+          setAgentConnectionState(AgentState.AGENT_REQUEST_FAILED);
+
+          if (agentConnectError.toString().indexOf('401') !== -1) {
+            Toast.show({
+              leadingIconName: 'alert',
+              type: 'error',
+              text1: 'Your session is expired. Please sign in to join call.',
+              text2: null,
+              visibilityTime: 5000,
+              primaryBtn: null,
+              secondaryBtn: null,
+              leadingIcon: null,
+            });
+          } else {
+            Toast.show({
+              leadingIconName: 'alert',
+              type: 'error',
+              text1: 'Uh oh! Agent failed to connect',
+              text2: null,
+              visibilityTime: 5000,
+              primaryBtn: null,
+              secondaryBtn: null,
+              leadingIcon: null,
+            });
+          }
+
+          throw agentConnectError;
+        }
+      }
+      // disconnect agent with agent is already connected or when earlier disconnect failed
+      if (
+        forceStop === true ||
+        agentConnectionState === AgentState.AGENT_CONNECTED ||
+        agentConnectionState === AgentState.AGENT_DISCONNECT_FAILED
+      ) {
+        try {
+          setAgentConnectionState(AgentState.AGENT_DISCONNECT_REQUEST);
+          await connectToAIAgent('stop', channel_name, localUid, store.token, {
+            agent_id: agentId,
+          });
+          setAgentConnectionState(AgentState.AWAITING_LEAVE);
+          if (!forceStop) {
+            Toast.show({
+              leadingIconName: 'tick-fill',
+              type: 'success',
+              text1: 'Agent disconnected',
+              text2: null,
+              visibilityTime: 3000,
+              primaryBtn: null,
+              secondaryBtn: null,
+              leadingIcon: null,
+            });
+          }
+          return Promise.resolve(true);
+        } catch (agentDisconnectError) {
+          setAgentConnectionState(AgentState.AGENT_DISCONNECT_FAILED);
+
+          Toast.show({
+            leadingIconName: 'alert',
+            type: 'error',
+            text1: 'Uh oh! Agent failed to disconnect',
+            text2: null,
+            visibilityTime: 5000,
+            primaryBtn: null,
+            secondaryBtn: null,
+            leadingIcon: null,
+          });
+
+          throw agentDisconnectError;
+        }
+      }
+    } catch (error) {
+      console.log(`Agent failed to connect/disconnect - ${error}`);
+    }
+  };
+
+  const value = {
+    toggleAgentConnection: handleConnectionToggle,
+  };
+
+  return (
+    <AgentConnectionContext.Provider value={value}>
+      {children}
+    </AgentConnectionContext.Provider>
+  );
+};
+
+export const connectToAIAgent = async (
+  agentAction: 'start' | 'stop',
+  channel_name: string,
+  localUid: UidType,
+  agentAuthToken: string,
+  data?: {agent_id: string; prompt?: string; voice?: string},
+): Promise<{}> => {
+  // const apiUrl = '/api/proxy';
+  const apiUrl = $config.BACKEND_ENDPOINT + '/v1/convoai';
+  const requestBody = {
+    channel_name: channel_name,
+    uid: localUid, // user uid // localUid or 0
+  };
+
+  if (data && data?.agent_id) {
+    requestBody['ai_agent_id'] = data.agent_id;
+  }
+  if (data && data?.voice) {
+    requestBody['voice'] = data.voice;
+  }
+  if (data && data?.prompt) {
+    requestBody['prompt'] = data.prompt;
+  }
+
+  const headers: HeadersInit = {
+    'Content-Type': 'application/json',
+    Authorization: `Bearer ${agentAuthToken}`,
+  };
+
+  try {
+    const response = await fetch(`${apiUrl}/${agentAction}`, {
+      method: 'POST',
+      headers: headers,
+      body: JSON.stringify(requestBody),
+    });
+
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}`);
+    }
+
+    const data = await response.json();
+
+    console.log(
+      `AI agent ${agentAction === 'start' ? 'connected' : 'disconnected'}`,
+      data,
+    );
+    if (agentAction === 'start') {
+      return data;
+    }
+  } catch (error) {
+    console.error(`Failed to ${agentAction} AI agent connection:`, error);
+    throw error;
+  }
+};

package/template/src/ai-agent/components/AgentControls/AgentContext.tsx

@@ -1,7 +1,7 @@
 import React, {createContext, useState} from 'react';
 import {AIAgentState, AgentState} from './const';
 import {UidType} from 'customization-api';
-
+import {AI_AGENT_VOICE} from './const';
 export interface ChatItem {
   id: string;
   uid: UidType;
@@ -22,6 +22,12 @@ export interface AgentContextInterface {
   setAgentUID: (uid: UidType | null) => void;
   chatItems: ChatItem[];
   addChatItem: (newItem: ChatItem) => void;
+  agentId: string;
+  setAgentId: (id: string) => void;
+  agentVoice?: keyof typeof AI_AGENT_VOICE | '';
+  setAgentVoice: (voice: keyof typeof AI_AGENT_VOICE) => void;
+  prompt?: string;
+  setPrompt: (prompt: string) => void;
 }
 
 export const AgentContext = createContext<AgentContextInterface>({
@@ -35,6 +41,12 @@ export const AgentContext = createContext<AgentContextInterface>({
   setAgentUID: () => {},
   chatItems: [],
   addChatItem: () => {}, // Default no-op
+  agentVoice: '',
+  setAgentVoice: () => {},
+  agentId: '',
+  setAgentId: () => {},
+  prompt: '',
+  setPrompt: () => {},
 });
 
 /**
@@ -74,6 +86,10 @@ export const AgentProvider: React.FC<{children: React.ReactNode}> = ({
   const [agentUID, setAgentUID] = useState<UidType | null>(null);
   const [isSubscribedForStreams, setIsSubscribedForStreams] = useState(false);
   const [chatItems, setChatItems] = useState<ChatItem[]>([]);
+  const [agentId, setAgentId] = useState('');
+  const [agentVoice, setAgentVoice] =
+    useState<AgentContextInterface['agentVoice']>('');
+  const [prompt, setPrompt] = useState('');
 
   /**
    * Adds a new chat item to the chat state while ensuring:
@@ -155,6 +171,12 @@ export const AgentProvider: React.FC<{children: React.ReactNode}> = ({
     setAgentUID,
     chatItems,
     addChatItem, // Expose the function in the context
+    agentId,
+    setAgentId,
+    agentVoice,
+    setAgentVoice,
+    prompt,
+    setPrompt,
   };
 
   return (

package/template/src/ai-agent/components/AgentControls/const.ts

@@ -1,16 +1,16 @@
 import {isMobileUA} from '../../../utils/common';
 
 export const AI_AGENT_STATE = {
-  NOT_CONNECTED: 'Start Call',
-  REQUEST_SENT: isMobileUA() ? 'Start Call' : 'Requesting agent join..', // loading - reg
-  AWAITING_JOIN: isMobileUA() ? 'Start Call' : 'Agent will join shortly..', // loading
+  NOT_CONNECTED: 'Join Call',
+  REQUEST_SENT: isMobileUA() ? 'Join Call' : 'Requesting agent join..', // loading - reg
+  AWAITING_JOIN: isMobileUA() ? 'Join Call' : 'Agent will join shortly..', // loading
   AGENT_CONNECTED: 'End Call',
-  AGENT_REQUEST_FAILED: 'Start Call',
+  AGENT_REQUEST_FAILED: 'Join Call',
   AGENT_DISCONNECT_REQUEST: isMobileUA()
     ? 'End Call'
     : 'Disconnecting agent...', // loading - req
   AGENT_DISCONNECT_FAILED: 'End Call',
-  AWAITING_LEAVE: 'Start Call', // loading
+  AWAITING_LEAVE: 'Join Call', // loading
 } as const;
 
 export type AIAgentState = keyof typeof AI_AGENT_STATE;
@@ -56,3 +56,36 @@ export const AGORA_SSO_LOGIN_PATH = '/api/v0/oauth/authorize';
 export const AGORA_SSO_LOGOUT_PATH = '/api/v0/logout';
 
 export const AGORA_SSO_CLIENT_ID = 'openai_agora';
+
+export const AI_AGENT_VOICE = {
+  'en-US-AvaMultilingualNeural': 'en-US-AvaMultilingualNeural',
+  'en-US-AndrewMultilingualNeural': 'en-US-AndrewMultilingualNeural',
+  'en-US-EmmaMultilingualNeural': 'en-US-EmmaMultilingualNeural',
+  'en-US-BrianMultilingualNeural': 'en-US-BrianMultilingualNeural',
+  'en-US-AvaNeural': 'en-US-AvaNeural',
+  'en-US-AndrewNeural': 'en-US-AndrewNeural',
+  'en-US-EmmaNeural': 'en-US-EmmaNeural',
+  'en-US-BrianNeural': 'en-US-BrianNeural',
+  'en-US-JennyNeural': 'en-US-JennyNeural',
+  'en-US-GuyNeural': 'en-US-GuyNeural',
+  'en-US-AriaNeural': 'en-US-AriaNeural',
+  'en-US-DavisNeural': 'en-US-DavisNeural',
+  'en-US-JaneNeural': 'en-US-JaneNeural',
+  'en-US-JasonNeural': 'en-US-JasonNeural',
+  'en-US-SaraNeural': 'en-US-SaraNeural',
+  'en-US-TonyNeural': 'en-US-TonyNeural',
+  'en-US-NancyNeural': 'en-US-NancyNeural',
+  'en-US-AmberNeural': 'en-US-AmberNeural',
+  'en-US-AnaNeural': 'en-US-AnaNeural',
+  'en-US-AshleyNeural': 'en-US-AshleyNeural',
+  'en-US-BrandonNeural': 'en-US-BrandonNeural',
+  'en-US-ChristopherNeural': 'en-US-ChristopherNeural',
+  'en-US-CoraNeural': 'en-US-CoraNeural',
+  'en-US-ElizabethNeural': 'en-US-ElizabethNeural',
+  'en-US-EricNeural': 'en-US-EricNeural',
+  'en-US-JacobNeural': 'en-US-JacobNeural',
+  'en-US-JennyMultilingualNeural4': 'en-US-JennyMultilingualNeural4',
+  'en-US-MichelleNeural': 'en-US-MichelleNeural',
+  'en-US-MonicaNeural': 'en-US-MonicaNeural',
+  'en-US-RogerNeural': 'en-US-RogerNeural',
+};