agora-appbuilder-core 4.0.35 → 4.1.0-beta-2

package/template/src/ai-agent/components/AgentControls/const.ts

@@ -0,0 +1,58 @@
+import {isMobileUA} from '../../../utils/common';
+
+export const AI_AGENT_STATE = {
+  NOT_CONNECTED: 'Start Call',
+  REQUEST_SENT: isMobileUA() ? 'Start Call' : 'Requesting agent join..', // loading - reg
+  AWAITING_JOIN: isMobileUA() ? 'Start Call' : 'Agent will join shortly..', // loading
+  AGENT_CONNECTED: 'End Call',
+  AGENT_REQUEST_FAILED: 'Start Call',
+  AGENT_DISCONNECT_REQUEST: isMobileUA()
+    ? 'End Call'
+    : 'Disconnecting agent...', // loading - req
+  AGENT_DISCONNECT_FAILED: 'End Call',
+  AWAITING_LEAVE: 'Start Call', // loading
+} as const;
+
+export type AIAgentState = keyof typeof AI_AGENT_STATE;
+
+export const AGENT_STATE_TO_API_ACTION = {
+  NOT_CONNECTED: 'start_agent',
+  AGENT_CONNECTED: 'stop_agent',
+} as const;
+
+export type AgentStateToApiAction = keyof typeof AGENT_STATE_TO_API_ACTION;
+
+export const enum AgentState {
+  NOT_CONNECTED = 'NOT_CONNECTED',
+  REQUEST_SENT = 'REQUEST_SENT',
+  AWAITING_JOIN = 'AWAITING_JOIN',
+  AGENT_CONNECTED = 'AGENT_CONNECTED',
+  AGENT_REQUEST_FAILED = 'AGENT_REQUEST_FAILED',
+  AGENT_DISCONNECT_REQUEST = 'AGENT_DISCONNECT_REQUEST',
+  AGENT_DISCONNECT_FAILED = 'AGENT_DISCONNECT_FAILED',
+  AWAITING_LEAVE = 'AWAITING_LEAVE',
+}
+
+export const AI_AGENT_UID = 123456;
+
+// export const AGENT_PROXY_URL = "http://localhost:3000/api/proxy"
+// export const AGENT_PROXY_URL = "https://conversational-ai-agent-git-testing-cors-agoraio.vercel.app/api/proxy"
+// export const AGENT_PROXY_URL = "https://nodejs-serverless-function-express-alpha-smoky.vercel.app/api/hello"
+// export const AGENT_PROXY_URL = "https://conversational-ai-agent-git-setmute-agoraio.vercel.app/api/proxy"
+
+// production router
+export const AGENT_PROXY_URL =
+  'https://agora-realtime-proxy-590d34bfeb04.herokuapp.com';
+// production sso
+export const AGORA_SSO_BASE = 'https://sso2.agora.io';
+
+// staging router
+// export const AGENT_PROXY_URL = "https://agora-realtime-proxy-dev-0af5192e12dd.herokuapp.com"
+// staging sso
+// export const AGORA_SSO_BASE = 'https://staging-sso.agora.io';
+
+export const AGORA_SSO_LOGIN_PATH = '/api/v0/oauth/authorize';
+
+export const AGORA_SSO_LOGOUT_PATH = '/api/v0/logout';
+
+export const AGORA_SSO_CLIENT_ID = 'openai_agora';

package/template/src/ai-agent/components/AgentControls/index.tsx

@@ -0,0 +1,293 @@
+import React, {useContext, useEffect} from 'react';
+import {AI_AGENT_STATE, AgentState} from './const';
+import {TouchableOpacity, Text, ActivityIndicator} from 'react-native';
+import {AgentContext} from './AgentContext';
+import {
+  ThemeConfig,
+  UidType,
+  useContent,
+  useEndCall,
+  useLocalUid,
+  useHistory,
+  useStorageContext,
+  Toast,
+} from 'customization-api';
+import {isMobileUA} from '../../../utils/common';
+import {CallIcon, EndCall} from '../icons';
+
+const connectToAIAgent = async (
+  agentAction: 'start' | 'stop',
+  channel_name: string,
+  localUid: UidType,
+  agentAuthToken: string,
+): Promise<{}> => {
+  // const apiUrl = '/api/proxy';
+  const apiUrl = $config.BACKEND_ENDPOINT + '/v1/convoai';
+  const requestBody = {
+    channel_name: channel_name,
+    uid: localUid, // user uid // localUid or 0
+  };
+  console.log({requestBody});
+  const headers: HeadersInit = {
+    'Content-Type': 'application/json',
+    Authorization: `Bearer ${agentAuthToken}`,
+  };
+
+  try {
+    const response = await fetch(`${apiUrl}/${agentAction}`, {
+      method: 'POST',
+      headers: headers,
+      body: JSON.stringify(requestBody),
+    });
+
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}`);
+    }
+
+    const data = await response.json();
+
+    // console.log({data}, "X-Client-ID start stop")
+    console.log(
+      `AI agent ${agentAction === 'start' ? 'connected' : 'disconnected'}`,
+      data,
+    );
+    if (agentAction === 'start') {
+      return data;
+    }
+  } catch (error) {
+    console.error(`Failed to ${agentAction} AI agent connection:`, error);
+    throw error;
+  }
+};
+
+export const AgentControl: React.FC<{channel_name: string}> = ({
+  channel_name,
+}) => {
+  const {
+    agentConnectionState,
+    setAgentConnectionState,
+    agentAuthToken,
+    setAgentAuthToken,
+    agentUID,
+    setAgentUID,
+  } = useContext(AgentContext);
+  // console.log("X-Client-ID state", clientId)
+  // const { users } = useContext(UserContext)
+  const {activeUids: users} = useContent();
+  const endcall = useEndCall();
+  const history = useHistory();
+  const {store} = useStorageContext();
+  const localUid = useLocalUid();
+
+  // stop_agent API is successful, but agent has not yet left the RTC channel
+  const isAwaitingLeave = agentConnectionState === AgentState.AWAITING_LEAVE;
+
+  console.log(
+    'Agent Control--',
+    {agentConnectionState},
+    {bth: AI_AGENT_STATE[agentConnectionState]},
+  );
+
+  const handleConnectionToggle = async () => {
+    try {
+      // connect to agent when agent is in not connected state or when earlier connect failed
+      if (
+        agentConnectionState === AgentState.NOT_CONNECTED ||
+        agentConnectionState === AgentState.AGENT_REQUEST_FAILED ||
+        isAwaitingLeave
+      ) {
+        try {
+          setAgentConnectionState(AgentState.REQUEST_SENT);
+          const data = await connectToAIAgent(
+            'start',
+            channel_name,
+            localUid,
+            store.token,
+          );
+          // console.log("response X-Client-ID", newClientId, typeof newClientId)
+          // @ts-ignore
+          const {agent_uid = null} = data;
+
+          //setClientId(agent_id);
+          setAgentUID(agent_uid);
+
+          setAgentConnectionState(AgentState.AWAITING_JOIN);
+
+          Toast.show({
+            leadingIconName: 'tick-fill',
+            type: 'success',
+            text1: 'Agent requested to join',
+            text2: null,
+            visibilityTime: 3000,
+            primaryBtn: null,
+            secondaryBtn: null,
+            leadingIcon: null,
+          });
+        } catch (agentConnectError) {
+          setAgentConnectionState(AgentState.AGENT_REQUEST_FAILED);
+
+          if (agentConnectError.toString().indexOf('401') !== -1) {
+            Toast.show({
+              leadingIconName: 'alert',
+              type: 'error',
+              text1: 'Your session is expired. Please sign in to join call.',
+              text2: null,
+              visibilityTime: 5000,
+              primaryBtn: null,
+              secondaryBtn: null,
+              leadingIcon: null,
+            });
+            // window.location.href = '/create'
+            await endcall();
+            setAgentAuthToken(null);
+            return;
+          } else {
+            Toast.show({
+              leadingIconName: 'alert',
+              type: 'error',
+              text1: 'Uh oh! Agent failed to connect',
+              text2: null,
+              visibilityTime: 5000,
+              primaryBtn: null,
+              secondaryBtn: null,
+              leadingIcon: null,
+            });
+          }
+
+          throw agentConnectError;
+        }
+      }
+      // disconnect agent with agent is already connected or when earlier disconnect failed
+      if (
+        agentConnectionState === AgentState.AGENT_CONNECTED ||
+        agentConnectionState === AgentState.AGENT_DISCONNECT_FAILED
+      ) {
+        if (isMobileUA()) {
+          await endcall();
+          setAgentConnectionState(AgentState.NOT_CONNECTED);
+          setAgentAuthToken(null);
+          return; // check later
+        }
+        try {
+          setAgentConnectionState(AgentState.AGENT_DISCONNECT_REQUEST);
+          await connectToAIAgent('stop', channel_name, localUid, store.token);
+          setAgentConnectionState(AgentState.AWAITING_LEAVE);
+
+          Toast.show({
+            leadingIconName: 'tick-fill',
+            type: 'success',
+            text1: 'Agent disconnected',
+            text2: null,
+            visibilityTime: 3000,
+            primaryBtn: null,
+            secondaryBtn: null,
+            leadingIcon: null,
+          });
+        } catch (agentDisconnectError) {
+          setAgentConnectionState(AgentState.AGENT_DISCONNECT_FAILED);
+
+          Toast.show({
+            leadingIconName: 'alert',
+            type: 'error',
+            text1: 'Uh oh! Agent failed to disconnect',
+            text2: null,
+            visibilityTime: 5000,
+            primaryBtn: null,
+            secondaryBtn: null,
+            leadingIcon: null,
+          });
+
+          throw agentDisconnectError;
+        }
+      }
+    } catch (error) {
+      console.log(`Agent failed to connect/disconnect - ${error}`);
+    }
+  };
+
+  useEffect(() => {
+    console.log('agent contrl', {users});
+    // welcome agent
+    const aiAgentUID = users.filter(item => item === agentUID);
+    if (
+      aiAgentUID.length &&
+      agentConnectionState === AgentState.AWAITING_JOIN
+    ) {
+      setAgentConnectionState(AgentState.AGENT_CONNECTED);
+
+      Toast.show({
+        leadingIconName: 'tick-fill',
+        type: 'success',
+        text1: 'Say Hi!!',
+        text2: null,
+        visibilityTime: 3000,
+        primaryBtn: null,
+        secondaryBtn: null,
+        leadingIcon: null,
+      });
+    }
+    // when agent leaves, show left toast, and set agent to not connected state
+    if (
+      !aiAgentUID.length &&
+      agentConnectionState === AgentState.AWAITING_LEAVE
+    ) {
+      setAgentConnectionState(AgentState.NOT_CONNECTED);
+    }
+  }, [users]);
+
+  const isLoading =
+    agentConnectionState === AgentState.REQUEST_SENT ||
+    agentConnectionState === AgentState.AGENT_DISCONNECT_REQUEST ||
+    // || agentConnectionState === AgentState.AWAITING_LEAVE
+    agentConnectionState === AgentState.AWAITING_JOIN;
+
+  const isStartAgent =
+    agentConnectionState === AgentState.NOT_CONNECTED ||
+    agentConnectionState === AgentState.AGENT_REQUEST_FAILED ||
+    isAwaitingLeave;
+  const isEndAgent =
+    agentConnectionState === AgentState.AGENT_CONNECTED ||
+    agentConnectionState === AgentState.AGENT_DISCONNECT_FAILED;
+
+  const backgroundColorStyle = isMobileUA()
+    ? {backgroundColor: isEndAgent ? '#FF414D' : '#00C2FF', height: 72}
+    : {};
+  const fontcolorStyle = isMobileUA()
+    ? {color: '#FFF'}
+    : {color: isEndAgent ? '#FF414D' : '#00C2FF'};
+  return (
+    <TouchableOpacity
+      style={{
+        display: 'flex',
+        height: 48,
+        padding: 20,
+        justifyContent: 'space-between',
+        alignItems: 'center',
+        gap: 8,
+        borderRadius: 40,
+        borderWidth: isMobileUA() ? 0 : 1,
+        borderColor: isEndAgent ? '#FF414D' : '#00C2FF',
+        flexDirection: 'row',
+        ...backgroundColorStyle,
+      }}
+      onPress={handleConnectionToggle}
+      disabled={isLoading}>
+      {isLoading ? (
+        <ActivityIndicator
+          size="small"
+          color={isMobileUA() ? '#FFFFFF' : '#00C2FF'}
+        />
+      ) : isStartAgent ? (
+        <CallIcon fill={isMobileUA() ? '#FFFFFF' : '#00C2FF'} />
+      ) : (
+        <EndCall fill={isMobileUA() ? '#FFFFFF' : '#FF414D'} />
+      )}
+
+      <Text
+        style={{
+          fontFamily: ThemeConfig.FontFamily.sansPro,
+          ...fontcolorStyle,
+        }}>{`${AI_AGENT_STATE[agentConnectionState]}`}</Text>
+    </TouchableOpacity>
+  );
+};

package/template/src/ai-agent/components/AudioVisualizer.tsx

@@ -0,0 +1,91 @@
+import React from 'react';
+import {Text, View} from 'react-native';
+import {LiveAudioVisualizer} from './react-audio-visualize';
+import {DisconnectedIconDesktop, DisconnectedIconMobile} from './icons';
+import {isMobileUA} from '../../utils/common';
+
+export const DisconnectedView = ({isConnected}) => {
+  return (
+    <View
+      style={{
+        flex: 1,
+        backgroundColor: '#222',
+        display: 'flex',
+        alignItems: 'center',
+        justifyContent: 'center',
+      }}>
+      {/* big circle that covers the parent view */}
+      {isMobileUA() ? <DisconnectedIconMobile /> : <DisconnectedIconDesktop />}
+      <Text
+        style={{
+          color: '#B3B3B3',
+          fontSize: 20,
+          fontWeight: '400',
+          marginTop: 20,
+        }}>
+        {isConnected ? '' : 'Not Joined'}
+      </Text>
+    </View>
+  );
+};
+
+function createSilentAudioTrack(): MediaStreamTrack {
+  const audioContext = new (window.AudioContext ||
+    (window as any).webkitAudioContext)();
+  const oscillator = audioContext.createOscillator();
+  const dst = audioContext.createMediaStreamDestination();
+  oscillator.connect(dst);
+  oscillator.start();
+  const track = dst.stream.getAudioTracks()[0];
+  return Object.assign(track, {enabled: false});
+}
+
+const emptyAudioTrack = {
+  getMediaStreamTrack: () => createSilentAudioTrack(),
+  getVolumeLevel: () => 0,
+  setVolume: () => {},
+  setEnabled: () => {},
+  play: () => {},
+  stop: () => {},
+  setPlaybackDevice: () => Promise.resolve(),
+  getStats: () => ({
+    receiveBytes: 0,
+    receivePackets: 0,
+    receivePacketsLost: 0,
+    state: 'stopped',
+  }),
+  isPlaying: false,
+  processTrack: {
+    on: () => {},
+    off: () => {},
+  },
+};
+
+const AudioVisualizer = ({audioTrack}) => {
+  return (
+    <View
+      style={{
+        flex: 1,
+        backgroundColor: '#222',
+        display: 'flex',
+        alignItems: 'center',
+        justifyContent: 'center',
+      }}>
+      <LiveAudioVisualizer
+        audioTrack={audioTrack || emptyAudioTrack}
+        width={300}
+        height={400}
+        fftSize={32}
+        barWidth={10}
+        minDecibels={-60}
+        maxDecibels={-10}
+        gap={2}
+        backgroundColor="transparent"
+        barColor="#00C2FF"
+        smoothingTimeConstant={0.9}
+      />
+    </View>
+  );
+};
+
+export default AudioVisualizer;

package/template/src/ai-agent/components/Bottombar.tsx

@@ -0,0 +1,91 @@
+import {
+  ToolbarPreset,
+  ToolbarComponents,
+  useSidePanel,
+  useRoomInfo,
+} from 'customization-api';
+import ThemeConfig from '../../theme';
+import {isMobileUA} from '../../utils/common';
+import React, {useEffect, useState} from 'react';
+import {Text, View} from 'react-native';
+import {AgentControl} from './AgentControls';
+import {LogoIcon} from './icons';
+
+export const LogoComponent = () => {
+  return (
+    <View
+      style={{
+        flexDirection: 'row',
+        alignItems: 'center',
+        justifyContent: 'center',
+        gap: 8,
+        marginRight: 20,
+      }}>
+      <LogoIcon />
+      <Text
+        style={{
+          color: '#C3C3C3',
+          textAlign: 'center',
+          fontSize: 18,
+          fontStyle: 'normal',
+          fontWeight: '600',
+          lineHeight: 18,
+          fontFamily: ThemeConfig.FontFamily.sansPro,
+        }}>
+        AI Builder Demo
+      </Text>
+    </View>
+  );
+};
+
+const Bottombar = () => {
+  const {MeetingTitleToolbarItem, ParticipantCountToolbarItem} =
+    ToolbarComponents;
+  const {setSidePanel} = useSidePanel();
+  const {data} = useRoomInfo();
+  const [clientId, setClientId] = useState<string | null>(null);
+  useEffect(() => {
+    !isMobileUA() && setSidePanel('agent-transcript-panel');
+  }, []);
+  return (
+    <ToolbarPreset
+      align="bottom"
+      items={{
+        layout: {hide: true},
+        invite: {hide: true},
+        more: {hide: true},
+        logo: {
+          align: 'start',
+          order: 0,
+          component: () => <LogoComponent />,
+        },
+        'meeting-title': {
+          align: 'start',
+          component: MeetingTitleToolbarItem,
+          order: 1,
+          hide: true,
+        },
+        'participant-count': {
+          align: 'start',
+          component: ParticipantCountToolbarItem,
+          order: 2,
+          hide: true,
+        },
+
+        'connect-agent': {
+          align: 'end',
+          label: 'Agent',
+          component: () => <AgentControl channel_name={data.channel} />,
+          order: 3,
+        },
+        'local-video': {hide: true},
+        screenshare: {hide: true},
+        recording: {hide: true},
+        'local-audio': {align: 'end', order: 1},
+        'end-call': {align: 'end', order: 2, hide: true},
+      }}
+    />
+  );
+};
+
+export default Bottombar;