agora-appbuilder-core 4.1.0-beta-8 → 4.1.0-beta-10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agora-appbuilder-core",
-  "version": "4.1.0-beta-8",
+  "version": "4.1.0-beta-10",
   "description": "React Native template for RTE app builder",
   "main": "index.js",
   "files": [

package/template/android/build.gradle

@@ -10,6 +10,8 @@ buildscript {
         // We use NDK 23 which has both M1 support and is the side-by-side NDK version from AGP.
         ndkVersion = "23.1.7779620"
         kotlinVersion = "1.8.21" // added this for datadog logs - https://docs.datadoghq.com/real_user_monitoring/mobile_and_tv_monitoring/setup/reactnative/#setup
+        androidXAnnotation = "1.2.0"
+        androidXBrowser = "1.3.0"
     }
     subprojects { subproject ->
         afterEvaluate{

package/template/defaultConfig.js

@@ -76,8 +76,8 @@ const DefaultConfig = {
   CHAT_ORG_NAME: '',
   CHAT_APP_NAME: '',
   CHAT_URL: '',
-  CLI_VERSION: '3.1.0-beta-8',
-  CORE_VERSION: '4.1.0-beta-8',
+  CLI_VERSION: '3.1.0-beta-10',
+  CORE_VERSION: '4.1.0-beta-10',
   DISABLE_LANDSCAPE_MODE: false,
   STT_AUTO_START: false,
   CLOUD_RECORDING_AUTO_START: false,
@@ -85,6 +85,7 @@ const DefaultConfig = {
   AUTO_CONNECT_RTM: false,
   ENABLE_CONVERSATIONAL_AI: false,
   CUSTOMIZE_AGENT: true,
+  AI_LAYOUT: 'DEFAULT_LAYOUT',
 };
 
 module.exports = DefaultConfig;

package/template/global.d.ts

@@ -172,6 +172,7 @@ interface ConfigInterface {
   AUTO_CONNECT_RTM: boolean;
   ENABLE_CONVERSATIONAL_AI: boolean;
   CUSTOMIZE_AGENT: boolean;
+  AI_LAYOUT: string;
 }
 declare var $config: ConfigInterface;
 declare module 'customization' {

package/template/package.json

@@ -57,7 +57,6 @@
     "@react-native-async-storage/async-storage": "1.19.2",
     "@react-native-community/checkbox": "0.5.16",
     "@react-native-community/clipboard": "1.5.1",
-    "@splinetool/react-spline": "^4.0.0",
     "@splinetool/runtime": "^1.9.69",
     "@supersami/rn-foreground-service": "^1.1.1",
     "add": "^2.0.6",
@@ -66,7 +65,7 @@
     "agora-extension-beauty-effect": "^1.0.2-beta",
     "agora-extension-virtual-background": "^1.1.3",
     "agora-react-native-rtm": "1.5.1",
-    "agora-rtc-sdk-ng": "4.23.0",
+    "agora-rtc-sdk-ng": "4.23.2",
     "agora-rtm-sdk": "1.5.1",
     "buffer": "^6.0.3",
     "electron-log": "4.3.5",
@@ -113,6 +112,7 @@
     "react-tooltip": "4.5.1",
     "rn-emoji-keyboard": "^1.7.0",
     "rn-fetch-blob": "0.12.0",
+    "text-encoding": "^0.7.0",
     "white-web-sdk": "2.16.42"
   },
   "devDependencies": {

package/template/src/ai-agent/components/AgentControls/AgentContext.tsx

@@ -10,6 +10,15 @@ import {
   Toast,
   useRtc,
 } from 'customization-api';
+import LocalEventEmitter, {
+  LocalEventsEnum,
+} from '../../../../src/rtm-events-api/LocalEvents';
+import {
+  messageService,
+  initializeMessageEngine,
+  closeMessageEngine,
+  IMessageListItem,
+} from './message';
 
 export interface ChatItem {
   id: string;
@@ -30,8 +39,6 @@ export interface AgentContextInterface {
   setIsSubscribedForStreams: (state: boolean) => void;
   agentUID: UidType | null;
   setAgentUID: (uid: UidType | null) => void;
-  chatItems: ChatItem[];
-  addChatItem: (newItem: ChatItem) => void;
   agentId: string;
   setAgentId: (id: string) => void;
   agentVoice?: keyof typeof AI_AGENT_VOICE | '';
@@ -42,6 +49,9 @@ export interface AgentContextInterface {
   setPrompt: (prompt: string) => void;
   isInterruptionHandlingEnabled: boolean;
   setIsInterruptionHandlingEnabled: (value: boolean) => void;
+  chatHistory: IMessageListItem[];
+  setChatHistory: (history: IMessageListItem[]) => void;
+  clearChatHistory: () => void;
 }
 
 export const AgentContext = createContext<AgentContextInterface>({
@@ -56,8 +66,6 @@ export const AgentContext = createContext<AgentContextInterface>({
   setIsSubscribedForStreams: () => {},
   agentUID: null,
   setAgentUID: () => {},
-  chatItems: [],
-  addChatItem: () => {}, // Default no-op
   agentVoice: '',
   setAgentVoice: () => {},
   agentId: '',
@@ -68,6 +76,9 @@ export const AgentContext = createContext<AgentContextInterface>({
   setIsInterruptionHandlingEnabled: () => {},
   language: '',
   setLanguage: () => {},
+  chatHistory: [],
+  setChatHistory: () => {},
+  clearChatHistory: () => {},
 });
 
 /**
@@ -106,7 +117,7 @@ export const AgentProvider: React.FC<{children: React.ReactNode}> = ({
   const [agentAuthToken, setAgentAuthToken] = useState<string | null>(null);
   const [agentUID, setAgentUID] = useState<UidType | null>(null);
   const [isSubscribedForStreams, setIsSubscribedForStreams] = useState(false);
-  const [chatItems, setChatItems] = useState<ChatItem[]>([]);
+  const [chatHistory, setChatHistory] = useState<IMessageListItem[]>([]);
   const [agentId, setAgentId] = useState('');
   const [agentVoice, setAgentVoice] =
     useState<AgentContextInterface['agentVoice']>('');
@@ -140,108 +151,32 @@ export const AgentProvider: React.FC<{children: React.ReactNode}> = ({
 
   React.useEffect(() => {
     if (!isSubscribedForStreams) {
-      RtcEngineUnsafe.addListener(
-        'onStreamMessage',
-        handleStreamMessageCallback,
-      );
+      RtcEngineUnsafe.addListener('onStreamMessage', (...args: any[]) => {
+        messageService?.handleStreamMessage(args[1]);
+      });
       setIsSubscribedForStreams(true);
     }
   }, []);
 
-  const handleStreamMessageCallback = (...args) => {
-    console.log('rec', args);
-    parseData(args[1]);
-  };
-
-  const parseData = data => {
-    let decoder = new TextDecoder('utf-8');
-    let decodedMessage = decoder.decode(data);
-    console.log('[test] textstream raw data', decodedMessage);
-    handleChunk(decodedMessage);
-  };
-  // Function to process received chunk via event emitter
-  const handleChunk = (formattedChunk: string) => {
-    try {
-      // Split the chunk by the delimiter "|"
-      const [message_id, partIndexStr, totalPartsStr, content] =
-        formattedChunk.split('|');
-
-      const part_index = parseInt(partIndexStr, 10);
-      const total_parts =
-        totalPartsStr === '???' ? -1 : parseInt(totalPartsStr, 10); // -1 means total parts unknown
-
-      // Ensure total_parts is known before processing further
-      if (total_parts === -1) {
-        console.warn(
-          `Total parts for message ${message_id} unknown, waiting for further parts.`,
-        );
-        return;
-      }
-
-      const chunkData = {
-        message_id,
-        part_index,
-        total_parts,
-        content,
-      };
-
-      // Check if we already have an entry for this message
-      if (!messageCache[message_id]) {
-        messageCache[message_id] = [];
-        // Set a timeout to discard incomplete messages
-        setTimeout(() => {
-          if (messageCache[message_id]?.length !== total_parts) {
-            console.warn(`Incomplete message with ID ${message_id} discarded`);
-            delete messageCache[message_id]; // Discard incomplete message
-          }
-        }, TIMEOUT_MS);
-      }
-
-      // Cache this chunk by message_id
-      messageCache[message_id].push(chunkData);
-
-      // If all parts are received, reconstruct the message
-      if (messageCache[message_id].length === total_parts) {
-        const completeMessage = reconstructMessage(messageCache[message_id]);
-        const data = atob(completeMessage);
-        const {stream_id, is_final, text, text_ts} = JSON.parse(data);
-        /** Data type of above object
-         * stream_id: number
-         * is_final: boolean
-         * text: string
-         * text_ts: number
-         */
-        const textItem = {
-          id: message_id,
-          uid: stream_id,
-          time: text_ts,
-          dataType: 'transcribe',
-          text: text,
-          isFinal: is_final,
-          isSelf: stream_id === 0 ? false : true,
-        };
-
-        if (text.trim().length > 0) {
-          //this.emit("textChanged", textItem);
-          console.warn('emit textChanged: ', textItem);
-          addChatItem(textItem);
-        }
-
-        // Clean up the cache
-        delete messageCache[message_id];
+  React.useEffect(() => {
+    const getChatHistoryFromEvent = (event: MessageEvent) => {
+      const {data} = event;
+      console.log('get chat history from event', data);
+      if (data.type === 'message') {
+        setChatHistory(prevChatHistory => [...(data?.chatHistory || [])]);
       }
-    } catch (error) {
-      console.error('Error processing chunk:', error);
-    }
-  };
-
-  const reconstructMessage = chunks => {
-    // Sort chunks by their part index
-    chunks.sort((a, b) => a.part_index - b.part_index);
-
-    // Concatenate all chunks to form the full message
-    return chunks.map(chunk => chunk.content).join('');
-  };
+    };
+    LocalEventEmitter.on(
+      LocalEventsEnum.AGENT_TRANSCRIPT_CHANGE,
+      getChatHistoryFromEvent,
+    );
+    return () => {
+      LocalEventEmitter.off(
+        LocalEventsEnum.AGENT_TRANSCRIPT_CHANGE,
+        getChatHistoryFromEvent,
+      );
+    };
+  }, []);
 
   useEffect(() => {
     console.log(
@@ -266,6 +201,7 @@ export const AgentProvider: React.FC<{children: React.ReactNode}> = ({
         agentConnectionState !== AgentState.AGENT_CONNECTED)
     ) {
       setAgentConnectionState(AgentState.AGENT_CONNECTED);
+      initializeMessageEngine();
       if (isStartAPICalled) {
         setStartAPICalled(false);
       }
@@ -294,6 +230,7 @@ export const AgentProvider: React.FC<{children: React.ReactNode}> = ({
         !aiAgentUID.length &&
         agentConnectionState !== AgentState.NOT_CONNECTED)
     ) {
+      closeMessageEngine(); // release message engine
       setAgentConnectionState(AgentState.NOT_CONNECTED);
       if (isStopAPICalled) {
         setStartAPICalled(true);
@@ -307,6 +244,10 @@ export const AgentProvider: React.FC<{children: React.ReactNode}> = ({
     isStopAPICalled,
   ]);
 
+  const clearChatHistory = () => {
+    setChatHistory([]);
+  };
+
   const handleConnectionToggle = async (forceStop: boolean = false) => {
     try {
       // connect to agent when agent is in not connected state or when earlier connect failed
@@ -324,7 +265,7 @@ export const AgentProvider: React.FC<{children: React.ReactNode}> = ({
             localUid,
             store.token,
             {
-              agent_id: agentId,
+              agent_id: agentId || agents?.length ? agents[0].id : null,
               prompt: prompt,
               voice: agents.find(a => a.id === agentId)?.config?.tts?.params
                 ?.voice_name,
@@ -400,7 +341,7 @@ export const AgentProvider: React.FC<{children: React.ReactNode}> = ({
           setAgentConnectionState(AgentState.AGENT_DISCONNECT_REQUEST);
           setStopAPICalled(true);
           await connectToAIAgent('stop', channel_name, localUid, store.token, {
-            agent_id: agentId,
+            agent_id: agentId || agents?.length ? agents[0].id : null,
           });
           setStore(prevState => {
             return {
@@ -444,75 +385,6 @@ export const AgentProvider: React.FC<{children: React.ReactNode}> = ({
     }
   };
 
-  /**
-   * Adds a new chat item to the chat state while ensuring:
-   * - Outdated messages are discarded.
-   * - Non-finalized messages are updated if a newer message is received.
-   * - Finalized messages are added without duplication.
-   * - Chat items remain sorted by their `time` property.
-   *
-   * @param newItem The new chat item to add.
-   */
-  const addChatItem = (newItem: ChatItem) => {
-    setChatItems(prevItems => {
-      // Find the index of the last finalized chat item for the same user
-      // Finalized messages are typically considered "complete" and should not be updated by non-final messages
-      const LastFinalIndex = prevItems.findLastIndex(
-        el => el.uid === newItem.uid && el.isFinal,
-      );
-
-      // Find the index of the last non-finalized chat item for the same user
-      // Non-finalized messages represent "in-progress" messages that can be updated or replaced
-      const LastNonFinalIndex = prevItems.findLastIndex(
-        el => el.uid === newItem.uid && !el.isFinal,
-      );
-
-      // Retrieve the actual items for the indices found above
-      const LastFinalItem =
-        LastFinalIndex !== -1 ? prevItems[LastFinalIndex] : null;
-      const LastNonFinalItem =
-        LastNonFinalIndex !== -1 ? prevItems[LastNonFinalIndex] : null;
-
-      // If the new message's timestamp is older than or equal to the last finalized message,
-      // it is considered outdated and discarded to prevent unnecessary overwrites.
-      if (LastFinalItem && newItem.time <= LastFinalItem.time) {
-        console.log(
-          '[AgentProvider] addChatItem - Discarded outdated message:',
-          newItem,
-        );
-        return prevItems; // Return the previous state without changes
-      }
-
-      // Create a new copy of the current chat items to maintain immutability
-      let updatedItems = [...prevItems];
-
-      // If there is a non-finalized message for the same user, replace it with the new message
-      if (LastNonFinalItem) {
-        console.log(
-          '[AgentProvider] addChatItem - Updating non-finalized message:',
-          newItem,
-        );
-        updatedItems[LastNonFinalIndex] = newItem; // Replace the non-finalized message
-      } else {
-        // If no non-finalized message exists, the new message is added to the array
-        console.log(
-          '[AgentProvider] addChatItem - Adding new message:',
-          newItem,
-        );
-
-        // Use binary search to find the correct insertion index for the new message
-        // This ensures the array remains sorted by the `time` property
-        const insertIndex = findInsertionIndex(updatedItems, newItem.time);
-
-        // Insert the new message at the correct position to maintain chronological order
-        updatedItems.splice(insertIndex, 0, newItem);
-      }
-
-      // Return the updated array, which will replace the previous state
-      return updatedItems;
-    });
-  };
-
   const value = {
     toggleAgentConnection: handleConnectionToggle,
     agentConnectionState,
@@ -523,8 +395,6 @@ export const AgentProvider: React.FC<{children: React.ReactNode}> = ({
     setIsSubscribedForStreams,
     agentUID,
     setAgentUID,
-    chatItems,
-    addChatItem, // Expose the function in the context
     agentId,
     setAgentId,
     agentVoice,
@@ -535,6 +405,9 @@ export const AgentProvider: React.FC<{children: React.ReactNode}> = ({
     setIsInterruptionHandlingEnabled,
     language,
     setLanguage,
+    setChatHistory,
+    chatHistory,
+    clearChatHistory,
   };
 
   return (