agora-appbuilder-core 4.1.10-beta.1 → 4.1.11

package/template/src/subComponents/caption/useStreamMessageUtils.ts

@@ -1,4 +1,3 @@
-import React from 'react';
 import {useCaption} from './useCaption';
 import protoRoot from './proto/ptoto';
 import PQueue from 'p-queue';
@@ -7,6 +6,16 @@ type StreamMessageCallback = (args: [number, Uint8Array]) => void;
 type FinalListType = {
   [key: string]: string[];
 };
+type TranslationData = {
+  lang: string;
+  text: string;
+  isFinal: boolean;
+};
+type FinalTranslationListType = {
+  [key: string]: {
+    [lang: string]: string[];
+  };
+};
 
 const useStreamMessageUtils = (): {
   streamMessageCallback: StreamMessageCallback;
@@ -16,30 +25,34 @@ const useStreamMessageUtils = (): {
     setMeetingTranscript,
     activeSpeakerRef,
     prevSpeakerRef,
+    // Use ref instead of state to avoid stale closure issues
+    // The ref always has the current value, even in callbacks created at mount time
+    selectedTranslationLanguageRef,
   } = useCaption();
 
   let captionStartTime: number = 0;
   const finalList: FinalListType = {};
   const finalTranscriptList: FinalListType = {};
+  const finalTranslationList: FinalTranslationListType = {};
   const queue = new PQueue({concurrency: 1});
-  let counter = 0;
-  let lastOfftime = 0;
 
   const streamMessageCallback: StreamMessageCallback = args => {
     const queueCallback = (args1: [number, Uint8Array]) => {
       /* uid - bot which sends stream message in channel
        payload - stream message in Uint8Array format
       */
-      const [, payload] = args1;
+      const [botUid, payload] = args1;
       let nonFinalText = ''; // holds intermediate results
       let finalText = ''; // holds final strings
       let currentFinalText = ''; // holds current caption
       let isInterjecting = false;
+      let translations: TranslationData[] = []; // holds translation data
 
       const textstream = protoRoot
-        .lookupType('Text')
+        .lookupType('agora.audio2text.Text')
         .decode(payload as Uint8Array) as any;
 
+      console.log('[STT_PER_USER_BOT] stt v7 textstream', botUid, textstream);
       //console.log('STT - Parsed Textstream : ', textstream);
       // console.log(
       //   `STT-callback(${++counter}): %c${textstream.uid} %c${textstream.words
@@ -84,7 +97,9 @@ const useStreamMessageUtils = (): {
 
     */
 
-      const finalWord = textstream.words.filter(word => word.isFinal === true);
+      const finalWord = textstream.words.filter(
+        (word: any) => word.isFinal === true,
+      );
       // when we only get final word for the previous speaker then don't flip previous speaker as active but update in place.
 
       if (
@@ -94,6 +109,14 @@ const useStreamMessageUtils = (): {
         // we have a speaker change so clear the context for prev speaker
         if (prevSpeakerRef.current !== '') {
           finalList[prevSpeakerRef.current] = [];
+          // Clear translations for previous speaker
+          if (finalTranslationList[prevSpeakerRef.current]) {
+            Object.keys(finalTranslationList[prevSpeakerRef.current]).forEach(
+              lang => {
+                finalTranslationList[prevSpeakerRef.current][lang] = [];
+              },
+            );
+          }
           isInterjecting = true;
           // console.log(
           //   '%cSTT-callback%c Interjection! ',
@@ -114,16 +137,63 @@ const useStreamMessageUtils = (): {
         finalTranscriptList[textstream.uid] = [];
       }
 
+      // Process translations if available
+      if (textstream.trans && textstream.trans.length > 0) {
+        for (const trans of textstream.trans) {
+          const lang = trans.lang;
+          const texts = trans.texts || [];
+          const isFinal = trans.isFinal || false;
+
+          if (!finalTranslationList[textstream.uid]) {
+            finalTranslationList[textstream.uid] = {};
+          }
+          if (!finalTranslationList[textstream.uid][lang]) {
+            finalTranslationList[textstream.uid][lang] = [];
+          }
+
+          const currentTranslationText = texts.join(' ');
+          if (currentTranslationText) {
+            if (isFinal) {
+              finalTranslationList[textstream.uid][lang].push(
+                currentTranslationText,
+              );
+            }
+
+            // Build complete translation text (final + current non-final)
+            const existingTranslationBuffer = isInterjecting
+              ? ''
+              : finalTranslationList[textstream.uid][lang]?.join(' ');
+            const latestTranslationString = isFinal
+              ? ''
+              : currentTranslationText;
+            const completeTranslationText =
+              existingTranslationBuffer.length > 0
+                ? latestTranslationString
+                  ? existingTranslationBuffer + ' ' + latestTranslationString
+                  : existingTranslationBuffer
+                : latestTranslationString;
+
+            if (completeTranslationText || isFinal) {
+              translations.push({
+                lang,
+                text: completeTranslationText,
+                isFinal,
+              });
+            }
+          }
+        }
+      }
+
       const words = textstream.words; //[Word,Word]
 
       /* categorize words into final & nonFinal objects per uid
-      Final Word Ex : {
-      "text": "Hello, are you doing?",
-      "durationMs": 960,
-      "isFinal": true,
-      "confidence": 0.8549408316612244
-      }
-  */
+        Final Word Ex : {
+        "text": "Hello, are you doing?",
+        "durationMs": 960,
+        "isFinal": true,
+        "confidence": 0.8549408316612244
+        }
+      */
 
       for (const word of words) {
         if (word.isFinal) {
@@ -150,7 +220,29 @@ const useStreamMessageUtils = (): {
       }
 
       /* Updating Meeting Transcript */
-      if (currentFinalText.length) {
+      // Update transcript when: (1) new text finalized OR (2) final translations arrived
+      const hasFinalTranslations = textstream.trans?.some(
+        (t: any) => t.isFinal === true,
+      );
+
+      if (currentFinalText.length || hasFinalTranslations) {
+        //  final translations for transcript - include ALL available final translations for this user
+        const finalTranslationsForTranscript: TranslationData[] = [];
+        if (finalTranslationList[textstream.uid]) {
+          Object.keys(finalTranslationList[textstream.uid]).forEach(lang => {
+            const translationText =
+              finalTranslationList[textstream.uid][lang]?.join(' ') || '';
+
+            if (translationText) {
+              finalTranslationsForTranscript.push({
+                lang: lang,
+                text: translationText,
+                isFinal: true,
+              });
+            }
+          });
+        }
+
         setMeetingTranscript(prevTranscript => {
           const lastTranscriptIndex = prevTranscript.length - 1;
           const lastTranscript =
@@ -160,21 +252,30 @@ const useStreamMessageUtils = (): {
 
           /*
             checking if the last item transcript matches with current uid
-            If yes then updating the last transcript msg with current text
-            If no then adding a new entry in the transcript
+            If yes then updating the last transcript msg with current text and translations
+            If no and we have new text, then adding a new entry in the transcript
+            If no new text and no matching uid, don't add empty item
           */
           if (lastTranscript && lastTranscript.uid === textstream.uid) {
+            // Update existing transcript item (text + translations or just translations)
             const updatedTranscript = {
               ...lastTranscript,
               //text: lastTranscript.text + ' ' + currentFinalText, // missing few updates with reading prev values
-              text: finalTranscriptList[textstream.uid].join(' '),
+              text: currentFinalText.length
+                ? finalTranscriptList[textstream.uid].join(' ')
+                : lastTranscript.text, // Keep existing text if no new text
+              translations: finalTranslationsForTranscript,
+              // preserve the original translation language from when this transcript was created
+              selectedTranslationLanguage:
+                lastTranscript.selectedTranslationLanguage,
             };
 
             return [
               ...prevTranscript.slice(0, lastTranscriptIndex),
               updatedTranscript,
             ];
-          } else {
+          } else if (currentFinalText.length) {
+            // Only create new item if we have new text
             finalTranscriptList[textstream.uid] = [currentFinalText];
 
             return [
@@ -183,8 +284,19 @@ const useStreamMessageUtils = (): {
                 uid: textstream.uid,
                 time: new Date().getTime(),
                 text: currentFinalText,
+                translations: finalTranslationsForTranscript,
+                // Store the current translation language with this transcript item
+                // This preserves which translation was active when this text was spoken
+                selectedTranslationLanguage:
+                  selectedTranslationLanguageRef.current,
               },
             ];
+          } else {
+            // No new text and uid doesn't match - don't modify transcript
+            // console.log(
+            //   '[TRANSCRIPT_DEBUG] Skipping transcript update - no new text and uid mismatch',
+            // );
+            return prevTranscript;
           }
         });
       }
@@ -197,22 +309,41 @@ const useStreamMessageUtils = (): {
         ? ''
         : finalList[textstream.uid]?.join(' ');
       const latestString = nonFinalText;
-      const captionText =
-        existingStringBuffer.length > 0
-          ? existingStringBuffer + ' ' + latestString
-          : latestString;
-
-      // updating the captions
-      captionText &&
-        setCaptionObj(prevState => {
-          return {
-            ...prevState,
-            [textstream.uid]: {
-              text: captionText,
-              lastUpdated: new Date().getTime(),
-            },
-          };
-        });
+      const captionText = isInterjecting
+        ? latestString
+        : existingStringBuffer.length > 0
+        ? existingStringBuffer + ' ' + latestString
+        : latestString;
+
+      // updating the captions with both transcription and translations
+      setCaptionObj(prevState => {
+        const existingTranslations =
+          prevState[textstream.uid]?.translations || [];
+
+        // Update existing translations or add new ones
+        const updatedTranslations = [...existingTranslations];
+
+        for (const newTrans of translations) {
+          const existingIndex = updatedTranslations.findIndex(
+            t => t.lang === newTrans.lang,
+          );
+
+          if (existingIndex >= 0) {
+            updatedTranslations[existingIndex] = newTrans;
+          } else {
+            updatedTranslations.push(newTrans);
+          }
+        }
+
+        return {
+          ...prevState,
+          [textstream.uid]: {
+            text: captionText || prevState[textstream.uid]?.text || '',
+            translations: updatedTranslations,
+            lastUpdated: new Date().getTime(),
+          },
+        };
+      });
 
       // console.group('STT-logs');
       // console.log('Recived uid =>', textstream.uid);

package/template/src/subComponents/caption/utils.ts

@@ -1,5 +1,5 @@
 import {ContentObjects} from '../../../agora-rn-uikit/src/Contexts/RtcContext';
-import {TranscriptItem} from './useCaption';
+import {TranscriptItem, LanguageTranslationConfig} from './useCaption';
 
 export function formatTime(timestamp: number): string {
   const d = new Date(timestamp);
@@ -11,6 +11,26 @@ export function formatTime(timestamp: number): string {
   return `${H}:${m} ${suffix}`;
 }
 
+/**
+ * Check if translation configuration has changed
+ * Compares source language and target languages between two configs
+ * @param prev - Previous translation configuration
+ * @param next - New translation configuration
+ * @returns true if config has changed, false otherwise
+ */
+export const hasConfigChanged = (
+  prev: LanguageTranslationConfig,
+  next: LanguageTranslationConfig,
+): boolean => {
+  const sourceChanged =
+    (prev.source || []).sort().join(',') !==
+    (next.source || []).sort().join(',');
+  const targetsChanged =
+    (prev.targets || []).sort().join(',') !==
+    (next.targets || []).sort().join(',');
+  return sourceChanged || targetsChanged;
+};
+
 export type LanguageType =
   | 'ar-EG'
   | 'ar-JO'
@@ -118,14 +138,56 @@ export const formatTranscriptContent = (
   meetingTitle: string,
   defaultContent: ContentObjects,
 ) => {
+  // Helper function to get display text based on stored translation language
+  // Uses the translation language that was active when this transcript item was created
+  const getDisplayText = (item: TranscriptItem): string => {
+    // Use the stored translation language from when this item was created
+    const storedTranslationLanguage = item.selectedTranslationLanguage;
+
+    if (!storedTranslationLanguage || !item.translations) {
+      return item.text; // no translation selected or no translations available, show original
+    }
+
+    // find translation for the stored language
+    const currentTranslation = item.translations.find(
+      t => t.lang === storedTranslationLanguage,
+    );
+    if (currentTranslation?.text) {
+      return currentTranslation.text;
+    }
+
+    // if stored language not available, show original
+    return item.text;
+  };
+
   const formattedContent = meetingTranscript
     .map(item => {
-      if (item.uid.toString().indexOf('langUpdate') !== -1) {
+      if (
+        item.uid.toString().indexOf('langUpdate') !== -1 ||
+        item.uid.toString().indexOf('translationUpdate') !== -1
+      ) {
         return `${defaultContent[item?.uid?.split('-')[1]]?.name} ${item.text}`;
       }
-      return `${defaultContent[item.uid].name} ${formatTime(
+
+      // Build transcript entry with original text and all translations
+      let transcriptEntry = `${defaultContent[item.uid]?.name} ${formatTime(
         Number(item?.time),
       )}:\n${item.text}`;
+
+      // Add all translations with language labels
+      if (item.translations && item.translations.length > 0) {
+        const translationLines = item.translations
+          .map(trans => {
+            const langLabel =
+              langData.find(l => l.value === trans.lang)?.label || trans.lang;
+            return `${langLabel}: ${trans.text}`;
+          })
+          .join('\n');
+
+        transcriptEntry += `\n${translationLines}`;
+      }
+
+      return transcriptEntry;
     })
     .join('\n\n');
 
@@ -160,3 +222,109 @@ export const formatTranscriptContent = (
 
   return [finalContent, fileName];
 };
+
+/**
+ * Get the appropriate caption text to display based on the user's source/spoken language
+ * For other users' captions: shows translation matching user's source/spoken language
+ * For current user's captions: shows original text
+ * Falls back to original text if translation is not available
+ *
+ * @param captionText - The original caption text
+ * @param translations - Array of available translations
+ * @param viewerSourceLanguage - The user's source (spoken) language
+ * @param speakerUid - The UID of the person speaking
+ * @param currentUserUid - The UID of the current user
+ * @returns The appropriate caption text to display
+ */
+export const getUserTranslatedText = (
+  captionText: string,
+  translations: Array<{lang: string; text: string; isFinal: boolean}> = [],
+  viewerSourceLanguage: LanguageType,
+  speakerUid: string | number,
+  currentUserUid: string | number,
+): {
+  value: string;
+  langCode: string;
+} => {
+  // console.log(
+  //   'getUserTranslatedText input params',
+  //   captionText,
+  //   translations,
+  //   viewerSourceLanguage,
+  //   speakerUid,
+  //   currentUserUid,
+  // );
+
+  // 1. If the speaker is the local user, always show their own source text
+  if (speakerUid === currentUserUid) {
+    return {
+      value: captionText,
+      langCode: getLanguageLabel([viewerSourceLanguage]) || '',
+    };
+  }
+  // For other users' captions, try to find translation matching viewer's source language
+  if (viewerSourceLanguage && translations && translations.length > 0) {
+    const matchingTranslation = translations.find(
+      t => t.lang === viewerSourceLanguage,
+    );
+    if (matchingTranslation) {
+      // Translation exists (even if empty)
+      // - If text is empty: show nothing (don’t fallback)
+      // - If text exists: show it
+      const translatedText = matchingTranslation.text?.trim() || '';
+      return {
+        value: translatedText,
+        langCode: getLanguageLabel([matchingTranslation.lang]) || '',
+      };
+    }
+  }
+
+  // Fallback to original text if no translation found
+  return {
+    value: captionText,
+    langCode: 'Original',
+  };
+};
+
+export interface TranslateConfig {
+  source_lang: string;
+  target_lang: string[];
+}
+
+export const mergeTranslationConfigs = (
+  existingTranslateConfig: TranslateConfig[],
+  userOwnLanguages: LanguageType[],
+  selectedTranslationLanguage: string,
+): TranslateConfig[] => {
+  // Create new translate_config for user's own languages
+  const newTranslateConfigs = userOwnLanguages.map(spokenLang => ({
+    source_lang: spokenLang,
+    target_lang: [selectedTranslationLanguage],
+  }));
+
+  // Merge with existing configuration
+  const mergedTranslateConfig = [...existingTranslateConfig];
+
+  newTranslateConfigs.forEach(newConfig => {
+    const existingIndex = mergedTranslateConfig.findIndex(
+      existing => existing.source_lang === newConfig.source_lang,
+    );
+
+    if (existingIndex !== -1) {
+      // Same source language - merge target languages
+      const existingTargets = mergedTranslateConfig[existingIndex].target_lang;
+      const mergedTargets = [
+        ...new Set([...existingTargets, ...newConfig.target_lang]),
+      ];
+      mergedTranslateConfig[existingIndex] = {
+        ...mergedTranslateConfig[existingIndex],
+        target_lang: mergedTargets,
+      };
+    } else {
+      // Different source language - add new config
+      mergedTranslateConfig.push(newConfig);
+    }
+  });
+
+  return mergedTranslateConfig;
+};

package/template/src/subComponents/chat/ChatSendButton.tsx

@@ -71,7 +71,6 @@ export const handleChatSend = ({
     return;
   }
 
-  // Text message sent update sender side ui
   const sendTextMessage = () => {
     const option = {
       chatType: selectedUserId

package/template/src/subComponents/screenshare/ScreenshareButton.tsx

@@ -25,7 +25,6 @@ import {
   toolbarItemShareText,
 } from '../../language/default-labels/videoCallScreenLabels';
 import {useToolbarProps} from '../../atoms/ToolbarItem';
-import {useBreakoutRoom} from '../../components/breakout-room/context/BreakoutRoomContext';
 /**
  * A component to start and stop screen sharing on web clients.
  * Screen sharing is not yet implemented on mobile platforms.
@@ -54,11 +53,6 @@ const ScreenshareButton = (props: ScreenshareButtonProps) => {
   const {setShowStartScreenSharePopup} = useVideoCall();
   const screenShareButtonLabel = useString<boolean>(toolbarItemShareText);
   const lstooltip = useString<boolean>(livestreamingShareTooltipText);
-  const {permissions} = useBreakoutRoom();
-  // In the main room (default case), permissions come from the main room state.
-  // If the user is in a breakout room, retrieve permissions from the breakout room instead.
-  const canScreenshareInBreakoutRoom = permissions.canScreenshare;
-
   const onPress = () => {
     if (isScreenshareActive) {
       stopScreenshare();
@@ -111,16 +105,6 @@ const ScreenshareButton = (props: ScreenshareButtonProps) => {
     iconButtonProps.disabled = true;
   }
 
-  if (!canScreenshareInBreakoutRoom) {
-    iconButtonProps.iconProps = {
-      ...iconButtonProps.iconProps,
-      tintColor: $config.SEMANTIC_NEUTRAL,
-      showWarningIcon: true,
-    };
-    iconButtonProps.toolTipMessage = 'cannot screenshare';
-    iconButtonProps.disabled = true;
-  }
-
   return props?.render ? (
     props.render(onPress, isScreenshareActive)
   ) : isToolbarMenuItem ? (

package/template/src/subComponents/screenshare/ScreenshareConfigure.native.tsx

@@ -319,7 +319,7 @@ export const ScreenshareConfigure = (props: {children: React.ReactNode}) => {
           'Trying to start native screenshare',
         );
         if (video) {
-          await localMute(MUTE_LOCAL_TYPE.video);
+          localMute(MUTE_LOCAL_TYPE.video);
         }
         try {
           await engine.current.startScreenCapture({

package/template/src/subComponents/waiting-rooms/WaitingRoomControls.tsx

@@ -14,14 +14,12 @@ import {
   peoplePanelWaitingRoomRequestApprovalBtnTxt,
   peoplePanelWaitingRoomRequestDenyBtnTxt,
 } from '../../../src/language/default-labels/videoCallScreenLabels';
-import ChatContext from '../../components/ChatContext';
 
 const WaitingRoomButton = props => {
   const {uid, screenUid, isAccept} = props;
   const {approval} = useWaitingRoomAPI();
   const localUid = useLocalUid();
   const {dispatch} = useContext(DispatchContext);
-  const {syncUserState} = useContext(ChatContext);
   const {waitingRoomRef} = useWaitingRoomContext();
   const admintext = useString(peoplePanelWaitingRoomRequestApprovalBtnTxt)();
   const denytext = useString(peoplePanelWaitingRoomRequestDenyBtnTxt)();
@@ -42,11 +40,10 @@ const WaitingRoomButton = props => {
       Toast.hide();
     }
 
-    //  dispatch({
-    //   type: 'UpdateRenderList',
-    //   value: [uid, {isInWaitingRoom: false}],
-    // });
-    syncUserState(uid, {isInWaitingRoom: false});
+    dispatch({
+      type: 'UpdateRenderList',
+      value: [uid, {isInWaitingRoom: false}],
+    });
 
     if (waitingRoomRef.current) {
       waitingRoomRef.current[uid] = approved ? 'APPROVED' : 'REJECTED';

package/template/src/utils/SdkEvents.ts

@@ -51,6 +51,9 @@ export interface userEventsMapInterface {
   'token-refreshed': () => void;
   'rtc-user-removed': (uid: UidType, channel: string) => void;
   unauthorized: (errorMessage) => void;
+  'waiting-room-approval-requested': () => void;
+  'waiting-room-approval-granted': () => void;
+  'waiting-room-approval-rejected': () => void;
 }
 
 const SDKEvents = createNanoEvents<userEventsMapInterface>();

package/template/src/utils/useEndCall.ts

@@ -4,11 +4,11 @@ import {
   useCaption,
   useContent,
   useRoomInfo,
-  useSTTAPI,
 } from 'customization-api';
 import {PropsContext, DispatchContext} from '../../agora-rn-uikit';
 import {useHistory} from '../components/Router';
 import {stopForegroundService} from '../subComponents/LocalEndCall';
+import RTMEngine from '../rtm/RTMEngine';
 import {ENABLE_AUTH} from '../auth/config';
 import {useAuth} from '../auth/AuthProvider';
 import {useChatConfigure} from '../components/chat/chatConfigure';
@@ -16,13 +16,12 @@ import {useChatConfigure} from '../components/chat/chatConfigure';
 const useEndCall = () => {
   const history = useHistory();
   const {defaultContent} = useContent();
-  const {isSTTActive} = useCaption();
+  const {isSTTActive, stopSTTBotSession} = useCaption();
   const {
     data: {isHost},
   } = useRoomInfo();
   const {authLogin} = useAuth();
   const {deleteChatUser} = useChatConfigure();
-  const {stop: stopSTTAPI} = useSTTAPI();
 
   const {rtcProps} = useContext(PropsContext);
   const {dispatch} = useContext(DispatchContext);
@@ -59,7 +58,7 @@ const useEndCall = () => {
     );
     if (usersInCall.length === 1 && isSTTActive) {
       console.log('Stopping stt api as only one host is in the call');
-      stopSTTAPI().catch(error => {
+      stopSTTBotSession().catch(error => {
         console.log('Error stopping stt', error);
       });
     }
@@ -68,6 +67,7 @@ const useEndCall = () => {
     if ($config.CHAT) {
       deleteChatUser();
     }
+    RTMEngine.getInstance().engine.leaveChannel(rtcProps.channel);
     if (!ENABLE_AUTH) {
       // await authLogout();
       await authLogin();

package/template/src/utils/useMuteToggleLocal.ts

@@ -86,16 +86,14 @@ function useMuteToggleLocal() {
                   );
 
               // Enable UI
-              const newAudioState =
-                localAudioState === ToggleState.enabled
-                  ? ToggleState.disabled
-                  : ToggleState.enabled;
-
               dispatch({
                 type: 'LocalMuteAudio',
-                value: [newAudioState],
+                value: [
+                  localAudioState === ToggleState.enabled
+                    ? ToggleState.disabled
+                    : ToggleState.enabled,
+                ],
               });
-
               handleQueue();
             } catch (e) {
               dispatch({
@@ -154,16 +152,14 @@ function useMuteToggleLocal() {
                 );
               }
               // Enable UI
-              const newVideoState =
-                localVideoState === ToggleState.enabled
-                  ? ToggleState.disabled
-                  : ToggleState.enabled;
-
               dispatch({
                 type: 'LocalMuteVideo',
-                value: [newVideoState],
+                value: [
+                  localVideoState === ToggleState.enabled
+                    ? ToggleState.disabled
+                    : ToggleState.enabled,
+                ],
               });
-
               handleQueue();
             } catch (e) {
               dispatch({