agora-appbuilder-core 4.1.9 → 4.1.11-beta.2

package/template/src/subComponents/caption/Caption.tsx

@@ -8,7 +8,11 @@ import {isWebInternal} from '../../utils/common';
 import useStreamMessageUtils from './useStreamMessageUtils';
 import hexadecimalTransparency from '../../utils/hexadecimalTransparency';
 import {useString} from '../../utils/useString';
-import {sttSettingSpokenLanguageText} from '../../language/default-labels/videoCallScreenLabels';
+import {useLocalUid} from '../../../agora-rn-uikit';
+import {
+  sttSettingSpokenLanguageText,
+  sttSettingTranslationLanguageText,
+} from '../../language/default-labels/videoCallScreenLabels';
 
 export type WebStreamMessageArgs = [number, Uint8Array];
 export type NativeStreamMessageArgs = [
@@ -33,13 +37,22 @@ const Caption: React.FC<CaptionProps> = ({
   const {RtcEngineUnsafe} = useRtc();
   const {
     isLangChangeInProgress,
+    isTranslationChangeInProgress,
     captionObj, //state for current live caption for all users
     isSTTListenerAdded,
     setIsSTTListenerAdded,
     activeSpeakerRef,
     prevSpeakerRef,
+    getBotOwnerUid,
+    isSTTActive,
+    translationConfig,
+    remoteSpokenLanguages,
   } = useCaption();
+  const currentUserUid = useLocalUid();
   const ssLabel = useString(sttSettingSpokenLanguageText)();
+  const stLabel = useString<boolean>(sttSettingTranslationLanguageText)(
+    isSTTActive,
+  );
   const {streamMessageCallback} = useStreamMessageUtils();
   const {defaultContent} = useContent();
 
@@ -66,18 +79,20 @@ const Caption: React.FC<CaptionProps> = ({
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, []);
 
-  if (isLangChangeInProgress)
+  if (isLangChangeInProgress || isTranslationChangeInProgress) {
     return (
       <Loading
-        text={ssLabel}
+        text={stLabel}
         background="transparent"
         indicatorColor={$config.FONT_COLOR + hexadecimalTransparency['70%']}
         textColor={$config.FONT_COLOR + hexadecimalTransparency['70%']}
       />
     );
+  }
 
-  console.log('current speaker uid', activeSpeakerRef.current);
-  console.log('prev current uid ', prevSpeakerRef.current);
+  console.log('[STT_PER_USER_BOT]  speaker uid', activeSpeakerRef.current);
+  console.log('[STT_PER_USER_BOT] prev current uid ', prevSpeakerRef.current);
+  console.log('[STT_PER_USER_BOT] captionObj ', captionObj);
 
   const speakerCount = prevSpeakerRef.current === '' ? 1 : 2;
 
@@ -104,8 +119,12 @@ const Caption: React.FC<CaptionProps> = ({
           {captionObj[prevSpeakerRef.current] &&
           captionObj[prevSpeakerRef.current].text ? (
             <CaptionText
-              user={defaultContent[prevSpeakerRef.current].name || 'Speaker'}
+              user={
+                defaultContent[getBotOwnerUid(prevSpeakerRef.current)]?.name ||
+                'Speaker'
+              }
               value={captionObj[prevSpeakerRef.current].text}
+              translations={captionObj[prevSpeakerRef.current].translations}
               activeSpeakersCount={speakerCount}
               isActiveSpeaker={false}
               activelinesAvailable={activelinesAvailable}
@@ -114,6 +133,15 @@ const Caption: React.FC<CaptionProps> = ({
               setInActiveLinesAvaialble={setInActiveLinesAvaialble}
               captionUserStyle={captionUserStyle}
               captionTextStyle={captionTextStyle}
+              speakerUid={getBotOwnerUid(prevSpeakerRef.current)}
+              userLocalUid={currentUserUid}
+              spokenLanguageCode={
+                getBotOwnerUid(prevSpeakerRef.current) === currentUserUid
+                  ? translationConfig.source[0]
+                  : remoteSpokenLanguages[
+                      getBotOwnerUid(prevSpeakerRef.current)
+                    ]
+              }
             />
           ) : (
             <></>
@@ -121,8 +149,12 @@ const Caption: React.FC<CaptionProps> = ({
           {captionObj[activeSpeakerRef.current] &&
           captionObj[activeSpeakerRef.current].text ? (
             <CaptionText
-              user={defaultContent[activeSpeakerRef.current].name || 'Speaker'}
+              user={
+                defaultContent[getBotOwnerUid(activeSpeakerRef.current)]
+                  ?.name || 'Speaker'
+              }
               value={captionObj[activeSpeakerRef.current].text}
+              translations={captionObj[activeSpeakerRef.current].translations}
               activeSpeakersCount={speakerCount}
               isActiveSpeaker={true}
               activelinesAvailable={activelinesAvailable}
@@ -131,6 +163,15 @@ const Caption: React.FC<CaptionProps> = ({
               setInActiveLinesAvaialble={setInActiveLinesAvaialble}
               captionUserStyle={captionUserStyle}
               captionTextStyle={captionTextStyle}
+              speakerUid={getBotOwnerUid(activeSpeakerRef.current)}
+              userLocalUid={currentUserUid}
+              spokenLanguageCode={
+                getBotOwnerUid(activeSpeakerRef.current) === currentUserUid
+                  ? translationConfig.source[0]
+                  : remoteSpokenLanguages[
+                      getBotOwnerUid(activeSpeakerRef.current)
+                    ]
+              }
             />
           ) : (
             <></>

package/template/src/subComponents/caption/CaptionContainer.tsx

@@ -9,7 +9,7 @@ import {
 import React from 'react';
 
 import Caption from './Caption';
-import {useCaption} from './useCaption';
+import {useCaption, LanguageTranslationConfig} from './useCaption';
 import ThemeConfig from '../../../src/theme';
 import {
   calculatePosition,
@@ -27,6 +27,8 @@ import LanguageSelectorPopup from './LanguageSelectorPopup';
 import useSTTAPI from './useSTTAPI';
 import useGetName from '../../utils/useGetName';
 import {useRoomInfo} from 'customization-api';
+import {useLocalUid} from '../../../agora-rn-uikit';
+import {getLanguageLabel} from './utils';
 import {
   SIDE_PANEL_MAX_WIDTH,
   SIDE_PANEL_GAP,
@@ -35,11 +37,13 @@ import {
   MOBILE_CAPTION_CONTAINER_HEIGHT,
 } from '../../../src/components/CommonStyles';
 import useCaptionWidth from './useCaptionWidth';
-import {LanguageType} from './utils';
+import {LanguageType, langData, mergeTranslationConfigs} from './utils';
 import hexadecimalTransparency from '../../utils/hexadecimalTransparency';
 import {useString} from '../../utils/useString';
 import {
   sttChangeSpokenLanguageText,
+  sttOriginalTranslatedText,
+  sttStopTranslationText,
   toolbarItemCaptionText,
 } from '../../language/default-labels/videoCallScreenLabels';
 import {logger, LogSource} from '../../logger/AppBuilderLogger';
@@ -56,8 +60,11 @@ const CaptionContainer: React.FC<CaptionContainerProps> = ({
   captionTextStyle = {},
 }) => {
   const moreIconRef = React.useRef<View>(null);
+  const langSelectIconRef = React.useRef<View>(null);
   const [actionMenuVisible, setActionMenuVisible] =
     React.useState<boolean>(false);
+  const [langActionMenuVisible, setLangActionMenuVisible] =
+    React.useState<boolean>(false);
   const [isHovered, setIsHovered] = React.useState<boolean>(false);
   const isDesktop = useIsDesktop();
   const isSmall = useIsSmall();
@@ -115,11 +122,23 @@ const CaptionContainer: React.FC<CaptionContainerProps> = ({
             btnRef={moreIconRef}
           />
 
+          {/* <TranslateActionMenu
+            actionMenuVisible={langActionMenuVisible}
+            setActionMenuVisible={setLangActionMenuVisible}
+            btnRef={langSelectIconRef}
+          /> */}
+
           {(isHovered || isMobileUA()) && !isLangChangeInProgress && (
-            <MoreMenu
-              ref={moreIconRef}
-              setActionMenuVisible={setActionMenuVisible}
-            />
+            <>
+              {/* <LanguageSelectMenu
+                ref={langSelectIconRef}
+                setActionMenuVisible={setLangActionMenuVisible}
+              /> */}
+              <MoreMenu
+                ref={moreIconRef}
+                setActionMenuVisible={setActionMenuVisible}
+              />
+            </>
           )}
 
           <Caption
@@ -199,6 +218,55 @@ const MoreMenu = React.forwardRef<View, MoreMenuProps>((props, ref) => {
   );
 });
 
+const LanguageSelectMenu = React.forwardRef<View, MoreMenuProps>(
+  (props, ref) => {
+    const {setActionMenuVisible} = props;
+    const isMobile = isMobileUA();
+    return (
+      <View
+        ref={ref}
+        collapsable={false}
+        style={{
+          width: 32,
+          height: 32,
+          alignSelf: 'center',
+          justifyContent: 'center',
+          alignItems: 'center',
+          borderRadius: 20,
+          position: 'absolute',
+          right: isMobile ? 45 : 50,
+          top: isMobile ? 3 : 8,
+          zIndex: 999,
+        }}>
+        <IconButton
+          hoverEffect={true}
+          hoverEffectStyle={{
+            backgroundColor:
+              $config.CARD_LAYER_5_COLOR + hexadecimalTransparency['25%'],
+            borderRadius: 20,
+          }}
+          iconProps={{
+            iconType: 'plain',
+            name: 'lang-select',
+            iconSize: isMobile ? 18 : 20,
+            tintColor: $config.SECONDARY_ACTION_COLOR,
+            iconContainerStyle: {
+              padding: isMobile ? 6 : 8,
+              borderRadius: 20,
+              backgroundColor: isMobile
+                ? $config.CARD_LAYER_5_COLOR + hexadecimalTransparency['25%']
+                : 'transparent',
+            },
+          }}
+          onPress={() => {
+            setActionMenuVisible(true);
+          }}
+        />
+      </View>
+    );
+  },
+);
+
 interface CaptionsActionMenuProps {
   actionMenuVisible: boolean;
   setActionMenuVisible: (actionMenuVisible: boolean) => void;
@@ -209,9 +277,15 @@ const CaptionsActionMenu = (props: CaptionsActionMenuProps) => {
   const {actionMenuVisible, setActionMenuVisible, btnRef} = props;
   const {
     setIsCaptionON,
-    language: prevLang,
+    // language: prevLang,
     isLangChangeInProgress,
-    setLanguage,
+    // setLanguage,
+    // selectedTranslationLanguage,
+    updateSTTBotSession,
+    translationConfig,
+    handleTranslateConfigChange,
+    captionViewMode,
+    setCaptionViewMode,
   } = useCaption();
   const actionMenuitems: ActionMenuItem[] = [];
   const [modalPosition, setModalPosition] = React.useState({});
@@ -219,32 +293,51 @@ const CaptionsActionMenu = (props: CaptionsActionMenuProps) => {
   const {width: globalWidth, height: globalHeight} = useWindowDimensions();
   const [isLanguagePopupOpen, setLanguagePopup] =
     React.useState<boolean>(false);
-  const {restart} = useSTTAPI();
-  const username = useGetName();
-  const {
-    data: {isHost},
-  } = useRoomInfo();
+  // const {restart} = useSTTAPI();
+  // const username = useGetName();
+  // const {
+  //   data: {isHost},
+  //   sttLanguage,
+  // } = useRoomInfo();
 
   const changeSpokenLangLabel = useString<boolean>(
     sttChangeSpokenLanguageText,
   )();
+  const sttStopTranslationLabel = useString<boolean>(sttStopTranslationText)();
+  const sttOriginalTranslatedLabel = useString(sttOriginalTranslatedText)();
 
   const hideCaptionLabel = useString<boolean>(toolbarItemCaptionText)(true);
 
-  // only Host is authorized to start/stop stt
-  isHost &&
-    actionMenuitems.push({
-      icon: 'lang-select',
-      iconColor: $config.SECONDARY_ACTION_COLOR,
-      textColor: $config.FONT_COLOR,
-      title: changeSpokenLangLabel + ' ',
-      disabled: isLangChangeInProgress,
-      onPress: () => {
-        setActionMenuVisible(false);
-        setLanguagePopup(true);
-      },
-    });
+  // Anyone can start/stop stt
+  actionMenuitems.push({
+    icon: 'globe',
+    iconColor: $config.SECONDARY_ACTION_COLOR,
+    textColor: $config.FONT_COLOR,
+    title: changeSpokenLangLabel + ' ',
+    disabled: isLangChangeInProgress,
+    onPress: () => {
+      setActionMenuVisible(false);
+      setLanguagePopup(true);
+    },
+  });
+
+  // Stop Translation (not STT bot, pass empty targets)
+  actionMenuitems.push({
+    icon: 'lang-select',
+    iconColor: $config.SECONDARY_ACTION_COLOR,
+    textColor: $config.FONT_COLOR,
+    title: sttStopTranslationLabel,
+    disabled: isLangChangeInProgress,
+    onPress: async () => {
+      setActionMenuVisible(false);
+      await updateSTTBotSession({
+        source: translationConfig.source,
+        targets: [], // Empty targets = no translation
+      });
+    },
+  });
 
+  // Hide Caption Panel
   actionMenuitems.push({
     icon: 'captions-off',
     iconColor: $config.SECONDARY_ACTION_COLOR,
@@ -256,32 +349,32 @@ const CaptionsActionMenu = (props: CaptionsActionMenuProps) => {
     },
   });
 
-  const onLanguageChange = (langChanged = false, language: LanguageType[]) => {
-    setLanguagePopup(false);
-    if (langChanged) {
-      logger.log(
-        LogSource.Internals,
-        'STT',
-        `Language changed to  ${language}. Restarting STT`,
+  // View Mode Options
+  actionMenuitems.push({
+    icon: 'lang-translate',
+    iconColor: $config.SECONDARY_ACTION_COLOR,
+    endIcon:
+      captionViewMode === 'original-and-translated' ? 'tick-fill' : undefined,
+    endIconColor: $config.SEMANTIC_SUCCESS,
+    textColor: $config.FONT_COLOR,
+    title: sttOriginalTranslatedLabel,
+    onPress: () => {
+      setCaptionViewMode(
+        captionViewMode === 'translated'
+          ? 'original-and-translated'
+          : 'translated',
       );
-      restart(language)
-        .then(() => {
-          logger.debug(
-            LogSource.Internals,
-            'STT',
-            'stt restarted successfully',
-          );
-        })
-        .catch(error => {
-          logger.error(
-            LogSource.Internals,
-            'STT',
-            'Error in restarting',
-            error,
-          );
-          // Handle the error case
-        });
-    }
+      setActionMenuVisible(false);
+    },
+  });
+
+  const onLanguageChange = async (
+    inputTranslateConfig: LanguageTranslationConfig,
+  ) => {
+    setLanguagePopup(false);
+    try {
+      await handleTranslateConfigChange(inputTranslateConfig);
+    } catch (error) {}
   };
 
   React.useEffect(() => {
@@ -310,6 +403,7 @@ const CaptionsActionMenu = (props: CaptionsActionMenuProps) => {
       );
     }
   }, [actionMenuVisible]);
+
   return (
     <>
       <ActionMenu
@@ -328,6 +422,185 @@ const CaptionsActionMenu = (props: CaptionsActionMenuProps) => {
   );
 };
 
+export interface TranslateActionMenuProps {
+  actionMenuVisible: boolean;
+  setActionMenuVisible: (actionMenuVisible: boolean) => void;
+  btnRef: React.RefObject<View>;
+}
+
+export const TranslateActionMenu = (props: TranslateActionMenuProps) => {
+  const {actionMenuVisible, setActionMenuVisible, btnRef} = props;
+  const [modalPosition, setModalPosition] = React.useState({});
+  const [isPosCalculated, setIsPosCalculated] = React.useState(false);
+  const {width: globalWidth, height: globalHeight} = useWindowDimensions();
+  const {
+    language: currentSpokenLanguages,
+    selectedTranslationLanguage,
+    setSelectedTranslationLanguage,
+    setMeetingTranscript,
+    translationConfig,
+  } = useCaption();
+  const {update} = useSTTAPI();
+  const localUid = useLocalUid();
+  const {sttLanguage} = useRoomInfo();
+
+  // Reset selected translation language if there are no targets configured
+  const targetLanguages = translationConfig?.targets || [];
+  React.useEffect(() => {
+    if (targetLanguages.length === 0 && selectedTranslationLanguage !== '') {
+      setSelectedTranslationLanguage('');
+    }
+  }, [
+    targetLanguages.length,
+    selectedTranslationLanguage,
+    setSelectedTranslationLanguage,
+  ]);
+
+  const actionMenuitems: ActionMenuItem[] = [];
+
+  actionMenuitems.push({
+    iconColor: $config.SECONDARY_ACTION_COLOR,
+    textColor: $config.FONT_COLOR,
+    title: 'Add Another Translation',
+    iconPosition: 'end',
+    disabled: true,
+    onPress: () => {},
+  });
+
+  const handleTranslationToggle = (targetLanguage: string) => {
+    // Simply update the selected translation language locally
+    // No API call needed - we're just switching between already-configured target languages
+    // const prevTranslationLanguage = selectedTranslationLanguage;
+    setSelectedTranslationLanguage(targetLanguage);
+
+    // // Add translation language change notification to transcript
+    // const getLanguageName = (langCode: string) => {
+    //   if (!langCode) return '';
+    //   const lang = langData.find(data => data.value === langCode);
+    //   return lang ? lang.label : langCode;
+    // };
+
+    // const actionText =
+    //   targetLanguage === ''
+    //     ? 'turned off translation'
+    //     : prevTranslationLanguage === ''
+    //     ? `set the translation language to "${getLanguageName(targetLanguage)}"`
+    //     : `changed the translation language from "${getLanguageName(
+    //         prevTranslationLanguage,
+    //       )}" to "${getLanguageName(targetLanguage)}"`;
+
+    // setMeetingTranscript(prev => [
+    //   ...prev,
+    //   {
+    //     name: 'translationUpdate',
+    //     time: new Date().getTime(),
+    //     uid: `translationUpdate-${localUid}`,
+    //     text: actionText,
+    //   },
+    // ]);
+
+    setActionMenuVisible(false);
+  };
+
+  // Check if there are any target languages configured
+  if (targetLanguages.length === 0) {
+    // No target languages - show a disabled message
+    actionMenuitems.push({
+      icon: undefined,
+      iconColor: $config.FONT_COLOR,
+      textColor: $config.FONT_COLOR + hexadecimalTransparency['50%'],
+      title: 'No languages configured',
+      iconPosition: 'end',
+      disabled: true,
+      onPress: () => {},
+    });
+  } else {
+    // Show "Off" option and target languages
+    actionMenuitems.push({
+      icon: selectedTranslationLanguage === '' ? 'tick-fill' : undefined,
+      iconColor: $config.PRIMARY_ACTION_BRAND_COLOR,
+      textColor: $config.FONT_COLOR,
+      title: 'Off',
+      iconPosition: 'end',
+      onPress: () => handleTranslationToggle(''),
+    });
+
+    // Add selected translation language right after "Off" if one is selected
+    if (selectedTranslationLanguage && selectedTranslationLanguage !== '') {
+      const selectedLanguage = langData.find(
+        lang => lang.value === selectedTranslationLanguage,
+      );
+      if (selectedLanguage) {
+        actionMenuitems.push({
+          icon: 'tick-fill',
+          iconColor: $config.PRIMARY_ACTION_BRAND_COLOR,
+          textColor: $config.FONT_COLOR,
+          title: selectedLanguage.label,
+          iconPosition: 'end',
+          onPress: () => handleTranslationToggle(selectedLanguage.value),
+        });
+      }
+    }
+
+    // Add remaining Translation language options from translationConfig.targets (excluding the selected one)
+    targetLanguages.forEach(targetLangCode => {
+      if (targetLangCode !== selectedTranslationLanguage) {
+        const language = langData.find(lang => lang.value === targetLangCode);
+        if (language) {
+          actionMenuitems.push({
+            icon: undefined,
+            iconColor: $config.PRIMARY_ACTION_BRAND_COLOR,
+            textColor: $config.FONT_COLOR,
+            title: language.label,
+            iconPosition: 'end',
+            onPress: () => handleTranslationToggle(language.value),
+          });
+        }
+      }
+    });
+  }
+
+  React.useEffect(() => {
+    if (actionMenuVisible) {
+      btnRef?.current?.measure(
+        (
+          _fx: number,
+          _fy: number,
+          localWidth: number,
+          localHeight: number,
+          px: number,
+          py: number,
+        ) => {
+          const data = calculatePosition({
+            px,
+            py,
+            localWidth,
+            localHeight,
+            globalHeight,
+            globalWidth,
+          });
+          setModalPosition(data);
+          setIsPosCalculated(true);
+        },
+      );
+    }
+  }, [actionMenuVisible]);
+
+  return (
+    <ActionMenu
+      from={'translation'}
+      actionMenuVisible={actionMenuVisible && isPosCalculated}
+      setActionMenuVisible={setActionMenuVisible}
+      modalPosition={modalPosition}
+      items={actionMenuitems}
+      containerStyle={{
+        maxHeight: Math.min(440, globalHeight * 0.6),
+        width: 220,
+      }}
+    />
+  );
+};
+
 export default CaptionContainer;
 
 const styles = StyleSheet.create({