agora-appbuilder-core 4.1.9 → 4.1.11-beta.2

package/template/src/subComponents/caption/Transcript.tsx

@@ -36,6 +36,7 @@ import DownloadTranscriptBtn from './DownloadTranscriptBtn';
 import {useString} from '../../../src/utils/useString';
 import {
   sttSettingSpokenLanguageText,
+  sttSettingTranslationLanguageText,
   sttTranscriptPanelNoSearchResultsFoundText,
   sttTranscriptPanelSearchText,
   sttTranscriptPanelViewLatestText,
@@ -50,11 +51,6 @@ type NativeStreamMessageArgs = [{}, number, number, Uint8Array, number, number];
 type StreamMessageArgs = WebStreamMessageArgs | NativeStreamMessageArgs;
 
 const Transcript = (props: TranscriptProps) => {
-  const settingSpokenLanguageLabel = useString(sttSettingSpokenLanguageText)();
-  const searchText = useString(sttTranscriptPanelSearchText)();
-  const noresults = useString(sttTranscriptPanelNoSearchResultsFoundText)();
-  const viewlatest = useString(sttTranscriptPanelViewLatestText)();
-
   const isSmall = useIsSmall();
   const {currentLayout} = useLayout();
   const {showHeader = true} = props;
@@ -63,9 +59,20 @@ const Transcript = (props: TranscriptProps) => {
     isLangChangeInProgress,
     isSTTListenerAdded,
     setIsSTTListenerAdded,
+    getBotOwnerUid,
+    isSTTActive,
   } = useCaption();
 
+  const settingSpokenLanguageLabel = useString(sttSettingSpokenLanguageText)();
+  const settingTranslationLanguageLabel = useString<boolean>(
+    sttSettingTranslationLanguageText,
+  )(isSTTActive);
+  const searchText = useString(sttTranscriptPanelSearchText)();
+  const noresults = useString(sttTranscriptPanelNoSearchResultsFoundText)();
+  const viewlatest = useString(sttTranscriptPanelViewLatestText)();
+
   const data = meetingTranscript; // Object.entries(transcript);
+  console.log('[STT_PER_USER_BOT] meetingTranscript data: ', data);
 
   const [showButton, setShowButton] = React.useState(false);
 
@@ -103,6 +110,19 @@ const Transcript = (props: TranscriptProps) => {
 
   const renderItem = ({item}) => {
     return item.uid.toString().indexOf('langUpdate') !== -1 ? (
+      <View style={styles.langChangeContainer}>
+        <ImageIcon
+          iconType="plain"
+          iconSize={20}
+          tintColor={$config.PRIMARY_ACTION_BRAND_COLOR}
+          name={'globe'}
+        />
+
+        <Text style={styles.langChange}>
+          {defaultContent[item?.uid?.split('-')[1]]?.name + ' ' + item.text}
+        </Text>
+      </View>
+    ) : item.uid.toString().indexOf('translationUpdate') !== -1 ? (
       <View style={styles.langChangeContainer}>
         <ImageIcon
           iconType="plain"
@@ -112,15 +132,17 @@ const Transcript = (props: TranscriptProps) => {
         />
 
         <Text style={styles.langChange}>
-          {defaultContent[item?.uid?.split('-')[1]].name + ' ' + item.text}
+          {defaultContent[item?.uid?.split('-')[1]]?.name + ' has ' + item.text}
         </Text>
       </View>
     ) : (
       <TranscriptText
-        user={defaultContent[item.uid].name}
+        user={defaultContent[getBotOwnerUid(item.uid)]?.name || 'Speaker'}
         time={item?.time}
         value={item.text}
+        translations={item.translations}
         searchQuery={searchQuery}
+        selectedTranslationLanguage={item.selectedTranslationLanguage}
       />
     );
   };
@@ -164,9 +186,20 @@ const Transcript = (props: TranscriptProps) => {
   const handleSearch = (text: string) => {
     setSearchQuery(text);
     // Filter the data based on the search query
-    const filteredResults = meetingTranscript.filter(item =>
-      item.text.toLowerCase().includes(text.toLowerCase()),
-    );
+    const filteredResults = meetingTranscript.filter(item => {
+      const searchText = text.toLowerCase();
+      // Search in original text
+      if (item.text.toLowerCase().includes(searchText)) {
+        return true;
+      }
+      // Search in translations if available
+      if (item.translations) {
+        return item.translations.some(translation =>
+          translation.text.toLowerCase().includes(searchText),
+        );
+      }
+      return false;
+    });
     setShowButton(false);
     setSearchResults(filteredResults);
     // Scroll to the top of the FlatList when searching
@@ -180,6 +213,7 @@ const Transcript = (props: TranscriptProps) => {
   };
 
   const handleStreamMessageCallback = (...args: StreamMessageArgs) => {
+    console.log('[STT_PER_USER_BOT] handleStreamMessageCallback', args);
     setIsSTTListenerAdded(true);
     if (isWebInternal()) {
       const [uid, data] = args as WebStreamMessageArgs;
@@ -260,7 +294,7 @@ const Transcript = (props: TranscriptProps) => {
       {isLangChangeInProgress ? (
         <View style={{flex: 1}}>
           <Loading
-            text={settingSpokenLanguageLabel}
+            text={settingTranslationLanguageLabel}
             background="transparent"
             indicatorColor={$config.FONT_COLOR + hexadecimalTransparency['70%']}
             textColor={$config.FONT_COLOR + hexadecimalTransparency['70%']}
@@ -438,6 +472,7 @@ export const styles = StyleSheet.create({
   langChangeContainer: {
     marginBottom: 20,
     flexDirection: 'row',
+    alignItems: 'center',
   },
   footer: {
     borderWidth: 1,

package/template/src/subComponents/caption/TranscriptIcon.tsx

@@ -3,7 +3,7 @@ import React from 'react';
 import {SidePanelType, useSidePanel} from 'customization-api';
 import IconButton, {IconButtonProps} from '../../atoms/IconButton';
 import LanguageSelectorPopup from './LanguageSelectorPopup';
-import {useCaption} from './useCaption';
+import {LanguageTranslationConfig, useCaption} from './useCaption';
 import useSTTAPI from './useSTTAPI';
 import {useString} from '../../utils/useString';
 import {toolbarItemTranscriptText} from '../../language/default-labels/videoCallScreenLabels';
@@ -30,28 +30,22 @@ const TranscriptIcon = (props: TranscriptIconProps) => {
     isMobileView = false,
   } = props;
 
-  const {start, restart, isAuthorizedTranscriptUser} = useSTTAPI();
-  const {isSTTActive, language: prevLang, isSTTError} = useCaption();
-  const isDisabled = !isAuthorizedTranscriptUser();
+  // const {start, restart, isAuthorizedTranscriptUser} = useSTTAPI();
+  const {isSTTActive, isSTTError, handleTranslateConfigChange} = useCaption();
+  // const isDisabled = !isAuthorizedTranscriptUser();
   const [isLanguagePopupOpen, setLanguagePopup] =
     React.useState<boolean>(false);
-  const isFirstTimePopupOpen = React.useRef(false);
+  // const isFirstTimePopupOpen = React.useRef(false);
 
   const isTranscriptON = sidePanel === SidePanelType.Transcript;
   const onPress = () => {
-    if (isSTTError) {
+    if (isSTTError || !isSTTActive) {
+      setLanguagePopup(true);
+    } else {
+      // isFirstTimePopupOpen.current = true;
       setSidePanel(
         isTranscriptON ? SidePanelType.None : SidePanelType.Transcript,
       );
-      return;
-    }
-    if (isSTTActive) {
-      setSidePanel(
-        isTranscriptON ? SidePanelType.None : SidePanelType.Transcript,
-      );
-    } else {
-      isFirstTimePopupOpen.current = true;
-      setLanguagePopup(true);
     }
   };
 
@@ -63,13 +57,11 @@ const TranscriptIcon = (props: TranscriptIconProps) => {
       iconBackgroundColor: isTranscriptON
         ? $config.PRIMARY_ACTION_BRAND_COLOR
         : '',
-      tintColor: isDisabled
-        ? $config.SEMANTIC_NEUTRAL
-        : isTranscriptON
+      tintColor: isTranscriptON
         ? $config.PRIMARY_ACTION_TEXT_COLOR
         : $config.SECONDARY_ACTION_COLOR,
     },
-    disabled: isDisabled,
+    disabled: false,
     btnTextProps: {
       text: showLabel
         ? isOnActionSheet
@@ -85,23 +77,23 @@ const TranscriptIcon = (props: TranscriptIconProps) => {
     iconButtonProps.toolTipMessage = label(isTranscriptON);
   }
 
-  const onConfirm = async (langChanged, language) => {
+  const onConfirm = async (inputTranslateConfig: LanguageTranslationConfig) => {
+    // isFirstTimePopupOpen.current = false;
+    // const method = isTranscriptON ? 'stop' : 'start';
+    // if (method === 'stop') return; // not closing the stt service as it will stop for whole channel
+    // if (method === 'start' && isSTTActive === true) return; // not triggering the start service if STT Service already started by anyone else in the channel
     setLanguagePopup(false);
-
-    isFirstTimePopupOpen.current = false;
-    const method = isTranscriptON ? 'stop' : 'start';
-    if (method === 'stop') return; // not closing the stt service as it will stop for whole channel
-    if (method === 'start' && isSTTActive === true) return; // not triggering the start service if STT Service already started by anyone else in the channel
-    if (!isTranscriptON) {
-      setSidePanel(SidePanelType.Transcript);
-    } else {
-      setSidePanel(SidePanelType.None);
-    }
     try {
-      const res = await start(language);
-      if (res?.message.includes('STARTED')) {
-        // channel is already started now restart
-        await restart(language);
+      // const res = await start(language, userOwnLanguages);
+      // if (res?.message.includes('STARTED')) {
+      //   // channel is already started now restart
+      //   await restart(language, userOwnLanguages);
+      // }
+      await handleTranslateConfigChange(inputTranslateConfig);
+      if (!isTranscriptON) {
+        setSidePanel(SidePanelType.Transcript);
+      } else {
+        setSidePanel(SidePanelType.None);
       }
     } catch (error) {
       console.log('eror in starting stt', error);
@@ -115,7 +107,7 @@ const TranscriptIcon = (props: TranscriptIconProps) => {
         modalVisible={isLanguagePopupOpen}
         setModalVisible={setLanguagePopup}
         onConfirm={onConfirm}
-        isFirstTimePopupOpen={isFirstTimePopupOpen.current}
+        //  isFirstTimePopupOpen={isFirstTimePopupOpen.current}
       />
     </View>
   );

package/template/src/subComponents/caption/TranscriptText.tsx

@@ -3,24 +3,64 @@ import React from 'react';
 
 import ThemeConfig from '../../../src/theme';
 import hexadecimalTransparency from '../../../src/utils/hexadecimalTransparency';
-import {formatTime} from './utils';
+import {formatTime, getLanguageLabel} from './utils';
+import {useLocalUid} from '../../../agora-rn-uikit';
+
+type TranslationItem = {
+  lang: string;
+  text: string;
+  isFinal: boolean;
+};
 
 interface TranscriptTextProps {
   user: string;
   time: number;
   value: string;
+  translations?: TranslationItem[];
   searchQuery?: string;
+  selectedTranslationLanguage?: string;
 }
 
 export const TranscriptText = ({
   user,
   time,
   value,
+  translations = [],
   searchQuery = '',
+  selectedTranslationLanguage: storedTranslationLanguage,
+  transcriptViewMode = 'translated',
+  speakerUid,
+  localUserSpokenLanguage,
 }: TranscriptTextProps) => {
   const t = time ? formatTime(Number(time)) : '';
+
+  //  text to display based on stored translation language
+  // const getDisplayText = () => {
+  //   if (!storedTranslationLanguage) {
+  //     return value; // no translation selected, show original
+  //   }
+
+  //   // find translation for the stored language
+  //   const currentTranslation = translations.find(
+  //     t => t.lang === storedTranslationLanguage,
+  //   );
+  //   if (currentTranslation?.text) {
+  //     return currentTranslation.text;
+  //   }
+
+  //   // if stored language not available, show original
+  //   return value;
+  // };
+
+  // const displayText = getDisplayText();
   const regex = searchQuery ? new RegExp(`(${searchQuery})`, 'gi') : ' ';
-  const parts = value.split(regex);
+  const originalParts = value.split(regex);
+
+  // Prepare all translations with their parts for search highlighting
+  const translationsParts = translations.map(trans => ({
+    lang: trans.lang,
+    parts: trans.text.split(regex),
+  }));
 
   return (
     <View key={user} style={styles.transcriptTextContainer}>
@@ -36,9 +76,10 @@ export const TranscriptText = ({
       </View>
 
       <View>
+        {/* Original Text */}
         <Text style={[styles.transciptText]}>
           {/* If substring matches search query then highlight it */}
-          {parts.map((part, index) =>
+          {originalParts.map((part, index) =>
             part.toLowerCase() === searchQuery.toLowerCase() &&
             searchQuery !== '' ? (
               <Text key={index} style={styles.highlightedText}>
@@ -49,6 +90,29 @@ export const TranscriptText = ({
             ),
           )}
         </Text>
+
+        {/* All Translations */}
+        {translationsParts.map((translation, translationIndex) => (
+          <Text
+            key={translation.lang}
+            style={[styles.transciptText, styles.translationText]}>
+            {/* lang code */}
+            <Text style={styles.languageLabel}>
+              ({getLanguageLabel([translation.lang])}):{' '}
+            </Text>
+            {/* lang */}
+            {translation.parts.map((part, index) =>
+              part.toLowerCase() === searchQuery.toLowerCase() &&
+              searchQuery !== '' ? (
+                <Text key={index} style={styles.highlightedText}>
+                  {searchQuery ? part : part + ' '}
+                </Text>
+              ) : (
+                <Text key={index}>{searchQuery ? part : part + ' '}</Text>
+              ),
+            )}
+          </Text>
+        ))}
       </View>
     </View>
   );
@@ -95,4 +159,15 @@ const styles = StyleSheet.create({
   highlightedText: {
     backgroundColor: $config.SEMANTIC_NEUTRAL,
   },
+  translationText: {
+    fontStyle: 'italic',
+    marginTop: 8,
+    fontWeight: '400',
+    lineHeight: 24,
+    fontSize: ThemeConfig.FontSize.normal,
+    color: $config.FONT_COLOR + ThemeConfig.EmphasisPlus.medium,
+  },
+  languageLabel: {
+    color: $config.FONT_COLOR + ThemeConfig.EmphasisPlus.low,
+  },
 });

package/template/src/subComponents/caption/proto/ptoto.js

@@ -1,9 +1,9 @@
 /*eslint-disable block-scoped-var, id-length, no-control-regex, no-magic-numbers, no-prototype-builtins, no-redeclare, no-shadow, no-var, sort-vars*/
 "use strict";
 
-var $protobuf = require("protobufjs/light");
+var $protobuf = require("protobufjs");
 
-var $root = ($protobuf.roots["default"] || ($protobuf.roots["default"] = new $protobuf.Root()))
+var $protobufRoot = ($protobuf.roots.default || ($protobuf.roots.default = new $protobuf.Root()))
     .addJSON({
         agora: {
             nested: {
@@ -55,7 +55,24 @@ var $root = ($protobuf.roots["default"] || ($protobuf.roots["default"] = new $pr
                                     rule: "repeated",
                                     type: "Word",
                                     id: 10
-                                }
+                                },
+                                end_of_segment: {
+                                    type: "bool",
+                                    id: 11
+                                },
+                                duration_ms: {
+                                    type: "int32",
+                                    id: 12
+                                },
+                                data_type: {
+                                    type: "string",
+                                    id: 13
+                                },
+                                trans: {
+                                    rule: "repeated",
+                                    type: "Translation",
+                                    id: 14
+                                },
                             }
                         },
                         Word: {
@@ -81,6 +98,23 @@ var $root = ($protobuf.roots["default"] || ($protobuf.roots["default"] = new $pr
                                     id: 5
                                 }
                             }
+                        },
+                        Translation: {
+                            fields: {
+                                isFinal: {
+                                    type: "bool",
+                                    id: 1
+                                },
+                                lang: {
+                                    type: "string",
+                                    id: 2
+                                },
+                                texts: {
+                                    rule: "repeated",
+                                    type: "string",
+                                    id: 3
+                                }
+                            }
                         }
                     }
                 }
@@ -88,4 +122,4 @@ var $root = ($protobuf.roots["default"] || ($protobuf.roots["default"] = new $pr
         }
     });
 
-module.exports = $root;
+module.exports = $protobufRoot;

package/template/src/subComponents/caption/proto/test.proto

@@ -1,23 +1,38 @@
 syntax = "proto3";
-package agora.audio2text;
-option java_package = "io.agora.rtc.audio2text";
-option java_outer_classname = "Audio2TextProtobuffer";
+
+package Agora.SpeechToText;
+option objc_class_prefix = "Stt";
+option csharp_namespace = "AgoraSTTSample.Protobuf";
+option java_package = "io.agora.rtc.speech2text";
+option java_outer_classname = "AgoraSpeech2TextProtobuffer";
+
 message Text {
- int32 vendor = 1;
- int32 version = 2;
- int32 seqnum = 3;
- uint32 uid = 4;
- int32 flag = 5;
- int64 time = 6;
- int32 lang = 7;
- int32 starttime = 8;
- int32 offtime = 9;
- repeated Word words = 10;
+    int32 vendor = 1;
+    int32 version = 2;
+    int32 seqnum = 3;
+    int64 uid = 4;
+    int32 flag = 5;
+    int64 time = 6;
+    int32 lang = 7;
+    int32 starttime = 8;
+    int32 offtime = 9;
+    repeated Word words = 10;
+    bool end_of_segment = 11;
+    int32 duration_ms = 12;
+    string data_type = 13;
+    repeated Translation trans = 14;
+    string culture = 15;
+    int64 text_ts = 16;
 }
 message Word {
- string text = 1;
- int32 start_ms = 2;
- int32 duration_ms = 3;
- bool is_final = 4;
- double confidence = 5;
-}
+    string text = 1;
+    int32 start_ms = 2;
+    int32 duration_ms = 3;
+    bool is_final = 4;
+    double confidence = 5;
+}
+message Translation {
+    bool is_final = 1;
+    string lang = 2;
+    repeated string texts = 3;
+}