agora-appbuilder-core 4.1.9 → 4.1.11-beta.2

package/template/src/subComponents/caption/useSTTAPI.tsx

@@ -1,64 +1,97 @@
 import React, {useContext} from 'react';
 import StorageContext from '../../components/StorageContext';
 import {useRoomInfo} from '../../components/room-info/useRoomInfo';
-import {useCaption} from './useCaption';
-import events, {PersistanceLevel} from '../../rtm-events-api';
-import {EventNames} from '../../rtm-events';
-import {getLanguageLabel, LanguageType} from './utils';
-import useGetName from '../../utils/useGetName';
-import {capitalizeFirstLetter} from '../../utils/common';
+import {LanguageTranslationConfig} from './useCaption';
 import {PropsContext, useLocalUid} from '../../../agora-rn-uikit';
 import {logger, LogSource} from '../../logger/AppBuilderLogger';
 import getUniqueID from '../../utils/getUniqueID';
 
+export interface STTAPIResponse {
+  success: boolean;
+  data?: any;
+  error?: {
+    message: string;
+    code?: number;
+  };
+}
+
 interface IuseSTTAPI {
-  start: (lang: LanguageType[]) => Promise<{message: string} | null>;
-  stop: () => Promise<void>;
-  restart: (lang: LanguageType[]) => Promise<void>;
-  isAuthorizedSTTUser: () => boolean;
-  isAuthorizedTranscriptUser: () => boolean;
+  start: (
+    botUid: number,
+    translationConfig: LanguageTranslationConfig,
+  ) => Promise<STTAPIResponse>;
+  update: (
+    botUid: number,
+    translationConfig: LanguageTranslationConfig,
+  ) => Promise<STTAPIResponse>;
+  stop: (botUid: number) => Promise<STTAPIResponse>;
 }
 
 const useSTTAPI = (): IuseSTTAPI => {
   const {store} = React.useContext(StorageContext);
   const {
-    data: {roomId, isHost},
+    data: {roomId},
   } = useRoomInfo();
-  const {
-    language,
-    isSTTActive,
-    setIsSTTActive,
-    setIsLangChangeInProgress,
-    setLanguage,
-    setMeetingTranscript,
-    setIsSTTError,
-  } = useCaption();
-
-  const currentLangRef = React.useRef<LanguageType[]>([]);
+  const {rtcProps} = useContext(PropsContext);
   const STT_API_URL = `${$config.BACKEND_ENDPOINT}/v1/stt`;
-  const username = useGetName();
   const localUid = useLocalUid();
-  const {rtcProps} = useContext(PropsContext);
-
-  React.useEffect(() => {
-    currentLangRef.current = language;
-  }, [language]);
 
-  const apiCall = async (method: string, lang: LanguageType[] = []) => {
+  const apiCall = async (
+    method: 'startv7' | 'update' | 'stopv7',
+    botUid: number,
+    translationConfig?: LanguageTranslationConfig,
+  ): Promise<STTAPIResponse> => {
     const requestId = getUniqueID();
     const startReqTs = Date.now();
-    logger.log(
-      LogSource.NetworkRest,
-      'stt',
-      `Trying to ${method} stt for lang ${lang}`,
-      {
-        method,
-        lang,
-        requestId,
-        startReqTs,
-      },
-    );
+
     try {
+      // Calculate which user this bot belongs to
+      const ownerUid = botUid - 900000000;
+
+      let requestBody: any = {
+        passphrase: roomId?.host || roomId?.attendee || '',
+        dataStream_uid: botUid,
+        encryption_mode: $config.ENCRYPTION_ENABLED
+          ? rtcProps.encryption.mode
+          : null,
+      };
+
+      console.log(
+        `[STT_BOT_SUBSCRIPTION] ${method.toUpperCase()} - Bot UID: ${botUid} will subscribe to User UID: ${ownerUid}`,
+        {
+          method,
+          botUid,
+          ownerUid,
+          translationConfig,
+        },
+      );
+      // Add translate_config only for start/update methods
+      if (translationConfig?.source?.[0]) {
+        requestBody.lang = translationConfig.source;
+        // Sanitize payload: remove source language from targets to avoid API errors
+        const sanitizedTargets =
+          translationConfig?.targets?.filter(
+            target => target !== translationConfig?.source[0],
+          ) || [];
+        const shouldTranslate = sanitizedTargets.length > 0;
+        // Add translate_config payload only if targets exist
+        if (shouldTranslate) {
+          requestBody.translate_config = [
+            {
+              source_lang: translationConfig.source[0],
+              target_lang: sanitizedTargets,
+            },
+          ];
+          if (method === 'update') {
+            requestBody.translate = true;
+          }
+        } else if (method === 'update') {
+          // If method is update and no targets are passed
+          requestBody.translate = false;
+        }
+        requestBody.subscribeAudioUids = [`${localUid}`];
+      }
+
       const response = await fetch(`${STT_API_URL}/${method}`, {
         method: 'POST',
         headers: {
@@ -67,22 +100,17 @@ const useSTTAPI = (): IuseSTTAPI => {
           'X-Request-Id': requestId,
           'X-Session-Id': logger.getSessionId(),
         },
-        body: JSON.stringify({
-          passphrase: roomId?.host || '',
-          lang: lang,
-          dataStream_uid: 111111, // bot ID
-          encryption_mode: $config.ENCRYPTION_ENABLED
-            ? rtcProps.encryption.mode
-            : null,
-        }),
+        body: JSON.stringify(requestBody),
       });
+
       const res = await response.json();
       const endReqTs = Date.now();
       const latency = endReqTs - startReqTs;
+
       logger.log(
         LogSource.NetworkRest,
         'stt',
-        `STT API Success - Called ${method} on stt with lang ${lang}`,
+        `STT API Success - Called ${method}`,
         {
           responseData: res,
           requestId,
@@ -91,14 +119,30 @@ const useSTTAPI = (): IuseSTTAPI => {
           latency,
         },
       );
-      return res;
+
+      // Check if response has error
+      if (res?.error?.message) {
+        return {
+          success: false,
+          error: {
+            message: res.error.message,
+            code: res.error.code,
+          },
+          data: res,
+        };
+      }
+
+      return {
+        success: true,
+        data: res,
+      };
     } catch (error) {
       const endReqTs = Date.now();
       const latency = endReqTs - startReqTs;
       logger.error(
         LogSource.NetworkRest,
         'stt',
-        `STT API Failure - Called ${method} on stt with lang ${lang}`,
+        `STT API Failure - Called ${method}`,
         error,
         {
           requestId,
@@ -107,170 +151,39 @@ const useSTTAPI = (): IuseSTTAPI => {
           latency,
         },
       );
-    }
-  };
-
-  const startWithDelay = (lang: LanguageType[]): Promise<string> =>
-    new Promise(resolve => {
-      setTimeout(async () => {
-        const res = await start(lang);
-        resolve(res);
-      }, 1000); // Delay of 1 seconds (1000 milliseconds) to allow existing stt service to fully stop
-    });
-
-  const start = async (lang: LanguageType[]) => {
-    try {
-      setIsLangChangeInProgress(true);
-      const res = await apiCall('startv7', lang);
-      // null means stt startred successfully
-      const isSTTAlreadyActive =
-        res?.error?.message
-          ?.toLowerCase()
-          .indexOf('current status is STARTED') !== -1 ||
-        res?.error?.code === 610 ||
-        false;
-
-      if (res?.error?.message && res?.error?.code !== 610) {
-        setIsSTTError(true);
-        logger.error(
-          LogSource.NetworkRest,
-          'stt',
-          `start stt for lang ${lang} failed`,
-          res?.error,
-        );
-      } else {
-        logger.log(
-          LogSource.NetworkRest,
-          'stt',
-          `start stt for lang ${lang} succesfull`,
-          res,
-        );
-        setIsSTTError(false);
-      }
-      if (res === null || isSTTAlreadyActive) {
-        // once STT is active in the channel , notify others so that they dont' trigger start again
-        events.send(
-          EventNames.STT_ACTIVE,
-          JSON.stringify({active: true}),
-          PersistanceLevel.Sender,
-        );
-        setIsSTTActive(true);
-        logger.debug(
-          LogSource.NetworkRest,
-          'stt',
-          `stt lang update from: ${language} to ${lang}`,
-        );
-        // inform about the language set for stt
-        events.send(
-          EventNames.STT_LANGUAGE,
-          JSON.stringify({
-            username: capitalizeFirstLetter(username),
-            uid: localUid,
-            prevLang: language,
-            newLang: lang,
-          }),
-          PersistanceLevel.Sender,
-        );
-        setLanguage(lang);
 
-        // updaing transcript for self
-        const actionText =
-          language.indexOf('') !== -1
-            ? `has set the spoken language to  "${getLanguageLabel(lang)}" `
-            : `changed the spoken language from "${getLanguageLabel(
-                language,
-              )}" to "${getLanguageLabel(lang)}" `;
-        //const msg = `${capitalizeFirstLetter(username)} ${actionText} `;
-        setMeetingTranscript(prev => {
-          return [
-            ...prev,
-            {
-              name: 'langUpdate',
-              time: new Date().getTime(),
-              uid: `langUpdate-${localUid}`,
-              text: actionText,
-            },
-          ];
-        });
-      }
-      return res;
-    } catch (errorMsg) {
-      logger.error(
-        LogSource.NetworkRest,
-        'stt',
-        'There was error in start stt',
-        errorMsg,
-      );
-      throw errorMsg;
-    } finally {
-      setIsLangChangeInProgress(false);
+      return {
+        success: false,
+        error: {
+          message: error?.message || 'Unknown error occurred',
+          code: error?.code,
+        },
+      };
     }
   };
 
-  const stop = async () => {
-    try {
-      const res = await apiCall('stopv7');
-      // once STT is non-active in the channel , notify others so that they dont' trigger start again
-      // events.send(
-      //   EventNames.STT_ACTIVE,
-      //   JSON.stringify({active: false}),
-      //   PersistanceLevel.Session,
-      // );
-      setIsSTTActive(false);
-      if (res?.error?.message) {
-        setIsSTTError(true);
-      } else {
-        logger.log(LogSource.NetworkRest, 'stt', 'stop stt succesfull', res);
-        setIsSTTError(false);
-      }
-      return res;
-    } catch (error) {
-      logger.error(
-        LogSource.NetworkRest,
-        'stt',
-        'There was error in stop stt',
-        error,
-      );
-      throw error;
-    }
-  };
-  const restart = async (lang: LanguageType[]) => {
-    try {
-      setIsLangChangeInProgress(true);
-      await stop();
-      await startWithDelay(lang);
-      return Promise.resolve();
-    } catch (error) {
-      logger.error(
-        LogSource.NetworkRest,
-        'stt',
-        'There was error error in re-starting STT',
-        error,
-      );
-      return Promise.reject(error);
-    } finally {
-      setIsLangChangeInProgress(false);
-    }
+  const start = async (
+    botUid: number,
+    translationConfig: LanguageTranslationConfig,
+  ): Promise<STTAPIResponse> => {
+    return await apiCall('startv7', botUid, translationConfig);
   };
 
-  // attendee can view option if any host has started STT
-  const isAuthorizedSTTUser = () =>
-    $config.ENABLE_STT &&
-    $config.ENABLE_CAPTION &&
-    (isHost || (!isHost && isSTTActive));
+  const update = async (
+    botUid: number,
+    translationConfig: LanguageTranslationConfig,
+  ): Promise<STTAPIResponse> => {
+    return await apiCall('update', botUid, translationConfig);
+  };
 
-  const isAuthorizedTranscriptUser = () =>
-    $config.ENABLE_STT &&
-    $config.ENABLE_CAPTION &&
-    $config.ENABLE_MEETING_TRANSCRIPT &&
-    (isHost || (!isHost && isSTTActive));
+  const stop = async (botUid: number): Promise<STTAPIResponse> => {
+    return await apiCall('stopv7', botUid);
+  };
 
   return {
     start,
     stop,
-    restart,
-    isAuthorizedSTTUser,
-    isAuthorizedTranscriptUser,
+    update,
   };
 };
 

package/template/src/subComponents/caption/useStreamMessageUtils.native.ts

@@ -6,10 +6,15 @@ type StreamMessageCallback = (args: [number, Uint8Array]) => void;
 type FinalListType = {
   [key: string]: string[];
 };
-type TranscriptItem = {
-  uid: string;
-  time: number;
+type TranslationData = {
+  lang: string;
   text: string;
+  isFinal: boolean;
+};
+type FinalTranslationListType = {
+  [key: string]: {
+    [lang: string]: string[];
+  };
 };
 
 const useStreamMessageUtils = (): {
@@ -20,27 +25,33 @@ const useStreamMessageUtils = (): {
     setMeetingTranscript,
     activeSpeakerRef,
     prevSpeakerRef,
+    // Use ref instead of state to avoid stale closure issues
+    // The ref always has the current value, even in callbacks created at mount time
+    selectedTranslationLanguageRef,
   } = useCaption();
 
   let captionStartTime: number = 0;
   const finalList: FinalListType = {};
   const finalTranscriptList: FinalListType = {};
+  const finalTranslationList: FinalTranslationListType = {};
 
   const streamMessageCallback: StreamMessageCallback = args => {
     /* uid - bot which sends stream message in channel
        payload - stream message in Uint8Array format
       */
-    const [uid, payload] = args;
+    const [botUid, payload] = args;
     let nonFinalText = ''; // holds intermediate results
     let finalText = ''; // holds final strings
     let currentFinalText = ''; // holds current caption
     let isInterjecting = false;
+    let translations: TranslationData[] = [];
 
     const textstream = protoRoot
-      .lookupType('Text')
+      .lookupType('agora.audio2text.Text')
       .decode(payload as Uint8Array) as any;
 
-    console.log('STT - Parsed Textstream : ', textstream);
+    console.log('[STT_PER_USER_BOT] stt v7 textstream', botUid, textstream);
+    // console.log('STT - Parsed Textstream : ', textstream);
 
     // Identifing Current & Prev Speakers for the Captions
     /*
@@ -81,6 +92,14 @@ const useStreamMessageUtils = (): {
       // we have a speaker change so clear the context for prev speaker
       if (prevSpeakerRef.current !== '') {
         finalList[prevSpeakerRef.current] = [];
+        // Clear translations for previous speaker
+        if (finalTranslationList[prevSpeakerRef.current]) {
+          Object.keys(finalTranslationList[prevSpeakerRef.current]).forEach(
+            lang => {
+              finalTranslationList[prevSpeakerRef.current][lang] = [];
+            },
+          );
+        }
         isInterjecting = true;
       }
       prevSpeakerRef.current = activeSpeakerRef.current;
@@ -96,6 +115,51 @@ const useStreamMessageUtils = (): {
       finalTranscriptList[textstream.uid] = [];
     }
 
+    /* Process translations if available */
+    if (textstream.trans && textstream.trans.length > 0) {
+      for (const trans of textstream.trans) {
+        const lang = trans.lang;
+        const texts = trans.texts || [];
+        const isFinal = trans.isFinal || false;
+
+        if (!finalTranslationList[textstream.uid]) {
+          finalTranslationList[textstream.uid] = {};
+        }
+        if (!finalTranslationList[textstream.uid][lang]) {
+          finalTranslationList[textstream.uid][lang] = [];
+        }
+
+        const currentTranslationText = texts.join(' ');
+        if (currentTranslationText) {
+          if (isFinal) {
+            finalTranslationList[textstream.uid][lang].push(
+              currentTranslationText,
+            );
+          }
+
+          // Build complete translation text (final + current non-final)
+          const existingTranslationBuffer = isInterjecting
+            ? ''
+            : finalTranslationList[textstream.uid][lang]?.join(' ');
+          const latestTranslationString = isFinal ? '' : currentTranslationText;
+          const completeTranslationText =
+            existingTranslationBuffer.length > 0
+              ? latestTranslationString
+                ? existingTranslationBuffer + ' ' + latestTranslationString
+                : existingTranslationBuffer
+              : latestTranslationString;
+
+          if (completeTranslationText || isFinal) {
+            translations.push({
+              lang,
+              text: completeTranslationText,
+              isFinal,
+            });
+          }
+        }
+      }
+    }
+
     const words = textstream.words; //[Word,Word]
 
     /* categorize words into final & nonFinal objects per uid
@@ -105,7 +169,7 @@ const useStreamMessageUtils = (): {
       "isFinal": true,
       "confidence": 0.8549408316612244
       }
-  */
+    */
     for (const word of words) {
       if (word.isFinal) {
         finalText = finalText + word.text;
@@ -130,32 +194,57 @@ const useStreamMessageUtils = (): {
     }
 
     /* Updating Meeting Transcript */
-    if (currentFinalText.length) {
+    // Update transcript when: (1) new text finalized OR (2) final translations arrived
+    const hasFinalTranslations = textstream.trans?.some(
+      (t: any) => t.isFinal === true,
+    );
+
+    if (currentFinalText.length || hasFinalTranslations) {
+      // Prepare final translations for transcript
+      const finalTranslationsForTranscript: TranslationData[] = [];
+      if (finalTranslationList[textstream.uid]) {
+        Object.keys(finalTranslationList[textstream.uid]).forEach(lang => {
+          const translationText =
+            finalTranslationList[textstream.uid][lang]?.join(' ') || '';
+
+          if (translationText) {
+            finalTranslationsForTranscript.push({
+              lang: lang,
+              text: translationText,
+              isFinal: true,
+            });
+          }
+        });
+      }
+
       setMeetingTranscript(prevTranscript => {
         const lastTranscriptIndex = prevTranscript.length - 1;
         const lastTranscript =
           lastTranscriptIndex >= 0 ? prevTranscript[lastTranscriptIndex] : null;
 
         /*
-            checking if the last item transcript matches with current uid 
-            If yes then updating the last transcript msg with current text
+            checking if the last item transcript matches with current uid
+            If yes then updating the last transcript msg with current text and translations
             If no then adding a new entry in the transcript
           */
         if (lastTranscript && lastTranscript.uid === textstream.uid) {
           const updatedTranscript = {
             ...lastTranscript,
             //text: lastTranscript.text + ' ' + currentFinalText, // missing few updates with reading prev values
-            text: finalTranscriptList[textstream.uid].join(' '),
+            text: currentFinalText.length
+              ? finalTranscriptList[textstream.uid].join(' ')
+              : lastTranscript.text, // Keep existing text if no new text
+            translations: finalTranslationsForTranscript,
+            // preserve the original translation language from when this transcript was created
+            selectedTranslationLanguage:
+              lastTranscript.selectedTranslationLanguage,
           };
 
           return [
             ...prevTranscript.slice(0, lastTranscriptIndex),
             updatedTranscript,
           ];
-        } else {
-          const isLangUpdate =
-            lastTranscript?.uid.toString().indexOf('langUpdate') > -1;
-
+        } else if (currentFinalText.length) {
           finalTranscriptList[textstream.uid] = [currentFinalText];
 
           return [
@@ -164,36 +253,66 @@ const useStreamMessageUtils = (): {
               uid: textstream.uid,
               time: new Date().getTime(),
               text: currentFinalText,
+              translations: finalTranslationsForTranscript,
+              // Store the current translation language with this transcript item
+              // This preserves which translation was active when this text was spoken
+              selectedTranslationLanguage:
+                selectedTranslationLanguageRef.current,
             },
           ];
+        } else {
+          // No new text and uid doesn't match - don't modify transcript
+          // console.log(
+          //   '[TRANSCRIPT_DEBUG] Skipping transcript update - no new text and uid mismatch',
+          // );
+          return prevTranscript;
         }
       });
     }
 
-    /* 
-     Previous final words of the uid are prepended and 
+    /*
+     Previous final words of the uid are prepended and
      then current non final words so that context of speech is not lost
     */
     const existingStringBuffer = isInterjecting
       ? ''
       : finalList[textstream.uid]?.join(' ');
     const latestString = nonFinalText;
-    const captionText =
-      existingStringBuffer.length > 0
-        ? existingStringBuffer + ' ' + latestString
-        : latestString;
-
-    // updating the captions
-    captionText &&
-      setCaptionObj(prevState => {
-        return {
-          ...prevState,
-          [textstream.uid]: {
-            text: captionText,
-            lastUpdated: new Date().getTime(),
-          },
-        };
-      });
+    const captionText = isInterjecting
+      ? latestString
+      : existingStringBuffer.length > 0
+      ? existingStringBuffer + ' ' + latestString
+      : latestString;
+
+    // updating the captions with translations
+    setCaptionObj(prevState => {
+      const existingTranslations =
+        prevState[textstream.uid]?.translations || [];
+
+      // Update existing translations or add new ones
+      const updatedTranslations = [...existingTranslations];
+
+      for (const newTrans of translations) {
+        const existingIndex = updatedTranslations.findIndex(
+          t => t.lang === newTrans.lang,
+        );
+
+        if (existingIndex >= 0) {
+          updatedTranslations[existingIndex] = newTrans;
+        } else {
+          updatedTranslations.push(newTrans);
+        }
+      }
+
+      return {
+        ...prevState,
+        [textstream.uid]: {
+          text: captionText || prevState[textstream.uid]?.text || '',
+          translations: updatedTranslations,
+          lastUpdated: new Date().getTime(),
+        },
+      };
+    });
 
     console.group('STT-logs');
     console.log('Recived uid =>', textstream.uid);