agora-appbuilder-core 4.0.0-api.5 → 4.0.0-api.6

package/template/src/subComponents/caption/proto/ptoto.js

@@ -0,0 +1,91 @@
+/*eslint-disable block-scoped-var, id-length, no-control-regex, no-magic-numbers, no-prototype-builtins, no-redeclare, no-shadow, no-var, sort-vars*/
+"use strict";
+
+var $protobuf = require("protobufjs/light");
+
+var $root = ($protobuf.roots["default"] || ($protobuf.roots["default"] = new $protobuf.Root()))
+    .addJSON({
+        agora: {
+            nested: {
+                audio2text: {
+                    options: {
+                        java_package: "io.agora.rtc.audio2text",
+                        java_outer_classname: "Audio2TextProtobuffer"
+                    },
+                    nested: {
+                        Text: {
+                            fields: {
+                                vendor: {
+                                    type: "int32",
+                                    id: 1
+                                },
+                                version: {
+                                    type: "int32",
+                                    id: 2
+                                },
+                                seqnum: {
+                                    type: "int32",
+                                    id: 3
+                                },
+                                uid: {
+                                    type: "uint32",
+                                    id: 4
+                                },
+                                flag: {
+                                    type: "int32",
+                                    id: 5
+                                },
+                                time: {
+                                    type: "int64",
+                                    id: 6
+                                },
+                                lang: {
+                                    type: "int32",
+                                    id: 7
+                                },
+                                starttime: {
+                                    type: "int32",
+                                    id: 8
+                                },
+                                offtime: {
+                                    type: "int32",
+                                    id: 9
+                                },
+                                words: {
+                                    rule: "repeated",
+                                    type: "Word",
+                                    id: 10
+                                }
+                            }
+                        },
+                        Word: {
+                            fields: {
+                                text: {
+                                    type: "string",
+                                    id: 1
+                                },
+                                startMs: {
+                                    type: "int32",
+                                    id: 2
+                                },
+                                durationMs: {
+                                    type: "int32",
+                                    id: 3
+                                },
+                                isFinal: {
+                                    type: "bool",
+                                    id: 4
+                                },
+                                confidence: {
+                                    type: "double",
+                                    id: 5
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    });
+
+module.exports = $root;

package/template/src/subComponents/caption/proto/test.proto

@@ -0,0 +1,23 @@
+syntax = "proto3";
+package agora.audio2text;
+option java_package = "io.agora.rtc.audio2text";
+option java_outer_classname = "Audio2TextProtobuffer";
+message Text {
+ int32 vendor = 1;
+ int32 version = 2;
+ int32 seqnum = 3;
+ uint32 uid = 4;
+ int32 flag = 5;
+ int64 time = 6;
+ int32 lang = 7;
+ int32 starttime = 8;
+ int32 offtime = 9;
+ repeated Word words = 10;
+}
+message Word {
+ string text = 1;
+ int32 start_ms = 2;
+ int32 duration_ms = 3;
+ bool is_final = 4;
+ double confidence = 5;
+}

package/template/src/subComponents/caption/useCaption.tsx

@@ -0,0 +1,123 @@
+import {createHook} from 'customization-implementation';
+import React from 'react';
+import {LanguageType} from './utils';
+
+export type TranscriptItem = {
+  uid: string;
+  time: number;
+  text: string;
+};
+
+type CaptionObj = {
+  [key: string]: {
+    text: string;
+    lastUpdated: number;
+  };
+};
+
+export const CaptionContext = React.createContext<{
+  // for caption btn state
+  isCaptionON: boolean;
+  setIsCaptionON: React.Dispatch<React.SetStateAction<boolean>>;
+
+  // for error state
+  isSTTError: boolean;
+  setIsSTTError: React.Dispatch<React.SetStateAction<boolean>>;
+
+  // to check if stt is active in the call
+  isSTTActive: boolean;
+  setIsSTTActive: React.Dispatch<React.SetStateAction<boolean>>;
+
+  // holds the language selection for stt
+  language: LanguageType[];
+  setLanguage: React.Dispatch<React.SetStateAction<LanguageType[]>>;
+
+  // holds meeting transcript
+  meetingTranscript: TranscriptItem[];
+  setMeetingTranscript: React.Dispatch<React.SetStateAction<TranscriptItem[]>>;
+
+  // holds status of stt language change process
+  isLangChangeInProgress: boolean;
+  setIsLangChangeInProgress: React.Dispatch<React.SetStateAction<boolean>>;
+
+  // holds live captions
+  captionObj: CaptionObj;
+  setCaptionObj: React.Dispatch<React.SetStateAction<CaptionObj>>;
+
+  // holds status of StreamMessageCallback listener added (caption/transcript)
+  isSTTListenerAdded: boolean;
+  setIsSTTListenerAdded: React.Dispatch<React.SetStateAction<boolean>>;
+
+  activeSpeakerRef: React.MutableRefObject<string>;
+  prevSpeakerRef: React.MutableRefObject<string>;
+}>({
+  isCaptionON: false,
+  setIsCaptionON: () => {},
+  isSTTError: false,
+  setIsSTTError: () => {},
+  isSTTActive: false,
+  setIsSTTActive: () => {},
+  language: ['en-US'],
+  setLanguage: () => {},
+  meetingTranscript: [],
+  setMeetingTranscript: () => {},
+  isLangChangeInProgress: false,
+  setIsLangChangeInProgress: () => {},
+  captionObj: {},
+  setCaptionObj: () => {},
+  isSTTListenerAdded: false,
+  setIsSTTListenerAdded: () => {},
+  activeSpeakerRef: {current: ''},
+  prevSpeakerRef: {current: ''},
+});
+
+const CaptionProvider = ({children}) => {
+  const [isSTTError, setIsSTTError] = React.useState<boolean>(false);
+  const [isCaptionON, setIsCaptionON] = React.useState<boolean>(false);
+  const [isSTTActive, setIsSTTActive] = React.useState<boolean>(false);
+  const [language, setLanguage] = React.useState<[LanguageType]>(['']);
+  const [isLangChangeInProgress, setIsLangChangeInProgress] =
+    React.useState<boolean>(false);
+  const [meetingTranscript, setMeetingTranscript] = React.useState<
+    TranscriptItem[]
+  >([]);
+  const [captionObj, setCaptionObj] = React.useState<CaptionObj>({});
+  const [isSTTListenerAdded, setIsSTTListenerAdded] =
+    React.useState<boolean>(false);
+  const [activeSpeakerUID, setActiveSpeakerUID] = React.useState<string>('');
+  const [prevActiveSpeakerUID, setPrevActiveSpeakerUID] =
+    React.useState<string>('');
+
+  const activeSpeakerRef = React.useRef('');
+  const prevSpeakerRef = React.useRef('');
+
+  return (
+    <CaptionContext.Provider
+      value={{
+        isCaptionON,
+        setIsCaptionON,
+        isSTTError,
+        setIsSTTError,
+        isSTTActive,
+        setIsSTTActive,
+        language,
+        setLanguage,
+        meetingTranscript,
+        setMeetingTranscript,
+        isLangChangeInProgress,
+        setIsLangChangeInProgress,
+        captionObj,
+        setCaptionObj,
+        isSTTListenerAdded,
+        setIsSTTListenerAdded,
+        activeSpeakerRef,
+        prevSpeakerRef,
+      }}>
+      {children}
+    </CaptionContext.Provider>
+  );
+};
+
+const useCaption = createHook(CaptionContext);
+
+export {CaptionProvider, useCaption};

package/template/src/subComponents/caption/useCaptionWidth.ts

@@ -0,0 +1,27 @@
+import React from 'react';
+import {useWindowDimensions} from 'react-native';
+import {useCaption} from './useCaption';
+import {useSidePanel, SidePanelType} from 'customization-api';
+import {CAPTION_CONTAINER_HEIGHT} from '../../../src/components/CommonStyles';
+
+interface CaptionWidthReturnType {
+  isCaptionNotFullWidth: boolean;
+  transcriptHeight: string;
+}
+
+const useCaptionWidth = (): CaptionWidthReturnType => {
+  const {isCaptionON} = useCaption();
+  const {width: windowWidth, height: windowHeight} = useWindowDimensions();
+  const {sidePanel} = useSidePanel();
+
+  const isCaptionNotFullWidth =
+    isCaptionON && sidePanel !== SidePanelType.None && windowWidth > 1200;
+
+  const transcriptHeight = isCaptionNotFullWidth
+    ? `calc(100% + ${CAPTION_CONTAINER_HEIGHT}px + 4px)` //10px is gap b/w <Video> & <Caption/>
+    : '';
+
+  return {isCaptionNotFullWidth, transcriptHeight};
+};
+
+export default useCaptionWidth;

package/template/src/subComponents/caption/useSTTAPI.tsx

@@ -0,0 +1,179 @@
+import React, {useContext} from 'react';
+import StorageContext from '../../components/StorageContext';
+import {useRoomInfo} from '../../components/room-info/useRoomInfo';
+import {useCaption} from './useCaption';
+import events, {PersistanceLevel} from '../../rtm-events-api';
+import {EventNames} from '../../rtm-events';
+import {getLanguageLabel, LanguageType} from './utils';
+import useGetName from '../../utils/useGetName';
+import {capitalizeFirstLetter} from '../../utils/common';
+import {PropsContext, useLocalUid} from '../../../agora-rn-uikit';
+
+interface IuseSTTAPI {
+  start: (lang: LanguageType[]) => Promise<{message: string} | null>;
+  stop: () => Promise<void>;
+  restart: (lang: LanguageType[]) => Promise<void>;
+  isAuthorizedSTTUser: () => boolean;
+}
+
+const useSTTAPI = (): IuseSTTAPI => {
+  const {store} = React.useContext(StorageContext);
+  const {
+    data: {roomId, isHost},
+  } = useRoomInfo();
+  const {
+    language,
+    isSTTActive,
+    setIsSTTActive,
+    setIsLangChangeInProgress,
+    setLanguage,
+    setMeetingTranscript,
+    setIsSTTError,
+  } = useCaption();
+
+  const currentLangRef = React.useRef<LanguageType[]>([]);
+  const STT_API_URL = `${$config.BACKEND_ENDPOINT}/v1/stt`;
+  const username = useGetName();
+  const localUid = useLocalUid();
+  const {rtcProps} = useContext(PropsContext);
+
+  React.useEffect(() => {
+    currentLangRef.current = language;
+  }, [language]);
+
+  const apiCall = async (method: string, lang: LanguageType[] = []) => {
+    const response = await fetch(`${STT_API_URL}/${method}`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        authorization: store.token ? `Bearer ${store.token}` : '',
+      },
+      body: JSON.stringify({
+        passphrase: roomId?.host || '',
+        lang: lang,
+        dataStream_uid: 111111, // bot ID
+        encryption_mode: $config.ENCRYPTION_ENABLED
+          ? rtcProps.encryption.mode
+          : null,
+      }),
+    });
+    const res = await response.json();
+    return res;
+  };
+
+  const startWithDelay = (lang: LanguageType[]): Promise<string> =>
+    new Promise(resolve => {
+      setTimeout(async () => {
+        const res = await start(lang);
+        resolve(res);
+      }, 1000); // Delay of 1 seconds (1000 milliseconds) to allow existing stt service to fully stop
+    });
+
+  const start = async (lang: LanguageType[]) => {
+    try {
+      setIsLangChangeInProgress(true);
+      const res = await apiCall('start', lang);
+      // null means stt startred successfully
+      const isSTTAlreadyActive =
+        res?.error?.message
+          ?.toLowerCase()
+          .indexOf('current status is started') !== -1 || false;
+
+      if (res?.error?.message) {
+        setIsSTTError(true);
+      } else {
+        setIsSTTError(false);
+      }
+      if (res === null || isSTTAlreadyActive) {
+        // once STT is active in the channel , notify others so that they dont' trigger start again
+        events.send(
+          EventNames.STT_ACTIVE,
+          JSON.stringify({active: true}),
+          PersistanceLevel.Sender,
+        );
+        setIsSTTActive(true);
+
+        console.log(`stt lang update from: ${language} to ${lang}`);
+        // inform about the language set for stt
+        events.send(
+          EventNames.STT_LANGUAGE,
+          JSON.stringify({
+            username: capitalizeFirstLetter(username),
+            uid: localUid,
+            prevLang: language,
+            newLang: lang,
+          }),
+          PersistanceLevel.Sender,
+        );
+        setLanguage(lang);
+
+        // updaing transcript for self
+        const actionText =
+          language.indexOf('') !== -1
+            ? `has set the spoken language to  "${getLanguageLabel(lang)}" `
+            : `changed the spoken language from "${getLanguageLabel(
+                language,
+              )}" to "${getLanguageLabel(lang)}" `;
+        //const msg = `${capitalizeFirstLetter(username)} ${actionText} `;
+        setMeetingTranscript(prev => {
+          return [
+            ...prev,
+            {
+              name: 'langUpdate',
+              time: new Date().getTime(),
+              uid: `langUpdate-${localUid}`,
+              text: actionText,
+            },
+          ];
+        });
+      }
+      return res;
+    } catch (errorMsg) {
+      throw errorMsg;
+    } finally {
+      setIsLangChangeInProgress(false);
+    }
+  };
+
+  const stop = async () => {
+    try {
+      const res = await apiCall('stop');
+      // once STT is non-active in the channel , notify others so that they dont' trigger start again
+      // events.send(
+      //   EventNames.STT_ACTIVE,
+      //   JSON.stringify({active: false}),
+      //   PersistanceLevel.Session,
+      // );
+      setIsSTTActive(false);
+      if (res?.error?.message) {
+        setIsSTTError(true);
+      } else {
+        setIsSTTError(false);
+      }
+      return res;
+    } catch (error) {
+      throw error;
+    }
+  };
+  const restart = async (lang: LanguageType[]) => {
+    try {
+      setIsLangChangeInProgress(true);
+      await stop();
+      await startWithDelay(lang);
+      return Promise.resolve();
+    } catch (error) {
+      console.log('error in re-starting STT', error);
+      return Promise.reject(error);
+    } finally {
+      setIsLangChangeInProgress(false);
+    }
+  };
+
+  // attendee can view option if any host has started STT
+  const isAuthorizedSTTUser = () =>
+    $config.ENABLE_STT && (isHost || (!isHost && isSTTActive));
+
+  return {start, stop, restart, isAuthorizedSTTUser};
+};
+
+export default useSTTAPI;

package/template/src/subComponents/caption/useStreamMessageUtils.native.ts

@@ -0,0 +1,211 @@
+import React from 'react';
+import {useCaption} from './useCaption';
+import protoRoot from './proto/ptoto';
+
+type StreamMessageCallback = (args: [number, Uint8Array]) => void;
+type FinalListType = {
+  [key: string]: string[];
+};
+type TranscriptItem = {
+  uid: string;
+  time: number;
+  text: string;
+};
+
+const useStreamMessageUtils = (): {
+  streamMessageCallback: StreamMessageCallback;
+} => {
+  const {
+    setCaptionObj,
+    setMeetingTranscript,
+    activeSpeakerRef,
+    prevSpeakerRef,
+  } = useCaption();
+
+  let captionStartTime: number = 0;
+  const finalList: FinalListType = {};
+  const finalTranscriptList: FinalListType = {};
+
+  const streamMessageCallback: StreamMessageCallback = args => {
+    /* uid - bot which sends stream message in channel
+       payload - stream message in Uint8Array format
+      */
+    const [uid, payload] = args;
+    let nonFinalText = ''; // holds intermediate results
+    let finalText = ''; // holds final strings
+    let currentFinalText = ''; // holds current caption
+    let isInterjecting = false;
+
+    const textstream = protoRoot
+      .lookupType('Text')
+      .decode(payload as Uint8Array) as any;
+
+    console.log('STT - Parsed Textstream : ', textstream);
+
+    // Identifing Current & Prev Speakers for the Captions
+    /*
+      t0 A : Hi All!
+      t1 B : Hi A
+      t2 A : Hi B
+      t3 C : What's Plan
+      t4 B : Goal Planning
+      t5 C : Yes Agree
+
+     Screen:
+     Time           :  t0       |   t1      |    t2     |      t3         |       t4          |    t5
+     Prev Speaker   :           | A:Hi All! |  B: Hi A  |  A: Hi  B       |  C: What's Plan   |  B: Goal Planning
+     Active Speaker : A:Hi All! | B: Hi A   |  A: Hi B  |  C: What's Plan |  B: Goal Planning |  C: Yes Agree
+     Clearing Required:                           A               B                A                     C
+
+     Logic:
+     time           :  t0  t1 t2 t3 t4 t5       
+     textstream.uid :   A  B  A  C  B  C
+     (B)prevSpeaker :   -  -  A  B  A  C
+     (B)activeSpeake:   -  A  B  A  C  B
+     Prev Data clear:   -  -  A  B  A  C 
+     ################## CHANGE ######################
+     (A)prevSpeaker :   -  A  B  A  C  B
+     (A)activeSpeake:   A  B  A  C  B  C 
+
+     Clear when textStream.uid == prevSpeakerRef.current
+
+    */
+
+    const finalWord = textstream.words.filter(word => word.isFinal === true);
+    // when we only get final word for the previous speaker then don't flip previous speaker as active but update in place.
+
+    if (
+      textstream.uid !== activeSpeakerRef.current &&
+      !(finalWord.length > 0 && textstream.uid === prevSpeakerRef.current)
+    ) {
+      // we have a speaker change so clear the context for prev speaker
+      if (prevSpeakerRef.current !== '') {
+        finalList[prevSpeakerRef.current] = [];
+        isInterjecting = true;
+      }
+      prevSpeakerRef.current = activeSpeakerRef.current;
+      activeSpeakerRef.current = textstream.uid;
+    }
+
+    /* creating [] for each user to store their complete transcripts
+       initializing captions state for cuurent speaker
+       ex: {282190954:[]}
+    */
+    if (!finalList[textstream.uid]) {
+      finalList[textstream.uid] = [];
+      finalTranscriptList[textstream.uid] = [];
+    }
+
+    const words = textstream.words; //[Word,Word]
+
+    /* categorize words into final & nonFinal objects per uid
+      Final Word Ex : {
+      "text": "Hello, are you doing?",
+      "durationMs": 960,
+      "isFinal": true,
+      "confidence": 0.8549408316612244
+      }
+  */
+    for (const word of words) {
+      if (word.isFinal) {
+        finalText = finalText + word.text;
+      } else {
+        nonFinalText =
+          word.text !== '.' ? nonFinalText + word.text : nonFinalText;
+        if (!captionStartTime) {
+          captionStartTime = performance.now();
+        }
+      }
+    }
+    if (finalText) {
+      finalList[textstream.uid].push(finalText);
+      finalTranscriptList[textstream.uid].push(finalText);
+      currentFinalText = finalText;
+      // log info to show measure the duration of passes in which a sentence gets finalized
+      const duration = performance.now() - captionStartTime;
+      console.log(
+        `stt-Time taken to finalize caption ${currentFinalText}: ${duration}ms`,
+      );
+      captionStartTime = null; // Reset start time
+    }
+
+    /* Updating Meeting Transcript */
+    if (currentFinalText.length) {
+      setMeetingTranscript(prevTranscript => {
+        const lastTranscriptIndex = prevTranscript.length - 1;
+        const lastTranscript =
+          lastTranscriptIndex >= 0 ? prevTranscript[lastTranscriptIndex] : null;
+
+        /*
+            checking if the last item transcript matches with current uid 
+            If yes then updating the last transcript msg with current text
+            If no then adding a new entry in the transcript
+          */
+        if (lastTranscript && lastTranscript.uid === textstream.uid) {
+          const updatedTranscript = {
+            ...lastTranscript,
+            //text: lastTranscript.text + ' ' + currentFinalText, // missing few updates with reading prev values
+            text: finalTranscriptList[textstream.uid].join(' '),
+          };
+
+          return [
+            ...prevTranscript.slice(0, lastTranscriptIndex),
+            updatedTranscript,
+          ];
+        } else {
+          const isLangUpdate =
+            lastTranscript?.uid.toString().indexOf('langUpdate') > -1;
+
+          finalTranscriptList[textstream.uid] = [currentFinalText];
+
+          return [
+            ...prevTranscript,
+            {
+              uid: textstream.uid,
+              time: new Date().getTime(),
+              text: currentFinalText,
+            },
+          ];
+        }
+      });
+    }
+
+    /* 
+     Previous final words of the uid are prepended and 
+     then current non final words so that context of speech is not lost
+    */
+    const existingStringBuffer = isInterjecting
+      ? ''
+      : finalList[textstream.uid]?.join(' ');
+    const latestString = nonFinalText;
+    const captionText =
+      existingStringBuffer.length > 0
+        ? existingStringBuffer + ' ' + latestString
+        : latestString;
+
+    // updating the captions
+    captionText &&
+      setCaptionObj(prevState => {
+        return {
+          ...prevState,
+          [textstream.uid]: {
+            text: captionText,
+            lastUpdated: new Date().getTime(),
+          },
+        };
+      });
+
+    console.group('STT-logs');
+    console.log('Recived uid =>', textstream.uid);
+    console.log('PrevSpeaker uid =>', prevSpeakerRef.current);
+    console.log('ActiveSpeaker uid=>', activeSpeakerRef.current);
+    console.log('final List =>', finalList);
+    console.groupEnd();
+  };
+
+  return {
+    streamMessageCallback,
+  };
+};
+
+export default useStreamMessageUtils;