agora-appbuilder-core 4.0.0 → 4.0.2

package/template/src/subComponents/caption/useStreamMessageUtils.native.ts

@@ -0,0 +1,211 @@
+import React from 'react';
+import {useCaption} from './useCaption';
+import protoRoot from './proto/ptoto';
+
+type StreamMessageCallback = (args: [number, Uint8Array]) => void;
+type FinalListType = {
+  [key: string]: string[];
+};
+type TranscriptItem = {
+  uid: string;
+  time: number;
+  text: string;
+};
+
+const useStreamMessageUtils = (): {
+  streamMessageCallback: StreamMessageCallback;
+} => {
+  const {
+    setCaptionObj,
+    setMeetingTranscript,
+    activeSpeakerRef,
+    prevSpeakerRef,
+  } = useCaption();
+
+  let captionStartTime: number = 0;
+  const finalList: FinalListType = {};
+  const finalTranscriptList: FinalListType = {};
+
+  const streamMessageCallback: StreamMessageCallback = args => {
+    /* uid - bot which sends stream message in channel
+       payload - stream message in Uint8Array format
+      */
+    const [uid, payload] = args;
+    let nonFinalText = ''; // holds intermediate results
+    let finalText = ''; // holds final strings
+    let currentFinalText = ''; // holds current caption
+    let isInterjecting = false;
+
+    const textstream = protoRoot
+      .lookupType('Text')
+      .decode(payload as Uint8Array) as any;
+
+    console.log('STT - Parsed Textstream : ', textstream);
+
+    // Identifing Current & Prev Speakers for the Captions
+    /*
+      t0 A : Hi All!
+      t1 B : Hi A
+      t2 A : Hi B
+      t3 C : What's Plan
+      t4 B : Goal Planning
+      t5 C : Yes Agree
+
+     Screen:
+     Time           :  t0       |   t1      |    t2     |      t3         |       t4          |    t5
+     Prev Speaker   :           | A:Hi All! |  B: Hi A  |  A: Hi  B       |  C: What's Plan   |  B: Goal Planning
+     Active Speaker : A:Hi All! | B: Hi A   |  A: Hi B  |  C: What's Plan |  B: Goal Planning |  C: Yes Agree
+     Clearing Required:                           A               B                A                     C
+
+     Logic:
+     time           :  t0  t1 t2 t3 t4 t5       
+     textstream.uid :   A  B  A  C  B  C
+     (B)prevSpeaker :   -  -  A  B  A  C
+     (B)activeSpeake:   -  A  B  A  C  B
+     Prev Data clear:   -  -  A  B  A  C 
+     ################## CHANGE ######################
+     (A)prevSpeaker :   -  A  B  A  C  B
+     (A)activeSpeake:   A  B  A  C  B  C 
+
+     Clear when textStream.uid == prevSpeakerRef.current
+
+    */
+
+    const finalWord = textstream.words.filter(word => word.isFinal === true);
+    // when we only get final word for the previous speaker then don't flip previous speaker as active but update in place.
+
+    if (
+      textstream.uid !== activeSpeakerRef.current &&
+      !(finalWord.length > 0 && textstream.uid === prevSpeakerRef.current)
+    ) {
+      // we have a speaker change so clear the context for prev speaker
+      if (prevSpeakerRef.current !== '') {
+        finalList[prevSpeakerRef.current] = [];
+        isInterjecting = true;
+      }
+      prevSpeakerRef.current = activeSpeakerRef.current;
+      activeSpeakerRef.current = textstream.uid;
+    }
+
+    /* creating [] for each user to store their complete transcripts
+       initializing captions state for cuurent speaker
+       ex: {282190954:[]}
+    */
+    if (!finalList[textstream.uid]) {
+      finalList[textstream.uid] = [];
+      finalTranscriptList[textstream.uid] = [];
+    }
+
+    const words = textstream.words; //[Word,Word]
+
+    /* categorize words into final & nonFinal objects per uid
+      Final Word Ex : {
+      "text": "Hello, are you doing?",
+      "durationMs": 960,
+      "isFinal": true,
+      "confidence": 0.8549408316612244
+      }
+  */
+    for (const word of words) {
+      if (word.isFinal) {
+        finalText = finalText + word.text;
+      } else {
+        nonFinalText =
+          word.text !== '.' ? nonFinalText + word.text : nonFinalText;
+        if (!captionStartTime) {
+          captionStartTime = performance.now();
+        }
+      }
+    }
+    if (finalText) {
+      finalList[textstream.uid].push(finalText);
+      finalTranscriptList[textstream.uid].push(finalText);
+      currentFinalText = finalText;
+      // log info to show measure the duration of passes in which a sentence gets finalized
+      const duration = performance.now() - captionStartTime;
+      console.log(
+        `stt-Time taken to finalize caption ${currentFinalText}: ${duration}ms`,
+      );
+      captionStartTime = null; // Reset start time
+    }
+
+    /* Updating Meeting Transcript */
+    if (currentFinalText.length) {
+      setMeetingTranscript(prevTranscript => {
+        const lastTranscriptIndex = prevTranscript.length - 1;
+        const lastTranscript =
+          lastTranscriptIndex >= 0 ? prevTranscript[lastTranscriptIndex] : null;
+
+        /*
+            checking if the last item transcript matches with current uid 
+            If yes then updating the last transcript msg with current text
+            If no then adding a new entry in the transcript
+          */
+        if (lastTranscript && lastTranscript.uid === textstream.uid) {
+          const updatedTranscript = {
+            ...lastTranscript,
+            //text: lastTranscript.text + ' ' + currentFinalText, // missing few updates with reading prev values
+            text: finalTranscriptList[textstream.uid].join(' '),
+          };
+
+          return [
+            ...prevTranscript.slice(0, lastTranscriptIndex),
+            updatedTranscript,
+          ];
+        } else {
+          const isLangUpdate =
+            lastTranscript?.uid.toString().indexOf('langUpdate') > -1;
+
+          finalTranscriptList[textstream.uid] = [currentFinalText];
+
+          return [
+            ...prevTranscript,
+            {
+              uid: textstream.uid,
+              time: new Date().getTime(),
+              text: currentFinalText,
+            },
+          ];
+        }
+      });
+    }
+
+    /* 
+     Previous final words of the uid are prepended and 
+     then current non final words so that context of speech is not lost
+    */
+    const existingStringBuffer = isInterjecting
+      ? ''
+      : finalList[textstream.uid]?.join(' ');
+    const latestString = nonFinalText;
+    const captionText =
+      existingStringBuffer.length > 0
+        ? existingStringBuffer + ' ' + latestString
+        : latestString;
+
+    // updating the captions
+    captionText &&
+      setCaptionObj(prevState => {
+        return {
+          ...prevState,
+          [textstream.uid]: {
+            text: captionText,
+            lastUpdated: new Date().getTime(),
+          },
+        };
+      });
+
+    console.group('STT-logs');
+    console.log('Recived uid =>', textstream.uid);
+    console.log('PrevSpeaker uid =>', prevSpeakerRef.current);
+    console.log('ActiveSpeaker uid=>', activeSpeakerRef.current);
+    console.log('final List =>', finalList);
+    console.groupEnd();
+  };
+
+  return {
+    streamMessageCallback,
+  };
+};
+
+export default useStreamMessageUtils;

package/template/src/subComponents/caption/useStreamMessageUtils.ts

@@ -0,0 +1,235 @@
+import React from 'react';
+import {useCaption} from './useCaption';
+import protoRoot from './proto/ptoto';
+import PQueue from 'p-queue';
+
+type StreamMessageCallback = (args: [number, Uint8Array]) => void;
+type FinalListType = {
+  [key: string]: string[];
+};
+
+const useStreamMessageUtils = (): {
+  streamMessageCallback: StreamMessageCallback;
+} => {
+  const {
+    setCaptionObj,
+    setMeetingTranscript,
+    activeSpeakerRef,
+    prevSpeakerRef,
+  } = useCaption();
+
+  let captionStartTime: number = 0;
+  const finalList: FinalListType = {};
+  const finalTranscriptList: FinalListType = {};
+  const queue = new PQueue({concurrency: 1});
+  let counter = 0;
+  let lastOfftime = 0;
+
+  const streamMessageCallback: StreamMessageCallback = args => {
+    const queueCallback = (args1: [number, Uint8Array]) => {
+      /* uid - bot which sends stream message in channel
+       payload - stream message in Uint8Array format
+      */
+      const [, payload] = args1;
+      let nonFinalText = ''; // holds intermediate results
+      let finalText = ''; // holds final strings
+      let currentFinalText = ''; // holds current caption
+      let isInterjecting = false;
+
+      const textstream = protoRoot
+        .lookupType('Text')
+        .decode(payload as Uint8Array) as any;
+
+      //console.log('STT - Parsed Textstream : ', textstream);
+      // console.log(
+      //   `STT-callback(${++counter}): %c${textstream.uid} %c${textstream.words
+      //     .map(word =>
+      //       word.text === '.'
+      //         ? ''
+      //         : word.isFinal
+      //         ? word.text + ' --Final--'
+      //         : word.text,
+      //     )
+      //     .join(' ')}`,
+      //   'color:red',
+      //   'color:blue',
+      // );
+
+      // Identifing Current & Prev Speakers for the Captions
+      /*
+      t0 A : Hi All!
+      t1 B : Hi A
+      t2 A : Hi B
+      t3 C : What's Plan
+      t4 B : Goal Planning
+      t5 C : Yes Agree
+
+     Screen:
+     Time           :  t0       |   t1      |    t2     |      t3         |       t4          |    t5
+     Prev Speaker   :           | A:Hi All! |  B: Hi A  |  A: Hi  B       |  C: What's Plan   |  B: Goal Planning
+     Active Speaker : A:Hi All! | B: Hi A   |  A: Hi B  |  C: What's Plan |  B: Goal Planning |  C: Yes Agree
+     Clearing Required:                           A               B                A                     C
+
+     Logic:
+     time           :  t0  t1 t2 t3 t4 t5
+     textstream.uid :   A  B  A  C  B  C
+     (B)prevSpeaker :   -  -  A  B  A  C
+     (B)activeSpeake:   -  A  B  A  C  B
+     Prev Data clear:   -  -  A  B  A  C
+     ################## CHANGE ######################
+     (A)prevSpeaker :   -  A  B  A  C  B
+     (A)activeSpeake:   A  B  A  C  B  C
+
+     Clear when textStream.uid == prevSpeakerRef.current
+
+    */
+
+      const finalWord = textstream.words.filter(word => word.isFinal === true);
+      // when we only get final word for the previous speaker then don't flip previous speaker as active but update in place.
+
+      if (
+        textstream.uid !== activeSpeakerRef.current &&
+        !(finalWord.length > 0 && textstream.uid === prevSpeakerRef.current)
+      ) {
+        // we have a speaker change so clear the context for prev speaker
+        if (prevSpeakerRef.current !== '') {
+          finalList[prevSpeakerRef.current] = [];
+          isInterjecting = true;
+          // console.log(
+          //   '%cSTT-callback%c Interjection! ',
+          //   'color:#fff',
+          //   'background: #222; color: #bada55',
+          // );
+        }
+        prevSpeakerRef.current = activeSpeakerRef.current;
+        activeSpeakerRef.current = textstream.uid;
+      }
+
+      /* creating [] for each user to store their complete transcripts
+       initializing captions state for cuurent speaker
+       ex: {282190954:[]}
+    */
+      if (!finalList[textstream.uid]) {
+        finalList[textstream.uid] = [];
+        finalTranscriptList[textstream.uid] = [];
+      }
+
+      const words = textstream.words; //[Word,Word]
+
+      /* categorize words into final & nonFinal objects per uid
+      Final Word Ex : {
+      "text": "Hello, are you doing?",
+      "durationMs": 960,
+      "isFinal": true,
+      "confidence": 0.8549408316612244
+      }
+  */
+
+      for (const word of words) {
+        if (word.isFinal) {
+          finalText = finalText + word.text;
+        } else {
+          nonFinalText =
+            word.text !== '.' ? nonFinalText + word.text : nonFinalText;
+          if (!captionStartTime) {
+            captionStartTime = performance.now();
+          }
+        }
+      }
+
+      if (finalText) {
+        finalList[textstream.uid].push(finalText);
+        finalTranscriptList[textstream.uid].push(finalText);
+        currentFinalText = finalText;
+        // log info to show measure the duration of passes in which a sentence gets finalized
+        const duration = performance.now() - captionStartTime;
+        console.log(
+          `stt-Time taken to finalize caption ${currentFinalText}: ${duration}ms`,
+        );
+        captionStartTime = null; // Reset start time
+      }
+
+      /* Updating Meeting Transcript */
+      if (currentFinalText.length) {
+        setMeetingTranscript(prevTranscript => {
+          const lastTranscriptIndex = prevTranscript.length - 1;
+          const lastTranscript =
+            lastTranscriptIndex >= 0
+              ? prevTranscript[lastTranscriptIndex]
+              : null;
+
+          /*
+            checking if the last item transcript matches with current uid
+            If yes then updating the last transcript msg with current text
+            If no then adding a new entry in the transcript
+          */
+          if (lastTranscript && lastTranscript.uid === textstream.uid) {
+            const updatedTranscript = {
+              ...lastTranscript,
+              //text: lastTranscript.text + ' ' + currentFinalText, // missing few updates with reading prev values
+              text: finalTranscriptList[textstream.uid].join(' '),
+            };
+
+            return [
+              ...prevTranscript.slice(0, lastTranscriptIndex),
+              updatedTranscript,
+            ];
+          } else {
+            finalTranscriptList[textstream.uid] = [currentFinalText];
+
+            return [
+              ...prevTranscript,
+              {
+                uid: textstream.uid,
+                time: new Date().getTime(),
+                text: currentFinalText,
+              },
+            ];
+          }
+        });
+      }
+
+      /*
+     Previous final words of the uid are prepended and
+     then current non final words so that context of speech is not lost
+    */
+      const existingStringBuffer = isInterjecting
+        ? ''
+        : finalList[textstream.uid]?.join(' ');
+      const latestString = nonFinalText;
+      const captionText =
+        existingStringBuffer.length > 0
+          ? existingStringBuffer + ' ' + latestString
+          : latestString;
+
+      // updating the captions
+      captionText &&
+        setCaptionObj(prevState => {
+          return {
+            ...prevState,
+            [textstream.uid]: {
+              text: captionText,
+              lastUpdated: new Date().getTime(),
+            },
+          };
+        });
+
+      // console.group('STT-logs');
+      // console.log('Recived uid =>', textstream.uid);
+      // console.log('PrevSpeaker uid =>', prevSpeakerRef.current);
+      // console.log('ActiveSpeaker uid=>', activeSpeakerRef.current);
+      // console.log('final List =>', finalList);
+      // console.groupEnd();
+    };
+    (async () => {
+      await queue.add(() => queueCallback(args));
+      console.log('stt- using pq queue');
+    })();
+  };
+
+  return {
+    streamMessageCallback,
+  };
+};
+
+export default useStreamMessageUtils;

package/template/src/subComponents/caption/useTranscriptDownload.native.ts

@@ -0,0 +1,63 @@
+import {formatTranscriptContent} from './utils';
+import {useCaption} from './useCaption';
+import RNFetchBlob from 'rn-fetch-blob';
+import {isAndroid, isIOS, useRoomInfo, useContent} from 'customization-api';
+import Share from 'react-native-share';
+
+const useTranscriptDownload = (): {
+  downloadTranscript: () => Promise<string | null>;
+} => {
+  const {meetingTranscript} = useCaption();
+  const {defaultContent} = useContent();
+  const {
+    data: {meetingTitle},
+  } = useRoomInfo();
+
+  const downloadTranscript = (): Promise<string | null> => {
+    return new Promise((resolve, reject) => {
+      try {
+        const [finalContent, fileName] = formatTranscriptContent(
+          meetingTranscript,
+          meetingTitle,
+          defaultContent,
+        );
+
+        // get path to the Documents directory, don't have access to Downloads folder so saving in documents 1
+        const documentsDir = RNFetchBlob.fs.dirs.DocumentDir;
+
+        // setting file path
+        const filePath = `${documentsDir}/${fileName}`;
+
+        // Writing content to the file
+        RNFetchBlob.fs
+          .writeFile(filePath, finalContent, 'utf8')
+          .then(() => {
+            console.warn('Content downloaded successfully on native.');
+            // need to show the preview of downloaded file
+            Share.open({url: `file://${filePath}`, type: 'text/plain'})
+              .then(res => {
+                //  console.warn('File shared successfully:', res);
+                resolve(filePath);
+              })
+              .catch(error => {
+                // console.error('Error sharing file:', error);
+                reject(error);
+              });
+
+            resolve(filePath);
+          })
+          .catch(error => {
+            console.error('Error downloading content:', error);
+            reject(error);
+          });
+      } catch (error) {
+        console.error('Error downloading content:', error);
+        reject(error);
+      }
+    });
+  };
+
+  return {downloadTranscript};
+};
+
+export default useTranscriptDownload;

package/template/src/subComponents/caption/useTranscriptDownload.ts

@@ -0,0 +1,52 @@
+import {formatTranscriptContent} from './utils';
+import {useCaption} from './useCaption';
+import {useRoomInfo, useContent} from 'customization-api';
+
+const useTranscriptDownload = (): {
+  downloadTranscript: () => Promise<string | null>;
+} => {
+  const {meetingTranscript} = useCaption();
+
+  const {defaultContent} = useContent();
+  const {
+    data: {meetingTitle},
+  } = useRoomInfo();
+
+  const downloadTranscript = (): Promise<string | null> => {
+    return new Promise((resolve, reject) => {
+      try {
+        const [finalContent, fileName] = formatTranscriptContent(
+          meetingTranscript,
+          meetingTitle,
+          defaultContent,
+        );
+
+        // blob with required content
+        const blob = new Blob([finalContent], {type: 'text/plain'});
+
+        // url to download content
+        const downloadUrl = URL.createObjectURL(blob);
+
+        // anchor ele to download
+        const anchor = document.createElement('a');
+        anchor.href = downloadUrl;
+        anchor.download = fileName;
+
+        // click to dowload the file
+        anchor.click();
+
+        // revoke download url
+        URL.revokeObjectURL(downloadUrl);
+        console.log('Content downloaded successfully.');
+        resolve(downloadUrl);
+      } catch (error) {
+        console.error('Error downloading content:', error);
+        reject(error);
+      }
+    });
+  };
+
+  return {downloadTranscript};
+};
+
+export default useTranscriptDownload;

package/template/src/subComponents/caption/utils.ts

@@ -0,0 +1,126 @@
+import {ContentObjects} from '../../../agora-rn-uikit/src/Contexts/RtcContext';
+import {TranscriptItem} from './useCaption';
+
+export function formatTime(timestamp: number): string {
+  const d = new Date(timestamp);
+  const h = d.getHours();
+  const m = d.getMinutes().toString().padStart(2, '0');
+  //const s = d.getSeconds().toString().padStart(2, '0');
+  const suffix = h >= 12 ? 'PM' : 'AM';
+  const H = h % 12 || 12;
+  return `${H}:${m} ${suffix}`;
+}
+
+export type LanguageType =
+  | 'en-US'
+  | 'hi-IN'
+  | 'zh-CN'
+  | 'zh-HK'
+  | 'fr-FR'
+  | 'de-DE'
+  | 'ko-KR'
+  | 'en-IN'
+  | 'ar'
+  | 'ja-JP'
+  | 'pt-PT'
+  | 'es-ES'
+  | 'it-IT'
+  | 'id-ID'
+  | '';
+
+interface LanguageData {
+  label: string;
+  value: LanguageType;
+}
+
+export const langData: LanguageData[] = [
+  {label: 'English (US)', value: 'en-US'},
+  {label: 'English (India)', value: 'en-IN'},
+  {label: 'Hindi', value: 'hi-IN'},
+  {label: 'Chinese (Simplified)', value: 'zh-CN'},
+  {label: 'Chinese (Traditional)', value: 'zh-HK'},
+  {label: 'Arabic', value: 'ar'},
+  {label: 'French', value: 'fr-FR'},
+  {label: 'German', value: 'de-DE'},
+  {label: 'Japanese', value: 'ja-JP'},
+  {label: 'Korean', value: 'ko-KR'},
+  {label: 'Portuguese', value: 'pt-PT'},
+  {label: 'Spanish', value: 'es-ES'},
+  {label: 'Italian', value: 'it-IT'},
+  {label: 'Indonesian', value: 'id-ID'},
+];
+
+export function getLanguageLabel(
+  languageCode: LanguageType[],
+): string | undefined {
+  const langLabels = languageCode.map(langCode => {
+    return langData.find(data => data.value === langCode).label;
+  });
+  return langLabels ? langLabels.join(', ') : undefined;
+}
+
+export function formatDateWithTimeZone(date: Date): string {
+  const day = String(date.getUTCDate()).padStart(2, '0');
+  const month = String(date.getUTCMonth() + 1).padStart(2, '0');
+  const year = date.getUTCFullYear();
+
+  const hours = String(date.getUTCHours()).padStart(2, '0');
+  const minutes = String(date.getUTCMinutes()).padStart(2, '0');
+
+  const timeZoneOffset = 5.5 * 60; // Offset in minutes for GMT +5:30
+  const offsetHours = Math.floor(timeZoneOffset / 60);
+  const offsetMinutes = timeZoneOffset % 60;
+  const offsetSign = '+';
+
+  return `${day}/${month}/${year} ${hours}:${minutes} GMT ${offsetSign}${offsetHours}:${String(
+    offsetMinutes,
+  ).padStart(2, '0')}`;
+}
+
+export const formatTranscriptContent = (
+  meetingTranscript: TranscriptItem[],
+  meetingTitle: string,
+  defaultContent: ContentObjects,
+) => {
+  const formattedContent = meetingTranscript
+    .map(item => {
+      if (item.uid.toString().indexOf('langUpdate') !== -1) {
+        return `${defaultContent[item?.uid?.split('-')[1]]?.name} ${item.text}`;
+      }
+      return `${defaultContent[item.uid].name} ${formatTime(
+        Number(item?.time),
+      )}:\n${item.text}`;
+    })
+    .join('\n\n');
+
+  const startTime = formatDateWithTimeZone(
+    new Date(meetingTranscript[0]?.time),
+  );
+
+  const attendees = Object.entries(defaultContent)
+    .filter(
+      arr =>
+        arr[1].type === 'rtc' &&
+        arr[0] !== '100000' && // exclude recording bot
+        (arr[1]?.isInWaitingRoom === true ? false : true),
+    )
+    .map(arr => arr[1].name)
+    .join(',');
+
+  const info =
+    'This editable transcript was computer generated and might contain errors. People can also change\nthe text after it was created.';
+  const finalContent = `${meetingTitle} (${startTime}) - Transcript \n\nParticipants\n${attendees} \n\nTranscript\n${info} \n\n${formattedContent}`;
+
+  // to give unique file name
+  const currentDate = new Date();
+  const formattedDate = currentDate.toISOString().slice(0, 10);
+  const formattedTime = currentDate
+    .toTimeString()
+    .slice(0, 8)
+    .replace(/:/g, '');
+
+  // adding the filename
+  const fileName = `MeetingTranscript_${formattedDate}_${formattedTime}.txt`;
+
+  return [finalContent, fileName];
+};