@100mslive/roomkit-react 0.3.8-alpha.1 → 0.3.8-alpha.2

package/src/Prebuilt/components/ConferenceScreen.tsx

@@ -1,4 +1,5 @@
 import React, { useEffect, useRef, useState } from 'react';
+import { useMedia } from 'react-use';
 import { DefaultConferencingScreen_Elements } from '@100mslive/types-prebuilt';
 import { v4 as uuid } from 'uuid';
 import {
@@ -18,6 +19,7 @@ import { ActivatedPIP } from './PIP/PIPComponent';
 import { PictureInPicture } from './PIP/PIPManager';
 import { RoleChangeRequestModal } from './RoleChangeRequest/RoleChangeRequestModal';
 import { Box, Flex } from '../../Layout';
+import { config } from '../../Theme';
 import { useHMSPrebuiltContext } from '../AppContext';
 import { VideoStreamingSection } from '../layouts/VideoStreamingSection';
 // @ts-ignore: No implicit Any
@@ -26,17 +28,21 @@ import FullPageProgress from './FullPageProgress';
 import { Header } from './Header';
 import { PreviousRoleInMetadata } from './PreviousRoleInMetadata';
 import { RaiseHand } from './RaiseHand';
+import { CaptionsViewer } from '../plugins/CaptionsViewer';
 import {
   useRoomLayoutConferencingScreen,
   useRoomLayoutPreviewScreen,
 } from '../provider/roomLayoutProvider/hooks/useRoomLayoutScreen';
 // @ts-ignore: No implicit Any
-import { useAuthToken, useSetAppDataByKey } from './AppData/useUISettings';
+import { useIsSidepaneTypeOpen } from './AppData/useSidepane';
+// @ts-ignore: No implicit Any
+import { useAuthToken, useIsCaptionEnabled, useSetAppDataByKey } from './AppData/useUISettings';
 import { useLandscapeHLSStream, useMobileHLSStream } from '../common/hooks';
-import { APP_DATA, isAndroid, isIOS, isIPadOS } from '../common/constants';
+import { APP_DATA, isAndroid, isIOS, isIPadOS, SIDE_PANE_OPTIONS } from '../common/constants';
 
 export const ConferenceScreen = () => {
   const { userName, endpoints, onJoin: onJoinFunc } = useHMSPrebuiltContext();
+  const isMobile = useMedia(config.media.md);
   const screenProps = useRoomLayoutConferencingScreen();
   const { isPreviewScreenEnabled } = useRoomLayoutPreviewScreen();
   const roomState = useHMSStore(selectRoomState);
@@ -57,6 +63,10 @@ export const ConferenceScreen = () => {
   const isMobileHLSStream = useMobileHLSStream();
   const isLandscapeHLSStream = useLandscapeHLSStream();
   const isMwebHLSStream = isMobileHLSStream || isLandscapeHLSStream;
+  const isCaptionEnabled = useIsCaptionEnabled();
+  const isChatOpen = useIsSidepaneTypeOpen(SIDE_PANE_OPTIONS.CHAT);
+
+  const showCaptionAtTop = screenProps.elements?.chat?.is_overlay && isChatOpen;
 
   const toggleControls = () => {
     if (dropdownListRef.current?.length === 0 && isMobileDevice && !isMwebHLSStream) {
@@ -124,6 +134,28 @@ export const ConferenceScreen = () => {
           <FullPageProgress text="Starting live stream..." css={{ opacity: 0.8, bg: '$background_dim' }} />
         </Box>
       ) : null}
+      {isCaptionEnabled && screenProps.screenType !== 'hls_live_streaming' && (
+        <Box
+          css={{
+            position: 'fixed',
+            maxWidth: isMobile ? '100%' : '40%',
+            bottom: showCaptionAtTop ? '' : hideControlsForStreaming ? '5%' : '10%',
+            top: showCaptionAtTop ? (hideControlsForStreaming ? '5%' : '10%') : '',
+            left: isMobile ? 0 : '50%',
+            transform: isMobile ? '' : 'translateX(-50%)',
+            background: '#000000A3',
+            overflow: 'clip',
+            zIndex: 10,
+            height: 'fit-content',
+            r: '$1',
+            p: '$6',
+            transition: 'bottom 0.3s ease-in-out',
+            '&:empty': { display: 'none' },
+          }}
+        >
+          <CaptionsViewer />
+        </Box>
+      )}
       <Flex css={{ size: '100%', overflow: 'hidden' }} direction="column">
         {!(screenProps.hideSections.includes('header') || isMwebHLSStream) && (
           <Box

package/src/Prebuilt/components/Footer/Footer.tsx

@@ -5,6 +5,7 @@ import { Chat_ChatState } from '@100mslive/types-prebuilt/elements/chat';
 import { config as cssConfig, Footer as AppFooter } from '../../..';
 // @ts-ignore: No implicit Any
 import { AudioVideoToggle } from '../AudioVideoToggle';
+import { CaptionIcon } from '../CaptionIcon';
 // @ts-ignore: No implicit Any
 import { EmojiReaction } from '../EmojiReaction';
 // @ts-ignore: No implicit Any
@@ -96,6 +97,7 @@ export const Footer = ({
           <>
             <ScreenshareToggle />
             <RaiseHand />
+            {screenType !== 'hls_live_streaming' && <CaptionIcon />}
             {elements?.emoji_reactions && <EmojiReaction />}
             <LeaveRoom screenType={screenType} />
           </>

package/src/Prebuilt/components/Footer/RoleAccordion.tsx

@@ -109,7 +109,7 @@ export const RoleAccordion = ({
           },
         }}
       >
-        <Flex justify="between" css={{ flexGrow: 1, pr: '$6' }}>
+        <Flex justify="between" align="center" css={{ flexGrow: 1, pr: '$6' }}>
           <Text
             variant="sm"
             css={{ fontWeight: '$semiBold', textTransform: 'capitalize', color: '$on_surface_medium' }}

package/src/Prebuilt/components/MoreSettings/SplitComponents/MwebOptions.tsx

@@ -5,6 +5,7 @@ import { match } from 'ts-pattern';
 import {
   selectIsConnectedToRoom,
   selectIsLocalVideoEnabled,
+  selectIsTranscriptionEnabled,
   selectPeerCount,
   selectPermissions,
   useHMSActions,
@@ -13,12 +14,14 @@ import {
 } from '@100mslive/react-sdk';
 import {
   BrbIcon,
+  ClosedCaptionIcon,
   CrossIcon,
   EmojiIcon,
   HamburgerMenuIcon,
   HandIcon,
   HandRaiseSlashedIcon,
   InfoIcon,
+  OpenCaptionIcon,
   PeopleIcon,
   QuizActiveIcon,
   QuizIcon,
@@ -49,7 +52,7 @@ import { useSheetToggle } from '../../AppData/useSheet';
 // @ts-ignore: No implicit any
 import { usePollViewToggle, useSidepaneToggle } from '../../AppData/useSidepane';
 // @ts-ignore: No implicit Any
-import { useShowPolls } from '../../AppData/useUISettings';
+import { useSetIsCaptionEnabled, useShowPolls } from '../../AppData/useUISettings';
 // @ts-ignore: No implicit any
 import { useDropdownList } from '../../hooks/useDropdownList';
 import { useMyMetadata } from '../../hooks/useMetadata';
@@ -102,6 +105,10 @@ export const MwebOptions = ({
   const toggleVB = useSidepaneToggle(SIDE_PANE_OPTIONS.VB);
   const isLocalVideoEnabled = useHMSStore(selectIsLocalVideoEnabled);
   const { startRecording, isRecordingLoading } = useRecordingHandler();
+
+  const isCaptionPresent = useHMSStore(selectIsTranscriptionEnabled);
+
+  const [isCaptionEnabled, setIsCaptionEnabled] = useSetIsCaptionEnabled();
   useDropdownList({ open: openModals.size > 0 || openOptionsSheet || openSettingsSheet, name: 'MoreSettings' });
 
   const updateState = (modalName: string, value: boolean) => {
@@ -186,6 +193,20 @@ export const MwebOptions = ({
                 <ActionTile.Title>{isHandRaised ? 'Lower' : 'Raise'} Hand</ActionTile.Title>
               </ActionTile.Root>
             ) : null}
+            {isCaptionPresent && screenType !== 'hls_live_streaming' ? (
+              <ActionTile.Root
+                onClick={() => {
+                  setIsCaptionEnabled(!isCaptionEnabled);
+                }}
+              >
+                {isCaptionEnabled ? (
+                  <ClosedCaptionIcon width="20" height="20px" />
+                ) : (
+                  <OpenCaptionIcon width="20" height="20px" />
+                )}
+                <ActionTile.Title>{isCaptionEnabled ? 'Hide Captions' : 'Captions Disabled'}</ActionTile.Title>
+              </ActionTile.Root>
+            ) : null}
 
             {isLocalVideoEnabled && !!elements?.virtual_background ? (
               <ActionTile.Root

package/src/Prebuilt/layouts/VideoStreamingSection.tsx

@@ -105,7 +105,6 @@ export const VideoStreamingSection = ({
             // @ts-ignore
             return <GridLayout {...(elements as DefaultConferencingScreen_Elements)?.video_tile_layout?.grid} />;
           })}
-
         <Box
           css={{
             flex: match({ isLandscapeHLSStream, isMobileHLSStream })

package/src/Prebuilt/plugins/CaptionsViewer.tsx

@@ -0,0 +1,191 @@
+import React, { useEffect, useState } from 'react';
+import { HMSTranscript, selectPeerNameByID, useHMSStore, useTranscript } from '@100mslive/react-sdk';
+import { Flex } from '../../Layout';
+import { Text } from '../../Text';
+
+interface CaptionQueueData extends HMSTranscript {
+  transcriptQueue: SimpleQueue;
+}
+
+interface TranscriptData extends HMSTranscript {
+  timeout?: NodeJS.Timeout | undefined;
+}
+class SimpleQueue {
+  private storage: TranscriptData[] = [];
+  constructor(private capacity: number = 3, private MAX_STORAGE_TIME: number = 5000) {}
+  enqueue(data: TranscriptData): void {
+    if (this.size() === this.capacity && this.storage[this.size() - 1].final) {
+      this.dequeue(this.storage[this.size() - 1]);
+    }
+    if (this.size() === 0) {
+      this.storage.push(data);
+      this.addTimeout(this.storage[this.size() - 1], data.final);
+      return;
+    }
+    if (this.size() > 0 && this.storage[this.size() - 1]?.final === true) {
+      this.storage.push(data);
+      this.addTimeout(this.storage[this.size() - 1], data.final);
+      return;
+    }
+    this.storage[this.size() - 1].transcript = data.transcript;
+    this.storage[this.size() - 1].final = data.final;
+    this.storage[this.size() - 1].end = data.end;
+    this.addTimeout(this.storage[this.size() - 1], data.final);
+  }
+  addTimeout(item: TranscriptData, isFinal: boolean) {
+    if (!isFinal) {
+      return;
+    }
+    item.timeout = setTimeout(() => {
+      this.dequeue(item);
+    }, this.MAX_STORAGE_TIME);
+  }
+  dequeue(item: TranscriptData): TranscriptData | undefined {
+    const index = this.storage.indexOf(item);
+    if (index === -1) {
+      return undefined;
+    }
+    const removedItem = this.storage.splice(index, 1);
+    if (removedItem.length <= 0) {
+      return undefined;
+    }
+    this.clearTimeout(removedItem[0]);
+    return item;
+  }
+  clearTimeout(item: TranscriptData) {
+    if (!item.timeout) {
+      return;
+    }
+    clearTimeout(item.timeout);
+  }
+  peek(): TranscriptData | undefined {
+    if (this.size() <= 0) {
+      return undefined;
+    }
+    return this.storage[0];
+  }
+  getTranscription(): string {
+    let script = '';
+    this.storage.forEach((value: TranscriptData) => (script += value.transcript + ' '));
+    return script;
+  }
+  reset() {
+    this.storage.length = 0;
+  }
+  size(): number {
+    return this.storage.length;
+  }
+}
+class Queue {
+  private storage: Record<string, CaptionQueueData> = {};
+  constructor(private capacity: number = 3) {}
+
+  enqueue(data: HMSTranscript): void {
+    if (this.size() === this.capacity) {
+      this.dequeue();
+    }
+    if (!this.storage[data.peer_id]) {
+      this.storage[data.peer_id] = {
+        peer_id: data.peer_id,
+        transcript: data.transcript,
+        final: data.final,
+        transcriptQueue: new SimpleQueue(),
+        start: data.start,
+        end: data.end,
+      };
+      this.storage[data.peer_id].transcriptQueue.enqueue(data as TranscriptData);
+      return;
+    }
+    this.storage[data.peer_id].transcriptQueue.enqueue(data as TranscriptData);
+  }
+  dequeue(): CaptionQueueData {
+    const key: string = Object.keys(this.storage).shift() || '';
+    const captionData = this.storage[key];
+    captionData.transcriptQueue.reset();
+    delete this.storage[key];
+    return captionData;
+  }
+
+  peek(): CaptionQueueData | undefined {
+    if (this.size() <= 0) return undefined;
+    const key: string = Object.keys(this.storage).shift() || '';
+    return this.storage[key];
+  }
+
+  findPeerData(): { [key: string]: string }[] {
+    const keys = Object.keys(this.storage);
+    const data = keys.map((key: string) => {
+      const data = this.storage[key];
+      const word = data.transcriptQueue.getTranscription();
+      return { [key]: word };
+    });
+    return data;
+  }
+  size(): number {
+    return Object.keys(this.storage).length;
+  }
+}
+
+class CaptionMaintainerQueue {
+  captionData: Queue = new Queue();
+  push(data: HMSTranscript[] = []) {
+    data.forEach((value: HMSTranscript) => {
+      this.captionData.enqueue(value);
+    });
+  }
+}
+const TranscriptView = ({ peer_id, data }: { peer_id: string; data: string }) => {
+  const peerName = useHMSStore(selectPeerNameByID(peer_id));
+  data = data.trim();
+  if (!data) return null;
+  return (
+    <Text
+      variant="body2"
+      css={{
+        fontWeight: '$normal',
+      }}
+    >
+      {`${peerName}: ${data}`}
+    </Text>
+  );
+};
+
+export const CaptionsViewer = () => {
+  const [captionQueue] = useState<CaptionMaintainerQueue>(new CaptionMaintainerQueue());
+  const [currentData, setCurrentData] = useState<{ [key: string]: string }[]>([]);
+
+  useEffect(() => {
+    const timeInterval = setInterval(() => {
+      if (!captionQueue) {
+        return;
+      }
+      const data = captionQueue.captionData?.findPeerData();
+      setCurrentData(data);
+    }, 1000);
+    return () => clearInterval(timeInterval);
+  }, [captionQueue]);
+
+  useTranscript({
+    onTranscript: (data: HMSTranscript[]) => {
+      captionQueue && captionQueue.push(data as HMSTranscript[]);
+    },
+  });
+  const dataToShow = currentData.filter((data: { [key: string]: string }) => {
+    const key = Object.keys(data)[0];
+    if (data[key]) {
+      return true;
+    }
+    return false;
+  });
+  if (dataToShow.length <= 0) {
+    return null;
+  }
+  return (
+    <Flex direction="column" gap={1}>
+      {dataToShow.map((data: { [key: string]: string }, index: number) => {
+        const key = Object.keys(data)[0];
+        return <TranscriptView key={index} peer_id={key} data={data[key]} />;
+      })}
+    </Flex>
+  );
+};