@100mslive/roomkit-react 0.4.3-alpha.2 → 0.4.3-alpha.4

package/src/Prebuilt/plugins/CaptionsViewer.tsx

@@ -1,261 +0,0 @@
-import React, { useEffect, useRef, useState } from 'react';
-import Draggable, { ControlPosition } from 'react-draggable';
-import { useMedia } from 'react-use';
-import {
-  HMSTranscript,
-  selectIsTranscriptionEnabled,
-  selectPeerNameByID,
-  useHMSStore,
-  useTranscript,
-} from '@100mslive/react-sdk';
-import { Box, Flex } from '../../Layout';
-import { Text } from '../../Text';
-import { config } from '../../Theme';
-// @ts-ignore: No implicit Any
-import { useIsSidepaneTypeOpen } from '../components/AppData/useSidepane';
-// @ts-ignore: No implicit Any
-import { useIsCaptionEnabled } from '../components/AppData/useUISettings';
-import { useRoomLayoutConferencingScreen } from '../provider/roomLayoutProvider/hooks/useRoomLayoutScreen';
-import { SIDE_PANE_OPTIONS } from '../common/constants';
-interface CaptionQueueData extends HMSTranscript {
-  transcriptQueue: SimpleQueue;
-}
-
-interface TranscriptData extends HMSTranscript {
-  timeout?: NodeJS.Timeout | undefined;
-}
-class SimpleQueue {
-  private storage: TranscriptData[] = [];
-  constructor(private capacity: number = 3, private MAX_STORAGE_TIME: number = 5000) {}
-  enqueue(data: TranscriptData): void {
-    if (!data.transcript.trim()) {
-      return;
-    }
-    if (this.size() === this.capacity && this.storage[this.size() - 1].final) {
-      this.dequeue(this.storage[this.size() - 1]);
-    }
-    if (this.size() === 0) {
-      this.storage.push(data);
-      this.addTimeout(this.storage[this.size() - 1], data.final);
-      return;
-    }
-    if (this.size() > 0 && this.storage[this.size() - 1]?.final === true) {
-      this.storage.push(data);
-      this.addTimeout(this.storage[this.size() - 1], data.final);
-      return;
-    }
-    this.storage[this.size() - 1].transcript = data.transcript;
-    this.storage[this.size() - 1].final = data.final;
-    this.storage[this.size() - 1].end = data.end;
-    this.addTimeout(this.storage[this.size() - 1], data.final);
-  }
-  addTimeout(item: TranscriptData, isFinal: boolean) {
-    if (!isFinal) {
-      return;
-    }
-    item.timeout = setTimeout(() => {
-      this.dequeue(item);
-    }, this.MAX_STORAGE_TIME);
-  }
-  dequeue(item: TranscriptData): TranscriptData | undefined {
-    const index = this.storage.indexOf(item);
-    if (index === -1) {
-      return undefined;
-    }
-    const removedItem = this.storage.splice(index, 1);
-    if (removedItem.length <= 0) {
-      return undefined;
-    }
-    this.clearTimeout(removedItem[0]);
-    return item;
-  }
-  clearTimeout(item: TranscriptData) {
-    if (!item.timeout) {
-      return;
-    }
-    clearTimeout(item.timeout);
-  }
-  peek(): TranscriptData | undefined {
-    if (this.size() <= 0) {
-      return undefined;
-    }
-    return this.storage[0];
-  }
-  getTranscription(): string {
-    let script = '';
-    this.storage.forEach((value: TranscriptData) => (script += value.transcript + ' '));
-    return script;
-  }
-  reset() {
-    this.storage.length = 0;
-  }
-  size(): number {
-    return this.storage.length;
-  }
-}
-class Queue {
-  private storage: Record<string, CaptionQueueData> = {};
-  constructor(private capacity: number = 3) {}
-
-  enqueue(data: HMSTranscript): void {
-    if (this.size() === this.capacity) {
-      this.dequeue();
-    }
-    if (!this.storage[data.peer_id]) {
-      this.storage[data.peer_id] = {
-        peer_id: data.peer_id,
-        transcript: data.transcript,
-        final: data.final,
-        transcriptQueue: new SimpleQueue(),
-        start: data.start,
-        end: data.end,
-      };
-      this.storage[data.peer_id].transcriptQueue.enqueue(data as TranscriptData);
-      return;
-    }
-    this.storage[data.peer_id].transcriptQueue.enqueue(data as TranscriptData);
-  }
-  dequeue(): CaptionQueueData {
-    const key: string = Object.keys(this.storage).shift() || '';
-    const captionData = this.storage[key];
-    captionData.transcriptQueue.reset();
-    delete this.storage[key];
-    return captionData;
-  }
-
-  peek(): CaptionQueueData | undefined {
-    if (this.size() <= 0) return undefined;
-    const key: string = Object.keys(this.storage).shift() || '';
-    return this.storage[key];
-  }
-
-  findPeerData(): { [key: string]: string }[] {
-    const keys = Object.keys(this.storage);
-    const data = keys.map((key: string) => {
-      const data = this.storage[key];
-      const word = data.transcriptQueue.getTranscription();
-      return { [key]: word };
-    });
-    return data;
-  }
-  size(): number {
-    return Object.keys(this.storage).length;
-  }
-}
-
-class CaptionMaintainerQueue {
-  captionData: Queue;
-  constructor() {
-    this.captionData = new Queue();
-  }
-  push(data: HMSTranscript[] = []) {
-    data.forEach((value: HMSTranscript) => {
-      this.captionData.enqueue(value);
-    });
-  }
-}
-const TranscriptView = ({ peer_id, data }: { peer_id: string; data: string }) => {
-  const peerName = useHMSStore(selectPeerNameByID(peer_id)) || 'Participant';
-  data = data.trim();
-  if (!data) return null;
-  return (
-    <Text
-      variant="body2"
-      css={{
-        fontWeight: '$normal',
-        color: 'inherit',
-      }}
-    >
-      <b>{peerName}: </b>
-      {data}
-    </Text>
-  );
-};
-
-export const CaptionsViewer = ({
-  defaultPosition,
-  setDefaultPosition,
-}: {
-  defaultPosition: ControlPosition;
-  setDefaultPosition: (position: ControlPosition) => void;
-}) => {
-  const { elements, screenType } = useRoomLayoutConferencingScreen();
-  const isMobile = useMedia(config.media.md);
-  const isChatOpen = useIsSidepaneTypeOpen(SIDE_PANE_OPTIONS.CHAT);
-
-  const showCaptionAtTop = elements?.chat?.is_overlay && isChatOpen && isMobile;
-
-  const [captionQueue] = useState<CaptionMaintainerQueue>(new CaptionMaintainerQueue());
-  const [currentData, setCurrentData] = useState<{ [key: string]: string }[]>([]);
-
-  const isCaptionEnabled = useIsCaptionEnabled();
-
-  const isTranscriptionEnabled = useHMSStore(selectIsTranscriptionEnabled);
-
-  const nodeRef = useRef<HTMLDivElement>(null);
-
-  useEffect(() => {
-    const timeInterval = setInterval(() => {
-      if (!captionQueue) {
-        return;
-      }
-      const data = captionQueue.captionData?.findPeerData();
-      setCurrentData(data);
-    }, 1000);
-    return () => clearInterval(timeInterval);
-  }, [captionQueue]);
-
-  useTranscript({
-    onTranscript: (data: HMSTranscript[]) => {
-      captionQueue && captionQueue.push(data as HMSTranscript[]);
-    },
-  });
-  const dataToShow = currentData.filter((data: { [key: string]: string }) => {
-    const key = Object.keys(data)[0];
-    if (data[key]) {
-      return true;
-    }
-    return false;
-  });
-  if (dataToShow.length <= 0 || screenType === 'hls_live_streaming' || !isCaptionEnabled || !isTranscriptionEnabled) {
-    return null;
-  }
-  return (
-    <Draggable
-      bounds="parent"
-      nodeRef={nodeRef}
-      defaultPosition={defaultPosition}
-      onStop={(_, data) => {
-        setDefaultPosition({ x: data.lastX, y: data.lastY });
-      }}
-    >
-      <Box
-        ref={nodeRef}
-        css={{
-          position: 'absolute',
-          w: isMobile ? '100%' : '40%',
-          bottom: showCaptionAtTop ? '' : '0',
-          top: showCaptionAtTop ? '0' : '',
-          left: isMobile ? 0 : '50%',
-          transform: isMobile ? '' : 'translateX(-50%)',
-          background: '#000000A3',
-          overflow: 'clip',
-          zIndex: 10,
-          color: '$on_primary_high',
-          height: 'fit-content',
-          r: '$1',
-          p: '$6',
-          transition: 'bottom 0.3s ease-in-out',
-          '&:empty': { display: 'none' },
-        }}
-      >
-        <Flex direction="column">
-          {dataToShow.map((data: { [key: string]: string }, index: number) => {
-            const key = Object.keys(data)[0];
-            return <TranscriptView key={index} peer_id={key} data={data[key]} />;
-          })}
-        </Flex>
-      </Box>
-    </Draggable>
-  );
-};

package/src/Prebuilt/plugins/FlyingEmoji.jsx

@@ -1,154 +0,0 @@
-import React, { useCallback, useEffect, useMemo, useState } from 'react';
-import { useMedia } from 'react-use';
-import {
-  selectIsLargeRoom,
-  selectLocalPeerID,
-  selectPeerNameByID,
-  useCustomEvent,
-  useHMSActions,
-  useHMSStore,
-  useHMSVanillaStore,
-} from '@100mslive/react-sdk';
-import { Box, Flex } from '../../Layout';
-import { Text } from '../../Text';
-import { config as cssConfig, keyframes } from '../../Theme';
-import { EMOJI_REACTION_TYPE } from '../common/constants';
-
-let emojiCount = 1;
-
-const flyAndFade = keyframes({
-  '20%': { opacity: 1 },
-  '100%': { bottom: '60%', opacity: 0 },
-});
-
-const wiggleLeftRight = keyframes({
-  '0%': { marginLeft: '-50px' },
-  '100%': { marginLeft: '50px' },
-});
-
-const wiggleRightLeft = keyframes({
-  '0%': { marginLeft: '50px' },
-  '100%': { marginLeft: '-50px' },
-});
-
-const getStartingPoints = isMobile => {
-  let arr = [];
-  const min = 5;
-  const max = isMobile ? 30 : 20;
-  const inc = isMobile ? 8 : 5;
-  for (let i = min; i <= max; i += inc) {
-    arr.push(i);
-  }
-  return arr;
-};
-
-export function FlyingEmoji() {
-  const localPeerId = useHMSStore(selectLocalPeerID);
-  const vanillaStore = useHMSVanillaStore();
-  const hmsActions = useHMSActions();
-  const [emojis, setEmojis] = useState([]);
-  const isMobile = useMedia(cssConfig.media.md);
-  const isLargeRoom = useHMSStore(selectIsLargeRoom);
-
-  const startingPoints = useMemo(() => getStartingPoints(isMobile), [isMobile]);
-
-  const showFlyingEmoji = useCallback(
-    async ({ emojiId, senderId }) => {
-      if (!emojiId || !senderId || document.hidden) {
-        return;
-      }
-      let senderPeerName = vanillaStore.getState(selectPeerNameByID(senderId));
-      if (!senderPeerName && isLargeRoom) {
-        const sender = await hmsActions.getPeer(senderId);
-        senderPeerName = sender?.name;
-      }
-      const nameToShow = localPeerId === senderId ? 'You' : senderPeerName;
-      const startingPoint = startingPoints[emojiCount % startingPoints.length];
-      const id = emojiCount++;
-
-      setEmojis(emojis => {
-        return [
-          ...emojis,
-          {
-            id: id,
-            emojiId: emojiId,
-            senderName: nameToShow,
-            startingPoint: `${startingPoint}%`,
-            wiggleType: Math.random() < 0.5 ? 0 : 1,
-          },
-        ];
-      });
-    },
-    [vanillaStore, isLargeRoom, localPeerId, startingPoints, hmsActions],
-  );
-
-  useCustomEvent({
-    type: EMOJI_REACTION_TYPE,
-    onEvent: showFlyingEmoji,
-  });
-
-  useEffect(() => {
-    window.showFlyingEmoji = showFlyingEmoji;
-  }, [showFlyingEmoji]);
-
-  return (
-    <Box
-      css={{
-        position: 'absolute',
-        top: 0,
-        bottom: 0,
-        left: 0,
-        right: 0,
-        overflow: 'hidden',
-        pointerEvents: 'none',
-        userSelect: 'none',
-        zIndex: 999,
-      }}
-    >
-      {emojis.map(emoji => {
-        return (
-          <Flex
-            key={emoji.id}
-            css={{
-              left: emoji.startingPoint,
-              flexDirection: 'column',
-              alignItems: 'center',
-              position: 'absolute',
-              bottom: 0,
-              animation: `${flyAndFade()} 5s forwards, ${
-                emoji.wiggleType === 0 ? wiggleLeftRight() : wiggleRightLeft()
-              } 1s ease-in-out infinite alternate`,
-            }}
-            onAnimationEnd={() => setEmojis(emojis.filter(item => item.id !== emoji.id))}
-          >
-            <Box>
-              <em-emoji id={emoji.emojiId} size="48px" set="apple" />
-            </Box>
-            {emoji.senderName ? (
-              <Box
-                css={{
-                  width: 'fit-content',
-                  padding: '$2 $4',
-                  background: '$surface_bright',
-                  borderRadius: '$1',
-                }}
-              >
-                <Text
-                  css={{
-                    fontSize: '$space$6',
-                    lineHeight: '$xs',
-                    color: '$on_surface_high',
-                  }}
-                >
-                  {emoji.senderName}
-                </Text>
-              </Box>
-            ) : (
-              ''
-            )}
-          </Flex>
-        );
-      })}
-    </Box>
-  );
-}

package/src/Prebuilt/plugins/RemoteStopScreenshare.jsx

@@ -1,18 +0,0 @@
-import React, { useCallback } from 'react';
-import { useCustomEvent, useHMSActions } from '@100mslive/react-sdk';
-import { REMOTE_STOP_SCREENSHARE_TYPE } from '../common/constants';
-
-export function RemoteStopScreenshare() {
-  const actions = useHMSActions();
-
-  const onRemoteStopScreenshare = useCallback(async () => {
-    await actions.setScreenShareEnabled(false);
-  }, [actions]);
-
-  useCustomEvent({
-    type: REMOTE_STOP_SCREENSHARE_TYPE,
-    onEvent: onRemoteStopScreenshare,
-  });
-
-  return <></>;
-}

package/src/Prebuilt/plugins/transcription/Transcriber.js

@@ -1,216 +0,0 @@
-import RecordRTC, { StereoAudioRecorder } from 'recordrtc';
-import { selectIsLocalAudioEnabled, selectLocalPeerName } from '@100mslive/react-sdk';
-
-export class Transcriber {
-  constructor({ hmsStore, setTranscriptAndSpeakingPeer, setIsTranscriptionEnabled }) {
-    this.hmsStore = hmsStore;
-    this.enabled = false;
-    this.audioSocket = null; // this is the socket that will be used to send audio to the STT server
-    this.setTranscriptAndSpeakingPeer = setTranscriptAndSpeakingPeer;
-    this.setIsTranscriptionEnabled = setIsTranscriptionEnabled;
-    this.sttTuningConfig = {
-      timeSlice: 250,
-      desiredSampRate: 8000,
-      numberOfAudioChannels: 1,
-      bufferSize: 256,
-    };
-    this.resetTextTimer = null; // used to reset the transcript after some time, if no new update comes
-    this.localPeerName = hmsStore.getState(selectLocalPeerName);
-    this.observingLocalPeerTrack = false;
-    this.trackIdBeingObserved = null;
-    this.recordRTCInstance = null;
-    this.unsubscribes = [];
-  }
-
-  async toggleTranscriptionState() {
-    await this.enableTranscription(!this.enabled);
-  }
-
-  async enableTranscription(enable) {
-    if (enable === this.enabled) {
-      return;
-    }
-    console.log('transcription enabled', enable);
-    if (enable) {
-      this.enabled = true;
-      await this.setIsTranscriptionEnabled(true);
-      await this.listen();
-    } else {
-      this.enabled = false;
-      await this.setIsTranscriptionEnabled(false);
-      this.cleanup();
-    }
-  }
-
-  setTranscriptAndPeerWithExpiry(transcript, peerName) {
-    if (!transcript) {
-      return;
-    }
-    this.setTranscriptAndSpeakingPeer(transcript, `[${peerName}]`);
-    // reset after some time if no new update comes
-    clearTimeout(this.resetTextTimer);
-    this.resetTextTimer = setTimeout(() => {
-      this.resetTranscriptAndPeer();
-    }, 5000);
-  }
-
-  resetTranscriptAndPeer() {
-    this.setTranscriptAndSpeakingPeer('', '');
-    clearTimeout(this.resetTextTimer);
-  }
-
-  async listen(retryCount = 0) {
-    if (retryCount > 5) {
-      console.error('transcription', 'Max retry count reached!!', retryCount);
-      this.cleanup();
-      return;
-    }
-    try {
-      let url = process.env.REACT_APP_DYNAMIC_STT_TOKEN_GENERATION_ENDPOINT;
-      let res = await fetch(url);
-      let body = await res.json();
-      const authToken = body.token;
-
-      if (authToken) {
-        this.audioSocket = await new WebSocket(
-          `wss://api.assemblyai.com/v2/realtime/ws?sample_rate=${this.sttTuningConfig.desiredSampRate}&token=${authToken}`,
-        );
-        this.resetTranscriptAndPeer();
-        this.audioSocket.onmessage = message => {
-          try {
-            const res = JSON.parse(message.data);
-            if (res.text && this.enabled) {
-              //Limiting the transcript size based on it's charecter length.
-              let messageText =
-                res.text.length >= 80
-                  ? res.text
-                      .split(' ')
-                      .slice(Math.max(res.text.split(' ').length - 10, 1))
-                      .join(' ')
-                  : res.text;
-              if (messageText) {
-                this.setTranscriptAndPeerWithExpiry(messageText, this.localPeerName);
-              }
-            }
-          } catch (err) {
-            console.error('transcription', err);
-          }
-        };
-
-        this.audioSocket.onerror = event => {
-          console.error('transcription', event);
-          this.audioSocket.close();
-        };
-
-        this.audioSocket.onclose = event => {
-          try {
-            console.log(event);
-            this.audioSocket = null;
-            if (this.enabled && event.code !== 4001) {
-              this.listen(retryCount++);
-            }
-          } catch (err) {
-            console.error('transcription', err);
-          }
-        };
-
-        this.audioSocket.onopen = () => {
-          this.observeLocalPeerTrack();
-        };
-      } else {
-        console.error('Unable to fetch dynamic token!!');
-      }
-    } catch (err) {
-      console.error('transcription', err);
-    }
-  }
-
-  async observeLocalPeerTrack() {
-    try {
-      if (this.observingLocalPeerTrack) {
-        return;
-      }
-      this.observingLocalPeerTrack = true;
-      console.log('transcription - observing local peer track');
-      let unsub = this.hmsStore.subscribe(this.getAndObserveStream, selectIsLocalAudioEnabled);
-      this.unsubscribes.push(unsub);
-      this.getAndObserveStream(); // call it once to start observing initially
-    } catch (err) {
-      console.error('transcription - observing local peer track', err);
-    }
-  }
-
-  /**
-   * This method is used to get the local peer's stream and observe it for changes.
-   * @returns {Promise<void>}
-   */
-  getAndObserveStream = async () => {
-    // a hacky way to get the local peer's stream till we have a better way
-    const localPeer = window.__hms.sdk.getLocalPeer();
-    const mediaTrack = localPeer.audioTrack.nativeTrack;
-    if (!mediaTrack || mediaTrack.id === this.trackIdBeingObserved) {
-      return;
-    }
-    this.trackIdBeingObserved = mediaTrack.id;
-    console.log('transcription - observing local peer track', mediaTrack.id);
-    try {
-      if (this.recordRTCInstance) {
-        console.log('transcription - destroying earlier instance');
-        this.recordRTCInstance.destroy();
-      }
-      this.recordRTCInstance = null;
-    } catch (err) {
-      console.error('transcription - in destroying earlier instance', err);
-    }
-    const stream = new MediaStream([mediaTrack]);
-    await this.observeStream(stream);
-  };
-
-  async observeStream(stream) {
-    this.recordRTCInstance = new RecordRTC(stream, {
-      ...this.sttTuningConfig,
-      type: 'audio',
-      mimeType: 'audio/webm;codecs=pcm',
-      recorderType: StereoAudioRecorder,
-      ondataavailable: blob => {
-        const reader = new FileReader();
-        reader.onload = () => {
-          const base64data = reader.result;
-          if (this.audioSocket && this.enabled && this.audioSocket.readyState && this.audioSocket.readyState === 1) {
-            try {
-              this.audioSocket.send(JSON.stringify({ audio_data: base64data.split('base64,')[1] }));
-            } catch (err) {
-              console.error('transcription', err);
-            }
-          }
-        };
-        reader.readAsDataURL(blob);
-      },
-    });
-    this.recordRTCInstance.startRecording();
-  }
-
-  cleanup() {
-    console.log('transcription - cleanup');
-    if (this.audioSocket) {
-      try {
-        this.audioSocket.close();
-        this.audioSocket = null;
-      } catch (err) {
-        console.error("transcription cleanup - couldn't close socket", err);
-      }
-    }
-    if (this.recordRTCInstance) {
-      try {
-        this.recordRTCInstance.destroy();
-        this.recordRTCInstance = null;
-      } catch (err) {
-        console.error("transcription cleanup - couldn't stop recording", err);
-      }
-    }
-    for (const unsub of this.unsubscribes) {
-      unsub();
-    }
-    this.resetTranscriptAndPeer();
-  }
-}