@stream-io/video-react-sdk 0.0.1-alpha.9

package/src/core/components/Video/Video.tsx

@@ -0,0 +1,238 @@
+import {
+  DetailedHTMLProps,
+  useCallback,
+  useEffect,
+  useRef,
+  useState,
+  VideoHTMLAttributes,
+} from 'react';
+import {
+  Call,
+  DebounceType,
+  SfuModels,
+  StreamVideoParticipant,
+  VisibilityState,
+} from '@stream-io/video-client';
+import clsx from 'clsx';
+import { VideoPlaceholder } from './VideoPlaceholder';
+import { BaseVideo } from './BaseVideo';
+
+export const Video = (
+  props: DetailedHTMLProps<
+    VideoHTMLAttributes<HTMLVideoElement>,
+    HTMLVideoElement
+  > & {
+    call: Call;
+    kind: 'video' | 'screen';
+    participant: StreamVideoParticipant;
+    setVideoElementRef?: (element: HTMLElement | null) => void;
+  },
+) => {
+  const { call, kind, participant, className, setVideoElementRef, ...rest } =
+    props;
+  const { sessionId, videoStream, screenShareStream, publishedTracks } =
+    participant;
+
+  const [videoElement, setVideoElement] = useState<HTMLVideoElement | null>(
+    null,
+  );
+  // const [videoTrackMuted, setVideoTrackMuted] = useState(false);
+  const [videoPlaying, setVideoPlaying] = useState(false);
+  const viewportVisibilityRef = useRef<VisibilityState | undefined>(
+    participant.viewportVisibilityState,
+  );
+
+  const stream = kind === 'video' ? videoStream : screenShareStream;
+
+  // TODO: handle track muting
+  // useEffect(() => {
+  //   if (!stream) return;
+
+  //   const [track] = stream.getVideoTracks();
+  //   setVideoTrackMuted(track.muted);
+
+  //   const handleMute = () => {
+  //     setVideoTrackMuted(true);
+  //   };
+  //   const handleUnmute = () => {
+  //     setVideoTrackMuted(false);
+  //   };
+
+  //   track.addEventListener('mute', handleMute);
+  //   track.addEventListener('unmute', handleUnmute);
+
+  //   return () => {
+  //     track.removeEventListener('mute', handleMute);
+  //     track.removeEventListener('unmute', handleUnmute);
+  //   };
+  // }, [stream]);
+
+  const isPublishingTrack = publishedTracks.includes(
+    kind === 'video'
+      ? SfuModels.TrackType.VIDEO
+      : SfuModels.TrackType.SCREEN_SHARE,
+  );
+
+  const displayPlaceholder =
+    !isPublishingTrack ||
+    (participant.viewportVisibilityState === VisibilityState.INVISIBLE &&
+      !screenShareStream) ||
+    !videoPlaying;
+
+  const lastDimensionRef = useRef<string | undefined>();
+  const updateSubscription = useCallback(
+    (
+      dimension?: SfuModels.VideoDimension,
+      type: DebounceType = DebounceType.SLOW,
+    ) => {
+      call.updateSubscriptionsPartial(
+        kind,
+        {
+          [sessionId]: {
+            dimension,
+          },
+        },
+        type,
+      );
+    },
+    [call, kind, sessionId],
+  );
+
+  // handle generic subscription updates
+
+  // handle visibility subscription updates
+  useEffect(() => {
+    viewportVisibilityRef.current = participant.viewportVisibilityState;
+
+    if (!videoElement || !isPublishingTrack || participant.isLoggedInUser)
+      return;
+
+    const isInvisibleVVS =
+      participant.viewportVisibilityState === VisibilityState.INVISIBLE;
+
+    updateSubscription(
+      isInvisibleVVS
+        ? undefined
+        : {
+            height: videoElement.clientHeight,
+            width: videoElement.clientWidth,
+          },
+      DebounceType.MEDIUM,
+    );
+  }, [
+    updateSubscription,
+    participant.viewportVisibilityState,
+    videoElement,
+    isPublishingTrack,
+    participant.isLoggedInUser,
+  ]);
+
+  // handle resize subscription updates
+  useEffect(() => {
+    if (!videoElement || !isPublishingTrack || participant.isLoggedInUser)
+      return;
+
+    const resizeObserver = new ResizeObserver(() => {
+      const currentDimensions = `${videoElement.clientWidth},${videoElement.clientHeight}`;
+
+      // skip initial trigger of the observer
+      if (!lastDimensionRef.current) {
+        return (lastDimensionRef.current = currentDimensions);
+      }
+
+      if (
+        lastDimensionRef.current === currentDimensions ||
+        viewportVisibilityRef.current === VisibilityState.INVISIBLE
+      )
+        return;
+
+      updateSubscription(
+        {
+          height: videoElement.clientHeight,
+          width: videoElement.clientWidth,
+        },
+        DebounceType.SLOW,
+      );
+      lastDimensionRef.current = currentDimensions;
+    });
+    resizeObserver.observe(videoElement);
+
+    return () => {
+      resizeObserver.disconnect();
+    };
+  }, [
+    updateSubscription,
+    videoElement,
+    participant.viewportVisibilityState,
+    isPublishingTrack,
+    participant.isLoggedInUser,
+  ]);
+
+  useEffect(() => {
+    if (!isPublishingTrack || !videoElement || participant.isLoggedInUser)
+      return;
+
+    updateSubscription(
+      {
+        height: videoElement.clientHeight,
+        width: videoElement.clientWidth,
+      },
+      DebounceType.FAST,
+    );
+
+    return () => {
+      updateSubscription(undefined, DebounceType.FAST);
+    };
+  }, [
+    updateSubscription,
+    videoElement,
+    isPublishingTrack,
+    participant.isLoggedInUser,
+  ]);
+
+  const [isWideMode, setIsWideMode] = useState(true);
+  useEffect(() => {
+    if (!stream || !videoElement) return;
+
+    setVideoPlaying(!videoElement.paused);
+
+    const calculateVideoRatio = () => {
+      setVideoPlaying(true);
+      const [track] = stream.getVideoTracks();
+      if (!track) return;
+
+      const { width = 0, height = 0 } = track.getSettings();
+      setIsWideMode(width > height);
+    };
+    videoElement.addEventListener('play', calculateVideoRatio);
+    return () => {
+      videoElement.removeEventListener('play', calculateVideoRatio);
+    };
+  }, [stream, videoElement]);
+
+  return (
+    <>
+      <BaseVideo
+        {...rest}
+        stream={stream}
+        className={clsx(className, {
+          'str-video__video--wide': isWideMode,
+          'str-video__video--tall': !isWideMode,
+        })}
+        data-user-id={participant.userId}
+        data-session-id={sessionId}
+        ref={(ref) => {
+          setVideoElement(ref);
+          setVideoElementRef?.(ref);
+        }}
+      />
+      {displayPlaceholder && (
+        <VideoPlaceholder
+          style={{ position: 'absolute' }}
+          participant={participant}
+          ref={setVideoElementRef}
+        />
+      )}
+    </>
+  );
+};

package/src/core/components/Video/VideoPlaceholder.tsx

@@ -0,0 +1,40 @@
+import { ComponentProps, forwardRef, useState } from 'react';
+import { clsx } from 'clsx';
+import { StreamVideoParticipant } from '@stream-io/video-client';
+
+export type VideoPlaceholderProps = {
+  participant: StreamVideoParticipant;
+} & ComponentProps<'div'>;
+
+export const VideoPlaceholder = forwardRef<
+  HTMLDivElement,
+  VideoPlaceholderProps
+>(({ participant, style }, ref) => {
+  const [error, setError] = useState(false);
+
+  const name = participant?.name || participant?.userId;
+
+  return (
+    <div className="str-video__participant-placeholder" style={style} ref={ref}>
+      {(!participant.image || error) &&
+        (name ? (
+          <div className="str-video__participant-placeholder--initials-fallback">
+            <div>{name[0]}</div>
+          </div>
+        ) : (
+          <div>Video is disabled</div>
+        ))}
+      {participant.image && !error && (
+        <img
+          onError={() => setError(true)}
+          alt="participant-placeholder"
+          className={clsx('str-video__participant-placeholder--avatar', {
+            'str-video__participant-placeholder--avatar-speaking':
+              participant.isSpeaking,
+          })}
+          src={participant.image}
+        />
+      )}
+    </div>
+  );
+});

package/src/core/components/Video/index.ts

@@ -0,0 +1,2 @@
+export * from './BaseVideo';
+export * from './Video';

package/src/core/components/index.ts

@@ -0,0 +1,7 @@
+export * from './Audio';
+export * from './ParticipantBox';
+
+export { Video } from './Video';
+export type { VideoProps } from './Video';
+
+export * from './CallLayout';

package/src/core/contexts/MediaDevicesContext.tsx

@@ -0,0 +1,373 @@
+import {
+  createContext,
+  PropsWithChildren,
+  useCallback,
+  useContext,
+  useEffect,
+  useState,
+} from 'react';
+import { map, pairwise, take } from 'rxjs';
+import {
+  CallingState,
+  checkIfAudioOutputChangeSupported,
+  disposeOfMediaStream,
+  getAudioDevices,
+  getAudioOutputDevices,
+  getAudioStream,
+  getVideoDevices,
+  getVideoStream,
+  SfuModels,
+  watchForDisconnectedAudioOutputDevice,
+} from '@stream-io/video-client';
+import {
+  useCall,
+  useCallCallingState,
+  useCallState,
+} from '@stream-io/video-react-bindings';
+
+import { useAudioPublisher, useVideoPublisher } from '../hooks';
+
+type EnabledStateType = 'starting' | 'playing';
+type DisabledStateType = 'uninitialized' | 'stopped';
+type ErrorStateType = 'error';
+type DeviceStateType = EnabledStateType | DisabledStateType | ErrorStateType;
+
+type EnabledDeviceState<T extends EnabledStateType> = {
+  type: T;
+  enabled: true;
+};
+type DisabledDeviceState<T extends DisabledStateType> = {
+  type: T;
+  enabled: false;
+};
+type ErrorDeviceState = {
+  type: 'error';
+  message: string;
+  enabled: false;
+};
+
+type DeviceState =
+  | EnabledDeviceState<EnabledStateType>
+  | DisabledDeviceState<DisabledStateType>
+  | ErrorDeviceState;
+
+const DEVICE_STATE_TOGGLE: Record<DeviceStateType, 'starting' | 'stopped'> = {
+  starting: 'stopped',
+  playing: 'stopped',
+  stopped: 'starting',
+  uninitialized: 'starting',
+  error: 'starting',
+};
+
+/**
+ * Exclude types from documentaiton site, but we should still add doc comments
+ * @internal
+ */
+export const DEVICE_STATE: {
+  starting: EnabledDeviceState<'starting'>;
+  playing: EnabledDeviceState<'playing'>;
+  stopped: DisabledDeviceState<'stopped'>;
+  uninitialized: DisabledDeviceState<'uninitialized'>;
+  error: ErrorDeviceState;
+} = {
+  starting: { type: 'starting', enabled: true },
+  playing: { type: 'playing', enabled: true },
+  stopped: { type: 'stopped', enabled: false },
+  uninitialized: { type: 'uninitialized', enabled: false },
+  error: { type: 'error', message: '', enabled: false },
+};
+
+/**
+ * Exclude types from documentaiton site, but we should still add doc comments
+ * @internal
+ */
+export type MediaDevicesContextAPI = {
+  audioInputDevices: MediaDeviceInfo[];
+  audioOutputDevices: MediaDeviceInfo[];
+  videoDevices: MediaDeviceInfo[];
+  disposeOfMediaStream: (stream: MediaStream) => void;
+  getAudioStream: (deviceId?: string) => Promise<MediaStream>;
+  getVideoStream: (deviceId?: string) => Promise<MediaStream>;
+  isAudioOutputChangeSupported: boolean;
+  initialAudioEnabled: boolean;
+  initialVideoState: DeviceState;
+  publishVideoStream: () => Promise<void>;
+  publishAudioStream: () => Promise<void>;
+  stopPublishingAudio: () => void;
+  stopPublishingVideo: () => void;
+  setInitialVideoState: (state: DeviceState) => void;
+  selectedAudioInputDeviceId?: string;
+  selectedAudioOutputDeviceId?: string;
+  selectedVideoDeviceId?: string;
+  switchDevice: (kind: MediaDeviceKind, deviceId?: string) => void;
+  toggleAudioMuteState: () => void;
+  toggleVideoMuteState: () => void;
+};
+
+const MediaDevicesContext = createContext<MediaDevicesContextAPI | null>(null);
+
+/**
+ * Exclude types from documentaiton site, but we should still add doc comments
+ * @internal
+ */
+export type MediaDevicesProviderProps = PropsWithChildren<{
+  enumerate?: boolean;
+  initialAudioEnabled?: boolean;
+  initialVideoEnabled?: boolean;
+  initialAudioInputDeviceId?: string;
+  initialAudioOutputDeviceId?: string;
+  initialVideoInputDeviceId?: string;
+}>;
+
+// todo: republish the stream, when a new default device connected
+/**
+ *
+ * @param param0
+ * @returns
+ *
+ * @category Device Management
+ */
+export const MediaDevicesProvider = ({
+  children,
+  enumerate = true,
+  initialAudioEnabled,
+  initialVideoEnabled,
+  initialVideoInputDeviceId = 'default',
+  initialAudioOutputDeviceId = 'default',
+  initialAudioInputDeviceId = 'default',
+}: MediaDevicesProviderProps) => {
+  const call = useCall();
+  const callingState = useCallCallingState();
+  const callState = useCallState();
+  const { localParticipant$ } = callState;
+
+  const [audioInputDevices, setAudioInputDevices] = useState<MediaDeviceInfo[]>(
+    [],
+  );
+  const [videoDevices, setVideoDevices] = useState<MediaDeviceInfo[]>([]);
+  const [audioOutputDevices, setAudioOutputDevices] = useState<
+    MediaDeviceInfo[]
+  >([]);
+
+  const [selectedAudioInputDeviceId, setSelectedAudioInputDeviceId] = useState<
+    MediaDevicesContextAPI['selectedAudioInputDeviceId']
+  >(initialAudioInputDeviceId);
+  const [selectedAudioOutputDeviceId, setSelectedAudioOutputDeviceId] =
+    useState<MediaDevicesContextAPI['selectedAudioOutputDeviceId']>(
+      initialAudioOutputDeviceId,
+    );
+  const [selectedVideoDeviceId, selectVideoDeviceId] = useState<
+    MediaDevicesContextAPI['selectedVideoDeviceId']
+  >(initialVideoInputDeviceId);
+
+  const [isAudioOutputChangeSupported] = useState<boolean>(() =>
+    checkIfAudioOutputChangeSupported(),
+  );
+  const [initAudioEnabled, setInitAudioEnabled] = useState<boolean>(
+    !!initialAudioEnabled,
+  );
+  const [initialVideoState, setInitialVideoState] = useState<DeviceState>(() =>
+    initialVideoEnabled ? DEVICE_STATE.starting : DEVICE_STATE.uninitialized,
+  );
+
+  const publishVideoStream = useVideoPublisher({
+    initialVideoMuted: !initialVideoState.enabled,
+    videoDeviceId: selectedVideoDeviceId,
+  });
+  const publishAudioStream = useAudioPublisher({
+    initialAudioMuted: !initAudioEnabled,
+    audioDeviceId: selectedAudioInputDeviceId,
+  });
+
+  const stopPublishingAudio = useCallback(() => {
+    if (
+      callingState === CallingState.IDLE ||
+      callingState === CallingState.RINGING
+    ) {
+      setInitAudioEnabled(false);
+    } else {
+      call?.stopPublish(SfuModels.TrackType.AUDIO);
+    }
+  }, [call, callingState]);
+
+  const stopPublishingVideo = useCallback(() => {
+    if (
+      callingState === CallingState.IDLE ||
+      callingState === CallingState.RINGING
+    ) {
+      setInitialVideoState(DEVICE_STATE.stopped);
+    } else {
+      call?.stopPublish(SfuModels.TrackType.VIDEO);
+    }
+  }, [call, callingState]);
+
+  const toggleAudioMuteState = useCallback(
+    () => setInitAudioEnabled((prev) => !prev),
+    [],
+  );
+  const toggleVideoMuteState = useCallback(
+    () =>
+      setInitialVideoState((prev) => {
+        const newType = DEVICE_STATE_TOGGLE[prev.type];
+        return DEVICE_STATE[newType];
+      }),
+    [],
+  );
+
+  const switchDevice = useCallback(
+    (kind: MediaDeviceKind, deviceId?: string) => {
+      if (kind === 'videoinput') {
+        selectVideoDeviceId(deviceId);
+      }
+      if (kind === 'audioinput') {
+        setSelectedAudioInputDeviceId(deviceId);
+      }
+      if (kind === 'audiooutput') {
+        setSelectedAudioOutputDeviceId(deviceId);
+      }
+    },
+    [],
+  );
+
+  useEffect(() => {
+    if (!enumerate) return;
+
+    const validateInitialInputDeviceId = getAudioDevices()
+      .pipe(take(1))
+      .subscribe((devices) => {
+        const initialDeviceFound = devices.find(
+          (device) => device.deviceId === initialAudioInputDeviceId,
+        );
+        if (!initialDeviceFound) {
+          setSelectedAudioInputDeviceId('default');
+        }
+      });
+
+    const subscription = getAudioDevices().subscribe(setAudioInputDevices);
+
+    return () => {
+      subscription.unsubscribe();
+      validateInitialInputDeviceId.unsubscribe();
+    };
+  }, [enumerate, initialAudioInputDeviceId]);
+
+  useEffect(() => {
+    if (!enumerate) return;
+
+    const validateInitialInputDeviceId = getVideoDevices()
+      .pipe(take(1))
+      .subscribe((devices) => {
+        const initialDeviceFound = devices.find(
+          (device) => device.deviceId === initialVideoInputDeviceId,
+        );
+        if (!initialDeviceFound) {
+          selectVideoDeviceId('default');
+        }
+      });
+
+    const subscription = getVideoDevices().subscribe(setVideoDevices);
+
+    return () => {
+      subscription.unsubscribe();
+      validateInitialInputDeviceId.unsubscribe();
+    };
+  }, [enumerate, initialVideoInputDeviceId]);
+
+  useEffect(() => {
+    if (!enumerate) return;
+
+    const validateInitialInputDeviceId = getAudioOutputDevices()
+      .pipe(take(1))
+      .subscribe((devices) => {
+        const initialDeviceFound = devices.find(
+          (device) => device.deviceId === initialAudioOutputDeviceId,
+        );
+        if (!initialDeviceFound) {
+          setSelectedAudioOutputDeviceId('default');
+        }
+      });
+
+    const subscription = getAudioOutputDevices().subscribe(
+      setAudioOutputDevices,
+    );
+    return () => {
+      subscription.unsubscribe();
+      validateInitialInputDeviceId.unsubscribe();
+    };
+  }, [enumerate, initialAudioOutputDeviceId]);
+
+  useEffect(() => {
+    const subscription = getVideoDevices()
+      .pipe(pairwise())
+      .subscribe(([prev, current]) => {
+        // When there are 0 video devices (e.g. when laptop lid closed),
+        // we do not restart the video automatically when the device is again available,
+        // but rather leave this to the user.
+        if (prev.length > 0 && current.length === 0) {
+          setInitialVideoState(DEVICE_STATE.stopped);
+        }
+      });
+
+    return () => subscription.unsubscribe();
+  }, [videoDevices.length]);
+
+  useEffect(() => {
+    if (!call || callingState !== CallingState.JOINED) return;
+    call.setAudioOutputDevice(selectedAudioOutputDeviceId);
+  }, [call, callingState, selectedAudioOutputDeviceId]);
+
+  useEffect(() => {
+    if (!localParticipant$) return;
+    const subscription = watchForDisconnectedAudioOutputDevice(
+      localParticipant$.pipe(map((p) => p?.audioOutputDeviceId)),
+    ).subscribe(async () => {
+      setSelectedAudioOutputDeviceId('default');
+    });
+    return () => {
+      subscription.unsubscribe();
+    };
+  }, [localParticipant$]);
+
+  const contextValue: MediaDevicesContextAPI = {
+    audioInputDevices,
+    videoDevices,
+    audioOutputDevices,
+    disposeOfMediaStream,
+    getAudioStream,
+    getVideoStream,
+    isAudioOutputChangeSupported,
+    selectedAudioInputDeviceId,
+    selectedAudioOutputDeviceId,
+    selectedVideoDeviceId,
+    switchDevice,
+    initialAudioEnabled: initAudioEnabled,
+    initialVideoState,
+    setInitialVideoState,
+    toggleAudioMuteState,
+    toggleVideoMuteState,
+    publishAudioStream,
+    publishVideoStream,
+    stopPublishingAudio,
+    stopPublishingVideo,
+  };
+
+  return (
+    <MediaDevicesContext.Provider value={contextValue}>
+      {children}
+    </MediaDevicesContext.Provider>
+  );
+};
+
+/**
+ *
+ * @returns
+ *
+ * @category Device Management
+ */
+export const useMediaDevices = () => {
+  const value = useContext(MediaDevicesContext);
+  if (!value) {
+    console.warn(`Null MediaDevicesContext`);
+  }
+  return value as MediaDevicesContextAPI;
+};

package/src/core/contexts/index.ts

@@ -0,0 +1 @@
+export * from './MediaDevicesContext';

package/src/core/hooks/index.ts

@@ -0,0 +1,2 @@
+export * from './useAudioPublisher';
+export * from './useVideoPublisher';