@stream-io/video-client 0.0.1-alpha.7

package/src/rtc/callEventHandlers.ts

@@ -0,0 +1,37 @@
+import { Call } from './Call';
+import { Dispatcher } from './Dispatcher';
+import { StreamVideoWriteableStateStore } from '../store';
+import {
+  watchParticipantJoined,
+  watchParticipantLeft,
+  watchTrackPublished,
+  watchTrackUnpublished,
+} from '../events/participant';
+import {
+  watchChangePublishQuality,
+  watchConnectionQualityChanged,
+} from '../events/internal';
+import {
+  watchAudioLevelChanged,
+  watchDominantSpeakerChanged,
+} from '../events/speaker';
+import { Batcher } from '../Batcher';
+
+export const registerEventHandlers = (
+  call: Call,
+  store: StreamVideoWriteableStateStore,
+  dispatcher: Dispatcher,
+  userBatcher: Batcher<string>,
+) => {
+  watchChangePublishQuality(dispatcher, call);
+  watchConnectionQualityChanged(dispatcher, store);
+
+  watchParticipantJoined(dispatcher, store, userBatcher);
+  watchParticipantLeft(dispatcher, store);
+
+  watchTrackPublished(dispatcher, store);
+  watchTrackUnpublished(dispatcher, store);
+
+  watchAudioLevelChanged(dispatcher, store);
+  watchDominantSpeakerChanged(dispatcher, store);
+};

package/src/rtc/codecs.ts

@@ -0,0 +1,61 @@
+export const getPreferredCodecs = (
+  kind: 'audio' | 'video',
+  videoCodec: string,
+) => {
+  if (!('getCapabilities' in RTCRtpSender)) {
+    console.warn('RTCRtpSender.getCapabilities is not supported');
+    return;
+  }
+  const cap = RTCRtpSender.getCapabilities(kind);
+  console.log('s4e');
+  if (!cap) return;
+  const matched: RTCRtpCodecCapability[] = [];
+  const partialMatched: RTCRtpCodecCapability[] = [];
+  const unmatched: RTCRtpCodecCapability[] = [];
+  cap.codecs.forEach((c) => {
+    const codec = c.mimeType.toLowerCase();
+    if (codec === 'audio/opus') {
+      matched.push(c);
+      return;
+    }
+    console.log(c);
+    const matchesVideoCodec = codec === `video/${videoCodec}`;
+    if (!matchesVideoCodec) {
+      unmatched.push(c);
+      return;
+    }
+    // for h264 codecs that have sdpFmtpLine available, use only if the
+    // profile-level-id is 42e01f for cross-browser compatibility
+    if (videoCodec === 'h264') {
+      if (c.sdpFmtpLine && c.sdpFmtpLine.includes('profile-level-id=42e01f')) {
+        matched.push(c);
+      } else {
+        partialMatched.push(c);
+      }
+      return;
+    }
+    console.log('matched', matched);
+    matched.push(c);
+  });
+
+  return [
+    ...matched,
+    ...partialMatched,
+    ...unmatched,
+  ] as RTCRtpCodecCapability[];
+};
+
+export const getGenericSdp = async (direction: RTCRtpTransceiverDirection) => {
+  const tempPc = new RTCPeerConnection();
+  tempPc.addTransceiver('audio', { direction });
+  tempPc.addTransceiver('video', { direction });
+
+  const offer = await tempPc.createOffer();
+  const sdp = offer.sdp;
+
+  tempPc.getTransceivers().forEach((t) => {
+    t.stop();
+  });
+  tempPc.close();
+  return sdp;
+};

package/src/rtc/helpers/iceCandidate.ts

@@ -0,0 +1,16 @@
+import { ICETrickle } from '../../gen/video/sfu/models/models';
+
+export function getIceCandidate(
+  candidate: RTCIceCandidate,
+): ICETrickle['iceCandidate'] {
+  if (!candidate.usernameFragment) {
+    // react-native-webrtc doesn't include usernameFragment in the candidate
+    const splittedCandidate = candidate.candidate.split(' ');
+    const ufragIndex =
+      splittedCandidate.findIndex((s: string) => s === 'ufrag') + 1;
+    const usernameFragment = splittedCandidate[ufragIndex];
+    return JSON.stringify({ ...candidate, usernameFragment });
+  } else {
+    return JSON.stringify(candidate.toJSON());
+  }
+}

package/src/rtc/helpers/tracks.ts

@@ -0,0 +1,18 @@
+import { TrackType } from '../../gen/video/sfu/models/models';
+import type { StreamVideoParticipant } from '../types';
+
+export const trackTypeToParticipantStreamKey = (
+  trackType: TrackType,
+): keyof StreamVideoParticipant | undefined => {
+  switch (trackType) {
+    case TrackType.SCREEN_SHARE:
+      return 'screenShareStream';
+    case TrackType.VIDEO:
+      return 'videoStream';
+    case TrackType.AUDIO:
+      return 'audioStream';
+    default:
+      console.error(`Unknown track type: ${trackType}`);
+      return undefined;
+  }
+};

package/src/rtc/publisher.ts

@@ -0,0 +1,305 @@
+import { StreamSfuClient } from '../StreamSfuClient';
+import {
+  PeerType,
+  TrackInfo,
+  TrackType,
+  VideoLayer,
+  VideoQuality,
+} from '../gen/video/sfu/models/models';
+import { getIceCandidate } from './helpers/iceCandidate';
+import {
+  findOptimalVideoLayers,
+  findOptimalScreenSharingLayers,
+} from './videoLayers';
+import { getPreferredCodecs } from './codecs';
+import { PublishOptions } from './types';
+
+export type PublisherOpts = {
+  rpcClient: StreamSfuClient;
+  connectionConfig?: RTCConfiguration;
+};
+
+/**
+ * @internal
+ * The `Publisher` is responsible for publishing/unpublishing media streams to/from the SFU
+ */
+export class Publisher {
+  private readonly publisher: RTCPeerConnection;
+  private readonly rpcClient: StreamSfuClient;
+  private readonly transceiverRegistry: {
+    [key in TrackType]: RTCRtpTransceiver | undefined;
+  } = {
+    [TrackType.AUDIO]: undefined,
+    [TrackType.VIDEO]: undefined,
+    [TrackType.SCREEN_SHARE]: undefined,
+    [TrackType.SCREEN_SHARE_AUDIO]: undefined,
+    [TrackType.UNSPECIFIED]: undefined,
+  };
+  private readonly trackKindRegistry: {
+    [key in TrackType]: 'video' | 'audio' | undefined;
+  } = {
+    [TrackType.AUDIO]: 'audio',
+    [TrackType.VIDEO]: 'video',
+    [TrackType.SCREEN_SHARE]: 'video',
+    [TrackType.SCREEN_SHARE_AUDIO]: undefined,
+    [TrackType.UNSPECIFIED]: undefined,
+  };
+
+  constructor({ connectionConfig, rpcClient }: PublisherOpts) {
+    const pc = new RTCPeerConnection(connectionConfig);
+    pc.addEventListener('icecandidate', this.onIceCandidate);
+    pc.addEventListener('negotiationneeded', this.onNegotiationNeeded);
+
+    pc.addEventListener('icecandidateerror', this.onIceCandidateError);
+    pc.addEventListener(
+      'iceconnectionstatechange',
+      this.onIceConnectionStateChange,
+    );
+    pc.addEventListener(
+      'icegatheringstatechange',
+      this.onIceGatheringStateChange,
+    );
+
+    this.publisher = pc;
+    this.rpcClient = rpcClient;
+  }
+
+  /**
+   * Starts publishing the given track of the given media stream.
+   *
+   * Consecutive calls to this method will replace the stream.
+   * The previous stream will be stopped.
+   * @param mediaStream
+   * @param track
+   * @param trackType
+   * @param opts
+   */
+  publishStream = async (
+    mediaStream: MediaStream,
+    track: MediaStreamTrack,
+    trackType: TrackType,
+    opts: PublishOptions = {},
+  ) => {
+    let transceiver = this.publisher
+      .getTransceivers()
+      .find(
+        (t) =>
+          t === this.transceiverRegistry[trackType] &&
+          t.sender.track &&
+          t.sender.track?.kind === this.trackKindRegistry[trackType],
+      );
+
+    if (!transceiver) {
+      let videoEncodings: RTCRtpEncodingParameters[] | undefined;
+      if (trackType === TrackType.VIDEO) {
+        videoEncodings = findOptimalVideoLayers(track);
+      }
+      transceiver = this.publisher.addTransceiver(track, {
+        direction: 'sendonly',
+        streams:
+          trackType === TrackType.VIDEO || trackType === TrackType.SCREEN_SHARE
+            ? [mediaStream]
+            : undefined,
+        sendEncodings:
+          trackType === TrackType.VIDEO ? videoEncodings : undefined,
+      });
+
+      this.transceiverRegistry[trackType] = transceiver;
+
+      if (trackType === TrackType.VIDEO) {
+        const codecPreferences = getPreferredCodecs(
+          'video',
+          opts.preferredCodec || 'vp8',
+        );
+
+        if ('setCodecPreferences' in transceiver && codecPreferences) {
+          console.log(`set codec preferences`, codecPreferences);
+          // @ts-ignore
+          transceiver.setCodecPreferences(codecPreferences);
+        }
+      }
+    } else {
+      transceiver.sender.track?.stop();
+      await transceiver.sender.replaceTrack(track);
+    }
+  };
+
+  /**
+   * Stops publishing the given track type to the SFU, if it is currently being published.
+   * Underlying track will be stopped and removed from the publisher.
+   * @param trackType
+   * @returns `true` if track with the given track type was found, otherwise `false`
+   */
+  unpublishStream = (trackType: TrackType) => {
+    const transceiver = this.publisher
+      .getTransceivers()
+      .find((t) => t === this.transceiverRegistry[trackType] && t.sender.track);
+    if (transceiver && transceiver.sender.track) {
+      transceiver.sender.track.stop();
+      return true;
+    } else {
+      return false;
+    }
+  };
+
+  /**
+   * Stops publishing all tracks and stop all tracks.
+   */
+  stopPublishing = () => {
+    this.publisher.getSenders().forEach((s) => {
+      if (s.track) {
+        s.track.stop();
+      }
+      if (this.publisher.signalingState !== 'closed') {
+        this.publisher.removeTrack(s);
+      }
+    });
+    this.publisher.close();
+  };
+
+  updateVideoPublishQuality = async (enabledRids: string[]) => {
+    console.log(
+      'Updating publish quality, qualities requested by SFU:',
+      enabledRids,
+    );
+
+    const videoSender = this.transceiverRegistry[TrackType.VIDEO]?.sender;
+
+    if (!videoSender) return;
+
+    const params = videoSender.getParameters();
+    let changed = false;
+    params.encodings.forEach((enc) => {
+      console.log(enc.rid, enc.active);
+      // flip 'active' flag only when necessary
+      const shouldEnable = enabledRids.includes(enc.rid!);
+      if (shouldEnable !== enc.active) {
+        enc.active = shouldEnable;
+        changed = true;
+      }
+    });
+    if (changed) {
+      if (params.encodings.length === 0) {
+        console.warn('No suitable video encoding quality found');
+      }
+      await videoSender.setParameters(params);
+    }
+  };
+
+  /**
+   * Returns the result of the `RTCPeerConnection.getStats()` method
+   * @param selector
+   * @returns
+   */
+  getStats(selector?: MediaStreamTrack | null | undefined) {
+    return this.publisher.getStats(selector);
+  }
+
+  private onIceCandidate = async (e: RTCPeerConnectionIceEvent) => {
+    const { candidate } = e;
+    if (!candidate) {
+      console.log('null ice candidate');
+      return;
+    }
+    await this.rpcClient.iceTrickle({
+      iceCandidate: getIceCandidate(candidate),
+      peerType: PeerType.PUBLISHER_UNSPECIFIED,
+    });
+  };
+
+  private onNegotiationNeeded = async () => {
+    console.log('AAA onNegotiationNeeded');
+    const offer = await this.publisher.createOffer();
+    await this.publisher.setLocalDescription(offer);
+
+    const trackInfos = this.publisher
+      .getTransceivers()
+      .filter((t) => t.direction === 'sendonly' && !!t.sender.track)
+      .map<TrackInfo>((transceiver) => {
+        const trackType = Number(
+          Object.keys(this.transceiverRegistry).find(
+            (key) =>
+              this.transceiverRegistry[key as any as TrackType] === transceiver,
+          ),
+        );
+        const track = transceiver.sender.track!;
+        const optimalLayers =
+          trackType === TrackType.VIDEO
+            ? findOptimalVideoLayers(track)
+            : trackType === TrackType.SCREEN_SHARE
+            ? findOptimalScreenSharingLayers(track)
+            : [];
+
+        const layers = optimalLayers.map<VideoLayer>((optimalLayer) => ({
+          rid: optimalLayer.rid || '',
+          bitrate: optimalLayer.maxBitrate || 0,
+          fps: optimalLayer.maxFramerate || 0,
+          quality: this.ridToVideoQuality(optimalLayer.rid || ''),
+          videoDimension: {
+            width: optimalLayer.width,
+            height: optimalLayer.height,
+          },
+        }));
+
+        return {
+          trackId: track.id,
+          layers: layers,
+          trackType,
+          mid: transceiver.mid || '',
+        };
+      });
+
+    // TODO debounce for 250ms
+    const response = await this.rpcClient.setPublisher({
+      sdp: offer.sdp || '',
+      tracks: trackInfos,
+    });
+
+    await this.publisher.setRemoteDescription({
+      type: 'answer',
+      sdp: response.response.sdp,
+    });
+
+    this.rpcClient.iceTrickleBuffer.publisherCandidates.subscribe(
+      async (candidate) => {
+        try {
+          const iceCandidate = JSON.parse(candidate.iceCandidate);
+          await this.publisher.addIceCandidate(iceCandidate);
+        } catch (e) {
+          console.error(`[Publisher] ICE candidate error`, e, candidate);
+        }
+      },
+    );
+  };
+
+  private onIceCandidateError = (e: Event) => {
+    const errorMessage =
+      e instanceof RTCPeerConnectionIceErrorEvent &&
+      `${e.errorCode}: ${e.errorText}`;
+    console.error(`Publisher: ICE Candidate error`, errorMessage, e);
+  };
+
+  private onIceConnectionStateChange = (e: Event) => {
+    console.log(
+      `Publisher: ICE Connection state changed`,
+      this.publisher.iceConnectionState,
+      e,
+    );
+  };
+
+  private onIceGatheringStateChange = (e: Event) => {
+    console.log(
+      `Publisher: ICE Gathering State`,
+      this.publisher.iceGatheringState,
+      e,
+    );
+  };
+
+  private ridToVideoQuality = (rid: string): VideoQuality => {
+    return rid === 'q'
+      ? VideoQuality.LOW_UNSPECIFIED
+      : rid === 'h'
+      ? VideoQuality.MID
+      : VideoQuality.HIGH; // default to HIGH
+  };
+}

package/src/rtc/signal.ts

@@ -0,0 +1,34 @@
+import { SfuEvent } from '../gen/video/sfu/event/events';
+
+export const createWebSocketSignalChannel = (opts: {
+  endpoint: string;
+  onMessage?: (message: SfuEvent) => void;
+}) => {
+  return new Promise<WebSocket>((resolve) => {
+    const { endpoint, onMessage } = opts;
+    const ws = new WebSocket(endpoint);
+    ws.binaryType = 'arraybuffer'; // do we need this?
+    ws.addEventListener('open', () => {
+      return resolve(ws);
+    });
+
+    ws.addEventListener('error', (e) => {
+      console.error('Error', e);
+    });
+
+    ws.addEventListener('close', (e) => {
+      console.warn('Signalling channel is closed', e);
+    });
+
+    if (onMessage) {
+      ws.addEventListener('message', (e) => {
+        const message =
+          e.data instanceof ArrayBuffer
+            ? SfuEvent.fromBinary(new Uint8Array(e.data))
+            : SfuEvent.fromJsonString(e.data);
+
+        onMessage(message);
+      });
+    }
+  });
+};

package/src/rtc/subscriber.ts

@@ -0,0 +1,85 @@
+import { StreamSfuClient } from '../StreamSfuClient';
+import { getIceCandidate } from './helpers/iceCandidate';
+import { PeerType } from '../gen/video/sfu/models/models';
+
+export type SubscriberOpts = {
+  rpcClient: StreamSfuClient;
+  connectionConfig?: RTCConfiguration;
+  onTrack?: (e: RTCTrackEvent) => void;
+};
+
+export const createSubscriber = ({
+  rpcClient,
+  connectionConfig,
+  onTrack,
+}: SubscriberOpts) => {
+  const subscriber = new RTCPeerConnection(connectionConfig);
+  subscriber.addEventListener('icecandidate', async (e) => {
+    const { candidate } = e;
+    if (!candidate) {
+      console.log('null ice candidate');
+      return;
+    }
+
+    await rpcClient.iceTrickle({
+      iceCandidate: getIceCandidate(candidate),
+      peerType: PeerType.SUBSCRIBER,
+    });
+  });
+  subscriber.addEventListener('icecandidateerror', (e) => {
+    const errorMessage =
+      e instanceof RTCPeerConnectionIceErrorEvent &&
+      `${e.errorCode}: ${e.errorText}`;
+    console.error(`Subscriber: ICE Candidate error`, errorMessage, e);
+  });
+  subscriber.addEventListener('iceconnectionstatechange', (e) => {
+    console.log(
+      `Subscriber: ICE Connection state changed`,
+      subscriber.iceConnectionState,
+      e,
+    );
+  });
+  subscriber.addEventListener('icegatheringstatechange', (e) => {
+    console.log(
+      `Subscriber: ICE Gathering State`,
+      subscriber.iceGatheringState,
+      e,
+    );
+  });
+
+  if (onTrack) {
+    subscriber.addEventListener('track', onTrack);
+  }
+
+  const { dispatcher, iceTrickleBuffer } = rpcClient;
+  dispatcher.on('subscriberOffer', async (message) => {
+    if (message.eventPayload.oneofKind !== 'subscriberOffer') return;
+    const { subscriberOffer } = message.eventPayload;
+    console.log(`Received subscriberOffer`, subscriberOffer);
+
+    await subscriber.setRemoteDescription({
+      type: 'offer',
+      sdp: subscriberOffer.sdp,
+    });
+
+    iceTrickleBuffer.subscriberCandidates.subscribe(async (candidate) => {
+      try {
+        const iceCandidate = JSON.parse(candidate.iceCandidate);
+        await subscriber.addIceCandidate(iceCandidate);
+      } catch (e) {
+        console.error(`[Subscriber] ICE candidate error`, e, candidate);
+      }
+    });
+
+    // apply ice candidates
+    const answer = await subscriber.createAnswer();
+    await subscriber.setLocalDescription(answer);
+
+    await rpcClient.sendAnswer({
+      peerType: PeerType.SUBSCRIBER,
+      sdp: answer.sdp || '',
+    });
+  });
+
+  return subscriber;
+};

package/src/rtc/types.ts

@@ -0,0 +1,105 @@
+import type { UserResponse } from '../gen/coordinator';
+import type {
+  Participant,
+  VideoDimension,
+} from '../gen/video/sfu/models/models';
+
+export interface StreamVideoParticipant extends Participant {
+  /**
+   * The participant's audio stream, if they are publishing audio and
+   * we have subscribed to it.
+   */
+  audioStream?: MediaStream;
+
+  /**
+   * The participant's video stream, if they are sharing their video,
+   * and we are subscribed to it.
+   */
+  videoStream?: MediaStream;
+
+  /**
+   * The participant's screen share stream, if they are sharing their screen,
+   * and we are subscribed to it.
+   */
+  screenShareStream?: MediaStream;
+
+  /**
+   * The preferred video dimensions for this participant.
+   * Set it to `undefined` to unsubscribe from this participant's video.
+   */
+  videoDimension?: VideoDimension;
+
+  /**
+   * The preferred screen share dimensions for this participant.
+   * Set it to `undefined` to unsubscribe from this participant's screen share.
+   */
+  screenShareDimension?: VideoDimension;
+
+  /**
+   * True if the participant is the local participant.
+   */
+  isLoggedInUser?: boolean;
+
+  /**
+   * True when the participant is pinned
+   */
+  isPinned?: boolean;
+
+  /**
+   * User metadata (profile picture, name...)
+   */
+  user?: UserResponse;
+}
+
+export interface StreamVideoLocalParticipant extends StreamVideoParticipant {
+  /**
+   * The device ID of the currently selected audio input device of the local participant (returned by the [MediaDevices API](https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia))
+   */
+  audioDeviceId?: string;
+  /**
+   * The device ID of the currently selected video input device of the local participant (returned by the [MediaDevices API](https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia))
+   */
+  videoDeviceId?: string;
+
+  /**
+   * The device ID of the currently selected audio output device of the local participant (returned by the [MediaDevices API](https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia))
+   *
+   * If the value is not defined, the user hasn't selected any device (in these cases the default system audio output could be used)
+   */
+  audioOutputDeviceId?: string;
+}
+
+/**
+ * A partial representation of the StreamVideoParticipant.
+ */
+export type StreamVideoParticipantPatch = Partial<
+  StreamVideoParticipant | StreamVideoLocalParticipant
+>;
+
+/**
+ * A collection of {@link StreamVideoParticipantPatch} organized by sessionId.
+ */
+export type StreamVideoParticipantPatches = {
+  [sessionId: string]: StreamVideoParticipantPatch;
+};
+
+export type SubscriptionChange = {
+  /**
+   * The video dimension to request.
+   * Set it to `undefined` in case you want to unsubscribe.
+   */
+  dimension: VideoDimension | undefined;
+};
+
+export type SubscriptionChanges = {
+  [sessionId: string]: SubscriptionChange;
+};
+
+export type CallOptions = {
+  connectionConfig?: RTCConfiguration;
+  edgeName?: string;
+};
+
+export type PublishOptions = {
+  preferredCodec?: string | null;
+};