@stream-io/video-client 0.0.1-alpha.10

package/src/rtc/videoLayers.ts

@@ -0,0 +1,103 @@
+export type OptimalVideoLayer = RTCRtpEncodingParameters & {
+  width: number;
+  height: number;
+
+  // defined here until we have this prop included in TypeScript
+  // https://github.com/microsoft/TypeScript-DOM-lib-generator/issues/1380
+  maxFramerate?: number;
+};
+
+export const findOptimalVideoLayers = (videoTrack: MediaStreamTrack) => {
+  const steps: [number, number, number][] = [
+    [1920, 1080, 3000000],
+    [1280, 720, 1250000],
+    [960, 540, 850000],
+    [640, 480, 500000],
+    [320, 240, 250000],
+    [160, 120, 125000],
+  ];
+
+  const optimalVideoLayers: OptimalVideoLayer[] = [];
+  const settings = videoTrack.getSettings();
+  for (let step = 0; step < steps.length; step++) {
+    const [w, h, maxBitrate] = steps[step];
+    // found ideal layer
+    if (w === settings.width && h === settings.height) {
+      let scaleFactor: number = 1;
+      ['f', 'h', 'q'].forEach((rid) => {
+        // Reversing the order [f, h, q] to [q, h, f] as Chrome uses encoding index
+        // when deciding which layer to disable when CPU or bandwidth is constrained.
+        // Encodings should be ordered in increasing spatial resolution order.
+        optimalVideoLayers.unshift({
+          active: true,
+          rid,
+          width: w / scaleFactor,
+          height: h / scaleFactor,
+          maxBitrate: maxBitrate / scaleFactor,
+          scaleResolutionDownBy: scaleFactor,
+          maxFramerate: {
+            f: 30,
+            h: 25,
+            q: 20,
+          }[rid],
+        });
+        scaleFactor *= 2;
+      });
+
+      break;
+    }
+  }
+  // for simplicity, we start with all layers enabled, then this function
+  // will clear/reassign the layers that are not needed
+  return withSimulcastConstraints(settings, optimalVideoLayers);
+};
+
+/**
+ * Browsers have different simulcast constraints for different video resolutions.
+ *
+ * This function modifies the provided list of video layers according to the
+ * current implementation of simulcast constraints in the Chromium based browsers.
+ *
+ * https://chromium.googlesource.com/external/webrtc/+/refs/heads/main/media/engine/simulcast.cc#90
+ */
+const withSimulcastConstraints = (
+  settings: MediaTrackSettings,
+  optimalVideoLayers: OptimalVideoLayer[],
+) => {
+  let layers;
+
+  const size = Math.max(settings.width || 0, settings.height || 0);
+  if (size <= 320) {
+    // provide only one layer 320x240 (q), the one with the highest quality
+    layers = optimalVideoLayers.filter((layer) => layer.rid === 'f');
+  } else if (size <= 640) {
+    // provide two layers, 160x120 (q) and 640x480 (h)
+    layers = optimalVideoLayers.filter((layer) => layer.rid !== 'h');
+  } else {
+    // provide three layers for sizes > 640x480
+    layers = optimalVideoLayers;
+  }
+
+  const ridMapping = ['q', 'h', 'f'];
+  return layers.map((layer, index) => ({
+    ...layer,
+    rid: ridMapping[index], // reassign rid
+  }));
+};
+
+export const findOptimalScreenSharingLayers = (
+  videoTrack: MediaStreamTrack,
+): OptimalVideoLayer[] => {
+  const settings = videoTrack.getSettings();
+  return [
+    {
+      active: true,
+      rid: 'q', // single track, start from 'q'
+      width: settings.width || 0,
+      height: settings.height || 0,
+      maxBitrate: 3000000,
+      scaleResolutionDownBy: 1,
+      maxFramerate: 30,
+    },
+  ];
+};

package/src/stats/coordinator-stats-reporter.ts

@@ -0,0 +1,167 @@
+import { StreamVideoReadOnlyStateStore } from '../store';
+import {
+  ReportCallStatEventRequest,
+  ReportCallStatEventResponse,
+  ReportCallStatsResponse,
+} from '../gen/video/coordinator/client_v1_rpc/client_rpc';
+import {
+  MediaStateChange,
+  MediaStateChangeReason,
+  MediaType,
+  MediaDirection,
+} from '../gen/video/coordinator/stat_v1/stat';
+import { pairwise, throttleTime } from 'rxjs';
+import { StreamVideoParticipant } from '../rtc/types';
+import { TrackType } from '../gen/video/sfu/models/models';
+
+/**
+ * Collects stat metrics and events from the state store and sends them to the Coordinator API
+ * @param readOnlyStateStore
+ * @param sendStatMetrics
+ * @param sendStatEvent
+ */
+export const reportStats = (
+  readOnlyStateStore: StreamVideoReadOnlyStateStore,
+  sendStatMetrics: (stats: Object) => Promise<ReportCallStatsResponse | void>,
+  sendStatEvent: (
+    statEvent: ReportCallStatEventRequest['event'],
+  ) => Promise<ReportCallStatEventResponse | void>,
+) => {
+  reportStatMetrics(readOnlyStateStore, sendStatMetrics);
+  reportStatEvents(readOnlyStateStore, sendStatEvent);
+};
+
+const reportStatMetrics = (
+  readOnlyStateStore: StreamVideoReadOnlyStateStore,
+  sendStatMetrics: (stats: Object) => Promise<ReportCallStatsResponse | void>,
+) => {
+  readOnlyStateStore.callStatsReport$
+    .pipe(throttleTime(15000))
+    .subscribe((report) => {
+      if (report?.publisherRawStats) {
+        const s: Record<string, any> = {};
+        report.publisherRawStats.forEach((v) => {
+          s[v.id] = v;
+        });
+        sendStatMetrics(s);
+      }
+      if (report?.subscriberRawStats) {
+        const s: Record<string, any> = {};
+        report.subscriberRawStats.forEach((v) => {
+          s[v.id] = v;
+        });
+        sendStatMetrics(s);
+      }
+    });
+};
+
+export const reportStatEvents = (
+  store: StreamVideoReadOnlyStateStore,
+  sendStatEvent: (
+    statEvent: ReportCallStatEventRequest['event'],
+  ) => Promise<ReportCallStatEventResponse | void>,
+) => {
+  store.localParticipant$
+    .pipe(pairwise())
+    .subscribe(([prevLocalParticipant, currentLocalParticipant]) => {
+      if (!prevLocalParticipant && currentLocalParticipant) {
+        const event: ReportCallStatEventRequest['event'] = {
+          oneofKind: 'participantConnected',
+          participantConnected: {},
+        };
+        sendStatEvent(event);
+      }
+      if (prevLocalParticipant && !currentLocalParticipant) {
+        const event: ReportCallStatEventRequest['event'] = {
+          oneofKind: 'participantDisconnected',
+          participantDisconnected: {},
+        };
+        sendStatEvent(event);
+      }
+      if (
+        (!prevLocalParticipant?.audioStream &&
+          currentLocalParticipant?.audioStream) ||
+        (prevLocalParticipant?.audioStream &&
+          !currentLocalParticipant?.audioStream)
+      ) {
+        const event: ReportCallStatEventRequest['event'] = {
+          oneofKind: 'mediaStateChanged',
+          mediaStateChanged: {
+            mediaType: MediaType.AUDIO,
+            change: currentLocalParticipant?.audioStream
+              ? MediaStateChange.STARTED
+              : MediaStateChange.ENDED,
+            reason: MediaStateChangeReason.CONNECTION,
+            direction: MediaDirection.SEND,
+          },
+        };
+        sendStatEvent(event);
+      }
+      if (
+        (!prevLocalParticipant?.videoStream &&
+          currentLocalParticipant?.videoStream) ||
+        (prevLocalParticipant?.videoStream &&
+          !currentLocalParticipant?.videoStream)
+      ) {
+        const event: ReportCallStatEventRequest['event'] = {
+          oneofKind: 'mediaStateChanged',
+          mediaStateChanged: {
+            mediaType: MediaType.VIDEO,
+            change: currentLocalParticipant?.videoStream
+              ? MediaStateChange.STARTED
+              : MediaStateChange.ENDED,
+            reason: MediaStateChangeReason.CONNECTION,
+            direction: MediaDirection.SEND,
+          },
+        };
+        sendStatEvent(event);
+      }
+      if (prevLocalParticipant && currentLocalParticipant) {
+        if (
+          isPublishingTrackOfType(prevLocalParticipant, TrackType.AUDIO) !==
+          isPublishingTrackOfType(currentLocalParticipant, TrackType.AUDIO)
+        ) {
+          const event: ReportCallStatEventRequest['event'] = {
+            oneofKind: 'mediaStateChanged',
+            mediaStateChanged: {
+              mediaType: MediaType.AUDIO,
+              change: isPublishingTrackOfType(
+                currentLocalParticipant,
+                TrackType.AUDIO,
+              )
+                ? MediaStateChange.ENDED
+                : MediaStateChange.STARTED,
+              reason: MediaStateChangeReason.MUTE,
+              direction: MediaDirection.SEND,
+            },
+          };
+          sendStatEvent(event);
+        }
+        if (
+          isPublishingTrackOfType(prevLocalParticipant, TrackType.VIDEO) !==
+          isPublishingTrackOfType(currentLocalParticipant, TrackType.VIDEO)
+        ) {
+          const event: ReportCallStatEventRequest['event'] = {
+            oneofKind: 'mediaStateChanged',
+            mediaStateChanged: {
+              mediaType: MediaType.VIDEO,
+              change: isPublishingTrackOfType(
+                currentLocalParticipant,
+                TrackType.VIDEO,
+              )
+                ? MediaStateChange.ENDED
+                : MediaStateChange.STARTED,
+              reason: MediaStateChangeReason.MUTE,
+              direction: MediaDirection.SEND,
+            },
+          };
+          sendStatEvent(event);
+        }
+      }
+    });
+};
+
+const isPublishingTrackOfType = (
+  participant: StreamVideoParticipant,
+  type: TrackType,
+) => participant.publishedTracks.includes(type);

package/src/stats/state-store-stats-reporter.ts

@@ -0,0 +1,364 @@
+import type {
+  AggregatedStatsReport,
+  BaseStats,
+  CallStatsReport,
+  ParticipantsStatsReport,
+  StatsReport,
+} from './types';
+import { StreamVideoWriteableStateStore } from '../store';
+import { Publisher } from '../rtc/publisher';
+
+export type StatsReporterOpts = {
+  subscriber: RTCPeerConnection;
+  publisher: Publisher;
+  store: StreamVideoWriteableStateStore;
+  pollingIntervalInMs?: number;
+  edgeName?: string;
+};
+
+export type StatsReporter = {
+  /**
+   * Will turn on stats reporting for a given sessionId.
+   *
+   * @param sessionId the session id.
+   */
+  startReportingStatsFor: (sessionId: string) => void;
+
+  /**
+   * Will turn off stats reporting for a given sessionId.
+   *
+   * @param sessionId the session id.
+   */
+  stopReportingStatsFor: (sessionId: string) => void;
+
+  /**
+   * Helper method for retrieving stats for a given peer connection kind
+   * and media stream flowing through it.
+   *
+   * @param kind the peer connection kind (subscriber or publisher).
+   * @param mediaStream the media stream.
+   */
+  getStatsForStream: (
+    kind: 'subscriber' | 'publisher',
+    mediaStream: MediaStream,
+  ) => Promise<StatsReport[]>;
+
+  /**
+   * Helper method for retrieving raw stats for a given peer connection kind.
+   *
+   * @param kind the peer connection kind (subscriber or publisher).
+   * @param selector the track selector. If not provided, stats for all tracks will be returned.
+   */
+  getRawStatsForTrack: (
+    kind: 'subscriber' | 'publisher',
+    selector?: MediaStreamTrack,
+  ) => Promise<RTCStatsReport | undefined>;
+
+  /**
+   * Stops the stats reporter and releases all resources.
+   */
+  stop: () => void;
+};
+
+/**
+ * Creates a new StatsReporter instance that collects metrics about the ongoing call and reports them to the state store
+ */
+export const createStatsReporter = ({
+  subscriber,
+  publisher,
+  store,
+  edgeName,
+  pollingIntervalInMs = 2000,
+}: StatsReporterOpts): StatsReporter => {
+  const getRawStatsForTrack = async (
+    kind: 'subscriber' | 'publisher',
+    selector?: MediaStreamTrack,
+  ) => {
+    if (kind === 'subscriber' && subscriber) {
+      return subscriber.getStats(selector);
+    } else if (kind === 'publisher' && publisher) {
+      return publisher.getStats(selector);
+    } else {
+      console.warn(`Can't retrieve RTC stats for`, kind);
+      return undefined;
+    }
+  };
+
+  const getStatsForStream = async (
+    kind: 'subscriber' | 'publisher',
+    mediaStream: MediaStream,
+  ) => {
+    const pc = kind === 'subscriber' ? subscriber : publisher;
+    const statsForStream: StatsReport[] = [];
+    for (let track of mediaStream.getTracks()) {
+      const report = await pc.getStats(track);
+      const stats = transform(report, {
+        // @ts-ignore
+        trackKind: track.kind,
+        kind,
+      });
+      statsForStream.push(stats);
+    }
+    return statsForStream;
+  };
+
+  const startReportingStatsFor = (sessionId: string) => {
+    sessionIdsToTrack.add(sessionId);
+    void run();
+  };
+
+  const stopReportingStatsFor = (sessionId: string) => {
+    sessionIdsToTrack.delete(sessionId);
+    void run();
+  };
+
+  const sessionIdsToTrack = new Set<string>();
+
+  /**
+   * The main stats reporting loop.
+   */
+  const run = async () => {
+    const participants = store.getCurrentValue(store.participantsSubject);
+    const participantStats: ParticipantsStatsReport = {};
+    const sessionIds = new Set(sessionIdsToTrack);
+    if (sessionIds.size > 0) {
+      for (let participant of participants) {
+        if (!sessionIds.has(participant.sessionId)) continue;
+        const kind = participant.isLoggedInUser ? 'publisher' : 'subscriber';
+        try {
+          const mergedStream = new MediaStream([
+            ...(participant.videoStream?.getVideoTracks() || []),
+            ...(participant.audioStream?.getAudioTracks() || []),
+          ]);
+          participantStats[participant.sessionId] = await getStatsForStream(
+            kind,
+            mergedStream,
+          );
+          mergedStream.getTracks().forEach((t) => {
+            mergedStream.removeTrack(t);
+          });
+        } catch (e) {
+          console.error(`Failed to collect stats for ${kind}`, participant, e);
+        }
+      }
+    }
+
+    const [subscriberStats, publisherStats] = await Promise.all([
+      subscriber
+        .getStats()
+        .then((report) =>
+          transform(report, {
+            kind: 'subscriber',
+            trackKind: 'video',
+          }),
+        )
+        .then(aggregate),
+      publisher
+        .getStats()
+        .then((report) =>
+          transform(report, {
+            kind: 'publisher',
+            trackKind: 'video',
+          }),
+        )
+        .then(aggregate),
+    ]);
+
+    const [subscriberRawStats, publisherRawStats] = await Promise.all([
+      getRawStatsForTrack('subscriber'),
+      getRawStatsForTrack('publisher'),
+    ]);
+
+    const statsReport: CallStatsReport = {
+      datacenter: edgeName || 'N/A',
+      publisherStats,
+      subscriberStats,
+      subscriberRawStats,
+      publisherRawStats,
+      participants: participantStats,
+      timestamp: Date.now(),
+    };
+
+    store.setCurrentValue(store.callStatsReportSubject, statsReport);
+  };
+
+  let timeoutId: NodeJS.Timeout | undefined;
+  if (pollingIntervalInMs > 0) {
+    const loop = async () => {
+      await run().catch((e) => {
+        console.log('Failed to collect stats', e);
+      });
+      timeoutId = setTimeout(loop, pollingIntervalInMs);
+    };
+    void loop();
+  }
+
+  const stop = () => {
+    if (timeoutId) {
+      clearTimeout(timeoutId);
+    }
+  };
+
+  return {
+    getRawStatsForTrack,
+    getStatsForStream,
+    startReportingStatsFor,
+    stopReportingStatsFor,
+    stop,
+  };
+};
+
+export type StatsTransformOpts = {
+  /**
+   * The kind of track we are transforming stats for.
+   */
+  trackKind: 'audio' | 'video';
+
+  /**
+   * The kind of peer connection we are transforming stats for.
+   */
+  kind: 'subscriber' | 'publisher';
+};
+
+/**
+ * Transforms raw RTC stats into a slimmer and uniform across browsers format.
+ *
+ * @param report the report to transform.
+ * @param opts the transform options.
+ */
+const transform = (
+  report: RTCStatsReport,
+  opts: StatsTransformOpts,
+): StatsReport => {
+  const { trackKind, kind } = opts;
+  const direction = kind === 'subscriber' ? 'inbound-rtp' : 'outbound-rtp';
+  const stats = flatten(report);
+  const streams = stats
+    .filter(
+      (stat) =>
+        stat.type === direction &&
+        (stat as RTCRtpStreamStats).kind === trackKind,
+    )
+    .map((stat): BaseStats => {
+      const rtcStreamStats = stat as RTCInboundRtpStreamStats &
+        RTCOutboundRtpStreamStats;
+
+      const codec = stats.find(
+        (s) => s.type === 'codec' && s.id === rtcStreamStats.codecId,
+      ) as { mimeType: string } | undefined; // FIXME OL: incorrect type!
+
+      const transport = stats.find(
+        (s) => s.type === 'transport' && s.id === rtcStreamStats.transportId,
+      ) as RTCTransportStats | undefined;
+
+      let roundTripTime: number | undefined;
+      if (transport && transport.dtlsState === 'connected') {
+        const candidatePair = stats.find(
+          (s) =>
+            s.type === 'candidate-pair' &&
+            s.id === transport.selectedCandidatePairId,
+        ) as RTCIceCandidatePairStats | undefined;
+        roundTripTime = candidatePair?.currentRoundTripTime;
+      }
+
+      return {
+        bytesSent: rtcStreamStats.bytesSent,
+        bytesReceived: rtcStreamStats.bytesReceived,
+        codec: codec?.mimeType,
+        currentRoundTripTime: roundTripTime,
+        frameHeight: rtcStreamStats.frameHeight,
+        frameWidth: rtcStreamStats.frameWidth,
+        framesPerSecond: rtcStreamStats.framesPerSecond,
+        jitter: rtcStreamStats.jitter,
+        kind: rtcStreamStats.kind,
+        // @ts-ignore: available in Chrome only, TS doesn't recognize this
+        qualityLimitationReason: rtcStreamStats.qualityLimitationReason,
+        rid: rtcStreamStats.rid,
+        ssrc: rtcStreamStats.ssrc,
+      };
+    });
+
+  return {
+    rawStats: report,
+    streams,
+    timestamp: Date.now(),
+  };
+};
+
+/**
+ * Aggregates generic stats.
+ *
+ * @param stats the stats to aggregate.
+ */
+const aggregate = (stats: StatsReport): AggregatedStatsReport => {
+  const aggregatedStats: AggregatedStatsReport = {
+    totalBytesSent: 0,
+    totalBytesReceived: 0,
+    averageJitterInMs: 0,
+    averageRoundTripTimeInMs: 0,
+    qualityLimitationReasons: 'none',
+    highestFrameWidth: 0,
+    highestFrameHeight: 0,
+    highestFramesPerSecond: 0,
+    timestamp: Date.now(),
+  };
+
+  let maxArea = -1;
+  const area = (w: number, h: number) => w * h;
+
+  const qualityLimitationReasons = new Set<string>();
+  const streams = stats.streams;
+  const report = streams.reduce((acc, stream) => {
+    acc.totalBytesSent += stream.bytesSent || 0;
+    acc.totalBytesReceived += stream.bytesReceived || 0;
+    acc.averageJitterInMs += stream.jitter || 0;
+    acc.averageRoundTripTimeInMs += stream.currentRoundTripTime || 0;
+
+    // naive calculation of the highest resolution
+    const streamArea = area(stream.frameWidth || 0, stream.frameHeight || 0);
+    if (streamArea > maxArea) {
+      acc.highestFrameWidth = stream.frameWidth || 0;
+      acc.highestFrameHeight = stream.frameHeight || 0;
+      acc.highestFramesPerSecond = stream.framesPerSecond || 0;
+      maxArea = streamArea;
+    }
+
+    qualityLimitationReasons.add(stream.qualityLimitationReason || '');
+    return acc;
+  }, aggregatedStats);
+
+  if (streams.length > 0) {
+    report.averageJitterInMs = Math.round(
+      (report.averageJitterInMs / streams.length) * 1000,
+    );
+    report.averageRoundTripTimeInMs = Math.round(
+      (report.averageRoundTripTimeInMs / streams.length) * 1000,
+    );
+  }
+
+  const qualityLimitationReason = [
+    qualityLimitationReasons.has('cpu') && 'cpu',
+    qualityLimitationReasons.has('bandwidth') && 'bandwidth',
+    qualityLimitationReasons.has('other') && 'other',
+  ]
+    .filter(Boolean)
+    .join(', ');
+  if (qualityLimitationReason) {
+    report.qualityLimitationReasons = qualityLimitationReason;
+  }
+
+  return report;
+};
+
+/**
+ * Flatten the stats report into an array of stats objects.
+ *
+ * @param report the report to flatten.
+ */
+const flatten = (report: RTCStatsReport) => {
+  const stats: RTCStats[] = [];
+  report.forEach((s) => {
+    stats.push(s);
+  });
+  return stats;
+};

package/src/stats/types.ts

@@ -0,0 +1,46 @@
+export type BaseStats = {
+  bytesSent?: number;
+  bytesReceived?: number;
+  codec?: string;
+  currentRoundTripTime?: number;
+  frameWidth?: number;
+  frameHeight?: number;
+  framesPerSecond?: number;
+  jitter?: number;
+  kind?: string;
+  qualityLimitationReason?: string;
+  rid?: string;
+  ssrc?: number;
+};
+
+export type StatsReport = {
+  rawStats?: RTCStatsReport;
+  streams: BaseStats[];
+  timestamp: number;
+};
+
+export type AggregatedStatsReport = {
+  totalBytesSent: number;
+  totalBytesReceived: number;
+  averageJitterInMs: number;
+  averageRoundTripTimeInMs: number;
+  qualityLimitationReasons: string;
+  highestFrameWidth: number;
+  highestFrameHeight: number;
+  highestFramesPerSecond: number;
+  timestamp: number;
+};
+
+export type ParticipantsStatsReport = {
+  [sessionId: string]: StatsReport[] | undefined;
+};
+
+export type CallStatsReport = {
+  datacenter: string;
+  publisherStats: AggregatedStatsReport;
+  publisherRawStats?: RTCStatsReport;
+  subscriberStats: AggregatedStatsReport;
+  subscriberRawStats?: RTCStatsReport;
+  participants: ParticipantsStatsReport;
+  timestamp: number;
+};

package/src/store/index.ts

@@ -0,0 +1,2 @@
+export * as RxUtils from './rxUtils';
+export * from './stateStore';