@stream-io/video-client 1.13.1 → 1.15.0

package/src/rtc/helpers/__tests__/rtcConfiguration.test.ts

@@ -0,0 +1,34 @@
+import { describe, expect, it } from 'vitest';
+import { ICEServer } from '../../../gen/coordinator';
+import { toRtcConfiguration } from '../rtcConfiguration';
+
+describe('rtcConfiguration', () => {
+  it('should map ICEServer configuration to RTCConfiguration', () => {
+    const config: ICEServer[] = [
+      {
+        urls: ['stun:stun.l.google.com:19302'],
+        username: 'user',
+        password: 'pass',
+      },
+      {
+        urls: ['turn:turn.example.com'],
+        username: 'user',
+        password: 'pass',
+      },
+    ];
+    expect(toRtcConfiguration(config)).toEqual({
+      iceServers: [
+        {
+          urls: ['stun:stun.l.google.com:19302'],
+          username: 'user',
+          credential: 'pass',
+        },
+        {
+          urls: ['turn:turn.example.com'],
+          username: 'user',
+          credential: 'pass',
+        },
+      ],
+    });
+  });
+});

package/src/rtc/helpers/__tests__/sdp.test.ts

@@ -0,0 +1,59 @@
+import '../../__tests__/mocks/webrtc.mocks';
+
+import { describe, expect, it, vi } from 'vitest';
+import { extractMid } from '../sdp';
+
+const sdp = `v=0
+o=- 8380609262679842857 2 IN IP4 127.0.0.1
+s=-
+t=0 0
+a=group:BUNDLE 0 1
+a=extmap-allow-mixed
+a=msid-semantic: WMS
+m=video 9 UDP/TLS/RTP/SAVPF 96
+c=IN IP4 0.0.0.0
+a=rtcp:9 IN IP4 0.0.0.0
+a=mid:100
+a=sendonly
+a=msid:- 8d240fd6-26a1-40f6-a769-4d7d24cfd286
+a=rtcp-mux
+a=rtcp-rsize
+a=rtpmap:96 VP8/90000
+a=rtcp-fb:96 goog-remb
+a=rtcp-fb:96 transport-cc
+a=rtcp-fb:96 ccm fir
+a=rtcp-fb:96 nack
+a=rtcp-fb:96 nack pli
+`;
+
+describe('sdp', () => {
+  it('should extract mid from transceiver', () => {
+    const transceiver = new RTCRtpTransceiver();
+    // @ts-ignore - mid is a readonly property
+    transceiver.mid = '10';
+    expect(extractMid(transceiver, -1, '')).toBe('10');
+  });
+
+  it('should use transceiverInitIndex (heuristic) when SDP is not present', () => {
+    expect(extractMid(new RTCRtpTransceiver(), 5, '')).toBe('5');
+  });
+
+  it('should extract mid from SDP', () => {
+    const track = new MediaStreamTrack();
+    // @ts-ignore - id is a readonly property
+    track.id = '8d240fd6-26a1-40f6-a769-4d7d24cfd286';
+    const transceiver = new RTCRtpTransceiver();
+    vi.spyOn(transceiver.sender, 'track', 'get').mockReturnValue(track);
+
+    expect(extractMid(transceiver, -1, sdp)).toBe('100');
+  });
+
+  it('should fallback to transceiverInitIndex when mid can not be found in SDP', () => {
+    const track = new MediaStreamTrack();
+    // @ts-ignore - id is a readonly property
+    track.id = 'not-known';
+    const transceiver = new RTCRtpTransceiver();
+    vi.spyOn(transceiver.sender, 'track', 'get').mockReturnValue(track);
+    expect(extractMid(transceiver, 3, sdp)).toBe('3');
+  });
+});

package/src/rtc/helpers/sdp.ts

@@ -0,0 +1,30 @@
+import { parse } from 'sdp-transform';
+
+/**
+ * Extracts the mid from the transceiver or the SDP.
+ *
+ * @param transceiver the transceiver.
+ * @param transceiverInitIndex the index of the transceiver in the transceiver's init array.
+ * @param sdp the SDP.
+ */
+export const extractMid = (
+  transceiver: RTCRtpTransceiver,
+  transceiverInitIndex: number,
+  sdp: string | undefined,
+): string => {
+  if (transceiver.mid) return transceiver.mid;
+  if (!sdp) return String(transceiverInitIndex);
+
+  const track = transceiver.sender.track!;
+  const parsedSdp = parse(sdp);
+  const media = parsedSdp.media.find((m) => {
+    return (
+      m.type === track.kind &&
+      // if `msid` is not present, we assume that the track is the first one
+      (m.msid?.includes(track.id) ?? true)
+    );
+  });
+  if (typeof media?.mid !== 'undefined') return String(media.mid);
+  if (transceiverInitIndex < 0) return '';
+  return String(transceiverInitIndex);
+};

package/src/rtc/helpers/tracks.ts

@@ -57,3 +57,6 @@ export const toTrackType = (trackType: string): TrackType | undefined => {
       return undefined;
   }
 };
+
+export const isAudioTrackType = (trackType: TrackType): boolean =>
+  trackType === TrackType.AUDIO || trackType === TrackType.SCREEN_SHARE_AUDIO;

package/src/rtc/index.ts

@@ -5,3 +5,7 @@ export * from './Publisher';
 export * from './Subscriber';
 export * from './signal';
 export * from './videoLayers';
+
+export * from './helpers/sdp';
+export * from './helpers/tracks';
+export * from './helpers/rtcConfiguration';

package/src/rtc/videoLayers.ts

@@ -1,8 +1,11 @@
-import { PreferredCodec, PublishOptions } from '../types';
-import { TargetResolutionResponse } from '../gen/shims';
 import { isSvcCodec } from './codecs';
-import { getOptimalBitrate } from './bitrateLookup';
-import { VideoQuality } from '../gen/video/sfu/models/models';
+import {
+  PublishOption,
+  VideoDimension,
+  VideoLayer,
+  VideoQuality,
+} from '../gen/video/sfu/models/models';
+import { isAudioTrackType } from './helpers/tracks';
 
 export type OptimalVideoLayer = RTCRtpEncodingParameters & {
   width: number;
@@ -11,17 +14,10 @@ export type OptimalVideoLayer = RTCRtpEncodingParameters & {
   scalabilityMode?: string;
 };
 
-const DEFAULT_BITRATE = 1250000;
-const defaultTargetResolution: TargetResolutionResponse = {
-  bitrate: DEFAULT_BITRATE,
-  width: 1280,
-  height: 720,
-};
-
 const defaultBitratePerRid: Record<string, number> = {
   q: 300000,
   h: 750000,
-  f: DEFAULT_BITRATE,
+  f: 1250000,
 };
 
 /**
@@ -31,9 +27,17 @@ const defaultBitratePerRid: Record<string, number> = {
  *
  * @param layers the layers to process.
  */
-export const toSvcEncodings = (layers: OptimalVideoLayer[] | undefined) => {
-  // we take the `f` layer, and we rename it to `q`.
-  return layers?.filter((l) => l.rid === 'f').map((l) => ({ ...l, rid: 'q' }));
+export const toSvcEncodings = (
+  layers: OptimalVideoLayer[] | undefined,
+): RTCRtpEncodingParameters[] | undefined => {
+  if (!layers) return;
+  // we take the highest quality layer, and we assign it to `q` encoder.
+  const withRid = (rid: string) => (l: OptimalVideoLayer) => l.rid === rid;
+  const highestLayer =
+    layers.find(withRid('f')) ||
+    layers.find(withRid('h')) ||
+    layers.find(withRid('q'));
+  return [{ ...highestLayer, rid: 'q' }];
 };
 
 /**
@@ -48,60 +52,80 @@ export const ridToVideoQuality = (rid: string): VideoQuality => {
 };
 
 /**
- * Determines the most optimal video layers for simulcasting
- * for the given track.
+ * Converts the given video layers to SFU video layers.
+ */
+export const toVideoLayers = (
+  layers: OptimalVideoLayer[] | undefined = [],
+): VideoLayer[] => {
+  return layers.map<VideoLayer>((layer) => ({
+    rid: layer.rid || '',
+    bitrate: layer.maxBitrate || 0,
+    fps: layer.maxFramerate || 0,
+    quality: ridToVideoQuality(layer.rid || ''),
+    videoDimension: { width: layer.width, height: layer.height },
+  }));
+};
+
+/**
+ * Converts the spatial and temporal layers to a scalability mode.
+ */
+const toScalabilityMode = (spatialLayers: number, temporalLayers: number) =>
+  `L${spatialLayers}T${temporalLayers}${spatialLayers > 1 ? '_KEY' : ''}`;
+
+/**
+ * Determines the most optimal video layers for the given track.
  *
  * @param videoTrack the video track to find optimal layers for.
- * @param targetResolution the expected target resolution.
- * @param codecInUse the codec in use.
- * @param publishOptions the publish options for the track.
+ * @param publishOption the publish options for the track.
  */
-export const findOptimalVideoLayers = (
+export const computeVideoLayers = (
   videoTrack: MediaStreamTrack,
-  targetResolution: TargetResolutionResponse = defaultTargetResolution,
-  codecInUse?: PreferredCodec,
-  publishOptions?: PublishOptions,
-) => {
+  publishOption: PublishOption,
+): OptimalVideoLayer[] | undefined => {
+  if (isAudioTrackType(publishOption.trackType)) return;
   const optimalVideoLayers: OptimalVideoLayer[] = [];
   const settings = videoTrack.getSettings();
   const { width = 0, height = 0 } = settings;
   const {
-    scalabilityMode,
-    bitrateDownscaleFactor = 2,
-    maxSimulcastLayers = 3,
-  } = publishOptions || {};
+    bitrate,
+    codec,
+    fps,
+    maxSpatialLayers = 3,
+    maxTemporalLayers = 3,
+    videoDimension = { width: 1280, height: 720 },
+  } = publishOption;
   const maxBitrate = getComputedMaxBitrate(
-    targetResolution,
+    videoDimension,
     width,
     height,
-    codecInUse,
-    publishOptions,
+    bitrate,
   );
   let downscaleFactor = 1;
   let bitrateFactor = 1;
-  const svcCodec = isSvcCodec(codecInUse);
-  const totalLayers = svcCodec ? 3 : Math.min(3, maxSimulcastLayers);
-  for (const rid of ['f', 'h', 'q'].slice(0, totalLayers)) {
+  const svcCodec = isSvcCodec(codec?.name);
+  for (const rid of ['f', 'h', 'q'].slice(0, maxSpatialLayers)) {
     const layer: OptimalVideoLayer = {
       active: true,
       rid,
       width: Math.round(width / downscaleFactor),
       height: Math.round(height / downscaleFactor),
-      maxBitrate:
-        Math.round(maxBitrate / bitrateFactor) || defaultBitratePerRid[rid],
-      maxFramerate: 30,
+      maxBitrate: maxBitrate / bitrateFactor || defaultBitratePerRid[rid],
+      maxFramerate: fps,
     };
     if (svcCodec) {
       // for SVC codecs, we need to set the scalability mode, and the
       // codec will handle the rest (layers, temporal layers, etc.)
-      layer.scalabilityMode = scalabilityMode || 'L3T2_KEY';
+      layer.scalabilityMode = toScalabilityMode(
+        maxSpatialLayers,
+        maxTemporalLayers,
+      );
     } else {
       // for non-SVC codecs, we need to downscale proportionally (simulcast)
       layer.scaleResolutionDownBy = downscaleFactor;
     }
 
     downscaleFactor *= 2;
-    bitrateFactor *= bitrateDownscaleFactor;
+    bitrateFactor *= 2;
 
     // Reversing the order [f, h, q] to [q, h, f] as Chrome uses encoding index
     // when deciding which layer to disable when CPU or bandwidth is constrained.
@@ -124,29 +148,17 @@ export const findOptimalVideoLayers = (
  * @param targetResolution the target resolution.
  * @param currentWidth the current width of the track.
  * @param currentHeight the current height of the track.
- * @param codecInUse the codec in use.
- * @param publishOptions the publish options.
+ * @param bitrate the target bitrate.
  */
 export const getComputedMaxBitrate = (
-  targetResolution: TargetResolutionResponse,
+  targetResolution: VideoDimension,
   currentWidth: number,
   currentHeight: number,
-  codecInUse?: PreferredCodec,
-  publishOptions?: PublishOptions,
+  bitrate: number,
 ): number => {
   // if the current resolution is lower than the target resolution,
   // we want to proportionally reduce the target bitrate
-  const {
-    width: targetWidth,
-    height: targetHeight,
-    bitrate: targetBitrate,
-  } = targetResolution;
-  const { preferredBitrate } = publishOptions || {};
-  const frameHeight =
-    currentWidth > currentHeight ? currentHeight : currentWidth;
-  const bitrate =
-    preferredBitrate ||
-    (codecInUse ? getOptimalBitrate(codecInUse, frameHeight) : targetBitrate);
+  const { width: targetWidth, height: targetHeight } = targetResolution;
   if (currentWidth < targetWidth || currentHeight < targetHeight) {
     const currentPixels = currentWidth * currentHeight;
     const targetPixels = targetWidth * targetHeight;
@@ -188,23 +200,3 @@ const withSimulcastConstraints = (
     rid: ridMapping[index], // reassign rid
   }));
 };
-
-export const findOptimalScreenSharingLayers = (
-  videoTrack: MediaStreamTrack,
-  publishOptions?: PublishOptions,
-  defaultMaxBitrate = 3000000,
-): OptimalVideoLayer[] => {
-  const { screenShareSettings: preferences } = publishOptions || {};
-  const settings = videoTrack.getSettings();
-  return [
-    {
-      active: true,
-      rid: 'q', // single track, start from 'q'
-      width: settings.width || 0,
-      height: settings.height || 0,
-      scaleResolutionDownBy: 1,
-      maxBitrate: preferences?.maxBitrate ?? defaultMaxBitrate,
-      maxFramerate: preferences?.maxFramerate ?? 30,
-    },
-  ];
-};

package/src/stats/{stateStoreStatsReporter.ts → CallStateStatsReporter.ts}

@@ -2,12 +2,15 @@ import type {
   AggregatedStatsReport,
   BaseStats,
   ParticipantsStatsReport,
+  RTCMediaSourceStats,
   StatsReport,
 } from './types';
 import { CallState } from '../store';
 import { Publisher, Subscriber } from '../rtc';
 import { getLogger } from '../logger';
 import { flatten } from './utils';
+import { TrackType } from '../gen/video/sfu/models/models';
+import { isFirefox } from '../helpers/browsers';
 
 export type StatsReporterOpts = {
   subscriber: Subscriber;
@@ -41,7 +44,7 @@ export type StatsReporter = {
    */
   getStatsForStream: (
     kind: 'subscriber' | 'publisher',
-    mediaStream: MediaStream,
+    tracks: MediaStreamTrack[],
   ) => Promise<StatsReport[]>;
 
   /**
@@ -87,12 +90,12 @@ export const createStatsReporter = ({
 
   const getStatsForStream = async (
     kind: 'subscriber' | 'publisher',
-    mediaStream: MediaStream,
+    tracks: MediaStreamTrack[],
   ) => {
     const pc = kind === 'subscriber' ? subscriber : publisher;
     if (!pc) return [];
     const statsForStream: StatsReport[] = [];
-    for (let track of mediaStream.getTracks()) {
+    for (const track of tracks) {
       const report = await pc.getStats(track);
       const stats = transform(report, {
         // @ts-ignore
@@ -121,31 +124,28 @@ export const createStatsReporter = ({
    */
   const run = async () => {
     const participantStats: ParticipantsStatsReport = {};
-    const sessionIds = new Set(sessionIdsToTrack);
-    if (sessionIds.size > 0) {
-      for (let participant of state.participants) {
+    if (sessionIdsToTrack.size > 0) {
+      const sessionIds = new Set(sessionIdsToTrack);
+      for (const participant of state.participants) {
         if (!sessionIds.has(participant.sessionId)) continue;
-        const kind = participant.isLocalParticipant
-          ? 'publisher'
-          : 'subscriber';
+        const {
+          audioStream,
+          isLocalParticipant,
+          sessionId,
+          userId,
+          videoStream,
+        } = participant;
+        const kind = isLocalParticipant ? 'publisher' : 'subscriber';
         try {
-          const mergedStream = new MediaStream([
-            ...(participant.videoStream?.getVideoTracks() || []),
-            ...(participant.audioStream?.getAudioTracks() || []),
-          ]);
-          participantStats[participant.sessionId] = await getStatsForStream(
-            kind,
-            mergedStream,
-          );
-          mergedStream.getTracks().forEach((t) => {
-            mergedStream.removeTrack(t);
-          });
+          const tracks = isLocalParticipant
+            ? publisher?.getPublishedTracks() || []
+            : [
+                ...(videoStream?.getVideoTracks() || []),
+                ...(audioStream?.getAudioTracks() || []),
+              ];
+          participantStats[sessionId] = await getStatsForStream(kind, tracks);
         } catch (e) {
-          logger(
-            'error',
-            `Failed to collect stats for ${kind} of ${participant.userId}`,
-            e,
-          );
+          logger('warn', `Failed to collect ${kind} stats for ${userId}`, e);
         }
       }
     }
@@ -157,6 +157,7 @@ export const createStatsReporter = ({
           transform(report, {
             kind: 'subscriber',
             trackKind: 'video',
+            publisher,
           }),
         )
         .then(aggregate),
@@ -167,6 +168,7 @@ export const createStatsReporter = ({
               transform(report, {
                 kind: 'publisher',
                 trackKind: 'video',
+                publisher,
               }),
             )
             .then(aggregate)
@@ -220,11 +222,14 @@ export type StatsTransformOpts = {
    * The kind of track we are transforming stats for.
    */
   trackKind: 'audio' | 'video';
-
   /**
    * The kind of peer connection we are transforming stats for.
    */
   kind: 'subscriber' | 'publisher';
+  /**
+   * The publisher instance.
+   */
+  publisher: Publisher | undefined;
 };
 
 /**
@@ -237,7 +242,7 @@ const transform = (
   report: RTCStatsReport,
   opts: StatsTransformOpts,
 ): StatsReport => {
-  const { trackKind, kind } = opts;
+  const { trackKind, kind, publisher } = opts;
   const direction = kind === 'subscriber' ? 'inbound-rtp' : 'outbound-rtp';
   const stats = flatten(report);
   const streams = stats
@@ -268,6 +273,20 @@ const transform = (
         roundTripTime = candidatePair?.currentRoundTripTime;
       }
 
+      let trackType: TrackType | undefined;
+      if (kind === 'publisher' && publisher) {
+        const firefox = isFirefox();
+        const mediaSource = stats.find(
+          (s) =>
+            s.type === 'media-source' &&
+            // Firefox doesn't have mediaSourceId, so we need to guess the media source
+            (firefox ? true : s.id === rtcStreamStats.mediaSourceId),
+        ) as RTCMediaSourceStats | undefined;
+        if (mediaSource) {
+          trackType = publisher.getTrackType(mediaSource.trackIdentifier);
+        }
+      }
+
       return {
         bytesSent: rtcStreamStats.bytesSent,
         bytesReceived: rtcStreamStats.bytesReceived,
@@ -278,10 +297,12 @@ const transform = (
         framesPerSecond: rtcStreamStats.framesPerSecond,
         jitter: rtcStreamStats.jitter,
         kind: rtcStreamStats.kind,
+        mediaSourceId: rtcStreamStats.mediaSourceId,
         // @ts-ignore: available in Chrome only, TS doesn't recognize this
         qualityLimitationReason: rtcStreamStats.qualityLimitationReason,
         rid: rtcStreamStats.rid,
         ssrc: rtcStreamStats.ssrc,
+        trackType,
       };
     });
 
@@ -304,6 +325,7 @@ const getEmptyStats = (stats?: StatsReport): AggregatedStatsReport => {
     highestFrameHeight: 0,
     highestFramesPerSecond: 0,
     codec: '',
+    codecPerTrackType: {},
     timestamp: Date.now(),
   };
 };
@@ -349,6 +371,15 @@ const aggregate = (stats: StatsReport): AggregatedStatsReport => {
     );
     // we take the first codec we find, as it should be the same for all streams
     report.codec = streams[0].codec || '';
+    report.codecPerTrackType = streams.reduce(
+      (acc, stream) => {
+        if (stream.trackType) {
+          acc[stream.trackType] = stream.codec || '';
+        }
+        return acc;
+      },
+      {} as Record<TrackType, string>,
+    );
   }
 
   const qualityLimitationReason = [

package/src/stats/SfuStatsReporter.ts

@@ -9,7 +9,10 @@ import {
   getWebRTCInfo,
   LocalClientDetailsType,
 } from '../client-details';
-import { InputDevices } from '../gen/video/sfu/models/models';
+import {
+  InputDevices,
+  WebsocketReconnectStrategy,
+} from '../gen/video/sfu/models/models';
 import { CameraManager, MicrophoneManager } from '../devices';
 import { createSubscription } from '../store/rxUtils';
 import { CallState } from '../store';
@@ -118,8 +121,33 @@ export class SfuStatsReporter {
     );
   };
 
-  sendTelemetryData = async (telemetryData: Telemetry) => {
-    return this.run(telemetryData);
+  sendConnectionTime = (connectionTimeSeconds: number) => {
+    this.sendTelemetryData({
+      data: {
+        oneofKind: 'connectionTimeSeconds',
+        connectionTimeSeconds,
+      },
+    });
+  };
+
+  sendReconnectionTime = (
+    strategy: WebsocketReconnectStrategy,
+    timeSeconds: number,
+  ) => {
+    this.sendTelemetryData({
+      data: {
+        oneofKind: 'reconnection',
+        reconnection: { strategy, timeSeconds },
+      },
+    });
+  };
+
+  private sendTelemetryData = (telemetryData: Telemetry) => {
+    // intentionally not awaiting the promise here
+    // to avoid impeding with the ongoing actions.
+    this.run(telemetryData).catch((err) => {
+      this.logger('warn', 'Failed to send telemetry data', err);
+    });
   };
 
   private run = async (telemetryData?: Telemetry) => {

package/src/stats/index.ts

@@ -1,3 +1,3 @@
-export * from './stateStoreStatsReporter';
+export * from './CallStateStatsReporter';
 export * from './SfuStatsReporter';
 export * from './types';

package/src/stats/types.ts

@@ -1,3 +1,5 @@
+import { TrackType } from '../gen/video/sfu/models/models';
+
 export type BaseStats = {
   audioLevel?: number;
   bytesSent?: number;
@@ -9,9 +11,11 @@ export type BaseStats = {
   framesPerSecond?: number;
   jitter?: number;
   kind?: string;
+  mediaSourceId?: string;
   qualityLimitationReason?: string;
   rid?: string;
   ssrc?: number;
+  trackType?: TrackType;
 };
 
 export type StatsReport = {
@@ -30,6 +34,7 @@ export type AggregatedStatsReport = {
   highestFrameHeight: number;
   highestFramesPerSecond: number;
   codec: string;
+  codecPerTrackType: Partial<Record<TrackType, string>>;
   timestamp: number;
   rawReport: StatsReport;
 };
@@ -48,3 +53,10 @@ export type CallStatsReport = {
   participants: ParticipantsStatsReport;
   timestamp: number;
 };
+
+// shim for RTCMediaSourceStats, not yet available in the standard types
+// https://www.w3.org/TR/webrtc-stats/#mediasourcestats-dict*
+export interface RTCMediaSourceStats {
+  kind: string;
+  trackIdentifier: string;
+}