@stream-io/video-client 1.8.4 → 1.9.1

package/src/rtc/__tests__/Publisher.test.ts

@@ -17,7 +17,8 @@ vi.mock('../../StreamSfuClient', () => {
   };
 });
 
-vi.mock('../codecs', () => {
+vi.mock('../codecs', async () => {
+  const codecs = await vi.importActual('../codecs');
   return {
     getPreferredCodecs: vi.fn((): RTCRtpCodecCapability[] => [
       {
@@ -27,6 +28,8 @@ vi.mock('../codecs', () => {
         sdpFmtpLine: 'profile-level-id=42e01f',
       },
     ]),
+    getOptimalVideoCodec: codecs.getOptimalVideoCodec,
+    isSvcCodec: codecs.isSvcCodec,
   };
 });
 
@@ -136,11 +139,7 @@ describe('Publisher', () => {
     vi.spyOn(transceiver.sender, 'track', 'get').mockReturnValue(newTrack);
 
     expect(track.stop).toHaveBeenCalled();
-    expect(track.removeEventListener).toHaveBeenCalledWith(
-      'ended',
-      expect.any(Function),
-    );
-    expect(newTrack.addEventListener).toHaveBeenCalledWith(
+    expect(newTrack.addEventListener).not.toHaveBeenCalledWith(
       'ended',
       expect.any(Function),
     );
@@ -154,7 +153,7 @@ describe('Publisher', () => {
     );
   });
 
-  it('can publish and un-pubish with just enabling and disabling tracks', async () => {
+  it('can publish and un-publish with just enabling and disabling tracks', async () => {
     const mediaStream = new MediaStream();
     const track = new MediaStreamTrack();
     mediaStream.addTrack(track);
@@ -277,4 +276,194 @@ describe('Publisher', () => {
       expect(publisher.restartIce).toHaveBeenCalled();
     });
   });
+
+  describe('changePublishQuality', () => {
+    it('can dynamically activate/deactivate simulcast layers', async () => {
+      const transceiver = new RTCRtpTransceiver();
+      const setParametersSpy = vi
+        .spyOn(transceiver.sender, 'setParameters')
+        .mockResolvedValue();
+      const getParametersSpy = vi
+        .spyOn(transceiver.sender, 'getParameters')
+        .mockReturnValue({
+          codecs: [
+            // @ts-expect-error incomplete data
+            { mimeType: 'video/VP8' },
+            // @ts-expect-error incomplete data
+            { mimeType: 'video/VP9' },
+            // @ts-expect-error incomplete data
+            { mimeType: 'video/H264' },
+            // @ts-expect-error incomplete data
+            { mimeType: 'video/AV1' },
+          ],
+          encodings: [
+            { rid: 'q', active: true },
+            { rid: 'h', active: true },
+            { rid: 'f', active: true },
+          ],
+        });
+
+      // inject the transceiver
+      publisher['transceiverCache'].set(TrackType.VIDEO, transceiver);
+
+      await publisher['changePublishQuality']([
+        {
+          name: 'q',
+          active: true,
+          maxBitrate: 100,
+          scaleResolutionDownBy: 4,
+          maxFramerate: 30,
+          scalabilityMode: '',
+        },
+        {
+          name: 'h',
+          active: false,
+          maxBitrate: 150,
+          scaleResolutionDownBy: 2,
+          maxFramerate: 30,
+          scalabilityMode: '',
+        },
+        {
+          name: 'f',
+          active: true,
+          maxBitrate: 200,
+          scaleResolutionDownBy: 1,
+          maxFramerate: 30,
+          scalabilityMode: '',
+        },
+      ]);
+
+      expect(getParametersSpy).toHaveBeenCalled();
+      expect(setParametersSpy).toHaveBeenCalled();
+      expect(setParametersSpy.mock.calls[0][0].encodings).toEqual([
+        {
+          rid: 'q',
+          active: true,
+          maxBitrate: 100,
+          scaleResolutionDownBy: 4,
+          maxFramerate: 30,
+        },
+        {
+          rid: 'h',
+          active: false,
+          maxBitrate: 150,
+          scaleResolutionDownBy: 2,
+          maxFramerate: 30,
+        },
+        {
+          rid: 'f',
+          active: true,
+          maxBitrate: 200,
+          scaleResolutionDownBy: 1,
+          maxFramerate: 30,
+        },
+      ]);
+    });
+
+    it('can dynamically update scalability mode in SVC', async () => {
+      const transceiver = new RTCRtpTransceiver();
+      const setParametersSpy = vi
+        .spyOn(transceiver.sender, 'setParameters')
+        .mockResolvedValue();
+      const getParametersSpy = vi
+        .spyOn(transceiver.sender, 'getParameters')
+        .mockReturnValue({
+          codecs: [
+            // @ts-expect-error incomplete data
+            { mimeType: 'video/VP9' },
+            // @ts-expect-error incomplete data
+            { mimeType: 'video/AV1' },
+            // @ts-expect-error incomplete data
+            { mimeType: 'video/VP8' },
+            // @ts-expect-error incomplete data
+            { mimeType: 'video/H264' },
+          ],
+          encodings: [
+            {
+              rid: 'q',
+              active: true,
+              maxBitrate: 100,
+              // @ts-expect-error not in the standard lib yet
+              scalabilityMode: 'L3T3_KEY',
+            },
+          ],
+        });
+
+      // inject the transceiver
+      publisher['transceiverCache'].set(TrackType.VIDEO, transceiver);
+
+      await publisher['changePublishQuality']([
+        {
+          name: 'q',
+          active: true,
+          maxBitrate: 50,
+          scaleResolutionDownBy: 1,
+          maxFramerate: 30,
+          scalabilityMode: 'L1T3',
+        },
+      ]);
+
+      expect(getParametersSpy).toHaveBeenCalled();
+      expect(setParametersSpy).toHaveBeenCalled();
+      expect(setParametersSpy.mock.calls[0][0].encodings).toEqual([
+        {
+          rid: 'q',
+          active: true,
+          maxBitrate: 50,
+          scaleResolutionDownBy: 1,
+          maxFramerate: 30,
+          scalabilityMode: 'L1T3',
+        },
+      ]);
+    });
+
+    it('supports empty rid in SVC', async () => {
+      const transceiver = new RTCRtpTransceiver();
+      const setParametersSpy = vi
+        .spyOn(transceiver.sender, 'setParameters')
+        .mockResolvedValue();
+      const getParametersSpy = vi
+        .spyOn(transceiver.sender, 'getParameters')
+        .mockReturnValue({
+          codecs: [
+            // @ts-expect-error incomplete data
+            { mimeType: 'video/VP9' },
+          ],
+          encodings: [
+            {
+              rid: undefined, // empty rid
+              active: true,
+              // @ts-expect-error not in the standard lib yet
+              scalabilityMode: 'L3T3_KEY',
+            },
+          ],
+        });
+
+      // inject the transceiver
+      publisher['transceiverCache'].set(TrackType.VIDEO, transceiver);
+
+      await publisher['changePublishQuality']([
+        {
+          name: 'q',
+          active: true,
+          maxBitrate: 50,
+          scaleResolutionDownBy: 1,
+          maxFramerate: 30,
+          scalabilityMode: 'L1T3',
+        },
+      ]);
+
+      expect(getParametersSpy).toHaveBeenCalled();
+      expect(setParametersSpy).toHaveBeenCalled();
+      expect(setParametersSpy.mock.calls[0][0].encodings).toEqual([
+        {
+          active: true,
+          maxBitrate: 50,
+          scaleResolutionDownBy: 1,
+          maxFramerate: 30,
+          scalabilityMode: 'L1T3',
+        },
+      ]);
+    });
+  });
 });

package/src/rtc/__tests__/bitrateLookup.test.ts

@@ -0,0 +1,12 @@
+import { describe, expect, it } from 'vitest';
+import { getOptimalBitrate } from '../bitrateLookup';
+
+describe('bitrateLookup', () => {
+  it('should return optimal bitrate', () => {
+    expect(getOptimalBitrate('vp9', 720)).toBe(1_250_000);
+  });
+
+  it('should return nearest bitrate for exotic dimensions', () => {
+    expect(getOptimalBitrate('vp9', 1000)).toBe(1_500_000);
+  });
+});

package/src/rtc/__tests__/mocks/webrtc.mocks.ts

@@ -45,6 +45,8 @@ const RTCRtpTransceiverMock = vi.fn((): Partial<RTCRtpTransceiver> => {
     sender: {
       track: null,
       replaceTrack: vi.fn(),
+      getParameters: vi.fn(),
+      setParameters: vi.fn(),
     },
     setCodecPreferences: vi.fn(),
   };

package/src/rtc/__tests__/videoLayers.test.ts

@@ -4,7 +4,11 @@ import {
   findOptimalScreenSharingLayers,
   findOptimalVideoLayers,
   getComputedMaxBitrate,
+  OptimalVideoLayer,
+  ridToVideoQuality,
+  toSvcEncodings,
 } from '../videoLayers';
+import { VideoQuality } from '../../gen/video/sfu/models/models';
 
 describe('videoLayers', () => {
   it('should find optimal screen sharing layers', () => {
@@ -135,15 +139,51 @@ describe('videoLayers', () => {
     expect(layers[2].rid).toBe('f');
   });
 
+  it('should announce only one layer for SVC codecs', () => {
+    const track = new MediaStreamTrack();
+    vi.spyOn(track, 'getSettings').mockReturnValue({
+      width: 1280,
+      height: 720,
+    });
+    const layers = findOptimalVideoLayers(track, undefined, 'vp9', {
+      preferredCodec: 'vp9',
+      scalabilityMode: 'L3T3',
+    });
+    expect(layers.length).toBe(3);
+    expect(layers[0].scalabilityMode).toBe('L3T3');
+    expect(layers[0].rid).toBe('q');
+    expect(layers[1].rid).toBe('h');
+    expect(layers[2].rid).toBe('f');
+  });
+
+  it('should map rids to VideoQuality', () => {
+    expect(ridToVideoQuality('q')).toBe(VideoQuality.LOW_UNSPECIFIED);
+    expect(ridToVideoQuality('h')).toBe(VideoQuality.MID);
+    expect(ridToVideoQuality('f')).toBe(VideoQuality.HIGH);
+    expect(ridToVideoQuality('')).toBe(VideoQuality.HIGH);
+  });
+
+  it('should map OptimalVideoLayer to SVC encodings', () => {
+    const layers: Array<Partial<OptimalVideoLayer>> = [
+      { rid: 'f', width: 1920, height: 1080, maxBitrate: 3000000 },
+      { rid: 'h', width: 960, height: 540, maxBitrate: 750000 },
+      { rid: 'q', width: 480, height: 270, maxBitrate: 187500 },
+    ];
+
+    const svcLayers = toSvcEncodings(layers as OptimalVideoLayer[]);
+    expect(svcLayers.length).toBe(1);
+    expect(svcLayers[0]).toEqual({
+      rid: 'q',
+      width: 1920,
+      height: 1080,
+      maxBitrate: 3000000,
+    });
+  });
+
   describe('getComputedMaxBitrate', () => {
     it('should scale target bitrate down if resolution is smaller than target resolution', () => {
       const targetResolution = { width: 1920, height: 1080, bitrate: 3000000 };
-      const scaledBitrate = getComputedMaxBitrate(
-        targetResolution,
-        1280,
-        720,
-        undefined,
-      );
+      const scaledBitrate = getComputedMaxBitrate(targetResolution, 1280, 720);
       expect(scaledBitrate).toBe(1333333);
     });
 
@@ -153,12 +193,7 @@ describe('videoLayers', () => {
       const targetBitrates = ['f', 'h', 'q'].map((rid) => {
         const width = targetResolution.width / downscaleFactor;
         const height = targetResolution.height / downscaleFactor;
-        const bitrate = getComputedMaxBitrate(
-          targetResolution,
-          width,
-          height,
-          undefined,
-        );
+        const bitrate = getComputedMaxBitrate(targetResolution, width, height);
         downscaleFactor *= 2;
         return {
           rid,
@@ -176,45 +211,25 @@ describe('videoLayers', () => {
 
     it('should not scale target bitrate if resolution is larger than target resolution', () => {
       const targetResolution = { width: 1280, height: 720, bitrate: 1000000 };
-      const scaledBitrate = getComputedMaxBitrate(
-        targetResolution,
-        2560,
-        1440,
-        undefined,
-      );
+      const scaledBitrate = getComputedMaxBitrate(targetResolution, 2560, 1440);
       expect(scaledBitrate).toBe(1000000);
     });
 
     it('should not scale target bitrate if resolution is equal to target resolution', () => {
       const targetResolution = { width: 1280, height: 720, bitrate: 1000000 };
-      const scaledBitrate = getComputedMaxBitrate(
-        targetResolution,
-        1280,
-        720,
-        undefined,
-      );
+      const scaledBitrate = getComputedMaxBitrate(targetResolution, 1280, 720);
       expect(scaledBitrate).toBe(1000000);
     });
 
     it('should handle 0 width and height', () => {
       const targetResolution = { width: 1280, height: 720, bitrate: 1000000 };
-      const scaledBitrate = getComputedMaxBitrate(
-        targetResolution,
-        0,
-        0,
-        undefined,
-      );
+      const scaledBitrate = getComputedMaxBitrate(targetResolution, 0, 0);
       expect(scaledBitrate).toBe(0);
     });
 
     it('should handle 4k target resolution', () => {
       const targetResolution = { width: 3840, height: 2160, bitrate: 15000000 };
-      const scaledBitrate = getComputedMaxBitrate(
-        targetResolution,
-        1280,
-        720,
-        undefined,
-      );
+      const scaledBitrate = getComputedMaxBitrate(targetResolution, 1280, 720);
       expect(scaledBitrate).toBe(1666667);
     });
   });

package/src/rtc/bitrateLookup.ts

@@ -0,0 +1,61 @@
+import { PreferredCodec } from '../types';
+
+const bitrateLookupTable: Record<
+  PreferredCodec,
+  Record<number | 'default', number | undefined> | undefined
+> = {
+  h264: {
+    2160: 5_000_000,
+    1440: 3_500_000,
+    1080: 2_750_000,
+    720: 1_250_000,
+    540: 750_000,
+    360: 400_000,
+    default: 1_250_000,
+  },
+  vp8: {
+    2160: 5_000_000,
+    1440: 2_750_000,
+    1080: 2_000_000,
+    720: 1_250_000,
+    540: 600_000,
+    360: 350_000,
+    default: 1_250_000,
+  },
+  vp9: {
+    2160: 3_000_000,
+    1440: 2_000_000,
+    1080: 1_500_000,
+    720: 1_250_000,
+    540: 500_000,
+    360: 275_000,
+    default: 1_250_000,
+  },
+  av1: {
+    2160: 2_000_000,
+    1440: 1_550_000,
+    1080: 1_000_000,
+    720: 600_000,
+    540: 350_000,
+    360: 200_000,
+    default: 600_000,
+  },
+};
+
+export const getOptimalBitrate = (
+  codec: PreferredCodec,
+  frameHeight: number,
+): number => {
+  const codecLookup = bitrateLookupTable[codec];
+  if (!codecLookup) throw new Error(`Unknown codec: ${codec}`);
+
+  let bitrate = codecLookup[frameHeight];
+  if (!bitrate) {
+    const keys = Object.keys(codecLookup).map(Number);
+    const nearest = keys.reduce((a, b) =>
+      Math.abs(b - frameHeight) < Math.abs(a - frameHeight) ? b : a,
+    );
+    bitrate = codecLookup[nearest];
+  }
+  return bitrate ?? codecLookup.default!;
+};

package/src/rtc/codecs.ts

@@ -1,4 +1,7 @@
 import { getOSInfo } from '../client-details';
+import { isReactNative } from '../helpers/platforms';
+import { isFirefox, isSafari } from '../helpers/browsers';
+import type { PreferredCodec } from '../types';
 
 /**
  * Returns back a list of sorted codecs, with the preferred codec first.
@@ -92,14 +95,58 @@ export const getGenericSdp = async (direction: RTCRtpTransceiverDirection) => {
 };
 
 /**
- * Returns the optimal codec for RN.
+ * Returns the optimal video codec for the device.
  */
-export const getRNOptimalCodec = () => {
-  const osName = getOSInfo()?.name.toLowerCase();
-  // in ipads it was noticed that if vp8 codec is used
-  // then the bytes sent is 0 in the outbound-rtp
-  // so we are forcing h264 codec for ipads
-  if (osName === 'ipados') return 'h264';
-  if (osName === 'android') return 'vp8';
-  return undefined;
+export const getOptimalVideoCodec = (
+  preferredCodec: PreferredCodec | undefined,
+): PreferredCodec => {
+  if (isReactNative()) {
+    const os = getOSInfo()?.name.toLowerCase();
+    if (os === 'android') return preferredOr(preferredCodec, 'vp8');
+    if (os === 'ios' || os === 'ipados') return 'h264';
+    return preferredOr(preferredCodec, 'h264');
+  }
+  if (isSafari()) return 'h264';
+  if (isFirefox()) return 'vp8';
+  return preferredOr(preferredCodec, 'vp8');
+};
+
+/**
+ * Determines if the platform supports the preferred codec.
+ * If not, it returns the fallback codec.
+ */
+const preferredOr = (
+  codec: PreferredCodec | undefined,
+  fallback: PreferredCodec,
+): PreferredCodec => {
+  if (!codec) return fallback;
+  if (!('getCapabilities' in RTCRtpSender)) return fallback;
+  const capabilities = RTCRtpSender.getCapabilities('video');
+  if (!capabilities) return fallback;
+
+  // Safari and Firefox do not have a good support encoding to SVC codecs,
+  // so we disable it for them.
+  if (isSvcCodec(codec) && (isSafari() || isFirefox())) return fallback;
+
+  const { codecs } = capabilities;
+  const codecMimeType = `video/${codec}`.toLowerCase();
+  return codecs.some((c) => c.mimeType.toLowerCase() === codecMimeType)
+    ? codec
+    : fallback;
+};
+
+/**
+ * Returns whether the codec is an SVC codec.
+ *
+ * @param codecOrMimeType the codec to check.
+ */
+export const isSvcCodec = (codecOrMimeType: string | undefined) => {
+  if (!codecOrMimeType) return false;
+  codecOrMimeType = codecOrMimeType.toLowerCase();
+  return (
+    codecOrMimeType === 'vp9' ||
+    codecOrMimeType === 'av1' ||
+    codecOrMimeType === 'video/vp9' ||
+    codecOrMimeType === 'video/av1'
+  );
 };

package/src/rtc/videoLayers.ts

@@ -1,9 +1,14 @@
-import { PublishOptions } from '../types';
+import { PreferredCodec, PublishOptions } from '../types';
 import { TargetResolutionResponse } from '../gen/shims';
+import { isSvcCodec } from './codecs';
+import { getOptimalBitrate } from './bitrateLookup';
+import { VideoQuality } from '../gen/video/sfu/models/models';
 
 export type OptimalVideoLayer = RTCRtpEncodingParameters & {
   width: number;
   height: number;
+  // NOTE OL: should be part of RTCRtpEncodingParameters
+  scalabilityMode?: string;
 };
 
 const DEFAULT_BITRATE = 1250000;
@@ -19,48 +24,85 @@ const defaultBitratePerRid: Record<string, number> = {
   f: DEFAULT_BITRATE,
 };
 
+/**
+ * In SVC, we need to send only one video encoding (layer).
+ * this layer will have the additional spatial and temporal layers
+ * defined via the scalabilityMode property.
+ *
+ * @param layers the layers to process.
+ */
+export const toSvcEncodings = (layers: OptimalVideoLayer[] | undefined) => {
+  // we take the `f` layer, and we rename it to `q`.
+  return layers?.filter((l) => l.rid === 'f').map((l) => ({ ...l, rid: 'q' }));
+};
+
+/**
+ * Converts the rid to a video quality.
+ */
+export const ridToVideoQuality = (rid: string): VideoQuality => {
+  return rid === 'q'
+    ? VideoQuality.LOW_UNSPECIFIED
+    : rid === 'h'
+      ? VideoQuality.MID
+      : VideoQuality.HIGH; // default to HIGH
+};
+
 /**
  * Determines the most optimal video layers for simulcasting
  * for the given track.
  *
  * @param videoTrack the video track to find optimal layers for.
  * @param targetResolution the expected target resolution.
+ * @param codecInUse the codec in use.
  * @param publishOptions the publish options for the track.
  */
 export const findOptimalVideoLayers = (
   videoTrack: MediaStreamTrack,
   targetResolution: TargetResolutionResponse = defaultTargetResolution,
+  codecInUse?: PreferredCodec,
   publishOptions?: PublishOptions,
 ) => {
   const optimalVideoLayers: OptimalVideoLayer[] = [];
   const settings = videoTrack.getSettings();
-  const { width: w = 0, height: h = 0 } = settings;
-  const { preferredBitrate, bitrateDownscaleFactor = 2 } = publishOptions || {};
+  const { width = 0, height = 0 } = settings;
+  const { scalabilityMode, bitrateDownscaleFactor = 2 } = publishOptions || {};
   const maxBitrate = getComputedMaxBitrate(
     targetResolution,
-    w,
-    h,
-    preferredBitrate,
+    width,
+    height,
+    codecInUse,
+    publishOptions,
   );
   let downscaleFactor = 1;
   let bitrateFactor = 1;
-  ['f', 'h', 'q'].forEach((rid) => {
-    // Reversing the order [f, h, q] to [q, h, f] as Chrome uses encoding index
-    // when deciding which layer to disable when CPU or bandwidth is constrained.
-    // Encodings should be ordered in increasing spatial resolution order.
-    optimalVideoLayers.unshift({
+  const svcCodec = isSvcCodec(codecInUse);
+  for (const rid of ['f', 'h', 'q']) {
+    const layer: OptimalVideoLayer = {
       active: true,
       rid,
-      width: Math.round(w / downscaleFactor),
-      height: Math.round(h / downscaleFactor),
+      width: Math.round(width / downscaleFactor),
+      height: Math.round(height / downscaleFactor),
       maxBitrate:
         Math.round(maxBitrate / bitrateFactor) || defaultBitratePerRid[rid],
-      scaleResolutionDownBy: downscaleFactor,
       maxFramerate: 30,
-    });
+    };
+    if (svcCodec) {
+      // for SVC codecs, we need to set the scalability mode, and the
+      // codec will handle the rest (layers, temporal layers, etc.)
+      layer.scalabilityMode = scalabilityMode || 'L3T2_KEY';
+    } else {
+      // for non-SVC codecs, we need to downscale proportionally (simulcast)
+      layer.scaleResolutionDownBy = downscaleFactor;
+    }
+
     downscaleFactor *= 2;
     bitrateFactor *= bitrateDownscaleFactor;
-  });
+
+    // Reversing the order [f, h, q] to [q, h, f] as Chrome uses encoding index
+    // when deciding which layer to disable when CPU or bandwidth is constrained.
+    // Encodings should be ordered in increasing spatial resolution order.
+    optimalVideoLayers.unshift(layer);
+  }
 
   // for simplicity, we start with all layers enabled, then this function
   // will clear/reassign the layers that are not needed
@@ -77,13 +119,15 @@ export const findOptimalVideoLayers = (
  * @param targetResolution the target resolution.
  * @param currentWidth the current width of the track.
  * @param currentHeight the current height of the track.
- * @param preferredBitrate the preferred bitrate for the track.
+ * @param codecInUse the codec in use.
+ * @param publishOptions the publish options.
  */
 export const getComputedMaxBitrate = (
   targetResolution: TargetResolutionResponse,
   currentWidth: number,
   currentHeight: number,
-  preferredBitrate: number | undefined,
+  codecInUse?: PreferredCodec,
+  publishOptions?: PublishOptions,
 ): number => {
   // if the current resolution is lower than the target resolution,
   // we want to proportionally reduce the target bitrate
@@ -92,7 +136,12 @@ export const getComputedMaxBitrate = (
     height: targetHeight,
     bitrate: targetBitrate,
   } = targetResolution;
-  const bitrate = preferredBitrate || targetBitrate;
+  const { preferredBitrate } = publishOptions || {};
+  const frameHeight =
+    currentWidth > currentHeight ? currentHeight : currentWidth;
+  const bitrate =
+    preferredBitrate ||
+    (codecInUse ? getOptimalBitrate(codecInUse, frameHeight) : targetBitrate);
   if (currentWidth < targetWidth || currentHeight < targetHeight) {
     const currentPixels = currentWidth * currentHeight;
     const targetPixels = targetWidth * targetHeight;