livekit-client 2.22.2 → 2.22.3

package/src/room/participant/publishUtils.ts

@@ -13,12 +13,12 @@ import type {
 import { ScreenSharePresets, VideoPreset, VideoPresets, VideoPresets43 } from '../track/options';
 import type { LoggerOptions } from '../types';
 import {
-  compareVersions,
   getReactNativeOs,
   isReactNative,
   isSVCCodec,
-  isSafariBased,
+  isSVCSimulcast,
   isSafariSvcApi,
+  usesLegacySVCEncodings,
 } from '../utils';
 
 /** @internal */
@@ -111,6 +111,8 @@ export function computeVideoEncodings(
   const useSimulcast = options?.simulcast;
   const scalabilityMode = options?.scalabilityMode;
   const videoCodec = options?.videoCodec;
+  // VP9/AV1 published as rid based simulcast rather than SVC, see isSVCSimulcast
+  const useSVCSimulcast = isSVCSimulcast(videoCodec, options);
 
   if ((!videoEncoding && !useSimulcast && !scalabilityMode) || !width || !height) {
     // when we aren't simulcasting or svc, will need to return a single encoding without
@@ -134,7 +136,7 @@ export function computeVideoEncodings(
     videoEncoding.priority,
   );
 
-  if (scalabilityMode && isSVCCodec(videoCodec)) {
+  if (scalabilityMode && isSVCCodec(videoCodec) && !useSVCSimulcast) {
     const sm = new ScalabilityMode(scalabilityMode);
 
     const encodings: RTCRtpEncodingParameters[] = [];
@@ -142,20 +144,10 @@ export function computeVideoEncodings(
     if (sm.spatial > 3) {
       throw new Error(`unsupported scalabilityMode: ${scalabilityMode}`);
     }
-    // Before M113 in Chrome, defining multiple encodings with an SVC codec indicated
-    // that SVC mode should be used. Safari still works this way.
-    // This is a bit confusing but is due to how libwebrtc interpreted the encodings field
-    // before M113.
-    // Announced here: https://groups.google.com/g/discuss-webrtc/c/-QQ3pxrl-fw?pli=1
+    // Browsers that read multiple encodings on an SVC codec as SVC rather than as
+    // simulcast, see usesLegacySVCEncodings
     const browser = getBrowser();
-    if (
-      isSafariBased() ||
-      // Even tho RN runs M114, it does not produce SVC layers when a single encoding
-      // is provided. So we'll use the legacy SVC specification for now.
-      // TODO: when we upstream libwebrtc, this will need additional verification
-      isReactNative() ||
-      (browser?.name === 'Chrome' && compareVersions(browser?.version, '113') < 0)
-    ) {
+    if (usesLegacySVCEncodings()) {
       const bitratesRatio = sm.suffix == 'h' ? 2 : 3;
       // safari 18.4 uses a different svc API that requires scaleResolutionDownBy to be set.
       const requireScale = isSafariSvcApi(browser);
@@ -193,6 +185,19 @@ export function computeVideoEncodings(
     return [videoEncoding];
   }
 
+  // Chrome M113+ only treats multiple encodings on an SVC capable codec as real
+  // simulcast when every encoding carries an explicit scalabilityMode next to its
+  // scaleResolutionDownBy. Without it the encodings are interpreted as legacy SVC.
+  const applySVCSimulcastMode = (encodings: RTCRtpEncodingParameters[]) => {
+    if (useSVCSimulcast) {
+      encodings.forEach((encoding) => {
+        /* @ts-ignore */
+        encoding.scalabilityMode = scalabilityMode;
+      });
+    }
+    return encodings;
+  };
+
   let presets: Array<VideoPreset>;
   if (isScreenShare) {
     presets =
@@ -220,13 +225,43 @@ export function computeVideoEncodings(
     //      based on other conditions.
     const size = Math.max(width, height);
     if (size >= 960 && midPreset) {
-      return encodingsFromPresets(width, height, [lowPreset, midPreset, original], sourceFramerate);
+      return applySVCSimulcastMode(
+        encodingsFromPresets(width, height, [lowPreset, midPreset, original], sourceFramerate),
+      );
     }
     if (size >= 480) {
-      return encodingsFromPresets(width, height, [lowPreset, original], sourceFramerate);
+      return applySVCSimulcastMode(
+        encodingsFromPresets(width, height, [lowPreset, original], sourceFramerate),
+      );
     }
   }
-  return encodingsFromPresets(width, height, [original]);
+  return applySVCSimulcastMode(encodingsFromPresets(width, height, [original]));
+}
+
+/**
+ * Bitrate to hint to the bandwidth estimator through `x-google-start-bitrate`, so that
+ * a publish does not spend its first seconds ramping up from a very low rate.
+ *
+ * It has to be the total the encoder will put on the wire, which means picking the
+ * encoding that carries the inclusive bitrate:
+ *  - SVC publishes a single stream with the layers built in. `encodings[0]` holds the
+ *    full bitrate — the legacy SVC shape orders its encodings `f`..`q`, so that holds
+ *    for both SVC shapes.
+ *  - Simulcast publishes independent streams ordered `q`..`f`, so the total is the sum.
+ *    This includes VP9/AV1 published as rid based simulcast, where `encodings[0]` is
+ *    the *smallest* layer even though the codec is SVC capable.
+ *
+ * @internal
+ */
+export function computeStartTargetBitrate(
+  codec: string,
+  options: TrackPublishOptions | undefined,
+  encodings: RTCRtpEncodingParameters[],
+): number {
+  if (isSVCCodec(codec) && !isSVCSimulcast(codec, options)) {
+    return encodings[0]?.maxBitrate ?? 0;
+  }
+  return encodings.reduce((sum, enc) => sum + (enc.maxBitrate ?? 0), 0);
 }
 
 export function computeTrackBackupEncodings(

package/src/room/track/LocalVideoTrack.ts

@@ -16,7 +16,7 @@ import {
 import type { VideoSenderStats } from '../stats';
 import { computeBitrate, monitorFrequency } from '../stats';
 import type { LoggerOptions } from '../types';
-import { isFireFox, isMobile, isSVCCodec, isWeb } from '../utils';
+import { isFireFox, isMobile, isSVCCodec, isSVCSimulcast, isWeb } from '../utils';
 import LocalTrack from './LocalTrack';
 import { Track, VideoQuality } from './Track';
 import type { TrackPublishOptions, VideoCaptureOptions, VideoCodec } from './options';
@@ -243,6 +243,16 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
     return items;
   }
 
+  /**
+   * Whether `codec` is being published as SVC (a single stream carrying all spatial
+   * layers) as opposed to rid based simulcast. VP9/AV1 are SVC unless the publisher
+   * opted into simulcast, in which case each rid is an independent stream and the
+   * layers can be enabled/disabled individually.
+   */
+  private isSvcPublish(codec?: string): boolean {
+    return isSVCCodec(codec) && !isSVCSimulcast(codec, this.publishOptions);
+  }
+
   setPublishingQuality(maxQuality: VideoQuality) {
     const qualities: SubscribedQuality[] = [];
     for (let q = VideoQuality.LOW; q <= VideoQuality.HIGH; q += 1) {
@@ -254,7 +264,7 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
       );
     }
     this.log.debug(`setting publishing quality. max quality ${maxQuality}`, this.logContext);
-    this.setPublishingLayers(isSVCCodec(this.codec), qualities);
+    this.setPublishingLayers(this.isSvcPublish(this.codec), qualities);
   }
 
   async restartTrack(options?: VideoCaptureOptions) {
@@ -491,7 +501,7 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
     });
     // only enable simulcast codec for preference codec setted
     if (!this.codec && codecs.length > 0) {
-      await this.setPublishingLayers(isSVCCodec(codecs[0].codec), codecs[0].qualities);
+      await this.setPublishingLayers(this.isSvcPublish(codecs[0].codec), codecs[0].qualities);
 
       return [];
     }
@@ -501,7 +511,7 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
     const newCodecs: VideoCodec[] = [];
     for await (const codec of codecs) {
       if (!this.codec || this.codec === codec.codec) {
-        await this.setPublishingLayers(isSVCCodec(codec.codec), codec.qualities);
+        await this.setPublishingLayers(this.isSvcPublish(codec.codec), codec.qualities);
       } else {
         const simulcastCodecInfo = this.simulcastCodecs.get(codec.codec as VideoCodec);
         this.log.debug(`try setPublishingCodec for ${codec.codec}`, {
@@ -522,7 +532,7 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
             simulcastCodecInfo.encodings!,
             codec.qualities,
             this.senderLock,
-            isSVCCodec(codec.codec),
+            this.isSvcPublish(codec.codec),
             this.log,
             this.logContext,
           );

package/src/room/track/options.ts

@@ -79,7 +79,7 @@ export interface TrackPublishDefaults {
 
   /**
    * scalability mode for svc codecs, defaults to 'L3T3_KEY'.
-   * for svc codecs, simulcast is disabled.
+   * for svc codecs, simulcast is disabled if more than one spatial layer is used ('L2Tx' or 'L3Tx').
    */
   scalabilityMode?: ScalabilityMode;
 

package/src/room/utils.test.ts

@@ -4,10 +4,13 @@ import {
   ddExtensionURI,
   extractMaxAgeFromRequestHeaders,
   getClientInfo,
+  isSVCSimulcast,
+  isSVCSimulcastSupportedByServer,
   negotiateDependencyDescriptor,
   splitUtf8,
   supportsAdaptiveStream,
   toWebsocketUrl,
+  usesLegacySVCEncodings,
 } from './utils';
 
 describe('toWebsocketUrl', () => {
@@ -280,3 +283,87 @@ describe('supportsAdaptiveStream', () => {
     expect(supportsAdaptiveStream()).toBe(false);
   });
 });
+
+describe('isSVCSimulcast', () => {
+  it('requires an svc capable codec, simulcast and a single spatial layer mode', () => {
+    expect(isSVCSimulcast('vp9', { simulcast: true, scalabilityMode: 'L1T2' })).toBe(true);
+    expect(isSVCSimulcast('av1', { simulcast: true, scalabilityMode: 'L1T3' })).toBe(true);
+  });
+
+  it('stays on svc without the opt in', () => {
+    expect(isSVCSimulcast('vp9', { simulcast: false, scalabilityMode: 'L1T2' })).toBe(false);
+    expect(isSVCSimulcast('vp9', { simulcast: true })).toBe(false);
+    // a multi spatial layer mode is svc by definition
+    expect(isSVCSimulcast('vp9', { simulcast: true, scalabilityMode: 'L3T3_KEY' })).toBe(false);
+    expect(isSVCSimulcast('vp9', undefined)).toBe(false);
+  });
+
+  it('does not apply to non svc codecs', () => {
+    expect(isSVCSimulcast('vp8', { simulcast: true, scalabilityMode: 'L1T2' })).toBe(false);
+    expect(isSVCSimulcast('h264', { simulcast: true, scalabilityMode: 'L1T2' })).toBe(false);
+    expect(isSVCSimulcast(undefined, { simulcast: true, scalabilityMode: 'L1T2' })).toBe(false);
+  });
+});
+
+describe('isSVCSimulcastSupportedByServer', () => {
+  it('requires a server newer than 1.13.6', () => {
+    expect(isSVCSimulcastSupportedByServer('1.13.7')).toBe(true);
+    expect(isSVCSimulcastSupportedByServer('1.14.0')).toBe(true);
+    expect(isSVCSimulcastSupportedByServer('2.0.0')).toBe(true);
+  });
+
+  it('rejects 1.13.6 and older', () => {
+    expect(isSVCSimulcastSupportedByServer('1.13.6')).toBe(false);
+    expect(isSVCSimulcastSupportedByServer('1.13.5')).toBe(false);
+    expect(isSVCSimulcastSupportedByServer('1.9.0')).toBe(false);
+    expect(isSVCSimulcastSupportedByServer('0.15.1')).toBe(false);
+  });
+
+  it('treats an unknown version as unsupported', () => {
+    expect(isSVCSimulcastSupportedByServer(undefined)).toBe(false);
+    expect(isSVCSimulcastSupportedByServer('')).toBe(false);
+  });
+});
+
+describe('usesLegacySVCEncodings', () => {
+  const stubUserAgent = (userAgent: string) =>
+    vi.stubGlobal('navigator', { userAgent, product: 'Gecko' });
+
+  afterEach(() => {
+    vi.unstubAllGlobals();
+  });
+
+  const chrome = (v: string) =>
+    `Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${v} Safari/537.36`;
+
+  it('is legacy on chrome before M113', () => {
+    stubUserAgent(chrome('112.0.0.0'));
+    expect(usesLegacySVCEncodings()).toBe(true);
+  });
+
+  it('is not legacy from chrome M113 onwards', () => {
+    stubUserAgent(chrome('113.0.0.0'));
+    expect(usesLegacySVCEncodings()).toBe(false);
+    stubUserAgent(chrome('120.0.0.0'));
+    expect(usesLegacySVCEncodings()).toBe(false);
+  });
+
+  it('is legacy on safari, which has no rid based svc simulcast', () => {
+    stubUserAgent(
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15',
+    );
+    expect(usesLegacySVCEncodings()).toBe(true);
+  });
+
+  it('is legacy on iOS, where every browser is webkit', () => {
+    stubUserAgent(
+      'Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) CriOS/120.0.0.0 Mobile/15E148 Safari/604.1',
+    );
+    expect(usesLegacySVCEncodings()).toBe(true);
+  });
+
+  it('is legacy on react native regardless of chrome version', () => {
+    vi.stubGlobal('navigator', { userAgent: chrome('120.0.0.0'), product: 'ReactNative' });
+    expect(usesLegacySVCEncodings()).toBe(true);
+  });
+});

package/src/room/utils.ts

@@ -185,6 +185,65 @@ export function negotiateDependencyDescriptor(transceiver: RTCRtpTransceiver): b
   }
 }
 
+/**
+ * VP9 and AV1 are published as SVC (a single RTP stream carrying every spatial layer)
+ * by default. They can instead be published as real, rid based simulcast — one
+ * independent stream per rid, each carrying a single spatial layer — when the caller
+ * opts in with `simulcast: true` and a single spatial layer scalability mode (`L1Tx`).
+ *
+ * The SFU has to be told about this: without an explicit
+ * `SimulcastCodec.videoLayerMode` it assumes `MULTIPLE_SPATIAL_LAYERS_PER_STREAM` for
+ * any SVC capable codec.
+ */
+export function isSVCSimulcast(
+  codec?: string,
+  options?: { simulcast?: boolean; scalabilityMode?: string },
+): boolean {
+  return isSVCCodec(codec) && !!options?.simulcast && !!options.scalabilityMode?.startsWith('L1T');
+}
+
+/**
+ * Whether the browser reads multiple encodings on an SVC capable codec as *legacy SVC*
+ * rather than as real simulcast.
+ *
+ * Before Chrome M113, supplying more than one encoding for VP9/AV1 selected SVC mode;
+ * only from M113 does libwebrtc treat such encodings as simulcast, and only when each
+ * one carries its own scalabilityMode. Safari (and anything WebKit based, i. e. every
+ * browser on iOS) still uses the old interpretation, as does React Native's libwebrtc.
+ * Announced at https://groups.google.com/g/discuss-webrtc/c/-QQ3pxrl-fw
+ *
+ * Where this is true the rids would not exist on the wire, so VP9/AV1 must be published
+ * as SVC no matter what the caller asked for.
+ */
+export function usesLegacySVCEncodings(): boolean {
+  const browser = getBrowser();
+  return (
+    isSafariBased() ||
+    // Even tho RN runs M114, it does not produce SVC layers when a single encoding
+    // is provided. So we'll use the legacy SVC specification for now.
+    // TODO: when we upstream libwebrtc, this will need additional verification
+    isReactNative() ||
+    (browser?.name === 'Chrome' && compareVersions(browser.version, '113') < 0)
+  );
+}
+
+/**
+ * Last server version that doesn't support vp9/av1 simulcast.
+ */
+const svcSimulcastMinServerVersion = '1.13.6';
+
+/**
+ * Whether the connected server honours `SimulcastCodec.videoLayerMode`, i. e. whether
+ * VP9/AV1 can be published as rid based simulcast. An unknown version is treated as
+ * unsupported so the publish falls back to SVC.
+ */
+export function isSVCSimulcastSupportedByServer(serverVersion?: string): boolean {
+  if (!serverVersion) {
+    return false;
+  }
+  return compareVersions(serverVersion, svcSimulcastMinServerVersion) > 0;
+}
+
 export function supportsSetSinkId(elm?: HTMLMediaElement): boolean {
   if (!document || isSafariBased()) {
     return false;