livekit-client 2.13.8 → 2.15.0

package/src/room/track/LocalVideoTrack.ts

@@ -7,6 +7,7 @@ import {
 } from '@livekit/protocol';
 import type { SignalClient } from '../../api/SignalClient';
 import type { StructuredLogger } from '../../logger';
+import { TrackEvent } from '../events';
 import { ScalabilityMode } from '../participant/publishUtils';
 import type { VideoSenderStats } from '../stats';
 import { computeBitrate, monitorFrequency } from '../stats';
@@ -56,6 +57,10 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
 
   private degradationPreference: RTCDegradationPreference = 'balanced';
 
+  private isCpuConstrained: boolean = false;
+
+  private optimizeForPerformance: boolean = false;
+
   get sender(): RTCRtpSender | undefined {
     return this._sender;
   }
@@ -251,6 +256,9 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
     }
     await this.restart(constraints);
 
+    // reset cpu constrained state after track is restarted
+    this.isCpuConstrained = false;
+
     for await (const sc of this.simulcastCodecs.values()) {
       if (sc.sender && sc.sender.transport?.state !== 'closed') {
         sc.mediaStreamTrack = this.mediaStreamTrack.clone();
@@ -334,6 +342,7 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
     // only enable simulcast codec for preference codec setted
     if (!this.codec && codecs.length > 0) {
       await this.setPublishingLayers(isSVCCodec(codecs[0].codec), codecs[0].qualities);
+
       return [];
     }
 
@@ -378,6 +387,13 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
    * Sets layers that should be publishing
    */
   async setPublishingLayers(isSvc: boolean, qualities: SubscribedQuality[]) {
+    if (this.optimizeForPerformance) {
+      this.log.info('skipping setPublishingLayers due to optimized publishing performance', {
+        ...this.logContext,
+        qualities,
+      });
+      return;
+    }
     this.log.debug('setting publishing layers', { ...this.logContext, qualities });
     if (!this.sender || !this.encodings) {
       return;
@@ -394,6 +410,49 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
     );
   }
 
+  /**
+   * Designed for lower powered devices, reduces video publishing quality and disables simulcast.
+   * @experimental
+   */
+  async prioritizePerformance() {
+    if (!this.sender) {
+      throw new Error('sender not found');
+    }
+
+    const unlock = await this.senderLock.lock();
+
+    try {
+      this.optimizeForPerformance = true;
+      const params = this.sender.getParameters();
+
+      params.encodings = params.encodings.map((e, idx) => ({
+        ...e,
+        active: idx === 0,
+        scaleResolutionDownBy: Math.max(
+          1,
+          Math.ceil((this.mediaStreamTrack.getSettings().height ?? 360) / 360),
+        ),
+        scalabilityMode: idx === 0 && isSVCCodec(this.codec) ? 'L1T3' : undefined,
+        maxFramerate: idx === 0 ? 15 : 0,
+        maxBitrate: idx === 0 ? e.maxBitrate : 0,
+      }));
+      this.log.debug('setting performance optimised encodings', {
+        ...this.logContext,
+        encodings: params.encodings,
+      });
+      this.encodings = params.encodings;
+      await this.sender.setParameters(params);
+    } catch (e) {
+      this.log.error('failed to set performance optimised encodings', {
+        ...this.logContext,
+        error: e,
+      });
+      this.optimizeForPerformance = false;
+    } finally {
+      unlock();
+    }
+  }
+
   protected monitorSender = async () => {
     if (!this.sender) {
       this._currentBitrate = 0;
@@ -404,11 +463,19 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
     try {
       stats = await this.getSenderStats();
     } catch (e) {
-      this.log.error('could not get audio sender stats', { ...this.logContext, error: e });
+      this.log.error('could not get video sender stats', { ...this.logContext, error: e });
       return;
     }
     const statsMap = new Map<string, VideoSenderStats>(stats.map((s) => [s.rid, s]));
 
+    const isCpuConstrained = stats.some((s) => s.qualityLimitationReason === 'cpu');
+    if (isCpuConstrained !== this.isCpuConstrained) {
+      this.isCpuConstrained = isCpuConstrained;
+      if (this.isCpuConstrained) {
+        this.emit(TrackEvent.CpuConstrained);
+      }
+    }
+
     if (this.prevStats) {
       let totalBitrate = 0;
       statsMap.forEach((s, key) => {

package/src/room/track/RemoteTrackPublication.ts

@@ -11,6 +11,7 @@ import { isRemoteVideoTrack } from '../utils';
 import type RemoteTrack from './RemoteTrack';
 import { Track, VideoQuality } from './Track';
 import { TrackPublication } from './TrackPublication';
+import { areDimensionsSmaller, layerDimensionsFor } from './utils';
 
 export default class RemoteTrackPublication extends TrackPublication {
   track?: RemoteTrack = undefined;
@@ -21,11 +22,15 @@ export default class RemoteTrackPublication extends TrackPublication {
   // keeps track of client's desire to subscribe to a track, also true if autoSubscribe is active
   protected subscribed?: boolean;
 
-  protected disabled: boolean = false;
+  protected requestedDisabled: boolean | undefined = undefined;
 
-  protected currentVideoQuality?: VideoQuality = VideoQuality.HIGH;
+  protected visible: boolean = true;
 
-  protected videoDimensions?: Track.Dimensions;
+  protected videoDimensionsAdaptiveStream?: Track.Dimensions;
+
+  protected requestedVideoDimensions?: Track.Dimensions;
+
+  protected requestedMaxQuality?: VideoQuality;
 
   protected fps?: number;
 
@@ -105,7 +110,11 @@ export default class RemoteTrackPublication extends TrackPublication {
   }
 
   get isEnabled(): boolean {
-    return !this.disabled;
+    return this.requestedDisabled !== undefined
+      ? !this.requestedDisabled
+      : this.isAdaptiveStream
+        ? this.visible
+        : true;
   }
 
   get isLocal() {
@@ -119,10 +128,10 @@ export default class RemoteTrackPublication extends TrackPublication {
    * @param enabled
    */
   setEnabled(enabled: boolean) {
-    if (!this.isManualOperationAllowed() || this.disabled === !enabled) {
+    if (!this.isManualOperationAllowed() || this.requestedDisabled === !enabled) {
       return;
     }
-    this.disabled = !enabled;
+    this.requestedDisabled = !enabled;
 
     this.emitTrackUpdate();
   }
@@ -135,29 +144,36 @@ export default class RemoteTrackPublication extends TrackPublication {
    * optimize for uninterrupted video
    */
   setVideoQuality(quality: VideoQuality) {
-    if (!this.isManualOperationAllowed() || this.currentVideoQuality === quality) {
+    if (!this.isManualOperationAllowed() || this.requestedMaxQuality === quality) {
       return;
     }
-    this.currentVideoQuality = quality;
-    this.videoDimensions = undefined;
+    this.requestedMaxQuality = quality;
+    this.requestedVideoDimensions = undefined;
 
     this.emitTrackUpdate();
   }
 
+  /**
+   * Explicitly set the video dimensions for this track.
+   *
+   * This will take precedence over adaptive stream dimensions.
+   *
+   * @param dimensions The video dimensions to set.
+   */
   setVideoDimensions(dimensions: Track.Dimensions) {
     if (!this.isManualOperationAllowed()) {
       return;
     }
     if (
-      this.videoDimensions?.width === dimensions.width &&
-      this.videoDimensions?.height === dimensions.height
+      this.requestedVideoDimensions?.width === dimensions.width &&
+      this.requestedVideoDimensions?.height === dimensions.height
     ) {
       return;
     }
     if (isRemoteVideoTrack(this.track)) {
-      this.videoDimensions = dimensions;
+      this.requestedVideoDimensions = dimensions;
     }
-    this.currentVideoQuality = undefined;
+    this.requestedMaxQuality = undefined;
 
     this.emitTrackUpdate();
   }
@@ -180,7 +196,7 @@ export default class RemoteTrackPublication extends TrackPublication {
   }
 
   get videoQuality(): VideoQuality | undefined {
-    return this.currentVideoQuality;
+    return this.requestedMaxQuality ?? VideoQuality.HIGH;
   }
 
   /** @internal */
@@ -260,13 +276,6 @@ export default class RemoteTrackPublication extends TrackPublication {
   }
 
   private isManualOperationAllowed(): boolean {
-    if (this.kind === Track.Kind.Video && this.isAdaptiveStream) {
-      this.log.warn(
-        'adaptive stream is enabled, cannot change video track settings',
-        this.logContext,
-      );
-      return false;
-    }
     if (!this.isDesired) {
       this.log.warn('cannot update track settings when not subscribed', this.logContext);
       return false;
@@ -288,7 +297,7 @@ export default class RemoteTrackPublication extends TrackPublication {
       `adaptivestream video visibility ${this.trackSid}, visible=${visible}`,
       this.logContext,
     );
-    this.disabled = !visible;
+    this.visible = visible;
     this.emitTrackUpdate();
   };
 
@@ -297,7 +306,7 @@ export default class RemoteTrackPublication extends TrackPublication {
       `adaptivestream video dimensions ${dimensions.width}x${dimensions.height}`,
       this.logContext,
     );
-    this.videoDimensions = dimensions;
+    this.videoDimensionsAdaptiveStream = dimensions;
     this.emitTrackUpdate();
   };
 
@@ -305,17 +314,67 @@ export default class RemoteTrackPublication extends TrackPublication {
   emitTrackUpdate() {
     const settings: UpdateTrackSettings = new UpdateTrackSettings({
       trackSids: [this.trackSid],
-      disabled: this.disabled,
+      disabled: !this.isEnabled,
       fps: this.fps,
     });
-    if (this.videoDimensions) {
-      settings.width = Math.ceil(this.videoDimensions.width);
-      settings.height = Math.ceil(this.videoDimensions.height);
-    } else if (this.currentVideoQuality !== undefined) {
-      settings.quality = this.currentVideoQuality;
-    } else {
-      // defaults to high quality
-      settings.quality = VideoQuality.HIGH;
+
+    if (this.kind === Track.Kind.Video) {
+      let minDimensions = this.requestedVideoDimensions;
+
+      if (this.videoDimensionsAdaptiveStream !== undefined) {
+        if (minDimensions) {
+          // check whether the adaptive stream dimensions are smaller than the requested dimensions and use smaller one
+          const smallerAdaptive = areDimensionsSmaller(
+            this.videoDimensionsAdaptiveStream,
+            minDimensions,
+          );
+          if (smallerAdaptive) {
+            this.log.debug('using adaptive stream dimensions instead of requested', {
+              ...this.logContext,
+              ...this.videoDimensionsAdaptiveStream,
+            });
+            minDimensions = this.videoDimensionsAdaptiveStream;
+          }
+        } else if (this.requestedMaxQuality !== undefined && this.trackInfo) {
+          // check whether adaptive stream dimensions are smaller than the max quality layer and use smaller one
+          const maxQualityLayer = layerDimensionsFor(this.trackInfo, this.requestedMaxQuality);
+
+          if (
+            maxQualityLayer &&
+            areDimensionsSmaller(this.videoDimensionsAdaptiveStream, maxQualityLayer)
+          ) {
+            this.log.debug('using adaptive stream dimensions instead of max quality layer', {
+              ...this.logContext,
+              ...this.videoDimensionsAdaptiveStream,
+            });
+            minDimensions = this.videoDimensionsAdaptiveStream;
+          }
+        } else {
+          this.log.debug('using adaptive stream dimensions', {
+            ...this.logContext,
+            ...this.videoDimensionsAdaptiveStream,
+          });
+          minDimensions = this.videoDimensionsAdaptiveStream;
+        }
+      }
+
+      if (minDimensions) {
+        settings.width = Math.ceil(minDimensions.width);
+        settings.height = Math.ceil(minDimensions.height);
+      } else if (this.requestedMaxQuality !== undefined) {
+        this.log.debug('using requested max quality', {
+          ...this.logContext,
+          quality: this.requestedMaxQuality,
+        });
+        settings.quality = this.requestedMaxQuality;
+      } else {
+        this.log.debug('using default quality', {
+          ...this.logContext,
+          quality: VideoQuality.HIGH,
+        });
+        // defaults to high quality
+        settings.quality = VideoQuality.HIGH;
+      }
     }
 
     this.emit(TrackEvent.UpdateSettings, settings);

package/src/room/track/Track.ts

@@ -529,4 +529,5 @@ export type TrackEventCallbacks = {
   audioTrackFeatureUpdate: (track: any, feature: AudioTrackFeature, enabled: boolean) => void;
   timeSyncUpdate: (update: { timestamp: number; rtpTimestamp: number }) => void;
   preConnectBufferFlushed: (buffer: Uint8Array[]) => void;
+  cpuConstrained: () => void;
 };

package/src/room/track/TrackPublication.ts

@@ -179,4 +179,5 @@ export type PublicationEventCallbacks = {
   subscriptionFailed: (error: SubscriptionError) => void;
   transcriptionReceived: (transcription: TranscriptionSegment[]) => void;
   timeSyncUpdate: (timestamp: number) => void;
+  cpuConstrained: (track: LocalVideoTrack) => void;
 };

package/src/room/track/create.ts

@@ -3,7 +3,7 @@ import { audioDefaults, videoDefaults } from '../defaults';
 import { DeviceUnsupportedError, TrackInvalidError } from '../errors';
 import { mediaTrackToLocalTrack } from '../participant/publishUtils';
 import type { LoggerOptions } from '../types';
-import { isAudioTrack, isSafari17, isVideoTrack, unwrapConstraint } from '../utils';
+import { isAudioTrack, isSafari17Based, isVideoTrack, unwrapConstraint } from '../utils';
 import LocalAudioTrack from './LocalAudioTrack';
 import type LocalTrack from './LocalTrack';
 import LocalVideoTrack from './LocalVideoTrack';
@@ -198,7 +198,7 @@ export async function createLocalScreenTracks(
   if (options === undefined) {
     options = {};
   }
-  if (options.resolution === undefined && !isSafari17()) {
+  if (options.resolution === undefined && !isSafari17Based()) {
     options.resolution = ScreenSharePresets.h1080fps30.resolution;
   }
 

package/src/room/track/options.ts

@@ -173,6 +173,11 @@ export interface VideoCaptureOptions {
    */
   deviceId?: ConstrainDOMString;
 
+  /**
+   * A ConstrainDouble specifying the frame rate or range of frame rates which are acceptable and/or required.
+   */
+  frameRate?: ConstrainDouble;
+
   /**
    * a facing or an array of facings which are acceptable and/or required.
    */
@@ -387,7 +392,7 @@ export interface AudioPreset {
 
 const backupCodecs = ['vp8', 'h264'] as const;
 
-export const videoCodecs = ['vp8', 'h264', 'vp9', 'av1'] as const;
+export const videoCodecs = ['vp8', 'h264', 'vp9', 'av1', 'h265'] as const;
 
 export type VideoCodec = (typeof videoCodecs)[number];
 

package/src/room/track/utils.ts

@@ -1,4 +1,4 @@
-import { TrackPublishedResponse, TrackSource } from '@livekit/protocol';
+import { TrackInfo, TrackPublishedResponse, TrackSource, VideoQuality } from '@livekit/protocol';
 import type { AudioProcessorOptions, TrackProcessor, VideoProcessorOptions } from '../..';
 import { cloneDeep } from '../../utils/cloneDeep';
 import { isSafari, sleep } from '../utils';
@@ -329,3 +329,14 @@ export function getTrackSourceFromProto(source: TrackSource): Track.Source {
       return Track.Source.Unknown;
   }
 }
+
+export function areDimensionsSmaller(a: Track.Dimensions, b: Track.Dimensions): boolean {
+  return a.width * a.height < b.width * b.height;
+}
+
+export function layerDimensionsFor(
+  trackInfo: TrackInfo,
+  quality: VideoQuality,
+): Track.Dimensions | undefined {
+  return trackInfo.layers?.find((l) => l.quality === quality);
+}

package/src/room/utils.ts

@@ -96,6 +96,14 @@ export function supportsVP9(): boolean {
       // Safari 16 and below does not support VP9
       return false;
     }
+    if (
+      browser?.os === 'iOS' &&
+      browser?.osVersion &&
+      compareVersions(browser.osVersion, '16') < 0
+    ) {
+      // Safari 16 and below on iOS does not support VP9 we need the iOS check to account for other browsers running webkit under the hood
+      return false;
+    }
   }
   const capabilities = RTCRtpSender.getCapabilities('video');
   let hasVP9 = false;
@@ -110,6 +118,24 @@ export function supportsVP9(): boolean {
   return hasVP9;
 }
 
+export function supportsH265(): boolean {
+  if (!('getCapabilities' in RTCRtpSender)) {
+    return false;
+  }
+
+  const capabilities = RTCRtpSender.getCapabilities('video');
+  let hasH265 = false;
+  if (capabilities) {
+    for (const codec of capabilities.codecs) {
+      if (codec.mimeType === 'video/H265') {
+        hasH265 = true;
+        break;
+      }
+    }
+  }
+  return hasH265;
+}
+
 export function isSVCCodec(codec?: string): boolean {
   return codec === 'av1' || codec === 'vp9';
 }
@@ -148,9 +174,12 @@ export function isSafariBased(): boolean {
   return b?.name === 'Safari' || b?.os === 'iOS';
 }
 
-export function isSafari17(): boolean {
+export function isSafari17Based(): boolean {
   const b = getBrowser();
-  return b?.name === 'Safari' && b.version.startsWith('17.');
+  return (
+    (b?.name === 'Safari' && b.version.startsWith('17.')) ||
+    (b?.os === 'iOS' && !!b?.osVersion && compareVersions(b.osVersion, '17') >= 0)
+  );
 }
 
 export function isSafariSvcApi(browser?: BrowserDetails): boolean {
@@ -158,7 +187,12 @@ export function isSafariSvcApi(browser?: BrowserDetails): boolean {
     browser = getBrowser();
   }
   // Safari 18.4 requires legacy svc api and scaleResolutionDown to be set
-  return browser?.name === 'Safari' && compareVersions(browser.version, '18.3') > 0;
+  return (
+    (browser?.name === 'Safari' && compareVersions(browser.version, '18.3') > 0) ||
+    (browser?.os === 'iOS' &&
+      !!browser?.osVersion &&
+      compareVersions(browser.osVersion, '18.3') > 0)
+  );
 }
 
 export function isMobile(): boolean {