livekit-client 2.13.8 → 2.14.0

package/src/room/PCTransport.ts

@@ -1,3 +1,4 @@
+import { Mutex } from '@livekit/mutex';
 import { EventEmitter } from 'events';
 import type { MediaDescription, SessionDescription } from 'sdp-transform';
 import { parse, write } from 'sdp-transform';
@@ -52,6 +53,8 @@ export default class PCTransport extends EventEmitter {
 
   private latestOfferId: number = 0;
 
+  private offerLock: Mutex;
+
   pendingCandidates: RTCIceCandidateInit[] = [];
 
   restartingIce: boolean = false;
@@ -86,6 +89,7 @@ export default class PCTransport extends EventEmitter {
     this.loggerOptions = loggerOptions;
     this.config = config;
     this._pc = this.createPC();
+    this.offerLock = new Mutex();
   }
 
   private createPC() {
@@ -251,101 +255,108 @@ export default class PCTransport extends EventEmitter {
   }, debounceInterval);
 
   async createAndSendOffer(options?: RTCOfferOptions) {
-    // increase the offer id at the start to ensure the offer is always > 0 so that we can use 0 as a default value for legacy behavior
-    const offerId = this.latestOfferId + 1;
-    this.latestOfferId = offerId;
-    if (this.onOffer === undefined) {
-      return;
-    }
+    const unlock = await this.offerLock.lock();
 
-    if (options?.iceRestart) {
-      this.log.debug('restarting ICE', this.logContext);
-      this.restartingIce = true;
-    }
+    try {
+      // increase the offer id at the start to ensure the offer is always > 0 so that we can use 0 as a default value for legacy behavior
+      const offerId = this.latestOfferId + 1;
+      this.latestOfferId = offerId;
 
-    if (this._pc && this._pc.signalingState === 'have-local-offer') {
-      // we're waiting for the peer to accept our offer, so we'll just wait
-      // the only exception to this is when ICE restart is needed
-      const currentSD = this._pc.remoteDescription;
-      if (options?.iceRestart && currentSD) {
-        // TODO: handle when ICE restart is needed but we don't have a remote description
-        // the best thing to do is to recreate the peerconnection
-        await this._pc.setRemoteDescription(currentSD);
-      } else {
-        this.renegotiate = true;
+      if (this.onOffer === undefined) {
         return;
       }
-    } else if (!this._pc || this._pc.signalingState === 'closed') {
-      this.log.warn('could not createOffer with closed peer connection', this.logContext);
-      return;
-    }
 
-    // actually negotiate
-    this.log.debug('starting to negotiate', this.logContext);
-    const offer = await this.pc.createOffer(options);
-    this.log.debug('original offer', { sdp: offer.sdp, ...this.logContext });
+      if (options?.iceRestart) {
+        this.log.debug('restarting ICE', this.logContext);
+        this.restartingIce = true;
+      }
 
-    const sdpParsed = parse(offer.sdp ?? '');
-    sdpParsed.media.forEach((media) => {
-      ensureIPAddrMatchVersion(media);
-      if (media.type === 'audio') {
-        ensureAudioNackAndStereo(media, [], []);
-      } else if (media.type === 'video') {
-        this.trackBitrates.some((trackbr): boolean => {
-          if (!media.msid || !trackbr.cid || !media.msid.includes(trackbr.cid)) {
-            return false;
-          }
+      if (this._pc && this._pc.signalingState === 'have-local-offer') {
+        // we're waiting for the peer to accept our offer, so we'll just wait
+        // the only exception to this is when ICE restart is needed
+        const currentSD = this._pc.remoteDescription;
+        if (options?.iceRestart && currentSD) {
+          // TODO: handle when ICE restart is needed but we don't have a remote description
+          // the best thing to do is to recreate the peerconnection
+          await this._pc.setRemoteDescription(currentSD);
+        } else {
+          this.renegotiate = true;
+          return;
+        }
+      } else if (!this._pc || this._pc.signalingState === 'closed') {
+        this.log.warn('could not createOffer with closed peer connection', this.logContext);
+        return;
+      }
 
-          let codecPayload = 0;
-          media.rtp.some((rtp): boolean => {
-            if (rtp.codec.toUpperCase() === trackbr.codec.toUpperCase()) {
-              codecPayload = rtp.payload;
-              return true;
+      // actually negotiate
+      this.log.debug('starting to negotiate', this.logContext);
+      const offer = await this.pc.createOffer(options);
+      this.log.debug('original offer', { sdp: offer.sdp, ...this.logContext });
+
+      const sdpParsed = parse(offer.sdp ?? '');
+      sdpParsed.media.forEach((media) => {
+        ensureIPAddrMatchVersion(media);
+        if (media.type === 'audio') {
+          ensureAudioNackAndStereo(media, [], []);
+        } else if (media.type === 'video') {
+          this.trackBitrates.some((trackbr): boolean => {
+            if (!media.msid || !trackbr.cid || !media.msid.includes(trackbr.cid)) {
+              return false;
             }
-            return false;
-          });
 
-          if (codecPayload === 0) {
-            return true;
-          }
+            let codecPayload = 0;
+            media.rtp.some((rtp): boolean => {
+              if (rtp.codec.toUpperCase() === trackbr.codec.toUpperCase()) {
+                codecPayload = rtp.payload;
+                return true;
+              }
+              return false;
+            });
 
-          if (isSVCCodec(trackbr.codec)) {
-            this.ensureVideoDDExtensionForSVC(media, sdpParsed);
-          }
+            if (codecPayload === 0) {
+              return true;
+            }
 
-          // TODO: av1 slow starting issue already fixed in chrome 124, clean this after some versions
-          // mung sdp for av1 bitrate setting that can't apply by sendEncoding
-          if (trackbr.codec !== 'av1') {
-            return true;
-          }
+            if (isSVCCodec(trackbr.codec)) {
+              this.ensureVideoDDExtensionForSVC(media, sdpParsed);
+            }
+
+            // TODO: av1 slow starting issue already fixed in chrome 124, clean this after some versions
+            // mung sdp for av1 bitrate setting that can't apply by sendEncoding
+            if (trackbr.codec !== 'av1') {
+              return true;
+            }
 
-          const startBitrate = Math.round(trackbr.maxbr * startBitrateForSVC);
+            const startBitrate = Math.round(trackbr.maxbr * startBitrateForSVC);
 
-          for (const fmtp of media.fmtp) {
-            if (fmtp.payload === codecPayload) {
-              // if another track's fmtp already is set, we cannot override the bitrate
-              // this has the unfortunate consequence of being forced to use the
-              // initial track's bitrate for all tracks
-              if (!fmtp.config.includes('x-google-start-bitrate')) {
-                fmtp.config += `;x-google-start-bitrate=${startBitrate}`;
+            for (const fmtp of media.fmtp) {
+              if (fmtp.payload === codecPayload) {
+                // if another track's fmtp already is set, we cannot override the bitrate
+                // this has the unfortunate consequence of being forced to use the
+                // initial track's bitrate for all tracks
+                if (!fmtp.config.includes('x-google-start-bitrate')) {
+                  fmtp.config += `;x-google-start-bitrate=${startBitrate}`;
+                }
+                break;
               }
-              break;
             }
-          }
-          return true;
+            return true;
+          });
+        }
+      });
+      if (this.latestOfferId > offerId) {
+        this.log.warn('latestOfferId mismatch', {
+          ...this.logContext,
+          latestOfferId: this.latestOfferId,
+          offerId,
         });
+        return;
       }
-    });
-    if (this.latestOfferId > offerId) {
-      this.log.warn('latestOfferId mismatch', {
-        ...this.logContext,
-        latestOfferId: this.latestOfferId,
-        offerId,
-      });
-      return;
+      await this.setMungedSDP(offer, write(sdpParsed));
+      this.onOffer(offer, this.latestOfferId);
+    } finally {
+      unlock();
     }
-    await this.setMungedSDP(offer, write(sdpParsed));
-    this.onOffer(offer, this.latestOfferId);
   }
 
   async createAndSetAnswer(): Promise<RTCSessionDescriptionInit> {

package/src/room/events.ts

@@ -429,6 +429,21 @@ export enum ParticipantEvent {
    */
   LocalTrackUnpublished = 'localTrackUnpublished',
 
+  /**
+   * A local track has been constrained by cpu.
+   * This event is useful to know when to reduce the capture resolution of the track.
+   *
+   * This event is emitted on the local participant.
+   *
+   * args: ([[LocalVideoTrack]], [[LocalTrackPublication]])
+   */
+  LocalTrackCpuConstrained = 'localTrackCpuConstrained',
+
+  /**
+   * @internal
+   */
+  LocalSenderCreated = 'localSenderCreated',
+
   /**
    * Participant metadata is a simple way for app-specific state to be pushed to
    * all users.
@@ -515,6 +530,11 @@ export enum ParticipantEvent {
    */
   TrackSubscriptionStatusChanged = 'trackSubscriptionStatusChanged',
 
+  /**
+   * a local track has been constrained by cpu
+   */
+  TrackCpuConstrained = 'trackCpuConstrained',
+
   // fired only on LocalParticipant
   /** @internal */
   MediaDevicesError = 'mediaDevicesError',
@@ -599,6 +619,7 @@ export enum TrackEvent {
   Ended = 'ended',
   Subscribed = 'subscribed',
   Unsubscribed = 'unsubscribed',
+  CpuConstrained = 'cpuConstrained',
   /** @internal */
   UpdateSettings = 'updateSettings',
   /** @internal */

package/src/room/participant/LocalParticipant.ts

@@ -93,7 +93,7 @@ import {
   isLocalTrack,
   isLocalVideoTrack,
   isSVCCodec,
-  isSafari17,
+  isSafari17Based,
   isVideoTrack,
   isWeb,
   numberToBigInt,
@@ -708,7 +708,7 @@ export default class LocalParticipant extends Participant {
       throw new DeviceUnsupportedError('getDisplayMedia not supported');
     }
 
-    if (options.resolution === undefined && !isSafari17()) {
+    if (options.resolution === undefined && !isSafari17Based()) {
       // we need to constrain the dimensions, otherwise it could lead to low bitrate
       // due to encoding a huge video. Encoding such large surfaces is really expensive
       // unfortunately Safari 17 has a but and cannot be constrained by default
@@ -1169,6 +1169,7 @@ export default class LocalParticipant extends Participant {
       }
 
       track.sender = await this.engine.createSender(track, opts, encodings);
+      this.emit(ParticipantEvent.LocalSenderCreated, track.sender, track);
 
       if (isLocalVideoTrack(track)) {
         opts.degradationPreference ??= getDefaultDegradationPreference(track);
@@ -1271,6 +1272,9 @@ export default class LocalParticipant extends Participant {
       loggerName: this.roomOptions.loggerName,
       loggerContextCb: () => this.logContext,
     });
+    publication.on(TrackEvent.CpuConstrained, (constrainedTrack) =>
+      this.onTrackCpuConstrained(constrainedTrack, publication),
+    );
     // save options for when it needs to be republished again
     publication.options = opts;
     track.sid = ti.sid;
@@ -2329,6 +2333,14 @@ export default class LocalParticipant extends Participant {
     this.engine.client.sendUpdateLocalAudioTrack(pub.trackSid, pub.getTrackFeatures());
   };
 
+  private onTrackCpuConstrained = (track: LocalVideoTrack, publication: LocalTrackPublication) => {
+    this.log.debug('track cpu constrained', {
+      ...this.logContext,
+      ...getLogContextFromTrack(publication),
+    });
+    this.emit(ParticipantEvent.LocalTrackCpuConstrained, track, publication);
+  };
+
   private handleSubscribedQualityUpdate = async (update: SubscribedQualityUpdate) => {
     if (!this.roomOptions?.dynacast) {
       return;

package/src/room/participant/Participant.ts

@@ -13,6 +13,7 @@ import type TypedEmitter from 'typed-emitter';
 import log, { LoggerNames, type StructuredLogger, getLogger } from '../../logger';
 import { ParticipantEvent, TrackEvent } from '../events';
 import type LocalTrackPublication from '../track/LocalTrackPublication';
+import type LocalVideoTrack from '../track/LocalVideoTrack';
 import type RemoteTrack from '../track/RemoteTrack';
 import type RemoteTrackPublication from '../track/RemoteTrackPublication';
 import { Track } from '../track/Track';
@@ -403,6 +404,8 @@ export type ParticipantEventCallbacks = {
   trackUnmuted: (publication: TrackPublication) => void;
   localTrackPublished: (publication: LocalTrackPublication) => void;
   localTrackUnpublished: (publication: LocalTrackPublication) => void;
+  localTrackCpuConstrained: (track: LocalVideoTrack, publication: LocalTrackPublication) => void;
+  localSenderCreated: (sender: RTCRtpSender, track: Track) => void;
   participantMetadataChanged: (prevMetadata: string | undefined, participant?: any) => void;
   participantNameChanged: (name: string) => void;
   dataReceived: (payload: Uint8Array, kind: DataPacket_Kind) => void;

package/src/room/participant/publishUtils.ts

@@ -18,7 +18,7 @@ import {
   isFireFox,
   isReactNative,
   isSVCCodec,
-  isSafari,
+  isSafariBased,
   isSafariSvcApi,
   unwrapConstraint,
 } from '../utils';
@@ -151,7 +151,7 @@ export function computeVideoEncodings(
     // Announced here: https://groups.google.com/g/discuss-webrtc/c/-QQ3pxrl-fw?pli=1
     const browser = getBrowser();
     if (
-      isSafari() ||
+      isSafariBased() ||
       // Even tho RN runs M114, it does not produce SVC layers when a single encoding
       // is provided. So we'll use the legacy SVC specification for now.
       // TODO: when we upstream libwebrtc, this will need additional verification

package/src/room/track/LocalTrackPublication.ts

@@ -1,7 +1,7 @@
 import { AudioTrackFeature, TrackInfo } from '@livekit/protocol';
 import { TrackEvent } from '../events';
 import type { LoggerOptions } from '../types';
-import { isAudioTrack } from '../utils';
+import { isAudioTrack, isVideoTrack } from '../utils';
 import LocalAudioTrack from './LocalAudioTrack';
 import type LocalTrack from './LocalTrack';
 import type LocalVideoTrack from './LocalVideoTrack';
@@ -28,12 +28,14 @@ export default class LocalTrackPublication extends TrackPublication {
   setTrack(track?: Track) {
     if (this.track) {
       this.track.off(TrackEvent.Ended, this.handleTrackEnded);
+      this.track.off(TrackEvent.CpuConstrained, this.handleCpuConstrained);
     }
 
     super.setTrack(track);
 
     if (track) {
       track.on(TrackEvent.Ended, this.handleTrackEnded);
+      track.on(TrackEvent.CpuConstrained, this.handleCpuConstrained);
     }
   }
 
@@ -116,4 +118,10 @@ export default class LocalTrackPublication extends TrackPublication {
   handleTrackEnded = () => {
     this.emit(TrackEvent.Ended);
   };
+
+  private handleCpuConstrained = () => {
+    if (this.track && isVideoTrack(this.track)) {
+      this.emit(TrackEvent.CpuConstrained, this.track);
+    }
+  };
 }

package/src/room/track/LocalVideoTrack.ts

@@ -7,6 +7,7 @@ import {
 } from '@livekit/protocol';
 import type { SignalClient } from '../../api/SignalClient';
 import type { StructuredLogger } from '../../logger';
+import { TrackEvent } from '../events';
 import { ScalabilityMode } from '../participant/publishUtils';
 import type { VideoSenderStats } from '../stats';
 import { computeBitrate, monitorFrequency } from '../stats';
@@ -56,6 +57,10 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
 
   private degradationPreference: RTCDegradationPreference = 'balanced';
 
+  private isCpuConstrained: boolean = false;
+
+  private optimizeForPerformance: boolean = false;
+
   get sender(): RTCRtpSender | undefined {
     return this._sender;
   }
@@ -251,6 +256,9 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
     }
     await this.restart(constraints);
 
+    // reset cpu constrained state after track is restarted
+    this.isCpuConstrained = false;
+
     for await (const sc of this.simulcastCodecs.values()) {
       if (sc.sender && sc.sender.transport?.state !== 'closed') {
         sc.mediaStreamTrack = this.mediaStreamTrack.clone();
@@ -334,6 +342,7 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
     // only enable simulcast codec for preference codec setted
     if (!this.codec && codecs.length > 0) {
       await this.setPublishingLayers(isSVCCodec(codecs[0].codec), codecs[0].qualities);
+
       return [];
     }
 
@@ -378,6 +387,13 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
    * Sets layers that should be publishing
    */
   async setPublishingLayers(isSvc: boolean, qualities: SubscribedQuality[]) {
+    if (this.optimizeForPerformance) {
+      this.log.info('skipping setPublishingLayers due to optimized publishing performance', {
+        ...this.logContext,
+        qualities,
+      });
+      return;
+    }
     this.log.debug('setting publishing layers', { ...this.logContext, qualities });
     if (!this.sender || !this.encodings) {
       return;
@@ -394,6 +410,49 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
     );
   }
 
+  /**
+   * Designed for lower powered devices, reduces video publishing quality and disables simulcast.
+   * @experimental
+   */
+  async prioritizePerformance() {
+    if (!this.sender) {
+      throw new Error('sender not found');
+    }
+
+    const unlock = await this.senderLock.lock();
+
+    try {
+      this.optimizeForPerformance = true;
+      const params = this.sender.getParameters();
+
+      params.encodings = params.encodings.map((e, idx) => ({
+        ...e,
+        active: idx === 0,
+        scaleResolutionDownBy: Math.max(
+          1,
+          Math.ceil((this.mediaStreamTrack.getSettings().height ?? 360) / 360),
+        ),
+        scalabilityMode: idx === 0 && isSVCCodec(this.codec) ? 'L1T3' : undefined,
+        maxFramerate: idx === 0 ? 15 : 0,
+        maxBitrate: idx === 0 ? e.maxBitrate : 0,
+      }));
+      this.log.debug('setting performance optimised encodings', {
+        ...this.logContext,
+        encodings: params.encodings,
+      });
+      this.encodings = params.encodings;
+      await this.sender.setParameters(params);
+    } catch (e) {
+      this.log.error('failed to set performance optimised encodings', {
+        ...this.logContext,
+        error: e,
+      });
+      this.optimizeForPerformance = false;
+    } finally {
+      unlock();
+    }
+  }
+
   protected monitorSender = async () => {
     if (!this.sender) {
       this._currentBitrate = 0;
@@ -404,11 +463,19 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
     try {
       stats = await this.getSenderStats();
     } catch (e) {
-      this.log.error('could not get audio sender stats', { ...this.logContext, error: e });
+      this.log.error('could not get video sender stats', { ...this.logContext, error: e });
       return;
     }
     const statsMap = new Map<string, VideoSenderStats>(stats.map((s) => [s.rid, s]));
 
+    const isCpuConstrained = stats.some((s) => s.qualityLimitationReason === 'cpu');
+    if (isCpuConstrained !== this.isCpuConstrained) {
+      this.isCpuConstrained = isCpuConstrained;
+      if (this.isCpuConstrained) {
+        this.emit(TrackEvent.CpuConstrained);
+      }
+    }
+
     if (this.prevStats) {
       let totalBitrate = 0;
       statsMap.forEach((s, key) => {

package/src/room/track/Track.ts

@@ -529,4 +529,5 @@ export type TrackEventCallbacks = {
   audioTrackFeatureUpdate: (track: any, feature: AudioTrackFeature, enabled: boolean) => void;
   timeSyncUpdate: (update: { timestamp: number; rtpTimestamp: number }) => void;
   preConnectBufferFlushed: (buffer: Uint8Array[]) => void;
+  cpuConstrained: () => void;
 };

package/src/room/track/TrackPublication.ts

@@ -179,4 +179,5 @@ export type PublicationEventCallbacks = {
   subscriptionFailed: (error: SubscriptionError) => void;
   transcriptionReceived: (transcription: TranscriptionSegment[]) => void;
   timeSyncUpdate: (timestamp: number) => void;
+  cpuConstrained: (track: LocalVideoTrack) => void;
 };

package/src/room/track/create.ts

@@ -3,7 +3,7 @@ import { audioDefaults, videoDefaults } from '../defaults';
 import { DeviceUnsupportedError, TrackInvalidError } from '../errors';
 import { mediaTrackToLocalTrack } from '../participant/publishUtils';
 import type { LoggerOptions } from '../types';
-import { isAudioTrack, isSafari17, isVideoTrack, unwrapConstraint } from '../utils';
+import { isAudioTrack, isSafari17Based, isVideoTrack, unwrapConstraint } from '../utils';
 import LocalAudioTrack from './LocalAudioTrack';
 import type LocalTrack from './LocalTrack';
 import LocalVideoTrack from './LocalVideoTrack';
@@ -198,7 +198,7 @@ export async function createLocalScreenTracks(
   if (options === undefined) {
     options = {};
   }
-  if (options.resolution === undefined && !isSafari17()) {
+  if (options.resolution === undefined && !isSafari17Based()) {
     options.resolution = ScreenSharePresets.h1080fps30.resolution;
   }
 

package/src/room/track/options.ts

@@ -173,6 +173,11 @@ export interface VideoCaptureOptions {
    */
   deviceId?: ConstrainDOMString;
 
+  /**
+   * A ConstrainDouble specifying the frame rate or range of frame rates which are acceptable and/or required.
+   */
+  frameRate?: ConstrainDouble;
+
   /**
    * a facing or an array of facings which are acceptable and/or required.
    */

package/src/room/utils.ts

@@ -96,6 +96,14 @@ export function supportsVP9(): boolean {
       // Safari 16 and below does not support VP9
       return false;
     }
+    if (
+      browser?.os === 'iOS' &&
+      browser?.osVersion &&
+      compareVersions(browser.osVersion, '16') < 0
+    ) {
+      // Safari 16 and below on iOS does not support VP9 we need the iOS check to account for other browsers running webkit under the hood
+      return false;
+    }
   }
   const capabilities = RTCRtpSender.getCapabilities('video');
   let hasVP9 = false;
@@ -148,9 +156,12 @@ export function isSafariBased(): boolean {
   return b?.name === 'Safari' || b?.os === 'iOS';
 }
 
-export function isSafari17(): boolean {
+export function isSafari17Based(): boolean {
   const b = getBrowser();
-  return b?.name === 'Safari' && b.version.startsWith('17.');
+  return (
+    (b?.name === 'Safari' && b.version.startsWith('17.')) ||
+    (b?.os === 'iOS' && !!b?.osVersion && compareVersions(b.osVersion, '17') >= 0)
+  );
 }
 
 export function isSafariSvcApi(browser?: BrowserDetails): boolean {
@@ -158,7 +169,12 @@ export function isSafariSvcApi(browser?: BrowserDetails): boolean {
     browser = getBrowser();
   }
   // Safari 18.4 requires legacy svc api and scaleResolutionDown to be set
-  return browser?.name === 'Safari' && compareVersions(browser.version, '18.3') > 0;
+  return (
+    (browser?.name === 'Safari' && compareVersions(browser.version, '18.3') > 0) ||
+    (browser?.os === 'iOS' &&
+      !!browser?.osVersion &&
+      compareVersions(browser.osVersion, '18.3') > 0)
+  );
 }
 
 export function isMobile(): boolean {