livekit-client 2.13.7 → 2.14.0

package/dist/livekit-client.esm.mjs

@@ -11025,6 +11025,19 @@ var ParticipantEvent;
    * args: ([[LocalTrackPublication]])
    */
   ParticipantEvent["LocalTrackUnpublished"] = "localTrackUnpublished";
+  /**
+   * A local track has been constrained by cpu.
+   * This event is useful to know when to reduce the capture resolution of the track.
+   *
+   * This event is emitted on the local participant.
+   *
+   * args: ([[LocalVideoTrack]], [[LocalTrackPublication]])
+   */
+  ParticipantEvent["LocalTrackCpuConstrained"] = "localTrackCpuConstrained";
+  /**
+   * @internal
+   */
+  ParticipantEvent["LocalSenderCreated"] = "localSenderCreated";
   /**
    * Participant metadata is a simple way for app-specific state to be pushed to
    * all users.
@@ -11101,6 +11114,10 @@ var ParticipantEvent;
    *
    */
   ParticipantEvent["TrackSubscriptionStatusChanged"] = "trackSubscriptionStatusChanged";
+  /**
+   * a local track has been constrained by cpu
+   */
+  ParticipantEvent["TrackCpuConstrained"] = "trackCpuConstrained";
   // fired only on LocalParticipant
   /** @internal */
   ParticipantEvent["MediaDevicesError"] = "mediaDevicesError";
@@ -11178,6 +11195,7 @@ var TrackEvent;
   TrackEvent["Ended"] = "ended";
   TrackEvent["Subscribed"] = "subscribed";
   TrackEvent["Unsubscribed"] = "unsubscribed";
+  TrackEvent["CpuConstrained"] = "cpuConstrained";
   /** @internal */
   TrackEvent["UpdateSettings"] = "updateSettings";
   /** @internal */
@@ -11331,7 +11349,7 @@ function getOSVersion(ua) {
   return ua.includes('mac os') ? getMatch(/\(.+?(\d+_\d+(:?_\d+)?)/, ua, 1).replace(/_/g, '.') : undefined;
 }
 
-var version$1 = "2.13.7";
+var version$1 = "2.14.0";
 
 const version = version$1;
 const protocolVersion = 16;
@@ -11902,6 +11920,10 @@ function supportsVP9() {
       // Safari 16 and below does not support VP9
       return false;
     }
+    if ((browser === null || browser === void 0 ? void 0 : browser.os) === 'iOS' && (browser === null || browser === void 0 ? void 0 : browser.osVersion) && compareVersions(browser.osVersion, '16') < 0) {
+      // Safari 16 and below on iOS does not support VP9 we need the iOS check to account for other browsers running webkit under the hood
+      return false;
+    }
   }
   const capabilities = RTCRtpSender.getCapabilities('video');
   let hasVP9 = false;
@@ -11945,16 +11967,16 @@ function isSafariBased() {
   const b = getBrowser();
   return (b === null || b === void 0 ? void 0 : b.name) === 'Safari' || (b === null || b === void 0 ? void 0 : b.os) === 'iOS';
 }
-function isSafari17() {
+function isSafari17Based() {
   const b = getBrowser();
-  return (b === null || b === void 0 ? void 0 : b.name) === 'Safari' && b.version.startsWith('17.');
+  return (b === null || b === void 0 ? void 0 : b.name) === 'Safari' && b.version.startsWith('17.') || (b === null || b === void 0 ? void 0 : b.os) === 'iOS' && !!(b === null || b === void 0 ? void 0 : b.osVersion) && compareVersions(b.osVersion, '17') >= 0;
 }
 function isSafariSvcApi(browser) {
   if (!browser) {
     browser = getBrowser();
   }
   // Safari 18.4 requires legacy svc api and scaleResolutionDown to be set
-  return (browser === null || browser === void 0 ? void 0 : browser.name) === 'Safari' && compareVersions(browser.version, '18.3') > 0;
+  return (browser === null || browser === void 0 ? void 0 : browser.name) === 'Safari' && compareVersions(browser.version, '18.3') > 0 || (browser === null || browser === void 0 ? void 0 : browser.os) === 'iOS' && !!(browser === null || browser === void 0 ? void 0 : browser.osVersion) && compareVersions(browser.osVersion, '18.3') > 0;
 }
 function isMobile() {
   var _a, _b;
@@ -12799,8 +12821,8 @@ class E2EEManager extends eventsExports.EventEmitter {
       });
       this.setParticipantCryptorEnabled(this.room.localParticipant.isE2EEEnabled, this.room.localParticipant.identity);
     });
-    room.localParticipant.on(ParticipantEvent.LocalTrackPublished, publication => __awaiter(this, void 0, void 0, function* () {
-      this.setupE2EESender(publication.track, publication.track.sender);
+    room.localParticipant.on(ParticipantEvent.LocalSenderCreated, (sender, track) => __awaiter(this, void 0, void 0, function* () {
+      this.setupE2EESender(track, sender);
     }));
     keyProvider.on(KeyProviderEvent.SetKey, keyInfo => this.postKey(keyInfo)).on(KeyProviderEvent.RatchetRequest, (participantId, keyIndex) => this.postRatchetRequest(participantId, keyIndex));
   }
@@ -12951,7 +12973,8 @@ class E2EEManager extends eventsExports.EventEmitter {
             writableStream: writable,
             trackId: trackId,
             codec,
-            participantIdentity: participantIdentity
+            participantIdentity: participantIdentity,
+            isReuse: E2EE_FLAG in receiver
           }
         };
         this.worker.postMessage(msg, [readable, writable]);
@@ -12994,7 +13017,8 @@ class E2EEManager extends eventsExports.EventEmitter {
           writableStream: senderStreams.writable,
           codec,
           trackId,
-          participantIdentity: this.room.localParticipant.identity
+          participantIdentity: this.room.localParticipant.identity,
+          isReuse: false
         }
       };
       this.worker.postMessage(msg, [senderStreams.readable, senderStreams.writable]);
@@ -14787,6 +14811,7 @@ class PCTransport extends eventsExports.EventEmitter {
     this.loggerOptions = loggerOptions;
     this.config = config;
     this._pc = this.createPC();
+    this.offerLock = new _();
   }
   createPC() {
     const pc = new RTCPeerConnection(this.config);
@@ -14925,92 +14950,97 @@ class PCTransport extends eventsExports.EventEmitter {
   createAndSendOffer(options) {
     return __awaiter(this, void 0, void 0, function* () {
       var _a;
-      // increase the offer id at the start to ensure the offer is always > 0 so that we can use 0 as a default value for legacy behavior
-      const offerId = this.latestOfferId + 1;
-      this.latestOfferId = offerId;
-      if (this.onOffer === undefined) {
-        return;
-      }
-      if (options === null || options === void 0 ? void 0 : options.iceRestart) {
-        this.log.debug('restarting ICE', this.logContext);
-        this.restartingIce = true;
-      }
-      if (this._pc && this._pc.signalingState === 'have-local-offer') {
-        // we're waiting for the peer to accept our offer, so we'll just wait
-        // the only exception to this is when ICE restart is needed
-        const currentSD = this._pc.remoteDescription;
-        if ((options === null || options === void 0 ? void 0 : options.iceRestart) && currentSD) {
-          // TODO: handle when ICE restart is needed but we don't have a remote description
-          // the best thing to do is to recreate the peerconnection
-          yield this._pc.setRemoteDescription(currentSD);
-        } else {
-          this.renegotiate = true;
+      const unlock = yield this.offerLock.lock();
+      try {
+        // increase the offer id at the start to ensure the offer is always > 0 so that we can use 0 as a default value for legacy behavior
+        const offerId = this.latestOfferId + 1;
+        this.latestOfferId = offerId;
+        if (this.onOffer === undefined) {
           return;
         }
-      } else if (!this._pc || this._pc.signalingState === 'closed') {
-        this.log.warn('could not createOffer with closed peer connection', this.logContext);
-        return;
-      }
-      // actually negotiate
-      this.log.debug('starting to negotiate', this.logContext);
-      const offer = yield this.pc.createOffer(options);
-      this.log.debug('original offer', Object.assign({
-        sdp: offer.sdp
-      }, this.logContext));
-      const sdpParsed = libExports.parse((_a = offer.sdp) !== null && _a !== void 0 ? _a : '');
-      sdpParsed.media.forEach(media => {
-        ensureIPAddrMatchVersion(media);
-        if (media.type === 'audio') {
-          ensureAudioNackAndStereo(media, [], []);
-        } else if (media.type === 'video') {
-          this.trackBitrates.some(trackbr => {
-            if (!media.msid || !trackbr.cid || !media.msid.includes(trackbr.cid)) {
-              return false;
-            }
-            let codecPayload = 0;
-            media.rtp.some(rtp => {
-              if (rtp.codec.toUpperCase() === trackbr.codec.toUpperCase()) {
-                codecPayload = rtp.payload;
+        if (options === null || options === void 0 ? void 0 : options.iceRestart) {
+          this.log.debug('restarting ICE', this.logContext);
+          this.restartingIce = true;
+        }
+        if (this._pc && this._pc.signalingState === 'have-local-offer') {
+          // we're waiting for the peer to accept our offer, so we'll just wait
+          // the only exception to this is when ICE restart is needed
+          const currentSD = this._pc.remoteDescription;
+          if ((options === null || options === void 0 ? void 0 : options.iceRestart) && currentSD) {
+            // TODO: handle when ICE restart is needed but we don't have a remote description
+            // the best thing to do is to recreate the peerconnection
+            yield this._pc.setRemoteDescription(currentSD);
+          } else {
+            this.renegotiate = true;
+            return;
+          }
+        } else if (!this._pc || this._pc.signalingState === 'closed') {
+          this.log.warn('could not createOffer with closed peer connection', this.logContext);
+          return;
+        }
+        // actually negotiate
+        this.log.debug('starting to negotiate', this.logContext);
+        const offer = yield this.pc.createOffer(options);
+        this.log.debug('original offer', Object.assign({
+          sdp: offer.sdp
+        }, this.logContext));
+        const sdpParsed = libExports.parse((_a = offer.sdp) !== null && _a !== void 0 ? _a : '');
+        sdpParsed.media.forEach(media => {
+          ensureIPAddrMatchVersion(media);
+          if (media.type === 'audio') {
+            ensureAudioNackAndStereo(media, [], []);
+          } else if (media.type === 'video') {
+            this.trackBitrates.some(trackbr => {
+              if (!media.msid || !trackbr.cid || !media.msid.includes(trackbr.cid)) {
+                return false;
+              }
+              let codecPayload = 0;
+              media.rtp.some(rtp => {
+                if (rtp.codec.toUpperCase() === trackbr.codec.toUpperCase()) {
+                  codecPayload = rtp.payload;
+                  return true;
+                }
+                return false;
+              });
+              if (codecPayload === 0) {
                 return true;
               }
-              return false;
-            });
-            if (codecPayload === 0) {
-              return true;
-            }
-            if (isSVCCodec(trackbr.codec)) {
-              this.ensureVideoDDExtensionForSVC(media, sdpParsed);
-            }
-            // TODO: av1 slow starting issue already fixed in chrome 124, clean this after some versions
-            // mung sdp for av1 bitrate setting that can't apply by sendEncoding
-            if (trackbr.codec !== 'av1') {
-              return true;
-            }
-            const startBitrate = Math.round(trackbr.maxbr * startBitrateForSVC);
-            for (const fmtp of media.fmtp) {
-              if (fmtp.payload === codecPayload) {
-                // if another track's fmtp already is set, we cannot override the bitrate
-                // this has the unfortunate consequence of being forced to use the
-                // initial track's bitrate for all tracks
-                if (!fmtp.config.includes('x-google-start-bitrate')) {
-                  fmtp.config += ";x-google-start-bitrate=".concat(startBitrate);
+              if (isSVCCodec(trackbr.codec)) {
+                this.ensureVideoDDExtensionForSVC(media, sdpParsed);
+              }
+              // TODO: av1 slow starting issue already fixed in chrome 124, clean this after some versions
+              // mung sdp for av1 bitrate setting that can't apply by sendEncoding
+              if (trackbr.codec !== 'av1') {
+                return true;
+              }
+              const startBitrate = Math.round(trackbr.maxbr * startBitrateForSVC);
+              for (const fmtp of media.fmtp) {
+                if (fmtp.payload === codecPayload) {
+                  // if another track's fmtp already is set, we cannot override the bitrate
+                  // this has the unfortunate consequence of being forced to use the
+                  // initial track's bitrate for all tracks
+                  if (!fmtp.config.includes('x-google-start-bitrate')) {
+                    fmtp.config += ";x-google-start-bitrate=".concat(startBitrate);
+                  }
+                  break;
                 }
-                break;
               }
-            }
-            return true;
-          });
+              return true;
+            });
+          }
+        });
+        if (this.latestOfferId > offerId) {
+          this.log.warn('latestOfferId mismatch', Object.assign(Object.assign({}, this.logContext), {
+            latestOfferId: this.latestOfferId,
+            offerId
+          }));
+          return;
         }
-      });
-      if (this.latestOfferId > offerId) {
-        this.log.warn('latestOfferId mismatch', Object.assign(Object.assign({}, this.logContext), {
-          latestOfferId: this.latestOfferId,
-          offerId
-        }));
-        return;
+        yield this.setMungedSDP(offer, libExports.write(sdpParsed));
+        this.onOffer(offer, this.latestOfferId);
+      } finally {
+        unlock();
       }
-      yield this.setMungedSDP(offer, libExports.write(sdpParsed));
-      this.onOffer(offer, this.latestOfferId);
     });
   }
   createAndSetAnswer() {
@@ -16675,7 +16705,7 @@ function computeVideoEncodings(isScreenShare, width, height, options) {
     // before M113.
     // Announced here: https://groups.google.com/g/discuss-webrtc/c/-QQ3pxrl-fw?pli=1
     const browser = getBrowser();
-    if (isSafari() ||
+    if (isSafariBased() ||
     // Even tho RN runs M114, it does not produce SVC layers when a single encoding
     // is provided. So we'll use the legacy SVC specification for now.
     // TODO: when we upstream libwebrtc, this will need additional verification
@@ -16962,6 +16992,8 @@ class LocalVideoTrack extends LocalTrack {
     /* @internal */
     this.simulcastCodecs = new Map();
     this.degradationPreference = 'balanced';
+    this.isCpuConstrained = false;
+    this.optimizeForPerformance = false;
     this.monitorSender = () => __awaiter(this, void 0, void 0, function* () {
       if (!this.sender) {
         this._currentBitrate = 0;
@@ -16971,12 +17003,19 @@ class LocalVideoTrack extends LocalTrack {
       try {
         stats = yield this.getSenderStats();
       } catch (e) {
-        this.log.error('could not get audio sender stats', Object.assign(Object.assign({}, this.logContext), {
+        this.log.error('could not get video sender stats', Object.assign(Object.assign({}, this.logContext), {
           error: e
         }));
         return;
       }
       const statsMap = new Map(stats.map(s => [s.rid, s]));
+      const isCpuConstrained = stats.some(s => s.qualityLimitationReason === 'cpu');
+      if (isCpuConstrained !== this.isCpuConstrained) {
+        this.isCpuConstrained = isCpuConstrained;
+        if (this.isCpuConstrained) {
+          this.emit(TrackEvent.CpuConstrained);
+        }
+      }
       if (this.prevStats) {
         let totalBitrate = 0;
         statsMap.forEach((s, key) => {
@@ -17212,6 +17251,8 @@ class LocalVideoTrack extends LocalTrack {
         }
       }
       yield this.restart(constraints);
+      // reset cpu constrained state after track is restarted
+      this.isCpuConstrained = false;
       try {
         for (var _e = true, _f = __asyncValues(this.simulcastCodecs.values()), _g; _g = yield _f.next(), _a = _g.done, !_a; _e = true) {
           _c = _g.value;
@@ -17381,6 +17422,12 @@ class LocalVideoTrack extends LocalTrack {
    */
   setPublishingLayers(isSvc, qualities) {
     return __awaiter(this, void 0, void 0, function* () {
+      if (this.optimizeForPerformance) {
+        this.log.info('skipping setPublishingLayers due to optimized publishing performance', Object.assign(Object.assign({}, this.logContext), {
+          qualities
+        }));
+        return;
+      }
       this.log.debug('setting publishing layers', Object.assign(Object.assign({}, this.logContext), {
         qualities
       }));
@@ -17390,6 +17437,44 @@ class LocalVideoTrack extends LocalTrack {
       yield setPublishingLayersForSender(this.sender, this.encodings, qualities, this.senderLock, isSvc, this.log, this.logContext);
     });
   }
+  /**
+   * Designed for lower powered devices, reduces video publishing quality and disables simulcast.
+   * @experimental
+   */
+  prioritizePerformance() {
+    return __awaiter(this, void 0, void 0, function* () {
+      if (!this.sender) {
+        throw new Error('sender not found');
+      }
+      const unlock = yield this.senderLock.lock();
+      try {
+        this.optimizeForPerformance = true;
+        const params = this.sender.getParameters();
+        params.encodings = params.encodings.map((e, idx) => {
+          var _a;
+          return Object.assign(Object.assign({}, e), {
+            active: idx === 0,
+            scaleResolutionDownBy: Math.max(1, Math.ceil(((_a = this.mediaStreamTrack.getSettings().height) !== null && _a !== void 0 ? _a : 360) / 360)),
+            scalabilityMode: idx === 0 && isSVCCodec(this.codec) ? 'L1T3' : undefined,
+            maxFramerate: idx === 0 ? 15 : 0,
+            maxBitrate: idx === 0 ? e.maxBitrate : 0
+          });
+        });
+        this.log.debug('setting performance optimised encodings', Object.assign(Object.assign({}, this.logContext), {
+          encodings: params.encodings
+        }));
+        this.encodings = params.encodings;
+        yield this.sender.setParameters(params);
+      } catch (e) {
+        this.log.error('failed to set performance optimised encodings', Object.assign(Object.assign({}, this.logContext), {
+          error: e
+        }));
+        this.optimizeForPerformance = false;
+      } finally {
+        unlock();
+      }
+    });
+  }
   handleAppVisibilityChanged() {
     const _super = Object.create(null, {
       handleAppVisibilityChanged: {
@@ -19816,16 +19901,23 @@ class LocalTrackPublication extends TrackPublication {
     this.handleTrackEnded = () => {
       this.emit(TrackEvent.Ended);
     };
+    this.handleCpuConstrained = () => {
+      if (this.track && isVideoTrack(this.track)) {
+        this.emit(TrackEvent.CpuConstrained, this.track);
+      }
+    };
     this.updateInfo(ti);
     this.setTrack(track);
   }
   setTrack(track) {
     if (this.track) {
       this.track.off(TrackEvent.Ended, this.handleTrackEnded);
+      this.track.off(TrackEvent.CpuConstrained, this.handleCpuConstrained);
     }
     super.setTrack(track);
     if (track) {
       track.on(TrackEvent.Ended, this.handleTrackEnded);
+      track.on(TrackEvent.CpuConstrained, this.handleCpuConstrained);
     }
   }
   get isMuted() {
@@ -20063,7 +20155,7 @@ function createLocalScreenTracks(options) {
     if (options === undefined) {
       options = {};
     }
-    if (options.resolution === undefined && !isSafari17()) {
+    if (options.resolution === undefined && !isSafari17Based()) {
       options.resolution = ScreenSharePresets.h1080fps30.resolution;
     }
     if (navigator.mediaDevices.getDisplayMedia === undefined) {
@@ -20496,6 +20588,10 @@ class LocalParticipant extends Participant {
       }
       this.engine.client.sendUpdateLocalAudioTrack(pub.trackSid, pub.getTrackFeatures());
     };
+    this.onTrackCpuConstrained = (track, publication) => {
+      this.log.debug('track cpu constrained', Object.assign(Object.assign({}, this.logContext), getLogContextFromTrack(publication)));
+      this.emit(ParticipantEvent.LocalTrackCpuConstrained, track, publication);
+    };
     this.handleSubscribedQualityUpdate = update => __awaiter(this, void 0, void 0, function* () {
       var _a, e_1, _b, _c;
       var _d;
@@ -20955,7 +21051,7 @@ class LocalParticipant extends Participant {
       if (navigator.mediaDevices.getDisplayMedia === undefined) {
         throw new DeviceUnsupportedError('getDisplayMedia not supported');
       }
-      if (options.resolution === undefined && !isSafari17()) {
+      if (options.resolution === undefined && !isSafari17Based()) {
         // we need to constrain the dimensions, otherwise it could lead to low bitrate
         // due to encoding a huge video. Encoding such large surfaces is really expensive
         // unfortunately Safari 17 has a but and cannot be constrained by default
@@ -21322,6 +21418,7 @@ class LocalParticipant extends Participant {
           throw new UnexpectedConnectionState('pcManager is not ready');
         }
         track.sender = yield this.engine.createSender(track, opts, encodings);
+        this.emit(ParticipantEvent.LocalSenderCreated, track.sender, track);
         if (isLocalVideoTrack(track)) {
           (_a = opts.degradationPreference) !== null && _a !== void 0 ? _a : opts.degradationPreference = getDefaultDegradationPreference(track);
           track.setDegradationPreference(opts.degradationPreference);
@@ -21407,6 +21504,7 @@ class LocalParticipant extends Participant {
         loggerName: this.roomOptions.loggerName,
         loggerContextCb: () => this.logContext
       });
+      publication.on(TrackEvent.CpuConstrained, constrainedTrack => this.onTrackCpuConstrained(constrainedTrack, publication));
       // save options for when it needs to be republished again
       publication.options = opts;
       track.sid = ti.sid;