livekit-client 0.16.1 → 0.16.5

package/src/room/participant/LocalParticipant.ts

@@ -416,8 +416,7 @@ export default class LocalParticipant extends Participant {
     // store RTPSender
     track.sender = transceiver.sender;
     if (track instanceof LocalVideoTrack) {
-      const disableLayerPause = this.roomOptions?.expDisableLayerPause ?? false;
-      track.startMonitor(this.engine.client, disableLayerPause);
+      track.startMonitor(this.engine.client);
     } else if (track instanceof LocalAudioTrack) {
       track.startMonitor();
     }

package/src/room/track/LocalAudioTrack.ts

@@ -73,7 +73,14 @@ export default class LocalAudioTrack extends LocalTrack {
       this._currentBitrate = 0;
       return;
     }
-    const stats = await this.getSenderStats();
+
+    let stats: AudioSenderStats | undefined;
+    try {
+      stats = await this.getSenderStats();
+    } catch (e) {
+      log.error('could not get audio sender stats', e);
+      return;
+    }
 
     if (stats && this.prevStats) {
       this._currentBitrate = computeBitrate(stats, this.prevStats);

package/src/room/track/LocalVideoTrack.ts

@@ -9,31 +9,14 @@ import { VideoCaptureOptions } from './options';
 import { Track } from './Track';
 import { constraintsForOptions } from './utils';
 
-// delay before attempting to upgrade
-const QUALITY_UPGRADE_DELAY = 60 * 1000;
-
-// avoid downgrading too quickly
-const QUALITY_DOWNGRADE_DELAY = 5 * 1000;
-
-const ridOrder = ['q', 'h', 'f'];
-
 export default class LocalVideoTrack extends LocalTrack {
   /* internal */
   signalClient?: SignalClient;
 
   private prevStats?: Map<string, VideoSenderStats>;
 
-  // last time it had a change in quality
-  private lastQualityChange?: number;
-
-  // last time we made an explicit change
-  private lastExplicitQualityChange?: number;
-
   private encodings?: RTCRtpEncodingParameters[];
 
-  // layers that are being subscribed to, and that we should publish
-  private activeQualities?: SubscribedQuality[];
-
   constructor(
     mediaTrack: MediaStreamTrack,
     constraints?: MediaTrackConstraints,
@@ -49,7 +32,7 @@ export default class LocalVideoTrack extends LocalTrack {
   }
 
   /* @internal */
-  startMonitor(signalClient: SignalClient, disableLayerPause: boolean) {
+  startMonitor(signalClient: SignalClient) {
     this.signalClient = signalClient;
     // save original encodings
     const params = this.sender?.getParameters();
@@ -58,7 +41,7 @@ export default class LocalVideoTrack extends LocalTrack {
     }
 
     setTimeout(() => {
-      this.monitorSender(disableLayerPause);
+      this.monitorSender();
     }, monitorFrequency);
   }
 
@@ -186,7 +169,6 @@ export default class LocalVideoTrack extends LocalTrack {
       return;
     }
 
-    this.activeQualities = qualities;
     let hasChanged = false;
     encodings.forEach((encoding, idx) => {
       let rid = encoding.rid ?? '';
@@ -227,17 +209,20 @@ export default class LocalVideoTrack extends LocalTrack {
     }
   }
 
-  private monitorSender = async (disableLayerPause: boolean) => {
+  private monitorSender = async () => {
     if (!this.sender) {
       this._currentBitrate = 0;
       return;
     }
-    const stats = await this.getSenderStats();
-    const statsMap = new Map<string, VideoSenderStats>(stats.map((s) => [s.rid, s]));
 
-    if (!disableLayerPause && this.prevStats && this.isSimulcast) {
-      this.checkAndUpdateSimulcast(statsMap);
+    let stats: VideoSenderStats[] | undefined;
+    try {
+      stats = await this.getSenderStats();
+    } catch (e) {
+      log.error('could not get audio sender stats', e);
+      return;
     }
+    const statsMap = new Map<string, VideoSenderStats>(stats.map((s) => [s.rid, s]));
 
     if (this.prevStats) {
       let totalBitrate = 0;
@@ -250,125 +235,9 @@ export default class LocalVideoTrack extends LocalTrack {
 
     this.prevStats = statsMap;
     setTimeout(() => {
-      this.monitorSender(disableLayerPause);
+      this.monitorSender();
     }, monitorFrequency);
   };
-
-  private checkAndUpdateSimulcast(statsMap: Map<string, VideoSenderStats>) {
-    if (!this.sender || this.isMuted || !this.encodings) {
-      return;
-    }
-
-    let bestEncoding: RTCRtpEncodingParameters | undefined;
-    const { encodings } = this.sender.getParameters();
-    encodings.forEach((encoding) => {
-      // skip inactive encodings
-      if (!encoding.active) return;
-
-      if (bestEncoding === undefined) {
-        bestEncoding = encoding;
-      } else if (
-        bestEncoding.rid
-        && encoding.rid
-        && ridOrder.indexOf(bestEncoding.rid) < ridOrder.indexOf(encoding.rid)
-      ) {
-        bestEncoding = encoding;
-      } else if (
-        bestEncoding.maxBitrate !== undefined
-        && encoding.maxBitrate !== undefined
-        && bestEncoding.maxBitrate < encoding.maxBitrate
-      ) {
-        bestEncoding = encoding;
-      }
-    });
-
-    if (!bestEncoding) {
-      return;
-    }
-    const rid: string = bestEncoding.rid ?? '';
-    const sendStats = statsMap.get(rid);
-    const lastStats = this.prevStats?.get(rid);
-    if (!sendStats || !lastStats) {
-      return;
-    }
-    const currentQuality = videoQualityForRid(rid);
-
-    // adaptive simulcast algorithm notes (davidzhao)
-    // Chrome (and other browsers) will automatically pause the highest layer
-    // when it runs into bandwidth limitations. When that happens, it would not
-    // be able to send any new frames between the two stats checks.
-    //
-    // We need to set that layer to inactive intentionally, because chrome tends
-    // to flicker, meaning it will attempt to send that layer again shortly
-    // afterwards, flip-flopping every few seconds. We want to avoid that.
-    //
-    // Note: even after bandwidth recovers, the flip-flopping behavior continues
-    // this is possibly due to SFU-side PLI generation and imperfect bandwidth estimation
-    if (sendStats.qualityLimitationResolutionChanges
-        - lastStats.qualityLimitationResolutionChanges > 0) {
-      this.lastQualityChange = new Date().getTime();
-    }
-
-    // log.debug('frameSent', sendStats.framesSent, 'lastSent', lastStats.framesSent,
-    //   'elapsed', sendStats.timestamp - lastStats.timestamp);
-    if (sendStats.framesSent - lastStats.framesSent > 0) {
-      // frames have been sending ok, consider upgrading quality
-      if (currentQuality === VideoQuality.HIGH || !this.lastQualityChange) return;
-
-      const nextQuality = currentQuality + 1;
-      if ((new Date()).getTime() - this.lastQualityChange < QUALITY_UPGRADE_DELAY) {
-        return;
-      }
-
-      if (this.activeQualities
-        && this.activeQualities.some((q) => q.quality === nextQuality && !q.enabled)
-      ) {
-        // quality has been disabled by the server, so we should skip
-        return;
-      }
-
-      // we are already at the highest layer
-      let bestQuality = VideoQuality.LOW;
-      encodings.forEach((encoding) => {
-        const quality = videoQualityForRid(encoding.rid ?? '');
-        if (quality > bestQuality) {
-          bestQuality = quality;
-        }
-      });
-      if (nextQuality > bestQuality) {
-        return;
-      }
-
-      log.debug('upgrading video quality to', nextQuality);
-      this.setPublishingQuality(nextQuality);
-      return;
-    }
-
-    // if best layer has not sent anything, do not downgrade till the
-    // best layer starts sending something. It is possible that the
-    // browser has not started some layer(s) due to cpu/bandwidth
-    // constraints
-    if (sendStats.framesSent === 0) return;
-
-    // if we've upgraded or downgraded recently, give it a bit of time before
-    // downgrading again
-    if (this.lastExplicitQualityChange
-      && ((new Date()).getTime() - this.lastExplicitQualityChange) < QUALITY_DOWNGRADE_DELAY) {
-      return;
-    }
-
-    if (currentQuality === VideoQuality.UNRECOGNIZED) {
-      return;
-    }
-
-    if (currentQuality === VideoQuality.LOW) {
-      // already the lowest quality, nothing we can do
-      return;
-    }
-
-    log.debug('downgrading video quality to', currentQuality - 1);
-    this.setPublishingQuality(currentQuality - 1);
-  }
 }
 
 export function videoQualityForRid(rid: string): VideoQuality {

package/src/room/track/RemoteVideoTrack.ts

@@ -75,6 +75,11 @@ export default class RemoteVideoTrack extends RemoteTrack {
 
       getIntersectionObserver().observe(element);
       getResizeObserver().observe(element);
+
+      // trigger the first resize update cycle
+      // if the tab is backgrounded, the initial resize event does not fire until
+      // the tab comes into focus for the first time.
+      this.debouncedHandleResize();
     }
     return element;
   }
@@ -190,9 +195,11 @@ export default class RemoteVideoTrack extends RemoteTrack {
     let maxWidth = 0;
     let maxHeight = 0;
     for (const info of this.elementInfos) {
-      if (info.element.clientWidth + info.element.clientHeight > maxWidth + maxHeight) {
-        maxWidth = info.element.clientWidth;
-        maxHeight = info.element.clientHeight;
+      const currentElementWidth = info.element.clientWidth * (window.devicePixelRatio ?? 1);
+      const currentElementHeight = info.element.clientHeight * (window.devicePixelRatio ?? 1);
+      if (currentElementWidth + currentElementHeight > maxWidth + maxHeight) {
+        maxWidth = currentElementWidth;
+        maxHeight = currentElementHeight;
       }
     }
 

package/src/room/track/Track.ts

@@ -2,7 +2,7 @@ import { EventEmitter } from 'events';
 import { TrackSource, TrackType } from '../../proto/livekit_models';
 import { StreamState as ProtoStreamState } from '../../proto/livekit_rtc';
 import { TrackEvent } from '../events';
-import { isFireFox } from '../utils';
+import { isFireFox, isSafari } from '../utils';
 
 // keep old audio elements when detached, we would re-use them since on iOS
 // Safari tracks which audio elements have been "blessed" by the user.
@@ -71,11 +71,6 @@ export class Track extends EventEmitter {
       }
     }
 
-    if (element instanceof HTMLVideoElement) {
-      element.playsInline = true;
-      element.autoplay = true;
-    }
-
     if (!this.attachedElements.includes(element)) {
       this.attachedElements.push(element);
     }
@@ -83,7 +78,6 @@ export class Track extends EventEmitter {
     // even if we believe it's already attached to the element, it's possible
     // the element's srcObject was set to something else out of band.
     // we'll want to re-attach it in that case
-
     attachToElement(this.mediaStreamTrack, element);
 
     if (element instanceof HTMLAudioElement) {
@@ -170,7 +164,6 @@ export function attachToElement(track: MediaStreamTrack, element: HTMLMediaEleme
     mediaStream = element.srcObject;
   } else {
     mediaStream = new MediaStream();
-    element.srcObject = mediaStream;
   }
 
   // check if track matches existing track
@@ -180,22 +173,31 @@ export function attachToElement(track: MediaStreamTrack, element: HTMLMediaEleme
   } else {
     existingTracks = mediaStream.getVideoTracks();
   }
-
-  if (existingTracks.includes(track)) {
-    return;
+  if (!existingTracks.includes(track)) {
+    existingTracks.forEach((et) => {
+      mediaStream.removeTrack(et);
+    });
+    mediaStream.addTrack(track);
   }
 
-  existingTracks.forEach((et) => {
-    mediaStream.removeTrack(et);
-  });
-
-  mediaStream.addTrack(track);
-  if (isFireFox()) {
-    // sometimes firefox doesn't render local video on the first try.
-    // It needs to be re-attached after a timeout.
-    setTimeout(() => {
-      element.srcObject = mediaStream;
-    }, 1);
+  // avoid flicker
+  if (element.srcObject !== mediaStream) {
+    element.srcObject = mediaStream;
+    if ((isSafari() || isFireFox()) && element instanceof HTMLVideoElement) {
+      // Firefox also has a timing issue where video doesn't actually get attached unless
+      // performed out-of-band
+      // Safari 15 has a bug where in certain layouts, video element renders
+      // black until the page is resized or other changes take place.
+      // Resetting the src triggers it to render.
+      // https://developer.apple.com/forums/thread/690523
+      setTimeout(() => {
+        element.srcObject = mediaStream;
+      }, 0);
+    }
+  }
+  element.autoplay = true;
+  if (element instanceof HTMLVideoElement) {
+    element.playsInline = true;
   }
 }
 

package/src/room/utils.ts

@@ -19,6 +19,10 @@ export function isFireFox(): boolean {
   return navigator.userAgent.indexOf('Firefox') !== -1;
 }
 
+export function isSafari(): boolean {
+  return /^((?!chrome|android).)*safari/i.test(navigator.userAgent);
+}
+
 function roDispatchCallback(entries: ResizeObserverEntry[]) {
   for (const entry of entries) {
     (entry.target as ObservableMediaElement).handleResize(entry);

package/src/version.ts

@@ -1,2 +1,2 @@
-export const version = '0.16.1';
+export const version = '0.16.5';
 export const protocolVersion = 6;