livekit-client 0.15.4 → 0.16.3

package/src/room/track/LocalVideoTrack.ts

@@ -9,31 +9,14 @@ import { VideoCaptureOptions } from './options';
 import { Track } from './Track';
 import { constraintsForOptions } from './utils';
 
-// delay before attempting to upgrade
-const QUALITY_UPGRADE_DELAY = 60 * 1000;
-
-// avoid downgrading too quickly
-const QUALITY_DOWNGRADE_DELAY = 5 * 1000;
-
-const ridOrder = ['q', 'h', 'f'];
-
 export default class LocalVideoTrack extends LocalTrack {
   /* internal */
   signalClient?: SignalClient;
 
   private prevStats?: Map<string, VideoSenderStats>;
 
-  // last time it had a change in quality
-  private lastQualityChange?: number;
-
-  // last time we made an explicit change
-  private lastExplicitQualityChange?: number;
-
   private encodings?: RTCRtpEncodingParameters[];
 
-  // layers that are being subscribed to, and that we should publish
-  private activeQualities?: SubscribedQuality[];
-
   constructor(
     mediaTrack: MediaStreamTrack,
     constraints?: MediaTrackConstraints,
@@ -49,7 +32,7 @@ export default class LocalVideoTrack extends LocalTrack {
   }
 
   /* @internal */
-  startMonitor(signalClient: SignalClient, disableLayerPause: boolean) {
+  startMonitor(signalClient: SignalClient) {
     this.signalClient = signalClient;
     // save original encodings
     const params = this.sender?.getParameters();
@@ -58,7 +41,7 @@ export default class LocalVideoTrack extends LocalTrack {
     }
 
     setTimeout(() => {
-      this.monitorSender(disableLayerPause);
+      this.monitorSender();
     }, monitorFrequency);
   }
 
@@ -186,7 +169,6 @@ export default class LocalVideoTrack extends LocalTrack {
       return;
     }
 
-    this.activeQualities = qualities;
     let hasChanged = false;
     encodings.forEach((encoding, idx) => {
       let rid = encoding.rid ?? '';
@@ -227,17 +209,20 @@ export default class LocalVideoTrack extends LocalTrack {
     }
   }
 
-  private monitorSender = async (disableLayerPause: boolean) => {
+  private monitorSender = async () => {
     if (!this.sender) {
       this._currentBitrate = 0;
       return;
     }
-    const stats = await this.getSenderStats();
-    const statsMap = new Map<string, VideoSenderStats>(stats.map((s) => [s.rid, s]));
 
-    if (!disableLayerPause && this.prevStats && this.isSimulcast) {
-      this.checkAndUpdateSimulcast(statsMap);
+    let stats: VideoSenderStats[] | undefined;
+    try {
+      stats = await this.getSenderStats();
+    } catch (e) {
+      log.error('could not get audio sender stats', e);
+      return;
     }
+    const statsMap = new Map<string, VideoSenderStats>(stats.map((s) => [s.rid, s]));
 
     if (this.prevStats) {
       let totalBitrate = 0;
@@ -250,125 +235,9 @@ export default class LocalVideoTrack extends LocalTrack {
 
     this.prevStats = statsMap;
     setTimeout(() => {
-      this.monitorSender(disableLayerPause);
+      this.monitorSender();
     }, monitorFrequency);
   };
-
-  private checkAndUpdateSimulcast(statsMap: Map<string, VideoSenderStats>) {
-    if (!this.sender || this.isMuted || !this.encodings) {
-      return;
-    }
-
-    let bestEncoding: RTCRtpEncodingParameters | undefined;
-    const { encodings } = this.sender.getParameters();
-    encodings.forEach((encoding) => {
-      // skip inactive encodings
-      if (!encoding.active) return;
-
-      if (bestEncoding === undefined) {
-        bestEncoding = encoding;
-      } else if (
-        bestEncoding.rid
-        && encoding.rid
-        && ridOrder.indexOf(bestEncoding.rid) < ridOrder.indexOf(encoding.rid)
-      ) {
-        bestEncoding = encoding;
-      } else if (
-        bestEncoding.maxBitrate !== undefined
-        && encoding.maxBitrate !== undefined
-        && bestEncoding.maxBitrate < encoding.maxBitrate
-      ) {
-        bestEncoding = encoding;
-      }
-    });
-
-    if (!bestEncoding) {
-      return;
-    }
-    const rid: string = bestEncoding.rid ?? '';
-    const sendStats = statsMap.get(rid);
-    const lastStats = this.prevStats?.get(rid);
-    if (!sendStats || !lastStats) {
-      return;
-    }
-    const currentQuality = videoQualityForRid(rid);
-
-    // adaptive simulcast algorithm notes (davidzhao)
-    // Chrome (and other browsers) will automatically pause the highest layer
-    // when it runs into bandwidth limitations. When that happens, it would not
-    // be able to send any new frames between the two stats checks.
-    //
-    // We need to set that layer to inactive intentionally, because chrome tends
-    // to flicker, meaning it will attempt to send that layer again shortly
-    // afterwards, flip-flopping every few seconds. We want to avoid that.
-    //
-    // Note: even after bandwidth recovers, the flip-flopping behavior continues
-    // this is possibly due to SFU-side PLI generation and imperfect bandwidth estimation
-    if (sendStats.qualityLimitationResolutionChanges
-        - lastStats.qualityLimitationResolutionChanges > 0) {
-      this.lastQualityChange = new Date().getTime();
-    }
-
-    // log.debug('frameSent', sendStats.framesSent, 'lastSent', lastStats.framesSent,
-    //   'elapsed', sendStats.timestamp - lastStats.timestamp);
-    if (sendStats.framesSent - lastStats.framesSent > 0) {
-      // frames have been sending ok, consider upgrading quality
-      if (currentQuality === VideoQuality.HIGH || !this.lastQualityChange) return;
-
-      const nextQuality = currentQuality + 1;
-      if ((new Date()).getTime() - this.lastQualityChange < QUALITY_UPGRADE_DELAY) {
-        return;
-      }
-
-      if (this.activeQualities
-        && this.activeQualities.some((q) => q.quality === nextQuality && !q.enabled)
-      ) {
-        // quality has been disabled by the server, so we should skip
-        return;
-      }
-
-      // we are already at the highest layer
-      let bestQuality = VideoQuality.LOW;
-      encodings.forEach((encoding) => {
-        const quality = videoQualityForRid(encoding.rid ?? '');
-        if (quality > bestQuality) {
-          bestQuality = quality;
-        }
-      });
-      if (nextQuality > bestQuality) {
-        return;
-      }
-
-      log.debug('upgrading video quality to', nextQuality);
-      this.setPublishingQuality(nextQuality);
-      return;
-    }
-
-    // if best layer has not sent anything, do not downgrade till the
-    // best layer starts sending something. It is possible that the
-    // browser has not started some layer(s) due to cpu/bandwidth
-    // constraints
-    if (sendStats.framesSent === 0) return;
-
-    // if we've upgraded or downgraded recently, give it a bit of time before
-    // downgrading again
-    if (this.lastExplicitQualityChange
-      && ((new Date()).getTime() - this.lastExplicitQualityChange) < QUALITY_DOWNGRADE_DELAY) {
-      return;
-    }
-
-    if (currentQuality === VideoQuality.UNRECOGNIZED) {
-      return;
-    }
-
-    if (currentQuality === VideoQuality.LOW) {
-      // already the lowest quality, nothing we can do
-      return;
-    }
-
-    log.debug('downgrading video quality to', currentQuality - 1);
-    this.setPublishingQuality(currentQuality - 1);
-  }
 }
 
 export function videoQualityForRid(rid: string): VideoQuality {

package/src/room/track/RemoteTrackPublication.ts

@@ -35,7 +35,12 @@ export default class RemoteTrackPublication extends TrackPublication {
     const sub: UpdateSubscription = {
       trackSids: [this.trackSid],
       subscribe: this.subscribed,
-      participantTracks: [],
+      participantTracks: [{
+        // sending an empty participant id since TrackPublication doesn't keep it
+        // this is filled in by the participant that receives this message
+        participantSid: '',
+        trackSids: [this.trackSid],
+      }],
     };
     this.emit(TrackEvent.UpdateSubscription, sub);
   }
@@ -172,7 +177,8 @@ export default class RemoteTrackPublication extends TrackPublication {
     this.emitTrackUpdate();
   };
 
-  protected emitTrackUpdate() {
+  /* @internal */
+  emitTrackUpdate() {
     const settings: UpdateTrackSettings = UpdateTrackSettings.fromPartial({
       trackSids: [this.trackSid],
       disabled: this.disabled,

package/src/room/track/RemoteVideoTrack.ts

@@ -58,7 +58,11 @@ export default class RemoteVideoTrack extends RemoteTrack {
       super.attach(element);
     }
 
-    if (this.adaptiveStream) {
+    // It's possible attach is called multiple times on an element. When that's
+    // the case, we'd want to avoid adding duplicate elementInfos
+    if (this.adaptiveStream
+      && this.elementInfos.find((info) => info.element === element) === undefined
+    ) {
       this.elementInfos.push({
         element,
         visible: true, // default visible
@@ -71,6 +75,11 @@ export default class RemoteVideoTrack extends RemoteTrack {
 
       getIntersectionObserver().observe(element);
       getResizeObserver().observe(element);
+
+      // trigger the first resize update cycle
+      // if the tab is backgrounded, the initial resize event does not fire until
+      // the tab comes into focus for the first time.
+      this.debouncedHandleResize();
     }
     return element;
   }
@@ -174,7 +183,7 @@ export default class RemoteVideoTrack extends RemoteTrack {
       // delay hidden events
       setTimeout(() => {
         this.updateVisibility();
-      }, Date.now() - lastVisibilityChange);
+      }, REACTION_DELAY);
       return;
     }
 
@@ -186,11 +195,9 @@ export default class RemoteVideoTrack extends RemoteTrack {
     let maxWidth = 0;
     let maxHeight = 0;
     for (const info of this.elementInfos) {
-      if (info.visible) {
-        if (info.element.clientWidth + info.element.clientHeight > maxWidth + maxHeight) {
-          maxWidth = info.element.clientWidth;
-          maxHeight = info.element.clientHeight;
-        }
+      if (info.element.clientWidth + info.element.clientHeight > maxWidth + maxHeight) {
+        maxWidth = info.element.clientWidth;
+        maxHeight = info.element.clientHeight;
       }
     }
 

package/src/room/track/Track.ts

@@ -1,14 +1,15 @@
 import { EventEmitter } from 'events';
+import type TypedEventEmitter from 'typed-emitter';
 import { TrackSource, TrackType } from '../../proto/livekit_models';
 import { StreamState as ProtoStreamState } from '../../proto/livekit_rtc';
 import { TrackEvent } from '../events';
-import { isFireFox } from '../utils';
+import { isFireFox, isSafari } from '../utils';
 
 // keep old audio elements when detached, we would re-use them since on iOS
 // Safari tracks which audio elements have been "blessed" by the user.
 const recycledElements: Array<HTMLAudioElement> = [];
 
-export class Track extends EventEmitter {
+export class Track extends (EventEmitter as new () => TypedEventEmitter<TrackEventCallbacks>) {
   kind: Track.Kind;
 
   mediaStreamTrack: MediaStreamTrack;
@@ -71,18 +72,14 @@ export class Track extends EventEmitter {
       }
     }
 
-    if (element instanceof HTMLVideoElement) {
-      element.playsInline = true;
-      element.autoplay = true;
-    }
-
-    // already attached
-    if (this.attachedElements.includes(element)) {
-      return element;
+    if (!this.attachedElements.includes(element)) {
+      this.attachedElements.push(element);
     }
 
+    // even if we believe it's already attached to the element, it's possible
+    // the element's srcObject was set to something else out of band.
+    // we'll want to re-attach it in that case
     attachToElement(this.mediaStreamTrack, element);
-    this.attachedElements.push(element);
 
     if (element instanceof HTMLAudioElement) {
       // manually play audio to detect audio playback status
@@ -168,28 +165,40 @@ export function attachToElement(track: MediaStreamTrack, element: HTMLMediaEleme
     mediaStream = element.srcObject;
   } else {
     mediaStream = new MediaStream();
-    element.srcObject = mediaStream;
   }
 
-  // remove existing tracks of same type from stream
+  // check if track matches existing track
   let existingTracks: MediaStreamTrack[];
   if (track.kind === 'audio') {
     existingTracks = mediaStream.getAudioTracks();
   } else {
     existingTracks = mediaStream.getVideoTracks();
   }
+  if (!existingTracks.includes(track)) {
+    existingTracks.forEach((et) => {
+      mediaStream.removeTrack(et);
+    });
+    mediaStream.addTrack(track);
+  }
 
-  existingTracks.forEach((et) => {
-    mediaStream.removeTrack(et);
-  });
-
-  mediaStream.addTrack(track);
-  if (isFireFox()) {
-    // sometimes firefox doesn't render local video on the first try.
-    // It needs to be re-attached after a timeout.
-    setTimeout(() => {
-      element.srcObject = mediaStream;
-    }, 1);
+  // avoid flicker
+  if (element.srcObject !== mediaStream) {
+    element.srcObject = mediaStream;
+    if ((isSafari() || isFireFox()) && element instanceof HTMLVideoElement) {
+      // Firefox also has a timing issue where video doesn't actually get attached unless
+      // performed out-of-band
+      // Safari 15 has a bug where in certain layouts, video element renders
+      // black until the page is resized or other changes take place.
+      // Resetting the src triggers it to render.
+      // https://developer.apple.com/forums/thread/690523
+      setTimeout(() => {
+        element.srcObject = mediaStream;
+      }, 0);
+    }
+  }
+  element.autoplay = true;
+  if (element instanceof HTMLVideoElement) {
+    element.playsInline = true;
   }
 }
 
@@ -299,3 +308,16 @@ export namespace Track {
     }
   }
 }
+
+export type TrackEventCallbacks = {
+  message: () => void,
+  muted: (track?: any) => void,
+  unmuted: (track?: any) => void,
+  ended: (track?: any) => void,
+  updateSettings: () => void,
+  updateSubscription: () => void,
+  audioPlaybackStarted: () => void,
+  audioPlaybackFailed: (error: Error) => void,
+  visibilityChanged: (visible: boolean, track?: any) => void,
+  videoDimensionsChanged: (dimensions: Track.Dimensions, track?: any) => void,
+};

package/src/room/track/types.ts

@@ -1,7 +1,7 @@
-import LocalAudioTrack from './LocalAudioTrack';
-import LocalVideoTrack from './LocalVideoTrack';
-import RemoteAudioTrack from './RemoteAudioTrack';
-import RemoteVideoTrack from './RemoteVideoTrack';
+import type LocalAudioTrack from './LocalAudioTrack';
+import type LocalVideoTrack from './LocalVideoTrack';
+import type RemoteAudioTrack from './RemoteAudioTrack';
+import type RemoteVideoTrack from './RemoteVideoTrack';
 
 export type RemoteTrack = RemoteAudioTrack | RemoteVideoTrack;
 export type AudioTrack = RemoteAudioTrack | LocalAudioTrack;

package/src/room/utils.ts

@@ -1,4 +1,3 @@
-import uaparser from 'ua-parser-js';
 import { ClientInfo, ClientInfo_SDK } from '../proto/livekit_models';
 import { protocolVersion, version } from '../version';
 
@@ -20,6 +19,10 @@ export function isFireFox(): boolean {
   return navigator.userAgent.indexOf('Firefox') !== -1;
 }
 
+export function isSafari(): boolean {
+  return /^((?!chrome|android).)*safari/i.test(navigator.userAgent);
+}
+
 function roDispatchCallback(entries: ResizeObserverEntry[]) {
   for (const entry of entries) {
     (entry.target as ObservableMediaElement).handleResize(entry);
@@ -50,25 +53,10 @@ export interface ObservableMediaElement extends HTMLMediaElement {
 }
 
 export function getClientInfo(): ClientInfo {
-  const ua = uaparser(navigator.userAgent);
   const info = ClientInfo.fromPartial({
     sdk: ClientInfo_SDK.JS,
     protocol: protocolVersion,
     version,
-    os: ua.os.name,
-    osVersion: ua.os.version,
-    browser: ua.browser.name,
-    browserVersion: ua.browser.version,
   });
-
-  let model = '';
-  if (ua.device.vendor) {
-    model += ua.device.vendor;
-  }
-  if (ua.device.model) {
-    if (model) model += ' ';
-    model += ua.device.model;
-  }
-  if (model) info.deviceModel = model;
   return info;
 }

package/src/version.ts

@@ -1,2 +1,2 @@
-export const version = '0.15.4';
-export const protocolVersion = 5;
+export const version = '0.16.3';
+export const protocolVersion = 6;