livekit-client 1.10.0 → 1.11.1

package/src/room/track/LocalTrack.ts

@@ -47,12 +47,16 @@ export default abstract class LocalTrack extends Track {
     userProvidedTrack = false,
   ) {
     super(mediaTrack, kind);
-    this._mediaStreamTrack.addEventListener('ended', this.handleEnded);
-    this.constraints = constraints ?? mediaTrack.getConstraints();
     this.reacquireTrack = false;
     this.providedByUser = userProvidedTrack;
     this.muteLock = new Mutex();
     this.pauseUpstreamLock = new Mutex();
+    // added to satisfy TS compiler, constraints are synced with MediaStreamTrack
+    this.constraints = mediaTrack.getConstraints();
+    this.setMediaStreamTrack(mediaTrack);
+    if (constraints) {
+      this.constraints = constraints;
+    }
   }
 
   get id(): string {
@@ -88,6 +92,50 @@ export default abstract class LocalTrack extends Track {
     return this.processor?.processedTrack ?? this._mediaStreamTrack;
   }
 
+  private async setMediaStreamTrack(newTrack: MediaStreamTrack) {
+    if (newTrack === this._mediaStreamTrack) {
+      return;
+    }
+    if (this._mediaStreamTrack) {
+      // detach
+      this.attachedElements.forEach((el) => {
+        detachTrack(this._mediaStreamTrack, el);
+      });
+      this._mediaStreamTrack.removeEventListener('ended', this.handleEnded);
+      this._mediaStreamTrack.removeEventListener('mute', this.pauseUpstream);
+      this._mediaStreamTrack.removeEventListener('unmute', this.resumeUpstream);
+      if (!this.providedByUser) {
+        this._mediaStreamTrack.stop();
+      }
+    }
+
+    this.mediaStream = new MediaStream([newTrack]);
+    if (newTrack) {
+      newTrack.addEventListener('ended', this.handleEnded);
+      // when underlying track emits mute, it indicates that the device is unable
+      // to produce media. In this case we'll need to signal with remote that
+      // the track is "muted"
+      // note this is different from LocalTrack.mute because we do not want to
+      // touch MediaStreamTrack.enabled
+      newTrack.addEventListener('mute', this.pauseUpstream);
+      newTrack.addEventListener('unmute', this.resumeUpstream);
+      this.constraints = newTrack.getConstraints();
+    }
+    if (this.sender) {
+      await this.sender.replaceTrack(newTrack);
+    }
+    this._mediaStreamTrack = newTrack;
+    if (newTrack) {
+      // sync muted state with the enabled state of the newly provided track
+      this._mediaStreamTrack.enabled = !this.isMuted;
+      // when a valid track is replace, we'd want to start producing
+      await this.resumeUpstream();
+      this.attachedElements.forEach((el) => {
+        attachToElement(newTrack, el);
+      });
+    }
+  }
+
   async waitForDimensions(timeout = defaultDimensionsTimeout): Promise<Track.Dimensions> {
     if (this.kind === Track.Kind.Audio) {
       throw new Error('cannot get dimensions for audio tracks');
@@ -133,37 +181,12 @@ export default abstract class LocalTrack extends Track {
       throw new TrackInvalidError('unable to replace an unpublished track');
     }
 
-    // detach
-    this.attachedElements.forEach((el) => {
-      detachTrack(this._mediaStreamTrack, el);
-    });
-    this._mediaStreamTrack.removeEventListener('ended', this.handleEnded);
-    // on Safari, the old audio track must be stopped before attempting to acquire
-    // the new track, otherwise the new track will stop with
-    // 'A MediaStreamTrack ended due to a capture failure`
-    if (!this.providedByUser) {
-      this._mediaStreamTrack.stop();
-    }
-
-    track.addEventListener('ended', this.handleEnded);
     log.debug('replace MediaStreamTrack');
-
-    if (this.sender) {
-      await this.sender.replaceTrack(track);
-    }
-    this._mediaStreamTrack = track;
-
-    // sync muted state with the enabled state of the newly provided track
-    this._mediaStreamTrack.enabled = !this.isMuted;
-
-    await this.resumeUpstream();
-
-    this.attachedElements.forEach((el) => {
-      attachToElement(track, el);
-    });
-
-    this.mediaStream = new MediaStream([track]);
+    this.setMediaStreamTrack(track);
+    // this must be synced *after* setting mediaStreamTrack above, since it relies
+    // on the previous state in order to cleanup
     this.providedByUser = userProvidedTrack;
+
     if (this.processor) {
       await this.stopProcessor();
     }
@@ -187,7 +210,8 @@ export default abstract class LocalTrack extends Track {
       streamConstraints.audio = constraints;
     }
 
-    // detach
+    // these steps are duplicated from setMediaStreamTrack because we must stop
+    // the previous tracks before new tracks can be acquired
     this.attachedElements.forEach((el) => {
       detachTrack(this.mediaStreamTrack, el);
     });
@@ -203,16 +227,7 @@ export default abstract class LocalTrack extends Track {
     newTrack.addEventListener('ended', this.handleEnded);
     log.debug('re-acquired MediaStreamTrack');
 
-    if (this.sender) {
-      // Track can be restarted after it's unpublished
-      await this.sender.replaceTrack(newTrack);
-    }
-
-    this._mediaStreamTrack = newTrack;
-
-    await this.resumeUpstream();
-
-    this.mediaStream = mediaStream;
+    this.setMediaStreamTrack(newTrack);
     this.constraints = constraints;
     if (this.processor) {
       const processor = this.processor;
@@ -263,11 +278,17 @@ export default abstract class LocalTrack extends Track {
     if (this.isInBackground) {
       this.reacquireTrack = true;
     }
+    this._mediaStreamTrack.removeEventListener('mute', this.pauseUpstream);
+    this._mediaStreamTrack.removeEventListener('unmute', this.resumeUpstream);
     this.emit(TrackEvent.Ended, this);
   };
 
   stop() {
     super.stop();
+
+    this._mediaStreamTrack.removeEventListener('ended', this.handleEnded);
+    this._mediaStreamTrack.removeEventListener('mute', this.pauseUpstream);
+    this._mediaStreamTrack.removeEventListener('unmute', this.resumeUpstream);
     this.processor?.destroy();
     this.processor = undefined;
   }
@@ -278,7 +299,7 @@ export default abstract class LocalTrack extends Track {
    * the server.
    * this API is unsupported on Safari < 12 due to a bug
    **/
-  async pauseUpstream() {
+  pauseUpstream = async () => {
     const unlock = await this.pauseUpstreamLock.lock();
     try {
       if (this._isUpstreamPaused === true) {
@@ -300,9 +321,9 @@ export default abstract class LocalTrack extends Track {
     } finally {
       unlock();
     }
-  }
+  };
 
-  async resumeUpstream() {
+  resumeUpstream = async () => {
     const unlock = await this.pauseUpstreamLock.lock();
     try {
       if (this._isUpstreamPaused === false) {
@@ -315,11 +336,12 @@ export default abstract class LocalTrack extends Track {
       this._isUpstreamPaused = false;
       this.emit(TrackEvent.UpstreamResumed, this);
 
+      // this operation is noop if mediastreamtrack is already being sent
       await this.sender.replaceTrack(this._mediaStreamTrack);
     } finally {
       unlock();
     }
-  }
+  };
 
   /**
    * Sets a processor on this track.

package/src/room/track/LocalVideoTrack.ts

@@ -351,8 +351,13 @@ async function setPublishingLayersForSender(
 
     let hasChanged = false;
 
+    /* disable closable spatial layer as it has video blur / frozen issue with current server / client
+    1. chrome 113: when switching to up layer with scalability Mode change, it will generate a 
+          low resolution frame and recover very quickly, but noticable
+    2. livekit sfu: additional pli request cause video frozen for a few frames, also noticable */
+    const closableSpatial = false;
     /* @ts-ignore */
-    if (encodings.length === 1 && encodings[0].scalabilityMode) {
+    if (closableSpatial && encodings[0].scalabilityMode) {
       // svc dynacast encodings
       const encoding = encodings[0];
       /* @ts-ignore */
@@ -372,10 +377,7 @@ async function setPublishingLayersForSender(
       } else if (!encoding.active /* || mode.spatial !== maxQuality + 1*/) {
         hasChanged = true;
         encoding.active = true;
-        /* disable closable spatial layer as it has video blur/frozen issue with current server/client
-          1. chrome 113: when switching to up layer with scalability Mode change, it will generate a 
-          low resolution frame and recover very quickly, but noticable
-          2. livekit sfu: additional pli request cause video frozen for a few frames, also noticable
+        /*
         @ts-ignore
         const originalMode = new ScalabilityMode(senderEncodings[0].scalabilityMode)
         mode.spatial = maxQuality + 1;
@@ -456,6 +458,7 @@ export function videoLayersFromEncodings(
   width: number,
   height: number,
   encodings?: RTCRtpEncodingParameters[],
+  svc?: boolean,
 ): VideoLayer[] {
   // default to a single layer, HQ
   if (!encodings) {
@@ -470,8 +473,7 @@ export function videoLayersFromEncodings(
     ];
   }
 
-  /* @ts-ignore */
-  if (encodings.length === 1 && encodings[0].scalabilityMode) {
+  if (svc) {
     // svc layers
     /* @ts-ignore */
     const sm = new ScalabilityMode(encodings[0].scalabilityMode);

package/src/room/track/RemoteVideoTrack.ts

@@ -1,11 +1,11 @@
 import { debounce } from 'ts-debounce';
 import log from '../../logger';
 import { TrackEvent } from '../events';
-import { computeBitrate } from '../stats';
 import type { VideoReceiverStats } from '../stats';
+import { computeBitrate } from '../stats';
 import CriticalTimers from '../timers';
-import { getDevicePixelRatio, getIntersectionObserver, getResizeObserver, isWeb } from '../utils';
 import type { ObservableMediaElement } from '../utils';
+import { getDevicePixelRatio, getIntersectionObserver, getResizeObserver, isWeb } from '../utils';
 import RemoteTrack from './RemoteTrack';
 import { Track, attachToElement, detachTrack } from './Track';
 import type { AdaptiveStreamSettings } from './types';
@@ -248,11 +248,10 @@ export default class RemoteVideoTrack extends RemoteTrack {
   private updateDimensions() {
     let maxWidth = 0;
     let maxHeight = 0;
+    const pixelDensity = this.getPixelDensity();
     for (const info of this.elementInfos) {
-      const pixelDensity = this.adaptiveStreamSettings?.pixelDensity ?? 1;
-      const pixelDensityValue = pixelDensity === 'screen' ? getDevicePixelRatio() : pixelDensity;
-      const currentElementWidth = info.width() * pixelDensityValue;
-      const currentElementHeight = info.height() * pixelDensityValue;
+      const currentElementWidth = info.width() * pixelDensity;
+      const currentElementHeight = info.height() * pixelDensity;
       if (currentElementWidth + currentElementHeight > maxWidth + maxHeight) {
         maxWidth = currentElementWidth;
         maxHeight = currentElementHeight;
@@ -270,6 +269,24 @@ export default class RemoteVideoTrack extends RemoteTrack {
 
     this.emit(TrackEvent.VideoDimensionsChanged, this.lastDimensions, this);
   }
+
+  private getPixelDensity(): number {
+    const pixelDensity = this.adaptiveStreamSettings?.pixelDensity;
+    if (pixelDensity === 'screen') {
+      return getDevicePixelRatio();
+    } else if (!pixelDensity) {
+      // when unset, we'll pick a sane default here.
+      // for higher pixel density devices (mobile phones, etc), we'll use 2
+      // otherwise it defaults to 1
+      const devicePixelRatio = getDevicePixelRatio();
+      if (devicePixelRatio > 2) {
+        return 2;
+      } else {
+        return 1;
+      }
+    }
+    return pixelDensity;
+  }
 }
 
 export interface ElementInfo {

package/src/room/track/Track.ts

@@ -1,5 +1,4 @@
-import { EventEmitter } from 'events';
-import type TypedEventEmitter from 'typed-emitter';
+import EventEmitter from 'eventemitter3';
 import type { SignalClient } from '../../api/SignalClient';
 import log from '../../logger';
 import { TrackSource, TrackType } from '../../proto/livekit_models';
@@ -13,7 +12,7 @@ const BACKGROUND_REACTION_DELAY = 5000;
 // Safari tracks which audio elements have been "blessed" by the user.
 const recycledElements: Array<HTMLAudioElement> = [];
 
-export abstract class Track extends (EventEmitter as new () => TypedEventEmitter<TrackEventCallbacks>) {
+export abstract class Track extends EventEmitter<TrackEventCallbacks> {
   kind: Track.Kind;
 
   attachedElements: HTMLMediaElement[] = [];
@@ -52,7 +51,6 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
 
   protected constructor(mediaTrack: MediaStreamTrack, kind: Track.Kind) {
     super();
-    this.setMaxListeners(100);
     this.kind = kind;
     this._mediaStreamTrack = mediaTrack;
     this._mediaStreamID = mediaTrack.id;

package/src/room/track/TrackPublication.ts

@@ -1,5 +1,4 @@
-import { EventEmitter } from 'events';
-import type TypedEventEmitter from 'typed-emitter';
+import EventEmitter from 'eventemitter3';
 import log from '../../logger';
 import type { SubscriptionError, TrackInfo } from '../../proto/livekit_models';
 import type { UpdateSubscription, UpdateTrackSettings } from '../../proto/livekit_rtc';
@@ -11,7 +10,7 @@ import type RemoteTrack from './RemoteTrack';
 import RemoteVideoTrack from './RemoteVideoTrack';
 import { Track } from './Track';
 
-export class TrackPublication extends (EventEmitter as new () => TypedEventEmitter<PublicationEventCallbacks>) {
+export class TrackPublication extends EventEmitter<PublicationEventCallbacks> {
   kind: Track.Kind;
 
   trackName: string;
@@ -38,7 +37,6 @@ export class TrackPublication extends (EventEmitter as new () => TypedEventEmitt
 
   constructor(kind: Track.Kind, id: string, name: string) {
     super();
-    this.setMaxListeners(100);
     this.kind = kind;
     this.trackSid = id;
     this.trackName = name;

package/src/room/track/options.ts

@@ -63,10 +63,19 @@ export interface TrackPublishDefaults {
   scalabilityMode?: ScalabilityMode;
 
   /**
-   * custom video simulcast layers for camera tracks, defaults to h180, h360
-   * You can specify up to two custom layers that will be used instead of
-   * the LiveKit default layers.
-   * Note: the layers need to be ordered from lowest to highest quality
+   * Up to two additional simulcast layers to publish in addition to the original
+   * Track.
+   * When left blank, it defaults to h180, h360.
+   * If a SVC codec is used (VP9 or AV1), this field has no effect.
+   *
+   * To publish three total layers, you would specify:
+   * {
+   *   videoEncoding: {...}, // encoding of the primary layer
+   *   videoSimulcastLayers: [
+   *     VideoPresets.h540,
+   *     VideoPresets.h216,
+   *   ],
+   * }
    */
   videoSimulcastLayers?: Array<VideoPreset>;
 

package/src/room/track/types.ts

@@ -8,7 +8,8 @@ export type VideoTrack = RemoteVideoTrack | LocalVideoTrack;
 
 export type AdaptiveStreamSettings = {
   /**
-   * Set a custom pixel density, defaults to 1
+   * Set a custom pixel density. Defaults to 2 for high density screens (3+) or
+   * 1 otherwise.
    * When streaming videos on a ultra high definition screen this setting
    * let's you account for the devicePixelRatio of those screens.
    * Set it to `screen` to use the actual pixel density of the screen

package/src/room/utils.ts

@@ -261,7 +261,7 @@ export function getEmptyVideoStreamTrack() {
   if (!emptyVideoStreamTrack) {
     emptyVideoStreamTrack = createDummyVideoStreamTrack();
   }
-  return emptyVideoStreamTrack;
+  return emptyVideoStreamTrack.clone();
 }
 
 export function createDummyVideoStreamTrack(
@@ -301,8 +301,11 @@ export function getEmptyAudioStreamTrack() {
     // implementation adapted from https://blog.mozilla.org/webrtc/warm-up-with-replacetrack/
     const ctx = new AudioContext();
     const oscillator = ctx.createOscillator();
+    const gain = ctx.createGain();
+    gain.gain.setValueAtTime(0, 0);
     const dst = ctx.createMediaStreamDestination();
-    oscillator.connect(dst);
+    oscillator.connect(gain);
+    gain.connect(dst);
     oscillator.start();
     [emptyAudioStreamTrack] = dst.stream.getAudioTracks();
     if (!emptyAudioStreamTrack) {
@@ -310,7 +313,7 @@ export function getEmptyAudioStreamTrack() {
     }
     emptyAudioStreamTrack.enabled = false;
   }
-  return emptyAudioStreamTrack;
+  return emptyAudioStreamTrack.clone();
 }
 
 export class Future<T> {