livekit-client 1.7.0 → 1.8.0

package/src/room/track/RemoteTrackPublication.ts

@@ -4,7 +4,7 @@ import { UpdateSubscription, UpdateTrackSettings } from '../../proto/livekit_rtc
 import { TrackEvent } from '../events';
 import type RemoteTrack from './RemoteTrack';
 import RemoteVideoTrack from './RemoteVideoTrack';
-import type { Track } from './Track';
+import { Track } from './Track';
 import { TrackPublication } from './TrackPublication';
 
 export default class RemoteTrackPublication extends TrackPublication {
@@ -181,6 +181,7 @@ export default class RemoteTrackPublication extends TrackPublication {
       prevTrack.off(TrackEvent.VisibilityChanged, this.handleVisibilityChange);
       prevTrack.off(TrackEvent.Ended, this.handleEnded);
       prevTrack.detach();
+      prevTrack.stopMonitor();
       this.emit(TrackEvent.Unsubscribed, prevTrack);
     }
     super.setTrack(track);
@@ -207,8 +208,13 @@ export default class RemoteTrackPublication extends TrackPublication {
   /** @internal */
   updateInfo(info: TrackInfo) {
     super.updateInfo(info);
+    const prevMetadataMuted = this.metadataMuted;
     this.metadataMuted = info.muted;
-    this.track?.setMuted(info.muted);
+    if (this.track) {
+      this.track.setMuted(info.muted);
+    } else if (prevMetadataMuted !== info.muted) {
+      this.emit(info.muted ? TrackEvent.Muted : TrackEvent.Unmuted);
+    }
   }
 
   private emitSubscriptionUpdateIfChanged(previousStatus: TrackPublication.SubscriptionStatus) {
@@ -233,8 +239,8 @@ export default class RemoteTrackPublication extends TrackPublication {
   }
 
   private isManualOperationAllowed(): boolean {
-    if (this.isAdaptiveStream) {
-      log.warn('adaptive stream is enabled, cannot change track settings', {
+    if (this.kind === Track.Kind.Video && this.isAdaptiveStream) {
+      log.warn('adaptive stream is enabled, cannot change video track settings', {
         trackSid: this.trackSid,
       });
       return false;

package/src/room/track/RemoteVideoTrack.test.ts

@@ -130,6 +130,8 @@ class MockElementInfo implements ElementInfo {
 
   visible = false;
 
+  pictureInPicture = false;
+
   setVisible = (visible: boolean) => {
     if (this.visible !== visible) {
       this.visible = visible;

package/src/room/track/RemoteVideoTrack.ts

@@ -3,7 +3,13 @@ import log from '../../logger';
 import { TrackEvent } from '../events';
 import { computeBitrate, VideoReceiverStats } from '../stats';
 import CriticalTimers from '../timers';
-import { getIntersectionObserver, getResizeObserver, ObservableMediaElement } from '../utils';
+import {
+  getDevicePixelRatio,
+  getIntersectionObserver,
+  getResizeObserver,
+  isWeb,
+  ObservableMediaElement,
+} from '../utils';
 import RemoteTrack from './RemoteTrack';
 import { attachToElement, detachTrack, Track } from './Track';
 import type { AdaptiveStreamSettings } from './types';
@@ -21,7 +27,7 @@ export default class RemoteVideoTrack extends RemoteTrack {
 
   private lastDimensions?: Track.Dimensions;
 
-  private hasUsedAttach: boolean = false;
+  private isObserved: boolean = false;
 
   constructor(
     mediaTrack: MediaStreamTrack,
@@ -38,7 +44,7 @@ export default class RemoteVideoTrack extends RemoteTrack {
   }
 
   get mediaStreamTrack() {
-    if (this.isAdaptiveStream && !this.hasUsedAttach) {
+    if (this.isAdaptiveStream && !this.isObserved) {
       log.warn(
         'When using adaptiveStream, you need to use remoteVideoTrack.attach() to add the track to a HTMLVideoElement, otherwise your video tracks might never start',
       );
@@ -78,7 +84,6 @@ export default class RemoteVideoTrack extends RemoteTrack {
       const elementInfo = new HTMLElementInfo(element);
       this.observeElementInfo(elementInfo);
     }
-    this.hasUsedAttach = true;
     return element;
   }
 
@@ -105,6 +110,7 @@ export default class RemoteVideoTrack extends RemoteTrack {
       // the tab comes into focus for the first time.
       this.debouncedHandleResize();
       this.updateVisibility();
+      this.isObserved = true;
     } else {
       log.warn('visibility resize observer not triggered');
     }
@@ -223,7 +229,9 @@ export default class RemoteVideoTrack extends RemoteTrack {
       this.adaptiveStreamSettings?.pauseVideoInBackground ?? true // default to true
         ? this.isInBackground
         : false;
-    const isVisible = this.elementInfos.some((info) => info.visible) && !backgroundPause;
+    const isPiPMode = this.elementInfos.some((info) => info.pictureInPicture);
+    const isVisible =
+      (this.elementInfos.some((info) => info.visible) && !backgroundPause) || isPiPMode;
 
     if (this.lastVisible === isVisible) {
       return;
@@ -246,7 +254,7 @@ export default class RemoteVideoTrack extends RemoteTrack {
     let maxHeight = 0;
     for (const info of this.elementInfos) {
       const pixelDensity = this.adaptiveStreamSettings?.pixelDensity ?? 1;
-      const pixelDensityValue = pixelDensity === 'screen' ? window.devicePixelRatio : pixelDensity;
+      const pixelDensityValue = pixelDensity === 'screen' ? getDevicePixelRatio() : pixelDensity;
       const currentElementWidth = info.width() * pixelDensityValue;
       const currentElementHeight = info.height() * pixelDensityValue;
       if (currentElementWidth + currentElementHeight > maxWidth + maxHeight) {
@@ -273,6 +281,7 @@ export interface ElementInfo {
   width(): number;
   height(): number;
   visible: boolean;
+  pictureInPicture: boolean;
   visibilityChangedAt: number | undefined;
 
   handleResize?: () => void;
@@ -284,7 +293,13 @@ export interface ElementInfo {
 class HTMLElementInfo implements ElementInfo {
   element: HTMLMediaElement;
 
-  visible: boolean;
+  get visible(): boolean {
+    return this.isPiP || this.isIntersecting;
+  }
+
+  get pictureInPicture(): boolean {
+    return this.isPiP;
+  }
 
   visibilityChangedAt: number | undefined;
 
@@ -292,9 +307,14 @@ class HTMLElementInfo implements ElementInfo {
 
   handleVisibilityChanged?: () => void;
 
+  private isPiP: boolean;
+
+  private isIntersecting: boolean;
+
   constructor(element: HTMLMediaElement, visible?: boolean) {
     this.element = element;
-    this.visible = visible ?? isElementInViewport(element);
+    this.isIntersecting = visible ?? isElementInViewport(element);
+    this.isPiP = isWeb() && document.pictureInPictureElement === element;
     this.visibilityChangedAt = 0;
   }
 
@@ -307,6 +327,10 @@ class HTMLElementInfo implements ElementInfo {
   }
 
   observe() {
+    // make sure we update the current visible state once we start to observe
+    this.isIntersecting = isElementInViewport(this.element);
+    this.isPiP = document.pictureInPictureElement === this.element;
+
     (this.element as ObservableMediaElement).handleResize = () => {
       this.handleResize?.();
     };
@@ -314,20 +338,40 @@ class HTMLElementInfo implements ElementInfo {
 
     getIntersectionObserver().observe(this.element);
     getResizeObserver().observe(this.element);
+    (this.element as HTMLVideoElement).addEventListener('enterpictureinpicture', this.onEnterPiP);
+    (this.element as HTMLVideoElement).addEventListener('leavepictureinpicture', this.onLeavePiP);
   }
 
   private onVisibilityChanged = (entry: IntersectionObserverEntry) => {
     const { target, isIntersecting } = entry;
     if (target === this.element) {
-      this.visible = isIntersecting;
+      this.isIntersecting = isIntersecting;
       this.visibilityChangedAt = Date.now();
       this.handleVisibilityChanged?.();
     }
   };
 
+  private onEnterPiP = () => {
+    this.isPiP = true;
+    this.handleVisibilityChanged?.();
+  };
+
+  private onLeavePiP = () => {
+    this.isPiP = false;
+    this.handleVisibilityChanged?.();
+  };
+
   stopObserving() {
     getIntersectionObserver()?.unobserve(this.element);
     getResizeObserver()?.unobserve(this.element);
+    (this.element as HTMLVideoElement).removeEventListener(
+      'enterpictureinpicture',
+      this.onEnterPiP,
+    );
+    (this.element as HTMLVideoElement).removeEventListener(
+      'leavepictureinpicture',
+      this.onLeavePiP,
+    );
   }
 }
 

package/src/room/track/Track.ts

@@ -42,7 +42,7 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
 
   protected _mediaStreamID: string;
 
-  protected isInBackground: boolean;
+  protected isInBackground: boolean = false;
 
   private backgroundTimeout: ReturnType<typeof setTimeout> | undefined;
 
@@ -57,12 +57,6 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
     this._mediaStreamTrack = mediaTrack;
     this._mediaStreamID = mediaTrack.id;
     this.source = Track.Source.Unknown;
-    if (isWeb()) {
-      this.isInBackground = document.visibilityState === 'hidden';
-      document.addEventListener('visibilitychange', this.appVisibilityChangedListener);
-    } else {
-      this.isInBackground = false;
-    }
   }
 
   /** current receive bits per second */
@@ -97,6 +91,9 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
     if (this.kind === Track.Kind.Video) {
       elementType = 'video';
     }
+    if (this.attachedElements.length === 0 && Track.Kind.Video) {
+      this.addAppVisibilityListener();
+    }
     if (!element) {
       if (elementType === 'audio') {
         recycledElements.forEach((e) => {
@@ -167,37 +164,40 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
    */
   detach(element: HTMLMediaElement): HTMLMediaElement;
   detach(element?: HTMLMediaElement): HTMLMediaElement | HTMLMediaElement[] {
-    // detach from a single element
-    if (element) {
-      detachTrack(this._mediaStreamTrack, element);
-      const idx = this.attachedElements.indexOf(element);
-      if (idx >= 0) {
-        this.attachedElements.splice(idx, 1);
-        this.recycleElement(element);
-        this.emit(TrackEvent.ElementDetached, element);
+    try {
+      // detach from a single element
+      if (element) {
+        detachTrack(this._mediaStreamTrack, element);
+        const idx = this.attachedElements.indexOf(element);
+        if (idx >= 0) {
+          this.attachedElements.splice(idx, 1);
+          this.recycleElement(element);
+          this.emit(TrackEvent.ElementDetached, element);
+        }
+        return element;
       }
-      return element;
-    }
 
-    const detached: HTMLMediaElement[] = [];
-    this.attachedElements.forEach((elm) => {
-      detachTrack(this._mediaStreamTrack, elm);
-      detached.push(elm);
-      this.recycleElement(elm);
-      this.emit(TrackEvent.ElementDetached, elm);
-    });
+      const detached: HTMLMediaElement[] = [];
+      this.attachedElements.forEach((elm) => {
+        detachTrack(this._mediaStreamTrack, elm);
+        detached.push(elm);
+        this.recycleElement(elm);
+        this.emit(TrackEvent.ElementDetached, elm);
+      });
 
-    // remove all tracks
-    this.attachedElements = [];
-    return detached;
+      // remove all tracks
+      this.attachedElements = [];
+      return detached;
+    } finally {
+      if (this.attachedElements.length === 0) {
+        this.removeAppVisibilityListener();
+      }
+    }
   }
 
   stop() {
     this.stopMonitor();
     this._mediaStreamTrack.stop();
-    if (isWeb()) {
-      document.removeEventListener('visibilitychange', this.appVisibilityChangedListener);
-    }
   }
 
   protected enable() {
@@ -212,7 +212,7 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
   abstract startMonitor(signalClient?: SignalClient): void;
 
   /* @internal */
-  protected stopMonitor() {
+  stopMonitor() {
     if (this.monitorInterval) {
       clearInterval(this.monitorInterval);
     }
@@ -253,6 +253,21 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
   protected async handleAppVisibilityChanged() {
     this.isInBackground = document.visibilityState === 'hidden';
   }
+
+  protected addAppVisibilityListener() {
+    if (isWeb()) {
+      this.isInBackground = document.visibilityState === 'hidden';
+      document.addEventListener('visibilitychange', this.appVisibilityChangedListener);
+    } else {
+      this.isInBackground = false;
+    }
+  }
+
+  protected removeAppVisibilityListener() {
+    if (isWeb()) {
+      document.removeEventListener('visibilitychange', this.appVisibilityChangedListener);
+    }
+  }
 }
 
 /** @internal */

package/src/room/track/utils.ts

@@ -105,8 +105,9 @@ export async function detectSilence(track: AudioTrack, timeOffset = 200): Promis
  * @internal
  */
 export function getNewAudioContext(): AudioContext | void {
-  // @ts-ignore
-  const AudioContext = window.AudioContext || window.webkitAudioContext;
+  const AudioContext =
+    // @ts-ignore
+    typeof window !== 'undefined' && (window.AudioContext || window.webkitAudioContext);
   if (AudioContext) {
     return new AudioContext({ latencyHint: 'interactive' });
   }

package/src/room/types.ts

@@ -20,3 +20,9 @@ export type DataPublishOptions = {
   /** the topic under which the message gets published */
   topic?: string;
 };
+
+export type LiveKitReactNativeInfo = {
+  // Corresponds to RN's PlatformOSType
+  platform: 'ios' | 'android' | 'windows' | 'macos' | 'web' | 'native';
+  devicePixelRatio: number;
+};

package/src/room/utils.ts

@@ -4,6 +4,7 @@ import { protocolVersion, version } from '../version';
 import type LocalAudioTrack from './track/LocalAudioTrack';
 import type RemoteAudioTrack from './track/RemoteAudioTrack';
 import { getNewAudioContext } from './track/utils';
+import type { LiveKitReactNativeInfo } from './types';
 
 const separator = '|';
 
@@ -122,6 +123,54 @@ export function isWeb(): boolean {
   return typeof document !== 'undefined';
 }
 
+export function isReactNative(): boolean {
+  // navigator.product is deprecated on browsers, but will be set appropriately for react-native.
+  return navigator.product == 'ReactNative';
+}
+
+export function isCloud(serverUrl: URL) {
+  return serverUrl.hostname.endsWith('.livekit.cloud');
+}
+
+function getLKReactNativeInfo(): LiveKitReactNativeInfo | undefined {
+  // global defined only for ReactNative.
+  // @ts-ignore
+  if (global && global.LiveKitReactNativeGlobal) {
+    // @ts-ignore
+    return global.LiveKitReactNativeGlobal as LiveKitReactNativeInfo;
+  }
+
+  return undefined;
+}
+
+export function getReactNativeOs(): string | undefined {
+  if (!isReactNative()) {
+    return undefined;
+  }
+
+  let info = getLKReactNativeInfo();
+  if (info) {
+    return info.platform;
+  }
+
+  return undefined;
+}
+
+export function getDevicePixelRatio(): number {
+  if (isWeb()) {
+    return window.devicePixelRatio;
+  }
+
+  if (isReactNative()) {
+    let info = getLKReactNativeInfo();
+    if (info) {
+      return info.devicePixelRatio;
+    }
+  }
+
+  return 1;
+}
+
 export function compareVersions(v1: string, v2: string): number {
   const parts1 = v1.split('.');
   const parts2 = v2.split('.');
@@ -174,6 +223,10 @@ export function getClientInfo(): ClientInfo {
     protocol: protocolVersion,
     version,
   });
+
+  if (isReactNative()) {
+    info.os = getReactNativeOs() ?? '';
+  }
   return info;
 }