livekit-client 0.16.6 → 0.17.2

package/src/room/track/LocalAudioTrack.ts

@@ -1,9 +1,10 @@
 import log from '../../logger';
+import { TrackEvent } from '../events';
 import { AudioSenderStats, computeBitrate, monitorFrequency } from '../stats';
 import LocalTrack from './LocalTrack';
 import { AudioCaptureOptions } from './options';
 import { Track } from './Track';
-import { constraintsForOptions } from './utils';
+import { constraintsForOptions, detectSilence } from './utils';
 
 export default class LocalAudioTrack extends LocalTrack {
   sender?: RTCRtpSender;
@@ -18,6 +19,7 @@ export default class LocalAudioTrack extends LocalTrack {
     constraints?: MediaTrackConstraints,
   ) {
     super(mediaTrack, Track.Kind.Audio, constraints);
+    this.checkForSilence();
   }
 
   async setDeviceId(deviceId: string) {
@@ -61,6 +63,12 @@ export default class LocalAudioTrack extends LocalTrack {
     await this.restart(constraints);
   }
 
+  protected async restart(constraints?: MediaTrackConstraints): Promise<LocalTrack> {
+    const track = await super.restart(constraints);
+    this.checkForSilence();
+    return track;
+  }
+
   /* @internal */
   startMonitor() {
     setTimeout(() => {
@@ -116,4 +124,14 @@ export default class LocalAudioTrack extends LocalTrack {
 
     return audioStats;
   }
+
+  async checkForSilence() {
+    const trackIsSilent = await detectSilence(this);
+    if (trackIsSilent) {
+      if (!this.isMuted) {
+        log.warn('silence detected on local audio track');
+      }
+      this.emit(TrackEvent.AudioSilenceDetected);
+    }
+  }
 }

package/src/room/track/LocalTrack.ts

@@ -2,6 +2,7 @@ import log from '../../logger';
 import DeviceManager from '../DeviceManager';
 import { TrackInvalidError } from '../errors';
 import { TrackEvent } from '../events';
+import { isMobile } from '../utils';
 import { attachToElement, detachTrack, Track } from './Track';
 
 export default class LocalTrack extends Track {
@@ -10,12 +11,18 @@ export default class LocalTrack extends Track {
 
   protected constraints: MediaTrackConstraints;
 
+  protected wasMuted: boolean;
+
+  protected reacquireTrack: boolean;
+
   protected constructor(
     mediaTrack: MediaStreamTrack, kind: Track.Kind, constraints?: MediaTrackConstraints,
   ) {
     super(mediaTrack, kind);
     this.mediaStreamTrack.addEventListener('ended', this.handleEnded);
     this.constraints = constraints ?? mediaTrack.getConstraints();
+    this.reacquireTrack = false;
+    this.wasMuted = false;
   }
 
   get id(): string {
@@ -104,6 +111,7 @@ export default class LocalTrack extends Track {
       attachToElement(newTrack, el);
     });
 
+    this.mediaStream = mediaStream;
     this.constraints = constraints;
     return this;
   }
@@ -118,7 +126,36 @@ export default class LocalTrack extends Track {
     this.emit(muted ? TrackEvent.Muted : TrackEvent.Unmuted, this);
   }
 
+  protected get needsReAcquisition(): boolean {
+    return this.mediaStreamTrack.readyState !== 'live'
+      || this.mediaStreamTrack.muted
+      || !this.mediaStreamTrack.enabled
+      || this.reacquireTrack;
+  }
+
+  protected async handleAppVisibilityChanged() {
+    await super.handleAppVisibilityChanged();
+    if (!isMobile()) return;
+    log.debug('visibility changed, is in Background: ', this.isInBackground);
+
+    if (!this.isInBackground && this.needsReAcquisition) {
+      log.debug('track needs to be reaquired, restarting', this.source);
+      await this.restart();
+      this.reacquireTrack = false;
+      // Restore muted state if had to be restarted
+      this.setTrackMuted(this.wasMuted);
+    }
+
+    // store muted state each time app goes to background
+    if (this.isInBackground) {
+      this.wasMuted = this.isMuted;
+    }
+  }
+
   private handleEnded = () => {
+    if (this.isInBackground) {
+      this.reacquireTrack = true;
+    }
     this.emit(TrackEvent.Ended, this);
   };
 }

package/src/room/track/LocalVideoTrack.ts

@@ -3,7 +3,7 @@ import log from '../../logger';
 import { VideoLayer, VideoQuality } from '../../proto/livekit_models';
 import { SubscribedQuality } from '../../proto/livekit_rtc';
 import { computeBitrate, monitorFrequency, VideoSenderStats } from '../stats';
-import { isFireFox } from '../utils';
+import { isFireFox, isMobile } from '../utils';
 import LocalTrack from './LocalTrack';
 import { VideoCaptureOptions } from './options';
 import { Track } from './Track';
@@ -238,6 +238,14 @@ export default class LocalVideoTrack extends LocalTrack {
       this.monitorSender();
     }, monitorFrequency);
   };
+
+  protected async handleAppVisibilityChanged() {
+    await super.handleAppVisibilityChanged();
+    if (!isMobile()) return;
+    if (this.isInBackground && this.source === Track.Source.Camera) {
+      this.mediaStreamTrack.enabled = false;
+    }
+  }
 }
 
 export function videoQualityForRid(rid: string): VideoQuality {

package/src/room/track/RemoteTrack.ts

@@ -6,6 +6,8 @@ export default abstract class RemoteTrack extends Track {
   /** @internal */
   receiver?: RTCRtpReceiver;
 
+  streamState: Track.StreamState = Track.StreamState.Active;
+
   constructor(
     mediaTrack: MediaStreamTrack,
     sid: string,
@@ -30,6 +32,7 @@ export default abstract class RemoteTrack extends Track {
     // this is needed to determine when the track is finished
     // we send each track down in its own MediaStream, so we can assume the
     // current track is the only one that can be removed.
+    this.mediaStream = stream;
     stream.onremovetrack = () => {
       this.receiver = undefined;
       this._currentBitrate = 0;

package/src/room/track/RemoteVideoTrack.ts

@@ -1,9 +1,12 @@
 import { debounce } from 'ts-debounce';
 import { TrackEvent } from '../events';
 import { computeBitrate, monitorFrequency, VideoReceiverStats } from '../stats';
-import { getIntersectionObserver, getResizeObserver, ObservableMediaElement } from '../utils';
+import {
+  getIntersectionObserver, getResizeObserver, isMobile, ObservableMediaElement,
+} from '../utils';
 import RemoteTrack from './RemoteTrack';
 import { attachToElement, detachTrack, Track } from './Track';
+import { AdaptiveStreamSettings } from './types';
 
 const REACTION_DELAY = 100;
 
@@ -15,7 +18,7 @@ export default class RemoteVideoTrack extends RemoteTrack {
 
   private elementInfos: ElementInfo[] = [];
 
-  private adaptiveStream?: boolean;
+  private adaptiveStreamSettings?: AdaptiveStreamSettings;
 
   private lastVisible?: boolean;
 
@@ -25,14 +28,14 @@ export default class RemoteVideoTrack extends RemoteTrack {
     mediaTrack: MediaStreamTrack,
     sid: string,
     receiver?: RTCRtpReceiver,
-    adaptiveStream?: boolean,
+    adaptiveStreamSettings?: AdaptiveStreamSettings,
   ) {
     super(mediaTrack, sid, Track.Kind.Video, receiver);
-    this.adaptiveStream = adaptiveStream;
+    this.adaptiveStreamSettings = adaptiveStreamSettings;
   }
 
   get isAdaptiveStream(): boolean {
-    return this.adaptiveStream ?? false;
+    return this.adaptiveStreamSettings !== undefined;
   }
 
   /** @internal */
@@ -60,7 +63,7 @@ export default class RemoteVideoTrack extends RemoteTrack {
 
     // It's possible attach is called multiple times on an element. When that's
     // the case, we'd want to avoid adding duplicate elementInfos
-    if (this.adaptiveStream
+    if (this.adaptiveStreamSettings
       && this.elementInfos.find((info) => info.element === element) === undefined
     ) {
       this.elementInfos.push({
@@ -164,6 +167,14 @@ export default class RemoteVideoTrack extends RemoteTrack {
     this.updateVisibility();
   };
 
+  protected async handleAppVisibilityChanged() {
+    await super.handleAppVisibilityChanged();
+    if (!this.isAdaptiveStream) return;
+    // on desktop don't pause when tab is backgrounded
+    if (!isMobile()) return;
+    this.updateVisibility();
+  }
+
   private readonly debouncedHandleResize = debounce(() => {
     this.updateDimensions();
   }, REACTION_DELAY);
@@ -173,7 +184,7 @@ export default class RemoteVideoTrack extends RemoteTrack {
       (prev, info) => Math.max(prev, info.visibilityChangedAt || 0),
       0,
     );
-    const isVisible = this.elementInfos.some((info) => info.visible);
+    const isVisible = this.elementInfos.some((info) => info.visible) && !this.isInBackground;
 
     if (this.lastVisible === isVisible) {
       return;
@@ -195,9 +206,13 @@ export default class RemoteVideoTrack extends RemoteTrack {
     let maxWidth = 0;
     let maxHeight = 0;
     for (const info of this.elementInfos) {
-      if (info.element.clientWidth + info.element.clientHeight > maxWidth + maxHeight) {
-        maxWidth = info.element.clientWidth;
-        maxHeight = info.element.clientHeight;
+      const pixelDensity = this.adaptiveStreamSettings?.pixelDensity ?? 1;
+      const pixelDensityValue = pixelDensity === 'screen' ? window.devicePixelRatio : pixelDensity;
+      const currentElementWidth = info.element.clientWidth * pixelDensityValue;
+      const currentElementHeight = info.element.clientHeight * pixelDensityValue;
+      if (currentElementWidth + currentElementHeight > maxWidth + maxHeight) {
+        maxWidth = currentElementWidth;
+        maxHeight = currentElementHeight;
       }
     }
 

package/src/room/track/Track.ts

@@ -12,16 +12,18 @@ const recycledElements: Array<HTMLAudioElement> = [];
 export class Track extends (EventEmitter as new () => TypedEventEmitter<TrackEventCallbacks>) {
   kind: Track.Kind;
 
+  mediaStream?: MediaStream;
+
   mediaStreamTrack: MediaStreamTrack;
 
   attachedElements: HTMLMediaElement[] = [];
 
   isMuted: boolean = false;
 
-  streamState: Track.StreamState = Track.StreamState.Active;
-
   source: Track.Source;
 
+  protected isInBackground: boolean;
+
   /**
    * sid is set after track is published to server, or if it's a remote track
    */
@@ -34,6 +36,8 @@ export class Track extends (EventEmitter as new () => TypedEventEmitter<TrackEve
     this.kind = kind;
     this.mediaStreamTrack = mediaTrack;
     this.source = Track.Source.Unknown;
+    this.isInBackground = document.visibilityState === 'hidden';
+    document.addEventListener('visibilitychange', this.appVisibilityChangedListener);
   }
 
   /** current receive bits per second */
@@ -131,6 +135,7 @@ export class Track extends (EventEmitter as new () => TypedEventEmitter<TrackEve
 
   stop() {
     this.mediaStreamTrack.stop();
+    document.removeEventListener('visibilitychange', this.appVisibilityChangedListener);
   }
 
   protected enable() {
@@ -156,6 +161,14 @@ export class Track extends (EventEmitter as new () => TypedEventEmitter<TrackEve
       }
     }
   }
+
+  appVisibilityChangedListener = () => {
+    this.handleAppVisibilityChanged();
+  };
+
+  protected async handleAppVisibilityChanged() {
+    this.isInBackground = document.visibilityState === 'hidden';
+  }
 }
 
 /** @internal */
@@ -318,6 +331,7 @@ export type TrackEventCallbacks = {
   updateSubscription: () => void,
   audioPlaybackStarted: () => void,
   audioPlaybackFailed: (error: Error) => void,
+  audioSilenceDetected: () => void,
   visibilityChanged: (visible: boolean, track?: any) => void,
   videoDimensionsChanged: (dimensions: Track.Dimensions, track?: any) => void,
 };

package/src/room/track/create.ts

@@ -47,6 +47,7 @@ export async function createLocalTracks(
     } else if (track.kind === Track.Kind.Audio) {
       track.source = Track.Source.Microphone;
     }
+    track.mediaStream = stream;
     return track;
   });
 }

package/src/room/track/defaults.ts

@@ -7,7 +7,7 @@ export const publishDefaults: TrackPublishDefaults = {
   audioBitrate: AudioPresets.speech.maxBitrate,
   dtx: true,
   simulcast: true,
-  screenShareEncoding: ScreenSharePresets.hd_15.encoding,
+  screenShareEncoding: ScreenSharePresets.h1080fps15.encoding,
   stopMicTrackOnMute: false,
 };
 
@@ -19,5 +19,5 @@ export const audioDefaults: AudioCaptureOptions = {
 };
 
 export const videoDefaults: VideoCaptureOptions = {
-  resolution: VideoPresets.qhd.resolution,
+  resolution: VideoPresets.h540.resolution,
 };

package/src/room/track/options.ts

@@ -33,6 +33,20 @@ export interface TrackPublishDefaults {
    */
   simulcast?: boolean;
 
+  /**
+   * custom video simulcast layers for camera tracks, defaults to h180, h360, h540
+   * You can specify up to two custom layers that will be used instead of
+   * the LiveKit default layers.
+   * Note: the layers need to be ordered from lowest to highest quality
+   */
+  videoSimulcastLayers?: Array<VideoPreset>;
+
+  /**
+   * custom video simulcast layers for screen tracks
+   * Note: the layers need to be ordered from lowest to highest quality
+   */
+  screenShareSimulcastLayers?: Array<VideoPreset>;
+
   /**
    * For local tracks, stop the underlying MediaStreamTrack when the track is muted (or paused)
    * on some platforms, this option is necessary to disable the microphone recording indicator.
@@ -202,28 +216,66 @@ export namespace AudioPresets {
  * Sane presets for video resolution/encoding
  */
 export const VideoPresets = {
+  h90: new VideoPreset(160, 90, 60_000, 15),
+  h180: new VideoPreset(320, 180, 120_000, 15),
+  h216: new VideoPreset(384, 216, 180_000, 15),
+  h360: new VideoPreset(640, 360, 300_000, 20),
+  h540: new VideoPreset(960, 540, 600_000, 25),
+  h720: new VideoPreset(1280, 720, 2_000_000, 30),
+  h1080: new VideoPreset(1920, 1080, 3_000_000, 30),
+  h1440: new VideoPreset(2560, 1440, 5_000_000, 30),
+  h2160: new VideoPreset(3840, 2160, 8_000_000, 30),
+  /** @deprecated */
   qvga: new VideoPreset(320, 180, 120_000, 10),
+  /** @deprecated */
   vga: new VideoPreset(640, 360, 300_000, 20),
+  /** @deprecated */
   qhd: new VideoPreset(960, 540, 600_000, 25),
+  /** @deprecated */
   hd: new VideoPreset(1280, 720, 2_000_000, 30),
+  /** @deprecated */
   fhd: new VideoPreset(1920, 1080, 3_000_000, 30),
-};
+} as const;
 
 /**
  * Four by three presets
  */
 export const VideoPresets43 = {
+  h120: new VideoPreset(160, 120, 80_000, 15),
+  h180: new VideoPreset(240, 180, 100_000, 15),
+  h240: new VideoPreset(320, 240, 150_000, 15),
+  h360: new VideoPreset(480, 360, 225_000, 20),
+  h480: new VideoPreset(640, 480, 300_000, 20),
+  h540: new VideoPreset(720, 540, 450_000, 25),
+  h720: new VideoPreset(960, 720, 1_500_000, 30),
+  h1080: new VideoPreset(1440, 1080, 2_500_000, 30),
+  h1440: new VideoPreset(1920, 1440, 3_500_000, 30),
+  /** @deprecated */
   qvga: new VideoPreset(240, 180, 90_000, 10),
+  /** @deprecated */
   vga: new VideoPreset(480, 360, 225_000, 20),
+  /** @deprecated */
   qhd: new VideoPreset(720, 540, 450_000, 25),
+  /** @deprecated */
   hd: new VideoPreset(960, 720, 1_500_000, 30),
+  /** @deprecated */
   fhd: new VideoPreset(1440, 1080, 2_800_000, 30),
-};
+} as const;
 
 export const ScreenSharePresets = {
+  h360fps3: new VideoPreset(640, 360, 200_000, 3),
+  h720fps5: new VideoPreset(1280, 720, 400_000, 5),
+  h720fps15: new VideoPreset(1280, 720, 1_000_000, 15),
+  h1080fps15: new VideoPreset(1920, 1080, 1_500_000, 15),
+  h1080fps30: new VideoPreset(1920, 1080, 3_000_000, 30),
+  /** @deprecated */
   vga: new VideoPreset(640, 360, 200_000, 3),
+  /** @deprecated */
   hd_8: new VideoPreset(1280, 720, 400_000, 5),
+  /** @deprecated */
   hd_15: new VideoPreset(1280, 720, 1_000_000, 15),
+  /** @deprecated */
   fhd_15: new VideoPreset(1920, 1080, 1_500_000, 15),
+  /** @deprecated */
   fhd_30: new VideoPreset(1920, 1080, 3_000_000, 30),
-};
+} as const;

package/src/room/track/types.ts

@@ -6,3 +6,15 @@ import type RemoteVideoTrack from './RemoteVideoTrack';
 export type RemoteTrack = RemoteAudioTrack | RemoteVideoTrack;
 export type AudioTrack = RemoteAudioTrack | LocalAudioTrack;
 export type VideoTrack = RemoteVideoTrack | LocalVideoTrack;
+
+export type AdaptiveStreamSettings = {
+  /**
+   * Set a custom pixel density, defaults to 1
+   * When streaming videos on a ultra high definition screen this setting
+   * let's you account for the devicePixelRatio of those screens.
+   * Set it to `screen` to use the actual pixel density of the screen
+   * Note: this might significantly increase the bandwidth consumed by people
+   * streaming on high definition screens.
+   */
+  pixelDensity?: number | 'screen'
+};

package/src/room/track/utils.ts

@@ -1,7 +1,9 @@
+import { sleep } from '../utils';
 import {
   AudioCaptureOptions, CreateLocalTracksOptions,
   VideoCaptureOptions,
 } from './options';
+import { AudioTrack } from './types';
 
 export function mergeDefaultOptions(
   options?: CreateLocalTracksOptions,
@@ -74,3 +76,40 @@ export function constraintsForOptions(options: CreateLocalTracksOptions): MediaS
   }
   return constraints;
 }
+/**
+ * This function detects silence on a given [[Track]] instance.
+ * Returns true if the track seems to be entirely silent.
+ */
+export async function detectSilence(
+  track: AudioTrack,
+  timeOffset = 200,
+): Promise<boolean> {
+  const ctx = getNewAudioContext();
+  if (ctx) {
+    const analyser = ctx.createAnalyser();
+    analyser.fftSize = 2048;
+
+    const bufferLength = analyser.frequencyBinCount;
+    const dataArray = new Uint8Array(bufferLength);
+    const source = ctx.createMediaStreamSource(new MediaStream([track.mediaStreamTrack]));
+
+    source.connect(analyser);
+    await sleep(timeOffset);
+    analyser.getByteTimeDomainData(dataArray);
+    const someNoise = dataArray.some((sample) => sample !== 128 && sample !== 0);
+    ctx.close();
+    return !someNoise;
+  }
+  return false;
+}
+
+/**
+ * @internal
+ */
+export function getNewAudioContext(): AudioContext | void {
+  // @ts-ignore
+  const AudioContext = window.AudioContext || window.webkitAudioContext;
+  if (AudioContext) {
+    return new AudioContext();
+  }
+}

package/src/room/utils.ts

@@ -23,6 +23,14 @@ export function isSafari(): boolean {
   return /^((?!chrome|android).)*safari/i.test(navigator.userAgent);
 }
 
+export function isMobile(): boolean {
+  return /Tablet|iPad|Mobile|Android|BlackBerry/.test(navigator.userAgent);
+}
+
+export function isWeb(): boolean {
+  return typeof document !== 'undefined';
+}
+
 function roDispatchCallback(entries: ResizeObserverEntry[]) {
   for (const entry of entries) {
     (entry.target as ObservableMediaElement).handleResize(entry);

package/src/version.ts

@@ -1,2 +1,2 @@
-export const version = '0.16.6';
+export const version = '0.17.2';
 export const protocolVersion = 6;