livekit-client 0.16.4 → 0.17.0

package/src/room/participant/publishUtils.ts

@@ -26,32 +26,38 @@ export function mediaTrackToLocalTrack(
 }
 
 /* @internal */
-export const presets169 = [
-  VideoPresets.qvga,
-  VideoPresets.vga,
-  VideoPresets.qhd,
-  VideoPresets.hd,
-  VideoPresets.fhd,
-];
+export const presets169 = Object.values(VideoPresets);
+
+/* @internal */
+export const presets43 = Object.values(VideoPresets43);
+
+/* @internal */
+export const presetsScreenShare = Object.values(ScreenSharePresets);
 
 /* @internal */
-export const presets43 = [
-  VideoPresets43.qvga,
-  VideoPresets43.vga,
-  VideoPresets43.qhd,
-  VideoPresets43.hd,
-  VideoPresets43.fhd,
+export const defaultSimulcastPresets169 = [
+  VideoPresets.h180,
+  VideoPresets.h360,
 ];
 
 /* @internal */
-export const presetsScreenShare = [
-  ScreenSharePresets.vga,
-  ScreenSharePresets.hd_8,
-  ScreenSharePresets.hd_15,
-  ScreenSharePresets.fhd_15,
-  ScreenSharePresets.fhd_30,
+export const defaultSimulcastPresets43 = [
+  VideoPresets43.h180,
+  VideoPresets43.h360,
 ];
 
+/* @internal */
+export const computeDefaultScreenShareSimulcastPresets = (fromPreset: VideoPreset) => {
+  const layers = [{ scaleResolutionDownBy: 2, fps: 3 }];
+  return layers.map((t) => new VideoPreset(
+    Math.floor(fromPreset.width / t.scaleResolutionDownBy),
+    Math.floor(fromPreset.height / t.scaleResolutionDownBy),
+    Math.max(150_000, Math.floor(fromPreset.encoding.maxBitrate
+      / (t.scaleResolutionDownBy ** 2 * ((fromPreset.encoding.maxFramerate ?? 30) / t.fps)))),
+    t.fps,
+  ));
+};
+
 const videoRids = ['q', 'h', 'f'];
 
 /* @internal */
@@ -65,7 +71,7 @@ export function computeVideoEncodings(
   if (isScreenShare) {
     videoEncoding = options?.screenShareEncoding;
   }
-  const useSimulcast = !isScreenShare && options?.simulcast;
+  const useSimulcast = options?.simulcast;
 
   if ((!videoEncoding && !useSimulcast) || !width || !height) {
     // when we aren't simulcasting, will need to return a single encoding without
@@ -82,16 +88,22 @@ export function computeVideoEncodings(
   if (!useSimulcast) {
     return [videoEncoding];
   }
-
-  const presets = presetsForResolution(isScreenShare, width, height);
+  const original = new VideoPreset(
+    width, height, videoEncoding.maxBitrate, videoEncoding.maxFramerate,
+  );
+  let presets: Array<VideoPreset> = [];
+  if (isScreenShare) {
+    presets = sortPresets(options?.screenShareSimulcastLayers)
+      ?? defaultSimulcastLayers(isScreenShare, original);
+  } else {
+    presets = sortPresets(options?.videoSimulcastLayers)
+      ?? defaultSimulcastLayers(isScreenShare, original);
+  }
   let midPreset: VideoPreset | undefined;
   const lowPreset = presets[0];
   if (presets.length > 1) {
-    [,midPreset] = presets;
+    [, midPreset] = presets;
   }
-  const original = new VideoPreset(
-    width, height, videoEncoding.maxBitrate, videoEncoding.maxFramerate,
-  );
 
   // NOTE:
   //   1. Ordering of these encodings is important. Chrome seems
@@ -108,7 +120,7 @@ export function computeVideoEncodings(
       lowPreset, midPreset, original,
     ]);
   }
-  if (size >= 500) {
+  if (size >= 480) {
     return encodingsFromPresets(width, height, [
       lowPreset, original,
     ]);
@@ -155,6 +167,21 @@ export function presetsForResolution(
   return presets43;
 }
 
+/* @internal */
+export function defaultSimulcastLayers(
+  isScreenShare: boolean, original: VideoPreset,
+): VideoPreset[] {
+  if (isScreenShare) {
+    return computeDefaultScreenShareSimulcastPresets(original);
+  }
+  const { width, height } = original;
+  const aspect = width > height ? width / height : height / width;
+  if (Math.abs(aspect - 16.0 / 9) < Math.abs(aspect - 4.0 / 3)) {
+    return defaultSimulcastPresets169;
+  }
+  return defaultSimulcastPresets43;
+}
+
 // presets should be ordered by low, medium, high
 function encodingsFromPresets(
   width: number,
@@ -178,3 +205,21 @@ function encodingsFromPresets(
   });
   return encodings;
 }
+
+/** @internal */
+export function sortPresets(presets: Array<VideoPreset> | undefined) {
+  if (!presets) return;
+  return presets.sort((a, b) => {
+    const { encoding: aEnc } = a;
+    const { encoding: bEnc } = b;
+
+    if (aEnc.maxBitrate > bEnc.maxBitrate) {
+      return 1;
+    }
+    if (aEnc.maxBitrate < bEnc.maxBitrate) return -1;
+    if (aEnc.maxBitrate === bEnc.maxBitrate && aEnc.maxFramerate && bEnc.maxFramerate) {
+      return aEnc.maxFramerate > bEnc.maxFramerate ? 1 : -1;
+    }
+    return 0;
+  });
+}

package/src/room/track/LocalAudioTrack.ts

@@ -1,9 +1,10 @@
 import log from '../../logger';
+import { TrackEvent } from '../events';
 import { AudioSenderStats, computeBitrate, monitorFrequency } from '../stats';
 import LocalTrack from './LocalTrack';
 import { AudioCaptureOptions } from './options';
 import { Track } from './Track';
-import { constraintsForOptions } from './utils';
+import { constraintsForOptions, detectSilence } from './utils';
 
 export default class LocalAudioTrack extends LocalTrack {
   sender?: RTCRtpSender;
@@ -18,6 +19,7 @@ export default class LocalAudioTrack extends LocalTrack {
     constraints?: MediaTrackConstraints,
   ) {
     super(mediaTrack, Track.Kind.Audio, constraints);
+    this.checkForSilence();
   }
 
   async setDeviceId(deviceId: string) {
@@ -61,6 +63,12 @@ export default class LocalAudioTrack extends LocalTrack {
     await this.restart(constraints);
   }
 
+  protected async restart(constraints?: MediaTrackConstraints): Promise<LocalTrack> {
+    const track = await super.restart(constraints);
+    this.checkForSilence();
+    return track;
+  }
+
   /* @internal */
   startMonitor() {
     setTimeout(() => {
@@ -116,4 +124,14 @@ export default class LocalAudioTrack extends LocalTrack {
 
     return audioStats;
   }
+
+  async checkForSilence() {
+    const trackIsSilent = await detectSilence(this);
+    if (trackIsSilent) {
+      if (!this.isMuted) {
+        log.warn('silence detected on local audio track');
+      }
+      this.emit(TrackEvent.AudioSilenceDetected);
+    }
+  }
 }

package/src/room/track/LocalTrack.ts

@@ -2,6 +2,7 @@ import log from '../../logger';
 import DeviceManager from '../DeviceManager';
 import { TrackInvalidError } from '../errors';
 import { TrackEvent } from '../events';
+import { isMobile } from '../utils';
 import { attachToElement, detachTrack, Track } from './Track';
 
 export default class LocalTrack extends Track {
@@ -10,12 +11,18 @@ export default class LocalTrack extends Track {
 
   protected constraints: MediaTrackConstraints;
 
+  protected wasMuted: boolean;
+
+  protected reacquireTrack: boolean;
+
   protected constructor(
     mediaTrack: MediaStreamTrack, kind: Track.Kind, constraints?: MediaTrackConstraints,
   ) {
     super(mediaTrack, kind);
     this.mediaStreamTrack.addEventListener('ended', this.handleEnded);
     this.constraints = constraints ?? mediaTrack.getConstraints();
+    this.reacquireTrack = false;
+    this.wasMuted = false;
   }
 
   get id(): string {
@@ -118,7 +125,36 @@ export default class LocalTrack extends Track {
     this.emit(muted ? TrackEvent.Muted : TrackEvent.Unmuted, this);
   }
 
+  protected get needsReAcquisition(): boolean {
+    return this.mediaStreamTrack.readyState !== 'live'
+      || this.mediaStreamTrack.muted
+      || !this.mediaStreamTrack.enabled
+      || this.reacquireTrack;
+  }
+
+  protected async handleAppVisibilityChanged() {
+    await super.handleAppVisibilityChanged();
+    if (!isMobile()) return;
+    log.debug('visibility changed, is in Background: ', this.isInBackground);
+
+    if (!this.isInBackground && this.needsReAcquisition) {
+      log.debug('track needs to be reaquired, restarting', this.source);
+      await this.restart();
+      this.reacquireTrack = false;
+      // Restore muted state if had to be restarted
+      this.setTrackMuted(this.wasMuted);
+    }
+
+    // store muted state each time app goes to background
+    if (this.isInBackground) {
+      this.wasMuted = this.isMuted;
+    }
+  }
+
   private handleEnded = () => {
+    if (this.isInBackground) {
+      this.reacquireTrack = true;
+    }
     this.emit(TrackEvent.Ended, this);
   };
 }

package/src/room/track/LocalVideoTrack.ts

@@ -3,7 +3,7 @@ import log from '../../logger';
 import { VideoLayer, VideoQuality } from '../../proto/livekit_models';
 import { SubscribedQuality } from '../../proto/livekit_rtc';
 import { computeBitrate, monitorFrequency, VideoSenderStats } from '../stats';
-import { isFireFox } from '../utils';
+import { isFireFox, isMobile } from '../utils';
 import LocalTrack from './LocalTrack';
 import { VideoCaptureOptions } from './options';
 import { Track } from './Track';
@@ -238,6 +238,14 @@ export default class LocalVideoTrack extends LocalTrack {
       this.monitorSender();
     }, monitorFrequency);
   };
+
+  protected async handleAppVisibilityChanged() {
+    await super.handleAppVisibilityChanged();
+    if (!isMobile()) return;
+    if (this.isInBackground && this.source === Track.Source.Camera) {
+      this.mediaStreamTrack.enabled = false;
+    }
+  }
 }
 
 export function videoQualityForRid(rid: string): VideoQuality {

package/src/room/track/RemoteTrack.ts

@@ -6,6 +6,8 @@ export default abstract class RemoteTrack extends Track {
   /** @internal */
   receiver?: RTCRtpReceiver;
 
+  streamState: Track.StreamState = Track.StreamState.Active;
+
   constructor(
     mediaTrack: MediaStreamTrack,
     sid: string,

package/src/room/track/RemoteVideoTrack.ts

@@ -4,6 +4,7 @@ import { computeBitrate, monitorFrequency, VideoReceiverStats } from '../stats';
 import { getIntersectionObserver, getResizeObserver, ObservableMediaElement } from '../utils';
 import RemoteTrack from './RemoteTrack';
 import { attachToElement, detachTrack, Track } from './Track';
+import { AdaptiveStreamSettings } from './types';
 
 const REACTION_DELAY = 100;
 
@@ -15,7 +16,7 @@ export default class RemoteVideoTrack extends RemoteTrack {
 
   private elementInfos: ElementInfo[] = [];
 
-  private adaptiveStream?: boolean;
+  private adaptiveStreamSettings?: AdaptiveStreamSettings;
 
   private lastVisible?: boolean;
 
@@ -25,14 +26,14 @@ export default class RemoteVideoTrack extends RemoteTrack {
     mediaTrack: MediaStreamTrack,
     sid: string,
     receiver?: RTCRtpReceiver,
-    adaptiveStream?: boolean,
+    adaptiveStreamSettings?: AdaptiveStreamSettings,
   ) {
     super(mediaTrack, sid, Track.Kind.Video, receiver);
-    this.adaptiveStream = adaptiveStream;
+    this.adaptiveStreamSettings = adaptiveStreamSettings;
   }
 
   get isAdaptiveStream(): boolean {
-    return this.adaptiveStream ?? false;
+    return this.adaptiveStreamSettings !== undefined;
   }
 
   /** @internal */
@@ -60,7 +61,7 @@ export default class RemoteVideoTrack extends RemoteTrack {
 
     // It's possible attach is called multiple times on an element. When that's
     // the case, we'd want to avoid adding duplicate elementInfos
-    if (this.adaptiveStream
+    if (this.adaptiveStreamSettings
       && this.elementInfos.find((info) => info.element === element) === undefined
     ) {
       this.elementInfos.push({
@@ -164,6 +165,12 @@ export default class RemoteVideoTrack extends RemoteTrack {
     this.updateVisibility();
   };
 
+  protected async handleAppVisibilityChanged() {
+    if (!this.isAdaptiveStream) return;
+    await super.handleAppVisibilityChanged();
+    this.updateVisibility();
+  }
+
   private readonly debouncedHandleResize = debounce(() => {
     this.updateDimensions();
   }, REACTION_DELAY);
@@ -173,7 +180,7 @@ export default class RemoteVideoTrack extends RemoteTrack {
       (prev, info) => Math.max(prev, info.visibilityChangedAt || 0),
       0,
     );
-    const isVisible = this.elementInfos.some((info) => info.visible);
+    const isVisible = this.elementInfos.some((info) => info.visible) && !this.isInBackground;
 
     if (this.lastVisible === isVisible) {
       return;
@@ -195,9 +202,13 @@ export default class RemoteVideoTrack extends RemoteTrack {
     let maxWidth = 0;
     let maxHeight = 0;
     for (const info of this.elementInfos) {
-      if (info.element.clientWidth + info.element.clientHeight > maxWidth + maxHeight) {
-        maxWidth = info.element.clientWidth;
-        maxHeight = info.element.clientHeight;
+      const pixelDensity = this.adaptiveStreamSettings?.pixelDensity ?? 1;
+      const pixelDensityValue = pixelDensity === 'screen' ? window.devicePixelRatio : pixelDensity;
+      const currentElementWidth = info.element.clientWidth * pixelDensityValue;
+      const currentElementHeight = info.element.clientHeight * pixelDensityValue;
+      if (currentElementWidth + currentElementHeight > maxWidth + maxHeight) {
+        maxWidth = currentElementWidth;
+        maxHeight = currentElementHeight;
       }
     }
 

package/src/room/track/Track.ts

@@ -18,10 +18,10 @@ export class Track extends (EventEmitter as new () => TypedEventEmitter<TrackEve
 
   isMuted: boolean = false;
 
-  streamState: Track.StreamState = Track.StreamState.Active;
-
   source: Track.Source;
 
+  protected isInBackground: boolean;
+
   /**
    * sid is set after track is published to server, or if it's a remote track
    */
@@ -34,6 +34,8 @@ export class Track extends (EventEmitter as new () => TypedEventEmitter<TrackEve
     this.kind = kind;
     this.mediaStreamTrack = mediaTrack;
     this.source = Track.Source.Unknown;
+    this.isInBackground = document.visibilityState === 'hidden';
+    document.addEventListener('visibilitychange', this.appVisibilityChangedListener);
   }
 
   /** current receive bits per second */
@@ -131,6 +133,7 @@ export class Track extends (EventEmitter as new () => TypedEventEmitter<TrackEve
 
   stop() {
     this.mediaStreamTrack.stop();
+    document.removeEventListener('visibilitychange', this.appVisibilityChangedListener);
   }
 
   protected enable() {
@@ -156,6 +159,14 @@ export class Track extends (EventEmitter as new () => TypedEventEmitter<TrackEve
       }
     }
   }
+
+  appVisibilityChangedListener = () => {
+    this.handleAppVisibilityChanged();
+  };
+
+  protected async handleAppVisibilityChanged() {
+    this.isInBackground = document.visibilityState === 'hidden';
+  }
 }
 
 /** @internal */
@@ -318,6 +329,7 @@ export type TrackEventCallbacks = {
   updateSubscription: () => void,
   audioPlaybackStarted: () => void,
   audioPlaybackFailed: (error: Error) => void,
+  audioSilenceDetected: () => void,
   visibilityChanged: (visible: boolean, track?: any) => void,
   videoDimensionsChanged: (dimensions: Track.Dimensions, track?: any) => void,
 };

package/src/room/track/defaults.ts

@@ -7,7 +7,7 @@ export const publishDefaults: TrackPublishDefaults = {
   audioBitrate: AudioPresets.speech.maxBitrate,
   dtx: true,
   simulcast: true,
-  screenShareEncoding: ScreenSharePresets.hd_15.encoding,
+  screenShareEncoding: ScreenSharePresets.h720fps15.encoding,
   stopMicTrackOnMute: false,
 };
 
@@ -19,5 +19,5 @@ export const audioDefaults: AudioCaptureOptions = {
 };
 
 export const videoDefaults: VideoCaptureOptions = {
-  resolution: VideoPresets.qhd.resolution,
+  resolution: VideoPresets.h540.resolution,
 };

package/src/room/track/options.ts

@@ -33,6 +33,20 @@ export interface TrackPublishDefaults {
    */
   simulcast?: boolean;
 
+  /**
+   * custom video simulcast layers for camera tracks, defaults to h180, h360, h540
+   * You can specify up to two custom layers that will be used instead of
+   * the LiveKit default layers.
+   * Note: the layers need to be ordered from lowest to highest quality
+   */
+  videoSimulcastLayers?: Array<VideoPreset>;
+
+  /**
+   * custom video simulcast layers for screen tracks
+   * Note: the layers need to be ordered from lowest to highest quality
+   */
+  screenShareSimulcastLayers?: Array<VideoPreset>;
+
   /**
    * For local tracks, stop the underlying MediaStreamTrack when the track is muted (or paused)
    * on some platforms, this option is necessary to disable the microphone recording indicator.
@@ -202,28 +216,66 @@ export namespace AudioPresets {
  * Sane presets for video resolution/encoding
  */
 export const VideoPresets = {
+  h90: new VideoPreset(160, 90, 60_000, 15),
+  h180: new VideoPreset(320, 180, 120_000, 15),
+  h216: new VideoPreset(384, 216, 180_000, 15),
+  h360: new VideoPreset(640, 360, 300_000, 20),
+  h540: new VideoPreset(960, 540, 600_000, 25),
+  h720: new VideoPreset(1280, 720, 2_000_000, 30),
+  h1080: new VideoPreset(1920, 1080, 3_000_000, 30),
+  h1440: new VideoPreset(2560, 1440, 5_000_000, 30),
+  h2160: new VideoPreset(3840, 2160, 8_000_000, 30),
+  /** @deprecated */
   qvga: new VideoPreset(320, 180, 120_000, 10),
+  /** @deprecated */
   vga: new VideoPreset(640, 360, 300_000, 20),
+  /** @deprecated */
   qhd: new VideoPreset(960, 540, 600_000, 25),
+  /** @deprecated */
   hd: new VideoPreset(1280, 720, 2_000_000, 30),
+  /** @deprecated */
   fhd: new VideoPreset(1920, 1080, 3_000_000, 30),
-};
+} as const;
 
 /**
  * Four by three presets
  */
 export const VideoPresets43 = {
+  h120: new VideoPreset(160, 120, 80_000, 15),
+  h180: new VideoPreset(240, 180, 100_000, 15),
+  h240: new VideoPreset(320, 240, 150_000, 15),
+  h360: new VideoPreset(480, 360, 225_000, 20),
+  h480: new VideoPreset(640, 480, 300_000, 20),
+  h540: new VideoPreset(720, 540, 450_000, 25),
+  h720: new VideoPreset(960, 720, 1_500_000, 30),
+  h1080: new VideoPreset(1440, 1080, 2_500_000, 30),
+  h1440: new VideoPreset(1920, 1440, 3_500_000, 30),
+  /** @deprecated */
   qvga: new VideoPreset(240, 180, 90_000, 10),
+  /** @deprecated */
   vga: new VideoPreset(480, 360, 225_000, 20),
+  /** @deprecated */
   qhd: new VideoPreset(720, 540, 450_000, 25),
+  /** @deprecated */
   hd: new VideoPreset(960, 720, 1_500_000, 30),
+  /** @deprecated */
   fhd: new VideoPreset(1440, 1080, 2_800_000, 30),
-};
+} as const;
 
 export const ScreenSharePresets = {
+  h360fps3: new VideoPreset(640, 360, 200_000, 3),
+  h720fps5: new VideoPreset(1280, 720, 400_000, 5),
+  h720fps15: new VideoPreset(1280, 720, 1_000_000, 15),
+  h1080fps15: new VideoPreset(1920, 1080, 1_500_000, 15),
+  h1080fps30: new VideoPreset(1920, 1080, 3_000_000, 30),
+  /** @deprecated */
   vga: new VideoPreset(640, 360, 200_000, 3),
+  /** @deprecated */
   hd_8: new VideoPreset(1280, 720, 400_000, 5),
+  /** @deprecated */
   hd_15: new VideoPreset(1280, 720, 1_000_000, 15),
+  /** @deprecated */
   fhd_15: new VideoPreset(1920, 1080, 1_500_000, 15),
+  /** @deprecated */
   fhd_30: new VideoPreset(1920, 1080, 3_000_000, 30),
-};
+} as const;

package/src/room/track/types.ts

@@ -6,3 +6,15 @@ import type RemoteVideoTrack from './RemoteVideoTrack';
 export type RemoteTrack = RemoteAudioTrack | RemoteVideoTrack;
 export type AudioTrack = RemoteAudioTrack | LocalAudioTrack;
 export type VideoTrack = RemoteVideoTrack | LocalVideoTrack;
+
+export type AdaptiveStreamSettings = {
+  /**
+   * Set a custom pixel density, defaults to 1
+   * When streaming videos on a ultra high definition screen this setting
+   * let's you account for the devicePixelRatio of those screens.
+   * Set it to `screen` to use the actual pixel density of the screen
+   * Note: this might significantly increase the bandwidth consumed by people
+   * streaming on high definition screens.
+   */
+  pixelDensity?: number | 'screen'
+};

package/src/room/track/utils.ts

@@ -1,7 +1,9 @@
+import { sleep } from '../utils';
 import {
   AudioCaptureOptions, CreateLocalTracksOptions,
   VideoCaptureOptions,
 } from './options';
+import { AudioTrack } from './types';
 
 export function mergeDefaultOptions(
   options?: CreateLocalTracksOptions,
@@ -74,3 +76,40 @@ export function constraintsForOptions(options: CreateLocalTracksOptions): MediaS
   }
   return constraints;
 }
+/**
+ * This function detects silence on a given [[Track]] instance.
+ * Returns true if the track seems to be entirely silent.
+ */
+export async function detectSilence(
+  track: AudioTrack,
+  timeOffset = 200,
+): Promise<boolean> {
+  const ctx = getNewAudioContext();
+  if (ctx) {
+    const analyser = ctx.createAnalyser();
+    analyser.fftSize = 2048;
+
+    const bufferLength = analyser.frequencyBinCount;
+    const dataArray = new Uint8Array(bufferLength);
+    const source = ctx.createMediaStreamSource(new MediaStream([track.mediaStreamTrack]));
+
+    source.connect(analyser);
+    await sleep(timeOffset);
+    analyser.getByteTimeDomainData(dataArray);
+    const someNoise = dataArray.some((sample) => sample !== 128 && sample !== 0);
+    ctx.close();
+    return !someNoise;
+  }
+  return false;
+}
+
+/**
+ * @internal
+ */
+export function getNewAudioContext(): AudioContext | void {
+  // @ts-ignore
+  const AudioContext = window.AudioContext || window.webkitAudioContext;
+  if (AudioContext) {
+    return new AudioContext();
+  }
+}

package/src/room/utils.ts

@@ -23,6 +23,10 @@ export function isSafari(): boolean {
   return /^((?!chrome|android).)*safari/i.test(navigator.userAgent);
 }
 
+export function isMobile(): boolean {
+  return /Tablet|iPad|Mobile|Android|BlackBerry/.test(navigator.userAgent);
+}
+
 function roDispatchCallback(entries: ResizeObserverEntry[]) {
   for (const entry of entries) {
     (entry.target as ObservableMediaElement).handleResize(entry);

package/src/version.ts

@@ -1,2 +1,2 @@
-export const version = '0.16.4';
+export const version = '0.16.6';
 export const protocolVersion = 6;