livekit-client 0.18.4-RC6 → 0.18.4

package/src/room/participant/publishUtils.test.ts

@@ -0,0 +1,144 @@
+import { ScreenSharePresets, VideoPreset, VideoPresets, VideoPresets43 } from '../track/options';
+import {
+  computeDefaultScreenShareSimulcastPresets,
+  computeVideoEncodings,
+  determineAppropriateEncoding,
+  presets169,
+  presets43,
+  presetsForResolution,
+  presetsScreenShare,
+  sortPresets,
+} from './publishUtils';
+
+describe('presetsForResolution', () => {
+  it('handles screenshare', () => {
+    expect(presetsForResolution(true, 600, 300)).toEqual(presetsScreenShare);
+  });
+
+  it('handles landscape', () => {
+    expect(presetsForResolution(false, 600, 300)).toEqual(presets169);
+    expect(presetsForResolution(false, 500, 500)).toEqual(presets43);
+  });
+
+  it('handles portrait', () => {
+    expect(presetsForResolution(false, 300, 600)).toEqual(presets169);
+    expect(presetsForResolution(false, 500, 500)).toEqual(presets43);
+  });
+});
+
+describe('determineAppropriateEncoding', () => {
+  it('uses higher encoding', () => {
+    expect(determineAppropriateEncoding(false, 600, 300)).toEqual(VideoPresets.vga.encoding);
+  });
+
+  it('handles portrait', () => {
+    expect(determineAppropriateEncoding(false, 300, 600)).toEqual(VideoPresets.vga.encoding);
+  });
+});
+
+describe('computeVideoEncodings', () => {
+  it('handles non-simulcast', () => {
+    const encodings = computeVideoEncodings(false, 640, 480, {
+      simulcast: false,
+    });
+    expect(encodings).toEqual([{}]);
+  });
+
+  it('respects client defined bitrate', () => {
+    const encodings = computeVideoEncodings(false, 640, 480, {
+      simulcast: false,
+      videoEncoding: {
+        maxBitrate: 1024,
+      },
+    });
+    expect(encodings).toHaveLength(1);
+    expect(encodings![0].maxBitrate).toBe(1024);
+  });
+
+  it('returns three encodings for high-res simulcast', () => {
+    const encodings = computeVideoEncodings(false, 960, 540, {
+      simulcast: true,
+    });
+    expect(encodings).toHaveLength(3);
+
+    // ensure they are what we expect
+    expect(encodings![0].rid).toBe('q');
+    expect(encodings![0].maxBitrate).toBe(VideoPresets.h180.encoding.maxBitrate);
+    expect(encodings![0].scaleResolutionDownBy).toBe(3);
+    expect(encodings![1].rid).toBe('h');
+    expect(encodings![1].scaleResolutionDownBy).toBe(1.5);
+    expect(encodings![2].rid).toBe('f');
+  });
+
+  it('handles portrait simulcast', () => {
+    const encodings = computeVideoEncodings(false, 540, 960, {
+      simulcast: true,
+    });
+    expect(encodings).toHaveLength(3);
+    expect(encodings![0].scaleResolutionDownBy).toBe(3);
+    expect(encodings![1].scaleResolutionDownBy).toBe(1.5);
+    expect(encodings![2].maxBitrate).toBe(VideoPresets.h540.encoding.maxBitrate);
+  });
+
+  it('returns two encodings for lower-res simulcast', () => {
+    const encodings = computeVideoEncodings(false, 640, 360, {
+      simulcast: true,
+    });
+    expect(encodings).toHaveLength(2);
+
+    // ensure they are what we expect
+    expect(encodings![0].rid).toBe('q');
+    expect(encodings![0].maxBitrate).toBe(VideoPresets.h180.encoding.maxBitrate);
+    expect(encodings![1].rid).toBe('h');
+    expect(encodings![1].maxBitrate).toBe(VideoPresets.h360.encoding.maxBitrate);
+  });
+
+  it('respects provided min resolution', () => {
+    const encodings = computeVideoEncodings(false, 100, 120, {
+      simulcast: true,
+    });
+    expect(encodings).toHaveLength(1);
+    expect(encodings![0].rid).toBe('q');
+    expect(encodings![0].maxBitrate).toBe(VideoPresets43.h120.encoding.maxBitrate);
+    expect(encodings![0].scaleResolutionDownBy).toBe(1);
+  });
+});
+
+describe('customSimulcastLayers', () => {
+  it('sorts presets from lowest to highest', () => {
+    const sortedPresets = sortPresets([
+      VideoPresets.h1440,
+      VideoPresets.h360,
+      VideoPresets.h1080,
+      VideoPresets.h90,
+    ]) as Array<VideoPreset>;
+    expect(sortPresets).not.toBeUndefined();
+    expect(sortedPresets[0]).toBe(VideoPresets.h90);
+    expect(sortedPresets[1]).toBe(VideoPresets.h360);
+    expect(sortedPresets[2]).toBe(VideoPresets.h1080);
+    expect(sortedPresets[3]).toBe(VideoPresets.h1440);
+  });
+  it('sorts presets from lowest to highest, even when dimensions are the same', () => {
+    const sortedPresets = sortPresets([
+      new VideoPreset(1920, 1080, 3_000_000, 20),
+      new VideoPreset(1920, 1080, 2_000_000, 15),
+      new VideoPreset(1920, 1080, 3_000_000, 15),
+    ]) as Array<VideoPreset>;
+    expect(sortPresets).not.toBeUndefined();
+    expect(sortedPresets[0].encoding.maxBitrate).toBe(2_000_000);
+    expect(sortedPresets[1].encoding.maxFramerate).toBe(15);
+    expect(sortedPresets[2].encoding.maxFramerate).toBe(20);
+  });
+});
+
+describe('screenShareSimulcastDefaults', () => {
+  it('computes appropriate bitrate from original preset', () => {
+    const defaultSimulcastLayers = computeDefaultScreenShareSimulcastPresets(
+      ScreenSharePresets.h720fps15,
+    );
+    expect(defaultSimulcastLayers[0].width).toBe(640);
+    expect(defaultSimulcastLayers[0].height).toBe(360);
+    expect(defaultSimulcastLayers[0].encoding.maxFramerate).toBe(3);
+    expect(defaultSimulcastLayers[0].encoding.maxBitrate).toBe(150_000);
+  });
+});

package/src/room/participant/publishUtils.ts

@@ -0,0 +1,229 @@
+import log from '../../logger';
+import { TrackInvalidError } from '../errors';
+import LocalAudioTrack from '../track/LocalAudioTrack';
+import LocalVideoTrack from '../track/LocalVideoTrack';
+import {
+  ScreenSharePresets,
+  TrackPublishOptions,
+  VideoEncoding,
+  VideoPreset,
+  VideoPresets,
+  VideoPresets43,
+} from '../track/options';
+
+/** @internal */
+export function mediaTrackToLocalTrack(
+  mediaStreamTrack: MediaStreamTrack,
+  constraints?: MediaTrackConstraints,
+): LocalVideoTrack | LocalAudioTrack {
+  switch (mediaStreamTrack.kind) {
+    case 'audio':
+      return new LocalAudioTrack(mediaStreamTrack, constraints);
+    case 'video':
+      return new LocalVideoTrack(mediaStreamTrack, constraints);
+    default:
+      throw new TrackInvalidError(`unsupported track type: ${mediaStreamTrack.kind}`);
+  }
+}
+
+/* @internal */
+export const presets169 = Object.values(VideoPresets);
+
+/* @internal */
+export const presets43 = Object.values(VideoPresets43);
+
+/* @internal */
+export const presetsScreenShare = Object.values(ScreenSharePresets);
+
+/* @internal */
+export const defaultSimulcastPresets169 = [VideoPresets.h180, VideoPresets.h360];
+
+/* @internal */
+export const defaultSimulcastPresets43 = [VideoPresets43.h180, VideoPresets43.h360];
+
+/* @internal */
+export const computeDefaultScreenShareSimulcastPresets = (fromPreset: VideoPreset) => {
+  const layers = [{ scaleResolutionDownBy: 2, fps: 3 }];
+  return layers.map(
+    (t) =>
+      new VideoPreset(
+        Math.floor(fromPreset.width / t.scaleResolutionDownBy),
+        Math.floor(fromPreset.height / t.scaleResolutionDownBy),
+        Math.max(
+          150_000,
+          Math.floor(
+            fromPreset.encoding.maxBitrate /
+              (t.scaleResolutionDownBy ** 2 * ((fromPreset.encoding.maxFramerate ?? 30) / t.fps)),
+          ),
+        ),
+        t.fps,
+      ),
+  );
+};
+
+const videoRids = ['q', 'h', 'f'];
+
+/* @internal */
+export function computeVideoEncodings(
+  isScreenShare: boolean,
+  width?: number,
+  height?: number,
+  options?: TrackPublishOptions,
+): RTCRtpEncodingParameters[] {
+  let videoEncoding: VideoEncoding | undefined = options?.videoEncoding;
+  if (isScreenShare) {
+    videoEncoding = options?.screenShareEncoding;
+  }
+  const useSimulcast = options?.simulcast;
+
+  if ((!videoEncoding && !useSimulcast) || !width || !height) {
+    // when we aren't simulcasting, will need to return a single encoding without
+    // capping bandwidth. we always require a encoding for dynacast
+    return [{}];
+  }
+
+  if (!videoEncoding) {
+    // find the right encoding based on width/height
+    videoEncoding = determineAppropriateEncoding(isScreenShare, width, height);
+    log.debug('using video encoding', videoEncoding);
+  }
+
+  if (!useSimulcast) {
+    return [videoEncoding];
+  }
+  const original = new VideoPreset(
+    width,
+    height,
+    videoEncoding.maxBitrate,
+    videoEncoding.maxFramerate,
+  );
+  let presets: Array<VideoPreset> = [];
+  if (isScreenShare) {
+    presets =
+      sortPresets(options?.screenShareSimulcastLayers) ??
+      defaultSimulcastLayers(isScreenShare, original);
+  } else {
+    presets =
+      sortPresets(options?.videoSimulcastLayers) ?? defaultSimulcastLayers(isScreenShare, original);
+  }
+  let midPreset: VideoPreset | undefined;
+  const lowPreset = presets[0];
+  if (presets.length > 1) {
+    [, midPreset] = presets;
+  }
+
+  // NOTE:
+  //   1. Ordering of these encodings is important. Chrome seems
+  //      to use the index into encodings to decide which layer
+  //      to disable when CPU constrained.
+  //      So encodings should be ordered in increasing spatial
+  //      resolution order.
+  //   2. ion-sfu translates rids into layers. So, all encodings
+  //      should have the base layer `q` and then more added
+  //      based on other conditions.
+  const size = Math.max(width, height);
+  if (size >= 960 && midPreset) {
+    return encodingsFromPresets(width, height, [lowPreset, midPreset, original]);
+  }
+  if (size >= 480) {
+    return encodingsFromPresets(width, height, [lowPreset, original]);
+  }
+  return encodingsFromPresets(width, height, [original]);
+}
+
+/* @internal */
+export function determineAppropriateEncoding(
+  isScreenShare: boolean,
+  width: number,
+  height: number,
+): VideoEncoding {
+  const presets = presetsForResolution(isScreenShare, width, height);
+  let { encoding } = presets[0];
+
+  // handle portrait by swapping dimensions
+  const size = Math.max(width, height);
+
+  for (let i = 0; i < presets.length; i += 1) {
+    const preset = presets[i];
+    encoding = preset.encoding;
+    if (preset.width >= size) {
+      break;
+    }
+  }
+
+  return encoding;
+}
+
+/* @internal */
+export function presetsForResolution(
+  isScreenShare: boolean,
+  width: number,
+  height: number,
+): VideoPreset[] {
+  if (isScreenShare) {
+    return presetsScreenShare;
+  }
+  const aspect = width > height ? width / height : height / width;
+  if (Math.abs(aspect - 16.0 / 9) < Math.abs(aspect - 4.0 / 3)) {
+    return presets169;
+  }
+  return presets43;
+}
+
+/* @internal */
+export function defaultSimulcastLayers(
+  isScreenShare: boolean,
+  original: VideoPreset,
+): VideoPreset[] {
+  if (isScreenShare) {
+    return computeDefaultScreenShareSimulcastPresets(original);
+  }
+  const { width, height } = original;
+  const aspect = width > height ? width / height : height / width;
+  if (Math.abs(aspect - 16.0 / 9) < Math.abs(aspect - 4.0 / 3)) {
+    return defaultSimulcastPresets169;
+  }
+  return defaultSimulcastPresets43;
+}
+
+// presets should be ordered by low, medium, high
+function encodingsFromPresets(
+  width: number,
+  height: number,
+  presets: VideoPreset[],
+): RTCRtpEncodingParameters[] {
+  const encodings: RTCRtpEncodingParameters[] = [];
+  presets.forEach((preset, idx) => {
+    if (idx >= videoRids.length) {
+      return;
+    }
+    const size = Math.min(width, height);
+    const rid = videoRids[idx];
+    encodings.push({
+      rid,
+      scaleResolutionDownBy: size / Math.min(preset.width, preset.height),
+      maxBitrate: preset.encoding.maxBitrate,
+      /* @ts-ignore */
+      maxFramerate: preset.encoding.maxFramerate,
+    });
+  });
+  return encodings;
+}
+
+/** @internal */
+export function sortPresets(presets: Array<VideoPreset> | undefined) {
+  if (!presets) return;
+  return presets.sort((a, b) => {
+    const { encoding: aEnc } = a;
+    const { encoding: bEnc } = b;
+
+    if (aEnc.maxBitrate > bEnc.maxBitrate) {
+      return 1;
+    }
+    if (aEnc.maxBitrate < bEnc.maxBitrate) return -1;
+    if (aEnc.maxBitrate === bEnc.maxBitrate && aEnc.maxFramerate && bEnc.maxFramerate) {
+      return aEnc.maxFramerate > bEnc.maxFramerate ? 1 : -1;
+    }
+    return 0;
+  });
+}

package/src/room/stats.ts

@@ -0,0 +1,134 @@
+export const monitorFrequency = 2000;
+
+// key stats for senders and receivers
+interface SenderStats {
+  /** number of packets sent */
+  packetsSent?: number;
+
+  /** number of bytes sent */
+  bytesSent?: number;
+
+  /** jitter as perceived by remote */
+  jitter?: number;
+
+  /** packets reported lost by remote */
+  packetsLost?: number;
+
+  /** RTT reported by remote */
+  roundTripTime?: number;
+
+  /** ID of the outbound stream */
+  streamId?: string;
+
+  timestamp: number;
+}
+
+export interface AudioSenderStats extends SenderStats {
+  type: 'audio';
+}
+
+export interface VideoSenderStats extends SenderStats {
+  type: 'video';
+
+  firCount: number;
+
+  pliCount: number;
+
+  nackCount: number;
+
+  rid: string;
+
+  frameWidth: number;
+
+  frameHeight: number;
+
+  framesSent: number;
+
+  // bandwidth, cpu, other, none
+  qualityLimitationReason: string;
+
+  qualityLimitationResolutionChanges: number;
+
+  retransmittedPacketsSent: number;
+}
+
+interface ReceiverStats {
+  jitterBufferDelay?: number;
+
+  /** packets reported lost by remote */
+  packetsLost?: number;
+
+  /** number of packets sent */
+  packetsReceived?: number;
+
+  bytesReceived?: number;
+
+  streamId?: string;
+
+  jitter?: number;
+
+  timestamp: number;
+}
+
+export interface AudioReceiverStats extends ReceiverStats {
+  type: 'audio';
+
+  concealedSamples?: number;
+
+  concealmentEvents?: number;
+
+  silentConcealedSamples?: number;
+
+  silentConcealmentEvents?: number;
+
+  totalAudioEnergy?: number;
+
+  totalSamplesDuration?: number;
+}
+
+export interface VideoReceiverStats extends ReceiverStats {
+  type: 'video';
+
+  framesDecoded: number;
+
+  framesDropped: number;
+
+  framesReceived: number;
+
+  frameWidth?: number;
+
+  frameHeight?: number;
+
+  firCount?: number;
+
+  pliCount?: number;
+
+  nackCount?: number;
+}
+
+export function computeBitrate<T extends ReceiverStats | SenderStats>(
+  currentStats: T,
+  prevStats?: T,
+): number {
+  if (!prevStats) {
+    return 0;
+  }
+  let bytesNow: number | undefined;
+  let bytesPrev: number | undefined;
+  if ('bytesReceived' in currentStats) {
+    bytesNow = (currentStats as ReceiverStats).bytesReceived;
+    bytesPrev = (prevStats as ReceiverStats).bytesReceived;
+  } else if ('bytesSent' in currentStats) {
+    bytesNow = (currentStats as SenderStats).bytesSent;
+    bytesPrev = (prevStats as SenderStats).bytesSent;
+  }
+  if (
+    bytesNow === undefined ||
+    bytesPrev === undefined ||
+    currentStats.timestamp === undefined ||
+    prevStats.timestamp === undefined
+  ) {
+    return 0;
+  }
+  return ((bytesNow - bytesPrev) * 8 * 1000) / (currentStats.timestamp - prevStats.timestamp);
+}

package/src/room/track/LocalAudioTrack.ts

@@ -0,0 +1,134 @@
+import log from '../../logger';
+import { TrackEvent } from '../events';
+import { AudioSenderStats, computeBitrate, monitorFrequency } from '../stats';
+import LocalTrack from './LocalTrack';
+import { AudioCaptureOptions } from './options';
+import { Track } from './Track';
+import { constraintsForOptions, detectSilence } from './utils';
+
+export default class LocalAudioTrack extends LocalTrack {
+  sender?: RTCRtpSender;
+
+  /** @internal */
+  stopOnMute: boolean = false;
+
+  private prevStats?: AudioSenderStats;
+
+  constructor(mediaTrack: MediaStreamTrack, constraints?: MediaTrackConstraints) {
+    super(mediaTrack, Track.Kind.Audio, constraints);
+    this.checkForSilence();
+  }
+
+  async setDeviceId(deviceId: string) {
+    if (this.constraints.deviceId === deviceId) {
+      return;
+    }
+    this.constraints.deviceId = deviceId;
+    if (!this.isMuted) {
+      await this.restartTrack();
+    }
+  }
+
+  async mute(): Promise<LocalAudioTrack> {
+    // disabled special handling as it will cause BT headsets to switch communication modes
+    if (this.source === Track.Source.Microphone && this.stopOnMute) {
+      log.debug('stopping mic track');
+      // also stop the track, so that microphone indicator is turned off
+      this.mediaStreamTrack.stop();
+    }
+    await super.mute();
+    return this;
+  }
+
+  async unmute(): Promise<LocalAudioTrack> {
+    if (this.source === Track.Source.Microphone && this.stopOnMute) {
+      log.debug('reacquiring mic track');
+      await this.restartTrack();
+    }
+    await super.unmute();
+    return this;
+  }
+
+  async restartTrack(options?: AudioCaptureOptions) {
+    let constraints: MediaTrackConstraints | undefined;
+    if (options) {
+      const streamConstraints = constraintsForOptions({ audio: options });
+      if (typeof streamConstraints.audio !== 'boolean') {
+        constraints = streamConstraints.audio;
+      }
+    }
+    await this.restart(constraints);
+  }
+
+  protected async restart(constraints?: MediaTrackConstraints): Promise<LocalTrack> {
+    const track = await super.restart(constraints);
+    this.checkForSilence();
+    return track;
+  }
+
+  /* @internal */
+  startMonitor() {
+    setTimeout(() => {
+      this.monitorSender();
+    }, monitorFrequency);
+  }
+
+  private monitorSender = async () => {
+    if (!this.sender) {
+      this._currentBitrate = 0;
+      return;
+    }
+
+    let stats: AudioSenderStats | undefined;
+    try {
+      stats = await this.getSenderStats();
+    } catch (e) {
+      log.error('could not get audio sender stats', e);
+      return;
+    }
+
+    if (stats && this.prevStats) {
+      this._currentBitrate = computeBitrate(stats, this.prevStats);
+    }
+
+    this.prevStats = stats;
+    setTimeout(() => {
+      this.monitorSender();
+    }, monitorFrequency);
+  };
+
+  async getSenderStats(): Promise<AudioSenderStats | undefined> {
+    if (!this.sender) {
+      return undefined;
+    }
+
+    const stats = await this.sender.getStats();
+    let audioStats: AudioSenderStats | undefined;
+    stats.forEach((v) => {
+      if (v.type === 'outbound-rtp') {
+        audioStats = {
+          type: 'audio',
+          streamId: v.id,
+          packetsSent: v.packetsSent,
+          packetsLost: v.packetsLost,
+          bytesSent: v.bytesSent,
+          timestamp: v.timestamp,
+          roundTripTime: v.roundTripTime,
+          jitter: v.jitter,
+        };
+      }
+    });
+
+    return audioStats;
+  }
+
+  async checkForSilence() {
+    const trackIsSilent = await detectSilence(this);
+    if (trackIsSilent) {
+      if (!this.isMuted) {
+        log.warn('silence detected on local audio track');
+      }
+      this.emit(TrackEvent.AudioSilenceDetected);
+    }
+  }
+}