livekit-client 0.15.1 → 0.16.0

package/src/room/participant/publishUtils.ts

@@ -0,0 +1,180 @@
+import log from '../../logger';
+import { TrackInvalidError } from '../errors';
+import LocalAudioTrack from '../track/LocalAudioTrack';
+import LocalVideoTrack from '../track/LocalVideoTrack';
+import {
+  ScreenSharePresets, TrackPublishOptions,
+  VideoEncoding, VideoPreset, VideoPresets,
+  VideoPresets43,
+} from '../track/options';
+
+/** @internal */
+export function mediaTrackToLocalTrack(
+  mediaStreamTrack: MediaStreamTrack,
+  constraints?: MediaTrackConstraints,
+): LocalVideoTrack | LocalAudioTrack {
+  switch (mediaStreamTrack.kind) {
+    case 'audio':
+      return new LocalAudioTrack(mediaStreamTrack, constraints);
+    case 'video':
+      return new LocalVideoTrack(mediaStreamTrack, constraints);
+    default:
+      throw new TrackInvalidError(
+        `unsupported track type: ${mediaStreamTrack.kind}`,
+      );
+  }
+}
+
+/* @internal */
+export const presets169 = [
+  VideoPresets.qvga,
+  VideoPresets.vga,
+  VideoPresets.qhd,
+  VideoPresets.hd,
+  VideoPresets.fhd,
+];
+
+/* @internal */
+export const presets43 = [
+  VideoPresets43.qvga,
+  VideoPresets43.vga,
+  VideoPresets43.qhd,
+  VideoPresets43.hd,
+  VideoPresets43.fhd,
+];
+
+/* @internal */
+export const presetsScreenShare = [
+  ScreenSharePresets.vga,
+  ScreenSharePresets.hd_8,
+  ScreenSharePresets.hd_15,
+  ScreenSharePresets.fhd_15,
+  ScreenSharePresets.fhd_30,
+];
+
+const videoRids = ['q', 'h', 'f'];
+
+/* @internal */
+export function computeVideoEncodings(
+  isScreenShare: boolean,
+  width?: number,
+  height?: number,
+  options?: TrackPublishOptions,
+): RTCRtpEncodingParameters[] {
+  let videoEncoding: VideoEncoding | undefined = options?.videoEncoding;
+  if (isScreenShare) {
+    videoEncoding = options?.screenShareEncoding;
+  }
+  const useSimulcast = !isScreenShare && options?.simulcast;
+
+  if ((!videoEncoding && !useSimulcast) || !width || !height) {
+    // when we aren't simulcasting, will need to return a single encoding without
+    // capping bandwidth. we always require a encoding for dynacast
+    return [{}];
+  }
+
+  if (!videoEncoding) {
+    // find the right encoding based on width/height
+    videoEncoding = determineAppropriateEncoding(isScreenShare, width, height);
+    log.debug('using video encoding', videoEncoding);
+  }
+
+  if (!useSimulcast) {
+    return [videoEncoding];
+  }
+
+  const presets = presetsForResolution(isScreenShare, width, height);
+  let midPreset: VideoPreset | undefined;
+  const lowPreset = presets[0];
+  if (presets.length > 1) {
+    [,midPreset] = presets;
+  }
+  const original = new VideoPreset(
+    width, height, videoEncoding.maxBitrate, videoEncoding.maxFramerate,
+  );
+
+  // NOTE:
+  //   1. Ordering of these encodings is important. Chrome seems
+  //      to use the index into encodings to decide which layer
+  //      to disable when CPU constrained.
+  //      So encodings should be ordered in increasing spatial
+  //      resolution order.
+  //   2. ion-sfu translates rids into layers. So, all encodings
+  //      should have the base layer `q` and then more added
+  //      based on other conditions.
+  const size = Math.max(width, height);
+  if (size >= 960 && midPreset) {
+    return encodingsFromPresets(width, height, [
+      lowPreset, midPreset, original,
+    ]);
+  }
+  if (size >= 500) {
+    return encodingsFromPresets(width, height, [
+      lowPreset, original,
+    ]);
+  }
+  return encodingsFromPresets(width, height, [
+    original,
+  ]);
+}
+
+/* @internal */
+export function determineAppropriateEncoding(
+  isScreenShare: boolean,
+  width: number,
+  height: number,
+): VideoEncoding {
+  const presets = presetsForResolution(isScreenShare, width, height);
+  let { encoding } = presets[0];
+
+  // handle portrait by swapping dimensions
+  const size = Math.max(width, height);
+
+  for (let i = 0; i < presets.length; i += 1) {
+    const preset = presets[i];
+    encoding = preset.encoding;
+    if (preset.width >= size) {
+      break;
+    }
+  }
+
+  return encoding;
+}
+
+/* @internal */
+export function presetsForResolution(
+  isScreenShare: boolean, width: number, height: number,
+): VideoPreset[] {
+  if (isScreenShare) {
+    return presetsScreenShare;
+  }
+  const aspect = width > height ? width / height : height / width;
+  if (Math.abs(aspect - 16.0 / 9) < Math.abs(aspect - 4.0 / 3)) {
+    return presets169;
+  }
+  return presets43;
+}
+
+// presets should be ordered by low, medium, high
+function encodingsFromPresets(
+  width: number,
+  height: number,
+  presets: VideoPreset[],
+): RTCRtpEncodingParameters[] {
+  const encodings: RTCRtpEncodingParameters[] = [];
+  presets.forEach((preset, idx) => {
+    if (idx >= videoRids.length) {
+      return;
+    }
+    const size = Math.min(width, height);
+    const rid = videoRids[idx];
+    encodings.push({
+      rid,
+      scaleResolutionDownBy: size / Math.min(preset.width, preset.height),
+      maxBitrate: preset.encoding.maxBitrate,
+      /* @ts-ignore */
+      maxFramerate: preset.encoding.maxFramerate,
+    });
+  });
+  return encodings;
+}

package/src/room/stats.ts

@@ -0,0 +1,130 @@
+export const monitorFrequency = 2000;
+
+// key stats for senders and receivers
+interface SenderStats {
+  /** number of packets sent */
+  packetsSent?: number;
+
+  /** number of bytes sent */
+  bytesSent?: number;
+
+  /** jitter as perceived by remote */
+  jitter?: number;
+
+  /** packets reported lost by remote */
+  packetsLost?: number;
+
+  /** RTT reported by remote */
+  roundTripTime?: number;
+
+  /** ID of the outbound stream */
+  streamId?: string;
+
+  timestamp: number;
+}
+
+export interface AudioSenderStats extends SenderStats {
+  type: 'audio';
+}
+
+export interface VideoSenderStats extends SenderStats {
+  type: 'video';
+
+  firCount: number;
+
+  pliCount: number;
+
+  nackCount: number;
+
+  rid: string;
+
+  frameWidth: number;
+
+  frameHeight: number;
+
+  framesSent: number;
+
+  // bandwidth, cpu, other, none
+  qualityLimitationReason: string;
+
+  qualityLimitationResolutionChanges: number;
+
+  retransmittedPacketsSent: number;
+}
+
+interface ReceiverStats {
+  jitterBufferDelay?: number;
+
+  /** packets reported lost by remote */
+  packetsLost?: number;
+
+  /** number of packets sent */
+  packetsReceived?: number;
+
+  bytesReceived?: number;
+
+  streamId?: string;
+
+  jitter?: number;
+
+  timestamp: number;
+}
+
+export interface AudioReceiverStats extends ReceiverStats {
+  type: 'audio';
+
+  concealedSamples?: number;
+
+  concealmentEvents?: number;
+
+  silentConcealedSamples?: number;
+
+  silentConcealmentEvents?: number;
+
+  totalAudioEnergy?: number;
+
+  totalSamplesDuration?: number;
+}
+
+export interface VideoReceiverStats extends ReceiverStats {
+  type: 'video';
+
+  framesDecoded: number;
+
+  framesDropped: number;
+
+  framesReceived: number;
+
+  frameWidth?: number;
+
+  frameHeight?: number;
+
+  firCount?: number;
+
+  pliCount?: number;
+
+  nackCount?: number;
+}
+
+export function computeBitrate<T extends ReceiverStats | SenderStats>(
+  currentStats: T,
+  prevStats?: T,
+): number {
+  if (!prevStats) {
+    return 0;
+  }
+  let bytesNow: number | undefined;
+  let bytesPrev: number | undefined;
+  if ('bytesReceived' in currentStats) {
+    bytesNow = (currentStats as ReceiverStats).bytesReceived;
+    bytesPrev = (prevStats as ReceiverStats).bytesReceived;
+  } else if ('bytesSent' in currentStats) {
+    bytesNow = (currentStats as SenderStats).bytesSent;
+    bytesPrev = (prevStats as SenderStats).bytesSent;
+  }
+  if (bytesNow === undefined || bytesPrev === undefined
+    || currentStats.timestamp === undefined || prevStats.timestamp === undefined) {
+    return 0;
+  }
+  return ((bytesNow - bytesPrev) * 8 * 1000) / (currentStats.timestamp - prevStats.timestamp);
+}

package/src/room/track/LocalAudioTrack.ts

@@ -0,0 +1,112 @@
+import log from '../../logger';
+import { AudioSenderStats, computeBitrate, monitorFrequency } from '../stats';
+import LocalTrack from './LocalTrack';
+import { AudioCaptureOptions } from './options';
+import { Track } from './Track';
+import { constraintsForOptions } from './utils';
+
+export default class LocalAudioTrack extends LocalTrack {
+  sender?: RTCRtpSender;
+
+  /** @internal */
+  stopOnMute: boolean = false;
+
+  private prevStats?: AudioSenderStats;
+
+  constructor(
+    mediaTrack: MediaStreamTrack,
+    constraints?: MediaTrackConstraints,
+  ) {
+    super(mediaTrack, Track.Kind.Audio, constraints);
+  }
+
+  async setDeviceId(deviceId: string) {
+    if (this.constraints.deviceId === deviceId) {
+      return;
+    }
+    this.constraints.deviceId = deviceId;
+    if (!this.isMuted) {
+      await this.restartTrack();
+    }
+  }
+
+  async mute(): Promise<LocalAudioTrack> {
+    // disabled special handling as it will cause BT headsets to switch communication modes
+    if (this.source === Track.Source.Microphone && this.stopOnMute) {
+      log.debug('stopping mic track');
+      // also stop the track, so that microphone indicator is turned off
+      this.mediaStreamTrack.stop();
+    }
+    await super.mute();
+    return this;
+  }
+
+  async unmute(): Promise<LocalAudioTrack> {
+    if (this.source === Track.Source.Microphone && this.stopOnMute) {
+      log.debug('reacquiring mic track');
+      await this.restartTrack();
+    }
+    await super.unmute();
+    return this;
+  }
+
+  async restartTrack(options?: AudioCaptureOptions) {
+    let constraints: MediaTrackConstraints | undefined;
+    if (options) {
+      const streamConstraints = constraintsForOptions({ audio: options });
+      if (typeof streamConstraints.audio !== 'boolean') {
+        constraints = streamConstraints.audio;
+      }
+    }
+    await this.restart(constraints);
+  }
+
+  /* @internal */
+  startMonitor() {
+    setTimeout(() => {
+      this.monitorSender();
+    }, monitorFrequency);
+  }
+
+  private monitorSender = async () => {
+    if (!this.sender) {
+      this._currentBitrate = 0;
+      return;
+    }
+    const stats = await this.getSenderStats();
+
+    if (stats && this.prevStats) {
+      this._currentBitrate = computeBitrate(stats, this.prevStats);
+    }
+
+    this.prevStats = stats;
+    setTimeout(() => {
+      this.monitorSender();
+    }, monitorFrequency);
+  };
+
+  async getSenderStats(): Promise<AudioSenderStats | undefined> {
+    if (!this.sender) {
+      return undefined;
+    }
+
+    const stats = await this.sender.getStats();
+    let audioStats: AudioSenderStats | undefined;
+    stats.forEach((v) => {
+      if (v.type === 'outbound-rtp') {
+        audioStats = {
+          type: 'audio',
+          streamId: v.id,
+          packetsSent: v.packetsSent,
+          packetsLost: v.packetsLost,
+          bytesSent: v.bytesSent,
+          timestamp: v.timestamp,
+          roundTripTime: v.roundTripTime,
+          jitter: v.jitter,
+        };
+      }
+    });
+
+    return audioStats;
+  }
+}

package/src/room/track/LocalTrack.ts

@@ -0,0 +1,124 @@
+import log from '../../logger';
+import DeviceManager from '../DeviceManager';
+import { TrackInvalidError } from '../errors';
+import { TrackEvent } from '../events';
+import { attachToElement, detachTrack, Track } from './Track';
+
+export default class LocalTrack extends Track {
+  /** @internal */
+  sender?: RTCRtpSender;
+
+  protected constraints: MediaTrackConstraints;
+
+  protected constructor(
+    mediaTrack: MediaStreamTrack, kind: Track.Kind, constraints?: MediaTrackConstraints,
+  ) {
+    super(mediaTrack, kind);
+    this.mediaStreamTrack.addEventListener('ended', this.handleEnded);
+    this.constraints = constraints ?? mediaTrack.getConstraints();
+  }
+
+  get id(): string {
+    return this.mediaStreamTrack.id;
+  }
+
+  get dimensions(): Track.Dimensions | undefined {
+    if (this.kind !== Track.Kind.Video) {
+      return undefined;
+    }
+
+    const { width, height } = this.mediaStreamTrack.getSettings();
+    if (width && height) {
+      return {
+        width,
+        height,
+      };
+    }
+    return undefined;
+  }
+
+  /**
+   * @returns DeviceID of the device that is currently being used for this track
+   */
+  async getDeviceId(): Promise<string | undefined> {
+    // screen share doesn't have a usable device id
+    if (this.source === Track.Source.ScreenShare) {
+      return;
+    }
+    const { deviceId, groupId } = this.mediaStreamTrack.getSettings();
+    const kind = this.kind === Track.Kind.Audio ? 'audioinput' : 'videoinput';
+
+    return DeviceManager.getInstance().normalizeDeviceId(kind, deviceId, groupId);
+  }
+
+  async mute(): Promise<LocalTrack> {
+    this.setTrackMuted(true);
+    return this;
+  }
+
+  async unmute(): Promise<LocalTrack> {
+    this.setTrackMuted(false);
+    return this;
+  }
+
+  protected async restart(constraints?: MediaTrackConstraints): Promise<LocalTrack> {
+    if (!this.sender) {
+      throw new TrackInvalidError('unable to restart an unpublished track');
+    }
+    if (!constraints) {
+      constraints = this.constraints;
+    }
+    log.debug('restarting track with constraints', constraints);
+
+    const streamConstraints: MediaStreamConstraints = {
+      audio: false,
+      video: false,
+    };
+
+    if (this.kind === Track.Kind.Video) {
+      streamConstraints.video = constraints;
+    } else {
+      streamConstraints.audio = constraints;
+    }
+
+    // detach
+    this.attachedElements.forEach((el) => {
+      detachTrack(this.mediaStreamTrack, el);
+    });
+    this.mediaStreamTrack.removeEventListener('ended', this.handleEnded);
+    // on Safari, the old audio track must be stopped before attempting to acquire
+    // the new track, otherwise the new track will stop with
+    // 'A MediaStreamTrack ended due to a capture failure`
+    this.mediaStreamTrack.stop();
+
+    // create new track and attach
+    const mediaStream = await navigator.mediaDevices.getUserMedia(streamConstraints);
+    const newTrack = mediaStream.getTracks()[0];
+    newTrack.addEventListener('ended', this.handleEnded);
+    log.debug('re-acquired MediaStreamTrack');
+
+    await this.sender.replaceTrack(newTrack);
+    this.mediaStreamTrack = newTrack;
+
+    this.attachedElements.forEach((el) => {
+      attachToElement(newTrack, el);
+    });
+
+    this.constraints = constraints;
+    return this;
+  }
+
+  protected setTrackMuted(muted: boolean) {
+    if (this.isMuted === muted) {
+      return;
+    }
+
+    this.isMuted = muted;
+    this.mediaStreamTrack.enabled = !muted;
+    this.emit(muted ? TrackEvent.Muted : TrackEvent.Unmuted, this);
+  }
+
+  private handleEnded = () => {
+    this.emit(TrackEvent.Ended, this);
+  };
+}

package/src/room/track/LocalTrackPublication.ts

@@ -0,0 +1,66 @@
+import { TrackInfo } from '../../proto/livekit_models';
+import { TrackEvent } from '../events';
+import LocalAudioTrack from './LocalAudioTrack';
+import LocalTrack from './LocalTrack';
+import LocalVideoTrack from './LocalVideoTrack';
+import { TrackPublishOptions } from './options';
+import { Track } from './Track';
+import { TrackPublication } from './TrackPublication';
+
+export default class LocalTrackPublication extends TrackPublication {
+  track?: LocalTrack;
+
+  options?: TrackPublishOptions;
+
+  constructor(kind: Track.Kind, ti: TrackInfo, track?: LocalTrack) {
+    super(kind, ti.sid, ti.name);
+
+    this.updateInfo(ti);
+    this.setTrack(track);
+  }
+
+  setTrack(track?: Track) {
+    if (this.track) {
+      this.track.off(TrackEvent.Ended, this.handleTrackEnded);
+    }
+
+    super.setTrack(track);
+
+    if (track) {
+      track.on(TrackEvent.Ended, this.handleTrackEnded);
+    }
+  }
+
+  get isMuted(): boolean {
+    if (this.track) {
+      return this.track.isMuted;
+    }
+    return super.isMuted;
+  }
+
+  get audioTrack(): LocalAudioTrack | undefined {
+    return super.audioTrack as LocalAudioTrack | undefined;
+  }
+
+  get videoTrack(): LocalVideoTrack | undefined {
+    return super.videoTrack as LocalVideoTrack | undefined;
+  }
+
+  /**
+   * Mute the track associated with this publication
+   */
+  async mute() {
+    return this.track?.mute();
+  }
+
+  /**
+   * Unmute track associated with this publication
+   */
+  async unmute() {
+    return this.track?.unmute();
+  }
+
+  handleTrackEnded = (track: LocalTrack) => {
+    this.emit(TrackEvent.Ended, track);
+  };
+}

package/src/room/track/LocalVideoTrack.test.ts

@@ -0,0 +1,70 @@
+import { VideoQuality } from '../../proto/livekit_models';
+import { videoLayersFromEncodings } from './LocalVideoTrack';
+
+describe('videoLayersFromEncodings', () => {
+  it('returns single layer for no encoding', () => {
+    const layers = videoLayersFromEncodings(640, 360);
+    expect(layers).toHaveLength(1);
+    expect(layers[0].quality).toBe(VideoQuality.HIGH);
+    expect(layers[0].width).toBe(640);
+    expect(layers[0].height).toBe(360);
+  });
+
+  it('returns single layer for explicit encoding', () => {
+    const layers = videoLayersFromEncodings(640, 360, [{
+      maxBitrate: 200_000,
+    }]);
+    expect(layers).toHaveLength(1);
+    expect(layers[0].quality).toBe(VideoQuality.HIGH);
+    expect(layers[0].bitrate).toBe(200_000);
+  });
+
+  it('returns three layers for simulcast', () => {
+    const layers = videoLayersFromEncodings(1280, 720, [
+      {
+        scaleResolutionDownBy: 4,
+        rid: 'q',
+        maxBitrate: 125_000,
+      },
+      {
+        scaleResolutionDownBy: 2,
+        rid: 'h',
+        maxBitrate: 500_000,
+      },
+      {
+        rid: 'f',
+        maxBitrate: 1_200_000,
+      },
+    ]);
+
+    expect(layers).toHaveLength(3);
+    expect(layers[0].quality).toBe(VideoQuality.LOW);
+    expect(layers[0].width).toBe(320);
+    expect(layers[2].quality).toBe(VideoQuality.HIGH);
+    expect(layers[2].height).toBe(720);
+  });
+
+  it('handles portrait', () => {
+    const layers = videoLayersFromEncodings(720, 1280, [
+      {
+        scaleResolutionDownBy: 4,
+        rid: 'q',
+        maxBitrate: 125_000,
+      },
+      {
+        scaleResolutionDownBy: 2,
+        rid: 'h',
+        maxBitrate: 500_000,
+      },
+      {
+        rid: 'f',
+        maxBitrate: 1_200_000,
+      },
+    ]);
+    expect(layers).toHaveLength(3);
+    expect(layers[0].quality).toBe(VideoQuality.LOW);
+    expect(layers[0].height).toBe(320);
+    expect(layers[2].quality).toBe(VideoQuality.HIGH);
+    expect(layers[2].width).toBe(720);
+  });
+});