livekit-client 0.18.4-RC7 → 0.18.5

package/src/room/track/LocalTrack.ts

@@ -0,0 +1,229 @@
+import log from '../../logger';
+import DeviceManager from '../DeviceManager';
+import { TrackInvalidError } from '../errors';
+import { TrackEvent } from '../events';
+import { getEmptyAudioStreamTrack, getEmptyVideoStreamTrack, isMobile } from '../utils';
+import { attachToElement, detachTrack, Track } from './Track';
+
+export default class LocalTrack extends Track {
+  /** @internal */
+  sender?: RTCRtpSender;
+
+  protected constraints: MediaTrackConstraints;
+
+  protected wasMuted: boolean;
+
+  protected reacquireTrack: boolean;
+
+  protected constructor(
+    mediaTrack: MediaStreamTrack,
+    kind: Track.Kind,
+    constraints?: MediaTrackConstraints,
+  ) {
+    super(mediaTrack, kind);
+    this._mediaStreamTrack.addEventListener('ended', this.handleEnded);
+    this.constraints = constraints ?? mediaTrack.getConstraints();
+    this.reacquireTrack = false;
+    this.wasMuted = false;
+  }
+
+  get id(): string {
+    return this._mediaStreamTrack.id;
+  }
+
+  get dimensions(): Track.Dimensions | undefined {
+    if (this.kind !== Track.Kind.Video) {
+      return undefined;
+    }
+
+    const { width, height } = this._mediaStreamTrack.getSettings();
+    if (width && height) {
+      return {
+        width,
+        height,
+      };
+    }
+    return undefined;
+  }
+
+  private _isUpstreamPaused: boolean = false;
+
+  get isUpstreamPaused() {
+    return this._isUpstreamPaused;
+  }
+
+  /**
+   * @returns DeviceID of the device that is currently being used for this track
+   */
+  async getDeviceId(): Promise<string | undefined> {
+    // screen share doesn't have a usable device id
+    if (this.source === Track.Source.ScreenShare) {
+      return;
+    }
+    const { deviceId, groupId } = this._mediaStreamTrack.getSettings();
+    const kind = this.kind === Track.Kind.Audio ? 'audioinput' : 'videoinput';
+
+    return DeviceManager.getInstance().normalizeDeviceId(kind, deviceId, groupId);
+  }
+
+  async mute(): Promise<LocalTrack> {
+    this.setTrackMuted(true);
+    return this;
+  }
+
+  async unmute(): Promise<LocalTrack> {
+    this.setTrackMuted(false);
+    return this;
+  }
+
+  async replaceTrack(track: MediaStreamTrack): Promise<LocalTrack> {
+    if (!this.sender) {
+      throw new TrackInvalidError('unable to replace an unpublished track');
+    }
+
+    // detach
+    this.attachedElements.forEach((el) => {
+      detachTrack(this._mediaStreamTrack, el);
+    });
+    this._mediaStreamTrack.removeEventListener('ended', this.handleEnded);
+    // on Safari, the old audio track must be stopped before attempting to acquire
+    // the new track, otherwise the new track will stop with
+    // 'A MediaStreamTrack ended due to a capture failure`
+    this._mediaStreamTrack.stop();
+
+    track.addEventListener('ended', this.handleEnded);
+    log.debug('replace MediaStreamTrack');
+
+    await this.sender.replaceTrack(track);
+    this._mediaStreamTrack = track;
+
+    this.attachedElements.forEach((el) => {
+      attachToElement(track, el);
+    });
+
+    this.mediaStream = new MediaStream([track]);
+    return this;
+  }
+
+  protected async restart(constraints?: MediaTrackConstraints): Promise<LocalTrack> {
+    if (!this.sender) {
+      throw new TrackInvalidError('unable to restart an unpublished track');
+    }
+    if (!constraints) {
+      constraints = this.constraints;
+    }
+    log.debug('restarting track with constraints', constraints);
+
+    const streamConstraints: MediaStreamConstraints = {
+      audio: false,
+      video: false,
+    };
+
+    if (this.kind === Track.Kind.Video) {
+      streamConstraints.video = constraints;
+    } else {
+      streamConstraints.audio = constraints;
+    }
+
+    // detach
+    this.attachedElements.forEach((el) => {
+      detachTrack(this._mediaStreamTrack, el);
+    });
+    this._mediaStreamTrack.removeEventListener('ended', this.handleEnded);
+    // on Safari, the old audio track must be stopped before attempting to acquire
+    // the new track, otherwise the new track will stop with
+    // 'A MediaStreamTrack ended due to a capture failure`
+    this._mediaStreamTrack.stop();
+
+    // create new track and attach
+    const mediaStream = await navigator.mediaDevices.getUserMedia(streamConstraints);
+    const newTrack = mediaStream.getTracks()[0];
+    newTrack.addEventListener('ended', this.handleEnded);
+    log.debug('re-acquired MediaStreamTrack');
+
+    await this.sender.replaceTrack(newTrack);
+    this._mediaStreamTrack = newTrack;
+
+    this.attachedElements.forEach((el) => {
+      attachToElement(newTrack, el);
+    });
+
+    this.mediaStream = mediaStream;
+    this.constraints = constraints;
+    return this;
+  }
+
+  protected setTrackMuted(muted: boolean) {
+    if (this.isMuted === muted) {
+      return;
+    }
+
+    this.isMuted = muted;
+    this._mediaStreamTrack.enabled = !muted;
+    this.emit(muted ? TrackEvent.Muted : TrackEvent.Unmuted, this);
+  }
+
+  protected get needsReAcquisition(): boolean {
+    return (
+      this._mediaStreamTrack.readyState !== 'live' ||
+      this._mediaStreamTrack.muted ||
+      !this._mediaStreamTrack.enabled ||
+      this.reacquireTrack
+    );
+  }
+
+  protected async handleAppVisibilityChanged() {
+    await super.handleAppVisibilityChanged();
+    if (!isMobile()) return;
+    log.debug(`visibility changed, is in Background: ${this.isInBackground}`);
+
+    if (!this.isInBackground && this.needsReAcquisition) {
+      log.debug(`track needs to be reaquired, restarting ${this.source}`);
+      await this.restart();
+      this.reacquireTrack = false;
+      // Restore muted state if had to be restarted
+      this.setTrackMuted(this.wasMuted);
+    }
+
+    // store muted state each time app goes to background
+    if (this.isInBackground) {
+      this.wasMuted = this.isMuted;
+    }
+  }
+
+  private handleEnded = () => {
+    if (this.isInBackground) {
+      this.reacquireTrack = true;
+    }
+    this.emit(TrackEvent.Ended, this);
+  };
+
+  async pauseUpstream() {
+    if (this._isUpstreamPaused === true) {
+      return;
+    }
+    if (!this.sender) {
+      log.warn('unable to pause upstream for an unpublished track');
+      return;
+    }
+    this._isUpstreamPaused = true;
+    this.emit(TrackEvent.UpstreamPaused, this);
+    const emptyTrack =
+      this.kind === Track.Kind.Audio ? getEmptyAudioStreamTrack() : getEmptyVideoStreamTrack();
+    await this.sender.replaceTrack(emptyTrack);
+  }
+
+  async resumeUpstream() {
+    if (this._isUpstreamPaused === false) {
+      return;
+    }
+    if (!this.sender) {
+      log.warn('unable to resume upstream for an unpublished track');
+      return;
+    }
+    this._isUpstreamPaused = false;
+    this.emit(TrackEvent.UpstreamResumed, this);
+
+    await this.sender.replaceTrack(this._mediaStreamTrack);
+  }
+}

package/src/room/track/LocalTrackPublication.ts

@@ -0,0 +1,87 @@
+import { TrackInfo } from '../../proto/livekit_models';
+import { TrackEvent } from '../events';
+import LocalAudioTrack from './LocalAudioTrack';
+import LocalTrack from './LocalTrack';
+import LocalVideoTrack from './LocalVideoTrack';
+import { TrackPublishOptions } from './options';
+import { Track } from './Track';
+import { TrackPublication } from './TrackPublication';
+
+export default class LocalTrackPublication extends TrackPublication {
+  track?: LocalTrack = undefined;
+
+  options?: TrackPublishOptions;
+
+  get isUpstreamPaused() {
+    return this.track?.isUpstreamPaused;
+  }
+
+  constructor(kind: Track.Kind, ti: TrackInfo, track?: LocalTrack) {
+    super(kind, ti.sid, ti.name);
+
+    this.updateInfo(ti);
+    this.setTrack(track);
+  }
+
+  setTrack(track?: Track) {
+    if (this.track) {
+      this.track.off(TrackEvent.Ended, this.handleTrackEnded);
+    }
+
+    super.setTrack(track);
+
+    if (track) {
+      track.on(TrackEvent.Ended, this.handleTrackEnded);
+    }
+  }
+
+  get isMuted(): boolean {
+    if (this.track) {
+      return this.track.isMuted;
+    }
+    return super.isMuted;
+  }
+
+  get audioTrack(): LocalAudioTrack | undefined {
+    return super.audioTrack as LocalAudioTrack | undefined;
+  }
+
+  get videoTrack(): LocalVideoTrack | undefined {
+    return super.videoTrack as LocalVideoTrack | undefined;
+  }
+
+  /**
+   * Mute the track associated with this publication
+   */
+  async mute() {
+    return this.track?.mute();
+  }
+
+  /**
+   * Unmute track associated with this publication
+   */
+  async unmute() {
+    return this.track?.unmute();
+  }
+
+  /**
+   * Pauses the media stream track associated with this publication from being sent to the server
+   * and signals "muted" event to other participants
+   * Useful if you want to pause the stream without pausing the local media stream track
+   */
+  async pauseUpstream() {
+    await this.track?.pauseUpstream();
+  }
+
+  /**
+   * Resumes sending the media stream track associated with this publication to the server after a call to [[pauseUpstream()]]
+   * and signals "unmuted" event to other participants (unless the track is explicitly muted)
+   */
+  async resumeUpstream() {
+    await this.track?.resumeUpstream();
+  }
+
+  handleTrackEnded = () => {
+    this.emit(TrackEvent.Ended);
+  };
+}

package/src/room/track/LocalVideoTrack.test.ts

@@ -0,0 +1,72 @@
+import { VideoQuality } from '../../proto/livekit_models';
+import { videoLayersFromEncodings } from './LocalVideoTrack';
+
+describe('videoLayersFromEncodings', () => {
+  it('returns single layer for no encoding', () => {
+    const layers = videoLayersFromEncodings(640, 360);
+    expect(layers).toHaveLength(1);
+    expect(layers[0].quality).toBe(VideoQuality.HIGH);
+    expect(layers[0].width).toBe(640);
+    expect(layers[0].height).toBe(360);
+  });
+
+  it('returns single layer for explicit encoding', () => {
+    const layers = videoLayersFromEncodings(640, 360, [
+      {
+        maxBitrate: 200_000,
+      },
+    ]);
+    expect(layers).toHaveLength(1);
+    expect(layers[0].quality).toBe(VideoQuality.HIGH);
+    expect(layers[0].bitrate).toBe(200_000);
+  });
+
+  it('returns three layers for simulcast', () => {
+    const layers = videoLayersFromEncodings(1280, 720, [
+      {
+        scaleResolutionDownBy: 4,
+        rid: 'q',
+        maxBitrate: 125_000,
+      },
+      {
+        scaleResolutionDownBy: 2,
+        rid: 'h',
+        maxBitrate: 500_000,
+      },
+      {
+        rid: 'f',
+        maxBitrate: 1_200_000,
+      },
+    ]);
+
+    expect(layers).toHaveLength(3);
+    expect(layers[0].quality).toBe(VideoQuality.LOW);
+    expect(layers[0].width).toBe(320);
+    expect(layers[2].quality).toBe(VideoQuality.HIGH);
+    expect(layers[2].height).toBe(720);
+  });
+
+  it('handles portrait', () => {
+    const layers = videoLayersFromEncodings(720, 1280, [
+      {
+        scaleResolutionDownBy: 4,
+        rid: 'q',
+        maxBitrate: 125_000,
+      },
+      {
+        scaleResolutionDownBy: 2,
+        rid: 'h',
+        maxBitrate: 500_000,
+      },
+      {
+        rid: 'f',
+        maxBitrate: 1_200_000,
+      },
+    ]);
+    expect(layers).toHaveLength(3);
+    expect(layers[0].quality).toBe(VideoQuality.LOW);
+    expect(layers[0].height).toBe(320);
+    expect(layers[2].quality).toBe(VideoQuality.HIGH);
+    expect(layers[2].width).toBe(720);
+  });
+});

package/src/room/track/LocalVideoTrack.ts

@@ -0,0 +1,295 @@
+import { SignalClient } from '../../api/SignalClient';
+import log from '../../logger';
+import { VideoLayer, VideoQuality } from '../../proto/livekit_models';
+import { SubscribedQuality } from '../../proto/livekit_rtc';
+import { computeBitrate, monitorFrequency, VideoSenderStats } from '../stats';
+import { isFireFox, isMobile } from '../utils';
+import LocalTrack from './LocalTrack';
+import { VideoCaptureOptions } from './options';
+import { Track } from './Track';
+import { constraintsForOptions } from './utils';
+
+export default class LocalVideoTrack extends LocalTrack {
+  /* internal */
+  signalClient?: SignalClient;
+
+  private prevStats?: Map<string, VideoSenderStats>;
+
+  private encodings?: RTCRtpEncodingParameters[];
+
+  constructor(mediaTrack: MediaStreamTrack, constraints?: MediaTrackConstraints) {
+    super(mediaTrack, Track.Kind.Video, constraints);
+  }
+
+  get isSimulcast(): boolean {
+    if (this.sender && this.sender.getParameters().encodings.length > 1) {
+      return true;
+    }
+    return false;
+  }
+
+  /* @internal */
+  startMonitor(signalClient: SignalClient) {
+    this.signalClient = signalClient;
+    // save original encodings
+    const params = this.sender?.getParameters();
+    if (params) {
+      this.encodings = params.encodings;
+    }
+
+    setTimeout(() => {
+      this.monitorSender();
+    }, monitorFrequency);
+  }
+
+  stop() {
+    this.sender = undefined;
+    this._mediaStreamTrack.getConstraints();
+    super.stop();
+  }
+
+  async mute(): Promise<LocalVideoTrack> {
+    if (this.source === Track.Source.Camera) {
+      log.debug('stopping camera track');
+      // also stop the track, so that camera indicator is turned off
+      this._mediaStreamTrack.stop();
+    }
+    await super.mute();
+    return this;
+  }
+
+  async unmute(): Promise<LocalVideoTrack> {
+    if (this.source === Track.Source.Camera) {
+      log.debug('reacquiring camera track');
+      await this.restartTrack();
+    }
+    await super.unmute();
+    return this;
+  }
+
+  async getSenderStats(): Promise<VideoSenderStats[]> {
+    if (!this.sender) {
+      return [];
+    }
+
+    const items: VideoSenderStats[] = [];
+
+    const stats = await this.sender.getStats();
+    stats.forEach((v) => {
+      if (v.type === 'outbound-rtp') {
+        const vs: VideoSenderStats = {
+          type: 'video',
+          streamId: v.id,
+          frameHeight: v.frameHeight,
+          frameWidth: v.frameWidth,
+          firCount: v.firCount,
+          pliCount: v.pliCount,
+          nackCount: v.nackCount,
+          packetsSent: v.packetsSent,
+          bytesSent: v.bytesSent,
+          framesSent: v.framesSent,
+          timestamp: v.timestamp,
+          rid: v.rid ?? '',
+          retransmittedPacketsSent: v.retransmittedPacketsSent,
+          qualityLimitationReason: v.qualityLimitationReason,
+          qualityLimitationResolutionChanges: v.qualityLimitationResolutionChanges,
+        };
+
+        // locate the appropriate remote-inbound-rtp item
+        const r = stats.get(v.remoteId);
+        if (r) {
+          vs.jitter = r.jitter;
+          vs.packetsLost = r.packetsLost;
+          vs.roundTripTime = r.roundTripTime;
+        }
+
+        items.push(vs);
+      }
+    });
+
+    return items;
+  }
+
+  setPublishingQuality(maxQuality: VideoQuality) {
+    const qualities: SubscribedQuality[] = [];
+    for (let q = VideoQuality.LOW; q <= VideoQuality.HIGH; q += 1) {
+      qualities.push({
+        quality: q,
+        enabled: q <= maxQuality,
+      });
+    }
+    log.debug(`setting publishing quality. max quality ${maxQuality}`);
+    this.setPublishingLayers(qualities);
+  }
+
+  async setDeviceId(deviceId: string) {
+    if (this.constraints.deviceId === deviceId) {
+      return;
+    }
+    this.constraints.deviceId = deviceId;
+    // when video is muted, underlying media stream track is stopped and
+    // will be restarted later
+    if (!this.isMuted) {
+      await this.restartTrack();
+    }
+  }
+
+  async restartTrack(options?: VideoCaptureOptions) {
+    let constraints: MediaTrackConstraints | undefined;
+    if (options) {
+      const streamConstraints = constraintsForOptions({ video: options });
+      if (typeof streamConstraints.video !== 'boolean') {
+        constraints = streamConstraints.video;
+      }
+    }
+    await this.restart(constraints);
+  }
+
+  /**
+   * @internal
+   * Sets layers that should be publishing
+   */
+  async setPublishingLayers(qualities: SubscribedQuality[]) {
+    log.debug('setting publishing layers', qualities);
+    if (!this.sender || !this.encodings) {
+      return;
+    }
+    const params = this.sender.getParameters();
+    const { encodings } = params;
+    if (!encodings) {
+      return;
+    }
+
+    if (encodings.length !== this.encodings.length) {
+      log.warn('cannot set publishing layers, encodings mismatch');
+      return;
+    }
+
+    let hasChanged = false;
+    encodings.forEach((encoding, idx) => {
+      let rid = encoding.rid ?? '';
+      if (rid === '') {
+        rid = 'q';
+      }
+      const quality = videoQualityForRid(rid);
+      const subscribedQuality = qualities.find((q) => q.quality === quality);
+      if (!subscribedQuality) {
+        return;
+      }
+      if (encoding.active !== subscribedQuality.enabled) {
+        hasChanged = true;
+        encoding.active = subscribedQuality.enabled;
+        log.debug(
+          `setting layer ${subscribedQuality.quality} to ${
+            encoding.active ? 'enabled' : 'disabled'
+          }`,
+        );
+
+        // FireFox does not support setting encoding.active to false, so we
+        // have a workaround of lowering its bitrate and resolution to the min.
+        if (isFireFox()) {
+          if (subscribedQuality.enabled) {
+            encoding.scaleResolutionDownBy = this.encodings![idx].scaleResolutionDownBy;
+            encoding.maxBitrate = this.encodings![idx].maxBitrate;
+            /* @ts-ignore */
+            encoding.maxFrameRate = this.encodings![idx].maxFrameRate;
+          } else {
+            encoding.scaleResolutionDownBy = 4;
+            encoding.maxBitrate = 10;
+            /* @ts-ignore */
+            encoding.maxFrameRate = 2;
+          }
+        }
+      }
+    });
+
+    if (hasChanged) {
+      params.encodings = encodings;
+      await this.sender.setParameters(params);
+    }
+  }
+
+  private monitorSender = async () => {
+    if (!this.sender) {
+      this._currentBitrate = 0;
+      return;
+    }
+
+    let stats: VideoSenderStats[] | undefined;
+    try {
+      stats = await this.getSenderStats();
+    } catch (e) {
+      log.error('could not get audio sender stats', { error: e });
+      return;
+    }
+    const statsMap = new Map<string, VideoSenderStats>(stats.map((s) => [s.rid, s]));
+
+    if (this.prevStats) {
+      let totalBitrate = 0;
+      statsMap.forEach((s, key) => {
+        const prev = this.prevStats?.get(key);
+        totalBitrate += computeBitrate(s, prev);
+      });
+      this._currentBitrate = totalBitrate;
+    }
+
+    this.prevStats = statsMap;
+    setTimeout(() => {
+      this.monitorSender();
+    }, monitorFrequency);
+  };
+
+  protected async handleAppVisibilityChanged() {
+    await super.handleAppVisibilityChanged();
+    if (!isMobile()) return;
+    if (this.isInBackground && this.source === Track.Source.Camera) {
+      this._mediaStreamTrack.enabled = false;
+    }
+  }
+}
+
+export function videoQualityForRid(rid: string): VideoQuality {
+  switch (rid) {
+    case 'f':
+      return VideoQuality.HIGH;
+    case 'h':
+      return VideoQuality.MEDIUM;
+    case 'q':
+      return VideoQuality.LOW;
+    default:
+      return VideoQuality.UNRECOGNIZED;
+  }
+}
+
+export function videoLayersFromEncodings(
+  width: number,
+  height: number,
+  encodings?: RTCRtpEncodingParameters[],
+): VideoLayer[] {
+  // default to a single layer, HQ
+  if (!encodings) {
+    return [
+      {
+        quality: VideoQuality.HIGH,
+        width,
+        height,
+        bitrate: 0,
+        ssrc: 0,
+      },
+    ];
+  }
+  return encodings.map((encoding) => {
+    const scale = encoding.scaleResolutionDownBy ?? 1;
+    let quality = videoQualityForRid(encoding.rid ?? '');
+    if (quality === VideoQuality.UNRECOGNIZED && encodings.length === 1) {
+      quality = VideoQuality.HIGH;
+    }
+    return {
+      quality,
+      width: width / scale,
+      height: height / scale,
+      bitrate: encoding.maxBitrate ?? 0,
+      ssrc: 0,
+    };
+  });
+}