livekit-client 0.18.4-RC8 → 0.18.6

package/src/room/track/RemoteAudioTrack.ts

@@ -0,0 +1,86 @@
+import { AudioReceiverStats, computeBitrate, monitorFrequency } from '../stats';
+import RemoteTrack from './RemoteTrack';
+import { Track } from './Track';
+
+export default class RemoteAudioTrack extends RemoteTrack {
+  private prevStats?: AudioReceiverStats;
+
+  private elementVolume: number;
+
+  constructor(mediaTrack: MediaStreamTrack, sid: string, receiver?: RTCRtpReceiver) {
+    super(mediaTrack, sid, Track.Kind.Audio, receiver);
+    this.elementVolume = 1;
+  }
+
+  /**
+   * sets the volume for all attached audio elements
+   */
+  setVolume(volume: number) {
+    for (const el of this.attachedElements) {
+      el.volume = volume;
+    }
+    this.elementVolume = volume;
+  }
+
+  /**
+   * gets the volume for all attached audio elements
+   */
+  getVolume(): number {
+    return this.elementVolume;
+  }
+
+  attach(): HTMLMediaElement;
+  attach(element: HTMLMediaElement): HTMLMediaElement;
+  attach(element?: HTMLMediaElement): HTMLMediaElement {
+    if (!element) {
+      element = super.attach();
+    } else {
+      super.attach(element);
+    }
+    element.volume = this.elementVolume;
+    return element;
+  }
+
+  protected monitorReceiver = async () => {
+    if (!this.receiver) {
+      this._currentBitrate = 0;
+      return;
+    }
+    const stats = await this.getReceiverStats();
+
+    if (stats && this.prevStats && this.receiver) {
+      this._currentBitrate = computeBitrate(stats, this.prevStats);
+    }
+
+    this.prevStats = stats;
+    setTimeout(() => {
+      this.monitorReceiver();
+    }, monitorFrequency);
+  };
+
+  protected async getReceiverStats(): Promise<AudioReceiverStats | undefined> {
+    if (!this.receiver) {
+      return;
+    }
+
+    const stats = await this.receiver.getStats();
+    let receiverStats: AudioReceiverStats | undefined;
+    stats.forEach((v) => {
+      if (v.type === 'inbound-rtp') {
+        receiverStats = {
+          type: 'audio',
+          timestamp: v.timestamp,
+          jitter: v.jitter,
+          bytesReceived: v.bytesReceived,
+          concealedSamples: v.concealedSamples,
+          concealmentEvents: v.concealmentEvents,
+          silentConcealedSamples: v.silentConcealedSamples,
+          silentConcealmentEvents: v.silentConcealmentEvents,
+          totalAudioEnergy: v.totalAudioEnergy,
+          totalSamplesDuration: v.totalSamplesDuration,
+        };
+      }
+    });
+    return receiverStats;
+  }
+}

package/src/room/track/RemoteTrack.ts

@@ -0,0 +1,62 @@
+import { TrackEvent } from '../events';
+import { monitorFrequency } from '../stats';
+import { Track } from './Track';
+
+export default abstract class RemoteTrack extends Track {
+  /** @internal */
+  receiver?: RTCRtpReceiver;
+
+  streamState: Track.StreamState = Track.StreamState.Active;
+
+  constructor(
+    mediaTrack: MediaStreamTrack,
+    sid: string,
+    kind: Track.Kind,
+    receiver?: RTCRtpReceiver,
+  ) {
+    super(mediaTrack, kind);
+    this.sid = sid;
+    this.receiver = receiver;
+  }
+
+  /** @internal */
+  setMuted(muted: boolean) {
+    if (this.isMuted !== muted) {
+      this.isMuted = muted;
+      this.emit(muted ? TrackEvent.Muted : TrackEvent.Unmuted, this);
+    }
+  }
+
+  /** @internal */
+  setMediaStream(stream: MediaStream) {
+    // this is needed to determine when the track is finished
+    // we send each track down in its own MediaStream, so we can assume the
+    // current track is the only one that can be removed.
+    this.mediaStream = stream;
+    stream.onremovetrack = () => {
+      this.receiver = undefined;
+      this._currentBitrate = 0;
+      this.emit(TrackEvent.Ended, this);
+    };
+  }
+
+  start() {
+    this.startMonitor();
+    // use `enabled` of track to enable re-use of transceiver
+    super.enable();
+  }
+
+  stop() {
+    // use `enabled` of track to enable re-use of transceiver
+    super.disable();
+  }
+
+  /* @internal */
+  startMonitor() {
+    setTimeout(() => {
+      this.monitorReceiver();
+    }, monitorFrequency);
+  }
+
+  protected abstract monitorReceiver(): void;
+}

package/src/room/track/RemoteTrackPublication.ts

@@ -0,0 +1,207 @@
+import log from '../../logger';
+import { TrackInfo, VideoQuality } from '../../proto/livekit_models';
+import { UpdateSubscription, UpdateTrackSettings } from '../../proto/livekit_rtc';
+import { TrackEvent } from '../events';
+import RemoteVideoTrack from './RemoteVideoTrack';
+import { Track } from './Track';
+import { TrackPublication } from './TrackPublication';
+import { RemoteTrack } from './types';
+
+export default class RemoteTrackPublication extends TrackPublication {
+  track?: RemoteTrack;
+
+  /** @internal */
+  _allowed = true;
+
+  // keeps track of client's desire to subscribe to a track
+  protected subscribed?: boolean;
+
+  protected disabled: boolean = false;
+
+  protected currentVideoQuality?: VideoQuality = VideoQuality.HIGH;
+
+  protected videoDimensions?: Track.Dimensions;
+
+  /**
+   * Subscribe or unsubscribe to this remote track
+   * @param subscribed true to subscribe to a track, false to unsubscribe
+   */
+  setSubscribed(subscribed: boolean) {
+    this.subscribed = subscribed;
+
+    const sub: UpdateSubscription = {
+      trackSids: [this.trackSid],
+      subscribe: this.subscribed,
+      participantTracks: [
+        {
+          // sending an empty participant id since TrackPublication doesn't keep it
+          // this is filled in by the participant that receives this message
+          participantSid: '',
+          trackSids: [this.trackSid],
+        },
+      ],
+    };
+    this.emit(TrackEvent.UpdateSubscription, sub);
+  }
+
+  get subscriptionStatus(): TrackPublication.SubscriptionStatus {
+    if (this.subscribed === false || !super.isSubscribed) {
+      return TrackPublication.SubscriptionStatus.Unsubscribed;
+    }
+    if (!this._allowed) {
+      return TrackPublication.SubscriptionStatus.NotAllowed;
+    }
+    return TrackPublication.SubscriptionStatus.Subscribed;
+  }
+
+  /**
+   * Returns true if track is subscribed, and ready for playback
+   */
+  get isSubscribed(): boolean {
+    if (this.subscribed === false) {
+      return false;
+    }
+    if (!this._allowed) {
+      return false;
+    }
+    return super.isSubscribed;
+  }
+
+  get isEnabled(): boolean {
+    return !this.disabled;
+  }
+
+  /**
+   * disable server from sending down data for this track. this is useful when
+   * the participant is off screen, you may disable streaming down their video
+   * to reduce bandwidth requirements
+   * @param enabled
+   */
+  setEnabled(enabled: boolean) {
+    if (!this.isManualOperationAllowed() || this.disabled === !enabled) {
+      return;
+    }
+    this.disabled = !enabled;
+
+    this.emitTrackUpdate();
+  }
+
+  /**
+   * for tracks that support simulcasting, adjust subscribed quality
+   *
+   * This indicates the highest quality the client can accept. if network
+   * bandwidth does not allow, server will automatically reduce quality to
+   * optimize for uninterrupted video
+   */
+  setVideoQuality(quality: VideoQuality) {
+    if (!this.isManualOperationAllowed() || this.currentVideoQuality === quality) {
+      return;
+    }
+    this.currentVideoQuality = quality;
+    this.videoDimensions = undefined;
+
+    this.emitTrackUpdate();
+  }
+
+  setVideoDimensions(dimensions: Track.Dimensions) {
+    if (!this.isManualOperationAllowed()) {
+      return;
+    }
+    if (
+      this.videoDimensions?.width === dimensions.width &&
+      this.videoDimensions?.height === dimensions.height
+    ) {
+      return;
+    }
+    if (this.track instanceof RemoteVideoTrack) {
+      this.videoDimensions = dimensions;
+    }
+    this.currentVideoQuality = undefined;
+
+    this.emitTrackUpdate();
+  }
+
+  get videoQuality(): VideoQuality | undefined {
+    return this.currentVideoQuality;
+  }
+
+  setTrack(track?: Track) {
+    if (this.track) {
+      // unregister listener
+      this.track.off(TrackEvent.VideoDimensionsChanged, this.handleVideoDimensionsChange);
+      this.track.off(TrackEvent.VisibilityChanged, this.handleVisibilityChange);
+      this.track.off(TrackEvent.Ended, this.handleEnded);
+    }
+    super.setTrack(track);
+    if (track) {
+      track.sid = this.trackSid;
+      track.on(TrackEvent.VideoDimensionsChanged, this.handleVideoDimensionsChange);
+      track.on(TrackEvent.VisibilityChanged, this.handleVisibilityChange);
+      track.on(TrackEvent.Ended, this.handleEnded);
+    }
+  }
+
+  /** @internal */
+  updateInfo(info: TrackInfo) {
+    super.updateInfo(info);
+    this.metadataMuted = info.muted;
+    this.track?.setMuted(info.muted);
+  }
+
+  private isManualOperationAllowed(): boolean {
+    if (this.isAdaptiveStream) {
+      log.warn('adaptive stream is enabled, cannot change track settings', {
+        trackSid: this.trackSid,
+      });
+      return false;
+    }
+    if (!this.isSubscribed) {
+      log.warn('cannot update track settings when not subscribed', { trackSid: this.trackSid });
+      return false;
+    }
+    return true;
+  }
+
+  protected handleEnded = (track: RemoteTrack) => {
+    this.emit(TrackEvent.Ended, track);
+  };
+
+  protected get isAdaptiveStream(): boolean {
+    return this.track instanceof RemoteVideoTrack && this.track.isAdaptiveStream;
+  }
+
+  protected handleVisibilityChange = (visible: boolean) => {
+    log.debug(`adaptivestream video visibility ${this.trackSid}, visible=${visible}`, {
+      trackSid: this.trackSid,
+    });
+    this.disabled = !visible;
+    this.emitTrackUpdate();
+  };
+
+  protected handleVideoDimensionsChange = (dimensions: Track.Dimensions) => {
+    log.debug(`adaptivestream video dimensions ${dimensions.width}x${dimensions.height}`, {
+      trackSid: this.trackSid,
+    });
+    this.videoDimensions = dimensions;
+    this.emitTrackUpdate();
+  };
+
+  /* @internal */
+  emitTrackUpdate() {
+    const settings: UpdateTrackSettings = UpdateTrackSettings.fromPartial({
+      trackSids: [this.trackSid],
+      disabled: this.disabled,
+    });
+    if (this.videoDimensions) {
+      settings.width = this.videoDimensions.width;
+      settings.height = this.videoDimensions.height;
+    } else if (this.currentVideoQuality !== undefined) {
+      settings.quality = this.currentVideoQuality;
+    } else {
+      // defaults to high quality
+      settings.quality = VideoQuality.HIGH;
+    }
+
+    this.emit(TrackEvent.UpdateSettings, settings);
+  }
+}

package/src/room/track/RemoteVideoTrack.ts

@@ -0,0 +1,253 @@
+import { debounce } from 'ts-debounce';
+import { TrackEvent } from '../events';
+import { computeBitrate, monitorFrequency, VideoReceiverStats } from '../stats';
+import {
+  getIntersectionObserver,
+  getResizeObserver,
+  isMobile,
+  ObservableMediaElement,
+} from '../utils';
+import RemoteTrack from './RemoteTrack';
+import { attachToElement, detachTrack, Track } from './Track';
+import { AdaptiveStreamSettings } from './types';
+import log from '../../logger';
+
+const REACTION_DELAY = 100;
+
+export default class RemoteVideoTrack extends RemoteTrack {
+  /** @internal */
+  receiver?: RTCRtpReceiver;
+
+  private prevStats?: VideoReceiverStats;
+
+  private elementInfos: ElementInfo[] = [];
+
+  private adaptiveStreamSettings?: AdaptiveStreamSettings;
+
+  private lastVisible?: boolean;
+
+  private lastDimensions?: Track.Dimensions;
+
+  private hasUsedAttach: boolean = false;
+
+  constructor(
+    mediaTrack: MediaStreamTrack,
+    sid: string,
+    receiver?: RTCRtpReceiver,
+    adaptiveStreamSettings?: AdaptiveStreamSettings,
+  ) {
+    super(mediaTrack, sid, Track.Kind.Video, receiver);
+    this.adaptiveStreamSettings = adaptiveStreamSettings;
+    if (this.isAdaptiveStream) {
+      this.streamState = Track.StreamState.Paused;
+    }
+  }
+
+  get isAdaptiveStream(): boolean {
+    return this.adaptiveStreamSettings !== undefined;
+  }
+
+  get mediaStreamTrack() {
+    if (this.isAdaptiveStream && !this.hasUsedAttach) {
+      log.warn(
+        'When using adaptiveStream, you need to use remoteVideoTrack.attach() to add the track to a HTMLVideoElement, otherwise your video tracks might never start',
+      );
+    }
+    return this._mediaStreamTrack;
+  }
+
+  /** @internal */
+  setMuted(muted: boolean) {
+    super.setMuted(muted);
+
+    this.attachedElements.forEach((element) => {
+      // detach or attach
+      if (muted) {
+        detachTrack(this._mediaStreamTrack, element);
+      } else {
+        attachToElement(this._mediaStreamTrack, element);
+      }
+    });
+  }
+
+  attach(): HTMLMediaElement;
+  attach(element: HTMLMediaElement): HTMLMediaElement;
+  attach(element?: HTMLMediaElement): HTMLMediaElement {
+    if (!element) {
+      element = super.attach();
+    } else {
+      super.attach(element);
+    }
+
+    // It's possible attach is called multiple times on an element. When that's
+    // the case, we'd want to avoid adding duplicate elementInfos
+    if (
+      this.adaptiveStreamSettings &&
+      this.elementInfos.find((info) => info.element === element) === undefined
+    ) {
+      this.elementInfos.push({
+        element,
+        visible: true, // default visible
+      });
+
+      (element as ObservableMediaElement).handleResize = this.debouncedHandleResize;
+      (element as ObservableMediaElement).handleVisibilityChanged = this.handleVisibilityChanged;
+
+      getIntersectionObserver().observe(element);
+      getResizeObserver().observe(element);
+
+      // trigger the first resize update cycle
+      // if the tab is backgrounded, the initial resize event does not fire until
+      // the tab comes into focus for the first time.
+      this.debouncedHandleResize();
+    }
+    this.hasUsedAttach = true;
+    return element;
+  }
+
+  detach(): HTMLMediaElement[];
+  detach(element: HTMLMediaElement): HTMLMediaElement;
+  detach(element?: HTMLMediaElement): HTMLMediaElement | HTMLMediaElement[] {
+    let detachedElements: HTMLMediaElement[] = [];
+    if (element) {
+      this.stopObservingElement(element);
+      return super.detach(element);
+    }
+    detachedElements = super.detach();
+
+    for (const e of detachedElements) {
+      this.stopObservingElement(e);
+    }
+
+    return detachedElements;
+  }
+
+  protected monitorReceiver = async () => {
+    if (!this.receiver) {
+      this._currentBitrate = 0;
+      return;
+    }
+    const stats = await this.getReceiverStats();
+
+    if (stats && this.prevStats && this.receiver) {
+      this._currentBitrate = computeBitrate(stats, this.prevStats);
+    }
+
+    this.prevStats = stats;
+    setTimeout(() => {
+      this.monitorReceiver();
+    }, monitorFrequency);
+  };
+
+  private async getReceiverStats(): Promise<VideoReceiverStats | undefined> {
+    if (!this.receiver) {
+      return;
+    }
+
+    const stats = await this.receiver.getStats();
+    let receiverStats: VideoReceiverStats | undefined;
+    stats.forEach((v) => {
+      if (v.type === 'inbound-rtp') {
+        receiverStats = {
+          type: 'video',
+          framesDecoded: v.framesDecoded,
+          framesDropped: v.framesDropped,
+          framesReceived: v.framesReceived,
+          packetsReceived: v.packetsReceived,
+          packetsLost: v.packetsLost,
+          frameWidth: v.frameWidth,
+          frameHeight: v.frameHeight,
+          pliCount: v.pliCount,
+          firCount: v.firCount,
+          nackCount: v.nackCount,
+          jitter: v.jitter,
+          timestamp: v.timestamp,
+          bytesReceived: v.bytesReceived,
+        };
+      }
+    });
+    return receiverStats;
+  }
+
+  private stopObservingElement(element: HTMLMediaElement) {
+    getIntersectionObserver()?.unobserve(element);
+    getResizeObserver()?.unobserve(element);
+    this.elementInfos = this.elementInfos.filter((info) => info.element !== element);
+  }
+
+  private handleVisibilityChanged = (entry: IntersectionObserverEntry) => {
+    const { target, isIntersecting } = entry;
+    const elementInfo = this.elementInfos.find((info) => info.element === target);
+    if (elementInfo) {
+      elementInfo.visible = isIntersecting;
+      elementInfo.visibilityChangedAt = Date.now();
+    }
+    this.updateVisibility();
+  };
+
+  protected async handleAppVisibilityChanged() {
+    await super.handleAppVisibilityChanged();
+    if (!this.isAdaptiveStream) return;
+    // on desktop don't pause when tab is backgrounded
+    if (!isMobile()) return;
+    this.updateVisibility();
+  }
+
+  private readonly debouncedHandleResize = debounce(() => {
+    this.updateDimensions();
+  }, REACTION_DELAY);
+
+  private updateVisibility() {
+    const lastVisibilityChange = this.elementInfos.reduce(
+      (prev, info) => Math.max(prev, info.visibilityChangedAt || 0),
+      0,
+    );
+    const isVisible = this.elementInfos.some((info) => info.visible) && !this.isInBackground;
+
+    if (this.lastVisible === isVisible) {
+      return;
+    }
+
+    if (!isVisible && Date.now() - lastVisibilityChange < REACTION_DELAY) {
+      // delay hidden events
+      setTimeout(() => {
+        this.updateVisibility();
+      }, REACTION_DELAY);
+      return;
+    }
+
+    this.lastVisible = isVisible;
+    this.emit(TrackEvent.VisibilityChanged, isVisible, this);
+  }
+
+  private updateDimensions() {
+    let maxWidth = 0;
+    let maxHeight = 0;
+    for (const info of this.elementInfos) {
+      const pixelDensity = this.adaptiveStreamSettings?.pixelDensity ?? 1;
+      const pixelDensityValue = pixelDensity === 'screen' ? window.devicePixelRatio : pixelDensity;
+      const currentElementWidth = info.element.clientWidth * pixelDensityValue;
+      const currentElementHeight = info.element.clientHeight * pixelDensityValue;
+      if (currentElementWidth + currentElementHeight > maxWidth + maxHeight) {
+        maxWidth = currentElementWidth;
+        maxHeight = currentElementHeight;
+      }
+    }
+
+    if (this.lastDimensions?.width === maxWidth && this.lastDimensions?.height === maxHeight) {
+      return;
+    }
+
+    this.lastDimensions = {
+      width: maxWidth,
+      height: maxHeight,
+    };
+    this.emit(TrackEvent.VideoDimensionsChanged, this.lastDimensions, this);
+  }
+}
+
+interface ElementInfo {
+  element: HTMLMediaElement;
+  visible: boolean;
+  visibilityChangedAt?: number;
+}