livekit-client 0.15.3 → 0.16.2

package/src/room/track/RemoteTrackPublication.ts

@@ -0,0 +1,198 @@
+import log from '../../logger';
+import { TrackInfo, VideoQuality } from '../../proto/livekit_models';
+import {
+  UpdateSubscription,
+  UpdateTrackSettings,
+} from '../../proto/livekit_rtc';
+import { TrackEvent } from '../events';
+import RemoteVideoTrack from './RemoteVideoTrack';
+import { Track } from './Track';
+import { TrackPublication } from './TrackPublication';
+import { RemoteTrack } from './types';
+
+export default class RemoteTrackPublication extends TrackPublication {
+  track?: RemoteTrack;
+
+  /** @internal */
+  _allowed = true;
+
+  // keeps track of client's desire to subscribe to a track
+  protected subscribed?: boolean;
+
+  protected disabled: boolean = false;
+
+  protected currentVideoQuality?: VideoQuality = VideoQuality.HIGH;
+
+  protected videoDimensions?: Track.Dimensions;
+
+  /**
+   * Subscribe or unsubscribe to this remote track
+   * @param subscribed true to subscribe to a track, false to unsubscribe
+   */
+  setSubscribed(subscribed: boolean) {
+    this.subscribed = subscribed;
+
+    const sub: UpdateSubscription = {
+      trackSids: [this.trackSid],
+      subscribe: this.subscribed,
+      participantTracks: [{
+        // sending an empty participant id since TrackPublication doesn't keep it
+        // this is filled in by the participant that receives this message
+        participantSid: '',
+        trackSids: [this.trackSid],
+      }],
+    };
+    this.emit(TrackEvent.UpdateSubscription, sub);
+  }
+
+  get subscriptionStatus(): TrackPublication.SubscriptionStatus {
+    if (this.subscribed === false || !super.isSubscribed) {
+      return TrackPublication.SubscriptionStatus.Unsubscribed;
+    }
+    if (!this._allowed) {
+      return TrackPublication.SubscriptionStatus.NotAllowed;
+    }
+    return TrackPublication.SubscriptionStatus.Subscribed;
+  }
+
+  /**
+   * Returns true if track is subscribed, and ready for playback
+   */
+  get isSubscribed(): boolean {
+    if (this.subscribed === false) {
+      return false;
+    }
+    if (!this._allowed) {
+      return false;
+    }
+    return super.isSubscribed;
+  }
+
+  get isEnabled(): boolean {
+    return !this.disabled;
+  }
+
+  /**
+   * disable server from sending down data for this track. this is useful when
+   * the participant is off screen, you may disable streaming down their video
+   * to reduce bandwidth requirements
+   * @param enabled
+   */
+  setEnabled(enabled: boolean) {
+    if (!this.isManualOperationAllowed() || this.disabled === !enabled) {
+      return;
+    }
+    this.disabled = !enabled;
+
+    this.emitTrackUpdate();
+  }
+
+  /**
+   * for tracks that support simulcasting, adjust subscribed quality
+   *
+   * This indicates the highest quality the client can accept. if network
+   * bandwidth does not allow, server will automatically reduce quality to
+   * optimize for uninterrupted video
+   */
+  setVideoQuality(quality: VideoQuality) {
+    if (!this.isManualOperationAllowed() || this.currentVideoQuality === quality) {
+      return;
+    }
+    this.currentVideoQuality = quality;
+    this.videoDimensions = undefined;
+
+    this.emitTrackUpdate();
+  }
+
+  setVideoDimensions(dimensions: Track.Dimensions) {
+    if (!this.isManualOperationAllowed()) {
+      return;
+    }
+    if (this.videoDimensions?.width === dimensions.width
+        && this.videoDimensions?.height === dimensions.height) {
+      return;
+    }
+    if (this.track instanceof RemoteVideoTrack) { this.videoDimensions = dimensions; }
+    this.currentVideoQuality = undefined;
+
+    this.emitTrackUpdate();
+  }
+
+  get videoQuality(): VideoQuality | undefined {
+    return this.currentVideoQuality;
+  }
+
+  setTrack(track?: Track) {
+    if (this.track) {
+      // unregister listener
+      this.track.off(TrackEvent.VideoDimensionsChanged, this.handleVideoDimensionsChange);
+      this.track.off(TrackEvent.VisibilityChanged, this.handleVisibilityChange);
+      this.track.off(TrackEvent.Ended, this.handleEnded);
+    }
+    super.setTrack(track);
+    if (track) {
+      track.sid = this.trackSid;
+      track.on(TrackEvent.VideoDimensionsChanged, this.handleVideoDimensionsChange);
+      track.on(TrackEvent.VisibilityChanged, this.handleVisibilityChange);
+      track.on(TrackEvent.Ended, this.handleEnded);
+    }
+  }
+
+  /** @internal */
+  updateInfo(info: TrackInfo) {
+    super.updateInfo(info);
+    this.metadataMuted = info.muted;
+    this.track?.setMuted(info.muted);
+  }
+
+  private isManualOperationAllowed(): boolean {
+    if (this.isAdaptiveStream) {
+      log.warn('adaptive stream is enabled, cannot change track settings', this.trackSid);
+      return false;
+    }
+    if (!this.isSubscribed) {
+      log.warn('cannot update track settings when not subscribed', this.trackSid);
+      return false;
+    }
+    return true;
+  }
+
+  protected handleEnded = (track: RemoteTrack) => {
+    this.emit(TrackEvent.Ended, track);
+  };
+
+  protected get isAdaptiveStream(): boolean {
+    return this.track instanceof RemoteVideoTrack && this.track.isAdaptiveStream;
+  }
+
+  protected handleVisibilityChange = (visible: boolean) => {
+    log.debug('adaptivestream video visibility', this.trackSid, `visible=${visible}`);
+    this.disabled = !visible;
+    this.emitTrackUpdate();
+  };
+
+  protected handleVideoDimensionsChange = (dimensions: Track.Dimensions) => {
+    log.debug('adaptivestream video dimensions', this.trackSid, `${dimensions.width}x${dimensions.height}`);
+    this.videoDimensions = dimensions;
+    this.emitTrackUpdate();
+  };
+
+  /* @internal */
+  emitTrackUpdate() {
+    const settings: UpdateTrackSettings = UpdateTrackSettings.fromPartial({
+      trackSids: [this.trackSid],
+      disabled: this.disabled,
+    });
+    if (this.videoDimensions) {
+      settings.width = this.videoDimensions.width;
+      settings.height = this.videoDimensions.height;
+    } else if (this.currentVideoQuality !== undefined) {
+      settings.quality = this.currentVideoQuality;
+    } else {
+      // defaults to high quality
+      settings.quality = VideoQuality.HIGH;
+    }
+
+    this.emit(TrackEvent.UpdateSettings, settings);
+  }
+}

package/src/room/track/RemoteVideoTrack.ts

@@ -0,0 +1,220 @@
+import { debounce } from 'ts-debounce';
+import { TrackEvent } from '../events';
+import { computeBitrate, monitorFrequency, VideoReceiverStats } from '../stats';
+import { getIntersectionObserver, getResizeObserver, ObservableMediaElement } from '../utils';
+import RemoteTrack from './RemoteTrack';
+import { attachToElement, detachTrack, Track } from './Track';
+
+const REACTION_DELAY = 100;
+
+export default class RemoteVideoTrack extends RemoteTrack {
+  /** @internal */
+  receiver?: RTCRtpReceiver;
+
+  private prevStats?: VideoReceiverStats;
+
+  private elementInfos: ElementInfo[] = [];
+
+  private adaptiveStream?: boolean;
+
+  private lastVisible?: boolean;
+
+  private lastDimensions?: Track.Dimensions;
+
+  constructor(
+    mediaTrack: MediaStreamTrack,
+    sid: string,
+    receiver?: RTCRtpReceiver,
+    adaptiveStream?: boolean,
+  ) {
+    super(mediaTrack, sid, Track.Kind.Video, receiver);
+    this.adaptiveStream = adaptiveStream;
+  }
+
+  get isAdaptiveStream(): boolean {
+    return this.adaptiveStream ?? false;
+  }
+
+  /** @internal */
+  setMuted(muted: boolean) {
+    super.setMuted(muted);
+
+    this.attachedElements.forEach((element) => {
+      // detach or attach
+      if (muted) {
+        detachTrack(this.mediaStreamTrack, element);
+      } else {
+        attachToElement(this.mediaStreamTrack, element);
+      }
+    });
+  }
+
+  attach(): HTMLMediaElement;
+  attach(element: HTMLMediaElement): HTMLMediaElement;
+  attach(element?: HTMLMediaElement): HTMLMediaElement {
+    if (!element) {
+      element = super.attach();
+    } else {
+      super.attach(element);
+    }
+
+    // It's possible attach is called multiple times on an element. When that's
+    // the case, we'd want to avoid adding duplicate elementInfos
+    if (this.adaptiveStream
+      && this.elementInfos.find((info) => info.element === element) === undefined
+    ) {
+      this.elementInfos.push({
+        element,
+        visible: true, // default visible
+      });
+
+      (element as ObservableMediaElement)
+        .handleResize = this.debouncedHandleResize;
+      (element as ObservableMediaElement)
+        .handleVisibilityChanged = this.handleVisibilityChanged;
+
+      getIntersectionObserver().observe(element);
+      getResizeObserver().observe(element);
+
+      // trigger the first resize update cycle
+      // if the tab is backgrounded, the initial resize event does not fire until
+      // the tab comes into focus for the first time.
+      this.debouncedHandleResize();
+    }
+    return element;
+  }
+
+  detach(): HTMLMediaElement[];
+  detach(element: HTMLMediaElement): HTMLMediaElement;
+  detach(element?: HTMLMediaElement): HTMLMediaElement | HTMLMediaElement[] {
+    let detachedElements: HTMLMediaElement[] = [];
+    if (element) {
+      this.stopObservingElement(element);
+      return super.detach(element);
+    }
+    detachedElements = super.detach();
+
+    for (const e of detachedElements) {
+      this.stopObservingElement(e);
+    }
+
+    return detachedElements;
+  }
+
+  protected monitorReceiver = async () => {
+    if (!this.receiver) {
+      this._currentBitrate = 0;
+      return;
+    }
+    const stats = await this.getReceiverStats();
+
+    if (stats && this.prevStats && this.receiver) {
+      this._currentBitrate = computeBitrate(stats, this.prevStats);
+    }
+
+    this.prevStats = stats;
+    setTimeout(() => {
+      this.monitorReceiver();
+    }, monitorFrequency);
+  };
+
+  private async getReceiverStats(): Promise<VideoReceiverStats | undefined> {
+    if (!this.receiver) {
+      return;
+    }
+
+    const stats = await this.receiver.getStats();
+    let receiverStats: VideoReceiverStats | undefined;
+    stats.forEach((v) => {
+      if (v.type === 'inbound-rtp') {
+        receiverStats = {
+          type: 'video',
+          framesDecoded: v.framesDecoded,
+          framesDropped: v.framesDropped,
+          framesReceived: v.framesReceived,
+          packetsReceived: v.packetsReceived,
+          packetsLost: v.packetsLost,
+          frameWidth: v.frameWidth,
+          frameHeight: v.frameHeight,
+          pliCount: v.pliCount,
+          firCount: v.firCount,
+          nackCount: v.nackCount,
+          jitter: v.jitter,
+          timestamp: v.timestamp,
+          bytesReceived: v.bytesReceived,
+        };
+      }
+    });
+    return receiverStats;
+  }
+
+  private stopObservingElement(element: HTMLMediaElement) {
+    getIntersectionObserver()?.unobserve(element);
+    getResizeObserver()?.unobserve(element);
+    this.elementInfos = this.elementInfos.filter((info) => info.element !== element);
+  }
+
+  private handleVisibilityChanged = (entry: IntersectionObserverEntry) => {
+    const { target, isIntersecting } = entry;
+    const elementInfo = this.elementInfos.find((info) => info.element === target);
+    if (elementInfo) {
+      elementInfo.visible = isIntersecting;
+      elementInfo.visibilityChangedAt = Date.now();
+    }
+    this.updateVisibility();
+  };
+
+  private readonly debouncedHandleResize = debounce(() => {
+    this.updateDimensions();
+  }, REACTION_DELAY);
+
+  private updateVisibility() {
+    const lastVisibilityChange = this.elementInfos.reduce(
+      (prev, info) => Math.max(prev, info.visibilityChangedAt || 0),
+      0,
+    );
+    const isVisible = this.elementInfos.some((info) => info.visible);
+
+    if (this.lastVisible === isVisible) {
+      return;
+    }
+
+    if (!isVisible && Date.now() - lastVisibilityChange < REACTION_DELAY) {
+      // delay hidden events
+      setTimeout(() => {
+        this.updateVisibility();
+      }, REACTION_DELAY);
+      return;
+    }
+
+    this.lastVisible = isVisible;
+    this.emit(TrackEvent.VisibilityChanged, isVisible, this);
+  }
+
+  private updateDimensions() {
+    let maxWidth = 0;
+    let maxHeight = 0;
+    for (const info of this.elementInfos) {
+      if (info.element.clientWidth + info.element.clientHeight > maxWidth + maxHeight) {
+        maxWidth = info.element.clientWidth;
+        maxHeight = info.element.clientHeight;
+      }
+    }
+
+    if (this.lastDimensions?.width === maxWidth && this.lastDimensions?.height === maxHeight) {
+      return;
+    }
+
+    this.lastDimensions = {
+      width: maxWidth,
+      height: maxHeight,
+    };
+    this.emit(TrackEvent.VideoDimensionsChanged, this.lastDimensions, this);
+  }
+}
+
+interface ElementInfo {
+  element: HTMLMediaElement;
+  visible: boolean;
+  visibilityChangedAt?: number;
+}