livekit-client 0.15.3 → 0.15.4

package/src/room/track/RemoteTrackPublication.ts

@@ -0,0 +1,192 @@
+import log from '../../logger';
+import { TrackInfo, VideoQuality } from '../../proto/livekit_models';
+import {
+  UpdateSubscription,
+  UpdateTrackSettings,
+} from '../../proto/livekit_rtc';
+import { TrackEvent } from '../events';
+import RemoteVideoTrack from './RemoteVideoTrack';
+import { Track } from './Track';
+import { TrackPublication } from './TrackPublication';
+import { RemoteTrack } from './types';
+
+export default class RemoteTrackPublication extends TrackPublication {
+  track?: RemoteTrack;
+
+  /** @internal */
+  _allowed = true;
+
+  // keeps track of client's desire to subscribe to a track
+  protected subscribed?: boolean;
+
+  protected disabled: boolean = false;
+
+  protected currentVideoQuality?: VideoQuality = VideoQuality.HIGH;
+
+  protected videoDimensions?: Track.Dimensions;
+
+  /**
+   * Subscribe or unsubscribe to this remote track
+   * @param subscribed true to subscribe to a track, false to unsubscribe
+   */
+  setSubscribed(subscribed: boolean) {
+    this.subscribed = subscribed;
+
+    const sub: UpdateSubscription = {
+      trackSids: [this.trackSid],
+      subscribe: this.subscribed,
+      participantTracks: [],
+    };
+    this.emit(TrackEvent.UpdateSubscription, sub);
+  }
+
+  get subscriptionStatus(): TrackPublication.SubscriptionStatus {
+    if (this.subscribed === false || !super.isSubscribed) {
+      return TrackPublication.SubscriptionStatus.Unsubscribed;
+    }
+    if (!this._allowed) {
+      return TrackPublication.SubscriptionStatus.NotAllowed;
+    }
+    return TrackPublication.SubscriptionStatus.Subscribed;
+  }
+
+  /**
+   * Returns true if track is subscribed, and ready for playback
+   */
+  get isSubscribed(): boolean {
+    if (this.subscribed === false) {
+      return false;
+    }
+    if (!this._allowed) {
+      return false;
+    }
+    return super.isSubscribed;
+  }
+
+  get isEnabled(): boolean {
+    return !this.disabled;
+  }
+
+  /**
+   * disable server from sending down data for this track. this is useful when
+   * the participant is off screen, you may disable streaming down their video
+   * to reduce bandwidth requirements
+   * @param enabled
+   */
+  setEnabled(enabled: boolean) {
+    if (!this.isManualOperationAllowed() || this.disabled === !enabled) {
+      return;
+    }
+    this.disabled = !enabled;
+
+    this.emitTrackUpdate();
+  }
+
+  /**
+   * for tracks that support simulcasting, adjust subscribed quality
+   *
+   * This indicates the highest quality the client can accept. if network
+   * bandwidth does not allow, server will automatically reduce quality to
+   * optimize for uninterrupted video
+   */
+  setVideoQuality(quality: VideoQuality) {
+    if (!this.isManualOperationAllowed() || this.currentVideoQuality === quality) {
+      return;
+    }
+    this.currentVideoQuality = quality;
+    this.videoDimensions = undefined;
+
+    this.emitTrackUpdate();
+  }
+
+  setVideoDimensions(dimensions: Track.Dimensions) {
+    if (!this.isManualOperationAllowed()) {
+      return;
+    }
+    if (this.videoDimensions?.width === dimensions.width
+        && this.videoDimensions?.height === dimensions.height) {
+      return;
+    }
+    if (this.track instanceof RemoteVideoTrack) { this.videoDimensions = dimensions; }
+    this.currentVideoQuality = undefined;
+
+    this.emitTrackUpdate();
+  }
+
+  get videoQuality(): VideoQuality | undefined {
+    return this.currentVideoQuality;
+  }
+
+  setTrack(track?: Track) {
+    if (this.track) {
+      // unregister listener
+      this.track.off(TrackEvent.VideoDimensionsChanged, this.handleVideoDimensionsChange);
+      this.track.off(TrackEvent.VisibilityChanged, this.handleVisibilityChange);
+      this.track.off(TrackEvent.Ended, this.handleEnded);
+    }
+    super.setTrack(track);
+    if (track) {
+      track.sid = this.trackSid;
+      track.on(TrackEvent.VideoDimensionsChanged, this.handleVideoDimensionsChange);
+      track.on(TrackEvent.VisibilityChanged, this.handleVisibilityChange);
+      track.on(TrackEvent.Ended, this.handleEnded);
+    }
+  }
+
+  /** @internal */
+  updateInfo(info: TrackInfo) {
+    super.updateInfo(info);
+    this.metadataMuted = info.muted;
+    this.track?.setMuted(info.muted);
+  }
+
+  private isManualOperationAllowed(): boolean {
+    if (this.isAdaptiveStream) {
+      log.warn('adaptive stream is enabled, cannot change track settings', this.trackSid);
+      return false;
+    }
+    if (!this.isSubscribed) {
+      log.warn('cannot update track settings when not subscribed', this.trackSid);
+      return false;
+    }
+    return true;
+  }
+
+  protected handleEnded = (track: RemoteTrack) => {
+    this.emit(TrackEvent.Ended, track);
+  };
+
+  protected get isAdaptiveStream(): boolean {
+    return this.track instanceof RemoteVideoTrack && this.track.isAdaptiveStream;
+  }
+
+  protected handleVisibilityChange = (visible: boolean) => {
+    log.debug('adaptivestream video visibility', this.trackSid, `visible=${visible}`);
+    this.disabled = !visible;
+    this.emitTrackUpdate();
+  };
+
+  protected handleVideoDimensionsChange = (dimensions: Track.Dimensions) => {
+    log.debug('adaptivestream video dimensions', this.trackSid, `${dimensions.width}x${dimensions.height}`);
+    this.videoDimensions = dimensions;
+    this.emitTrackUpdate();
+  };
+
+  protected emitTrackUpdate() {
+    const settings: UpdateTrackSettings = UpdateTrackSettings.fromPartial({
+      trackSids: [this.trackSid],
+      disabled: this.disabled,
+    });
+    if (this.videoDimensions) {
+      settings.width = this.videoDimensions.width;
+      settings.height = this.videoDimensions.height;
+    } else if (this.currentVideoQuality !== undefined) {
+      settings.quality = this.currentVideoQuality;
+    } else {
+      // defaults to high quality
+      settings.quality = VideoQuality.HIGH;
+    }
+
+    this.emit(TrackEvent.UpdateSettings, settings);
+  }
+}

package/src/room/track/RemoteVideoTrack.ts

@@ -0,0 +1,213 @@
+import { debounce } from 'ts-debounce';
+import { TrackEvent } from '../events';
+import { computeBitrate, monitorFrequency, VideoReceiverStats } from '../stats';
+import { getIntersectionObserver, getResizeObserver, ObservableMediaElement } from '../utils';
+import RemoteTrack from './RemoteTrack';
+import { attachToElement, detachTrack, Track } from './Track';
+
+const REACTION_DELAY = 100;
+
+export default class RemoteVideoTrack extends RemoteTrack {
+  /** @internal */
+  receiver?: RTCRtpReceiver;
+
+  private prevStats?: VideoReceiverStats;
+
+  private elementInfos: ElementInfo[] = [];
+
+  private adaptiveStream?: boolean;
+
+  private lastVisible?: boolean;
+
+  private lastDimensions?: Track.Dimensions;
+
+  constructor(
+    mediaTrack: MediaStreamTrack,
+    sid: string,
+    receiver?: RTCRtpReceiver,
+    adaptiveStream?: boolean,
+  ) {
+    super(mediaTrack, sid, Track.Kind.Video, receiver);
+    this.adaptiveStream = adaptiveStream;
+  }
+
+  get isAdaptiveStream(): boolean {
+    return this.adaptiveStream ?? false;
+  }
+
+  /** @internal */
+  setMuted(muted: boolean) {
+    super.setMuted(muted);
+
+    this.attachedElements.forEach((element) => {
+      // detach or attach
+      if (muted) {
+        detachTrack(this.mediaStreamTrack, element);
+      } else {
+        attachToElement(this.mediaStreamTrack, element);
+      }
+    });
+  }
+
+  attach(): HTMLMediaElement;
+  attach(element: HTMLMediaElement): HTMLMediaElement;
+  attach(element?: HTMLMediaElement): HTMLMediaElement {
+    if (!element) {
+      element = super.attach();
+    } else {
+      super.attach(element);
+    }
+
+    if (this.adaptiveStream) {
+      this.elementInfos.push({
+        element,
+        visible: true, // default visible
+      });
+
+      (element as ObservableMediaElement)
+        .handleResize = this.debouncedHandleResize;
+      (element as ObservableMediaElement)
+        .handleVisibilityChanged = this.handleVisibilityChanged;
+
+      getIntersectionObserver().observe(element);
+      getResizeObserver().observe(element);
+    }
+    return element;
+  }
+
+  detach(): HTMLMediaElement[];
+  detach(element: HTMLMediaElement): HTMLMediaElement;
+  detach(element?: HTMLMediaElement): HTMLMediaElement | HTMLMediaElement[] {
+    let detachedElements: HTMLMediaElement[] = [];
+    if (element) {
+      this.stopObservingElement(element);
+      return super.detach(element);
+    }
+    detachedElements = super.detach();
+
+    for (const e of detachedElements) {
+      this.stopObservingElement(e);
+    }
+
+    return detachedElements;
+  }
+
+  protected monitorReceiver = async () => {
+    if (!this.receiver) {
+      this._currentBitrate = 0;
+      return;
+    }
+    const stats = await this.getReceiverStats();
+
+    if (stats && this.prevStats && this.receiver) {
+      this._currentBitrate = computeBitrate(stats, this.prevStats);
+    }
+
+    this.prevStats = stats;
+    setTimeout(() => {
+      this.monitorReceiver();
+    }, monitorFrequency);
+  };
+
+  private async getReceiverStats(): Promise<VideoReceiverStats | undefined> {
+    if (!this.receiver) {
+      return;
+    }
+
+    const stats = await this.receiver.getStats();
+    let receiverStats: VideoReceiverStats | undefined;
+    stats.forEach((v) => {
+      if (v.type === 'inbound-rtp') {
+        receiverStats = {
+          type: 'video',
+          framesDecoded: v.framesDecoded,
+          framesDropped: v.framesDropped,
+          framesReceived: v.framesReceived,
+          packetsReceived: v.packetsReceived,
+          packetsLost: v.packetsLost,
+          frameWidth: v.frameWidth,
+          frameHeight: v.frameHeight,
+          pliCount: v.pliCount,
+          firCount: v.firCount,
+          nackCount: v.nackCount,
+          jitter: v.jitter,
+          timestamp: v.timestamp,
+          bytesReceived: v.bytesReceived,
+        };
+      }
+    });
+    return receiverStats;
+  }
+
+  private stopObservingElement(element: HTMLMediaElement) {
+    getIntersectionObserver()?.unobserve(element);
+    getResizeObserver()?.unobserve(element);
+    this.elementInfos = this.elementInfos.filter((info) => info.element !== element);
+  }
+
+  private handleVisibilityChanged = (entry: IntersectionObserverEntry) => {
+    const { target, isIntersecting } = entry;
+    const elementInfo = this.elementInfos.find((info) => info.element === target);
+    if (elementInfo) {
+      elementInfo.visible = isIntersecting;
+      elementInfo.visibilityChangedAt = Date.now();
+    }
+    this.updateVisibility();
+  };
+
+  private readonly debouncedHandleResize = debounce(() => {
+    this.updateDimensions();
+  }, REACTION_DELAY);
+
+  private updateVisibility() {
+    const lastVisibilityChange = this.elementInfos.reduce(
+      (prev, info) => Math.max(prev, info.visibilityChangedAt || 0),
+      0,
+    );
+    const isVisible = this.elementInfos.some((info) => info.visible);
+
+    if (this.lastVisible === isVisible) {
+      return;
+    }
+
+    if (!isVisible && Date.now() - lastVisibilityChange < REACTION_DELAY) {
+      // delay hidden events
+      setTimeout(() => {
+        this.updateVisibility();
+      }, Date.now() - lastVisibilityChange);
+      return;
+    }
+
+    this.lastVisible = isVisible;
+    this.emit(TrackEvent.VisibilityChanged, isVisible, this);
+  }
+
+  private updateDimensions() {
+    let maxWidth = 0;
+    let maxHeight = 0;
+    for (const info of this.elementInfos) {
+      if (info.visible) {
+        if (info.element.clientWidth + info.element.clientHeight > maxWidth + maxHeight) {
+          maxWidth = info.element.clientWidth;
+          maxHeight = info.element.clientHeight;
+        }
+      }
+    }
+
+    if (this.lastDimensions?.width === maxWidth && this.lastDimensions?.height === maxHeight) {
+      return;
+    }
+
+    this.lastDimensions = {
+      width: maxWidth,
+      height: maxHeight,
+    };
+    this.emit(TrackEvent.VideoDimensionsChanged, this.lastDimensions, this);
+  }
+}
+
+interface ElementInfo {
+  element: HTMLMediaElement;
+  visible: boolean;
+  visibilityChangedAt?: number;
+}

package/src/room/track/Track.ts

@@ -0,0 +1,301 @@
+import { EventEmitter } from 'events';
+import { TrackSource, TrackType } from '../../proto/livekit_models';
+import { StreamState as ProtoStreamState } from '../../proto/livekit_rtc';
+import { TrackEvent } from '../events';
+import { isFireFox } from '../utils';
+
+// keep old audio elements when detached, we would re-use them since on iOS
+// Safari tracks which audio elements have been "blessed" by the user.
+const recycledElements: Array<HTMLAudioElement> = [];
+
+export class Track extends EventEmitter {
+  kind: Track.Kind;
+
+  mediaStreamTrack: MediaStreamTrack;
+
+  attachedElements: HTMLMediaElement[] = [];
+
+  isMuted: boolean = false;
+
+  streamState: Track.StreamState = Track.StreamState.Active;
+
+  source: Track.Source;
+
+  /**
+   * sid is set after track is published to server, or if it's a remote track
+   */
+  sid?: Track.SID;
+
+  protected _currentBitrate: number = 0;
+
+  protected constructor(mediaTrack: MediaStreamTrack, kind: Track.Kind) {
+    super();
+    this.kind = kind;
+    this.mediaStreamTrack = mediaTrack;
+    this.source = Track.Source.Unknown;
+  }
+
+  /** current receive bits per second */
+  get currentBitrate(): number {
+    return this._currentBitrate;
+  }
+
+  /**
+   * creates a new HTMLAudioElement or HTMLVideoElement, attaches to it, and returns it
+   */
+  attach(): HTMLMediaElement;
+
+  /**
+   * attaches track to an existing HTMLAudioElement or HTMLVideoElement
+   */
+  attach(element: HTMLMediaElement): HTMLMediaElement;
+  attach(element?: HTMLMediaElement): HTMLMediaElement {
+    let elementType = 'audio';
+    if (this.kind === Track.Kind.Video) {
+      elementType = 'video';
+    }
+    if (!element) {
+      if (elementType === 'audio') {
+        recycledElements.forEach((e) => {
+          if (e.parentElement === null && !element) {
+            element = e;
+          }
+        });
+        if (element) {
+          // remove it from pool
+          recycledElements.splice(recycledElements.indexOf(element), 1);
+        }
+      }
+      if (!element) {
+        element = <HTMLMediaElement>document.createElement(elementType);
+      }
+    }
+
+    if (element instanceof HTMLVideoElement) {
+      element.playsInline = true;
+      element.autoplay = true;
+    }
+
+    // already attached
+    if (this.attachedElements.includes(element)) {
+      return element;
+    }
+
+    attachToElement(this.mediaStreamTrack, element);
+    this.attachedElements.push(element);
+
+    if (element instanceof HTMLAudioElement) {
+      // manually play audio to detect audio playback status
+      element.play()
+        .then(() => {
+          this.emit(TrackEvent.AudioPlaybackStarted);
+        })
+        .catch((e) => {
+          this.emit(TrackEvent.AudioPlaybackFailed, e);
+        });
+    }
+
+    return element;
+  }
+
+  /**
+   * Detaches from all attached elements
+   */
+  detach(): HTMLMediaElement[];
+
+  /**
+   * Detach from a single element
+   * @param element
+   */
+  detach(element: HTMLMediaElement): HTMLMediaElement;
+  detach(element?: HTMLMediaElement): HTMLMediaElement | HTMLMediaElement[] {
+    // detach from a single element
+    if (element) {
+      detachTrack(this.mediaStreamTrack, element);
+      const idx = this.attachedElements.indexOf(element);
+      if (idx >= 0) {
+        this.attachedElements.splice(idx, 1);
+        this.recycleElement(element);
+      }
+      return element;
+    }
+
+    const detached: HTMLMediaElement[] = [];
+    this.attachedElements.forEach((elm) => {
+      detachTrack(this.mediaStreamTrack, elm);
+      detached.push(elm);
+      this.recycleElement(elm);
+    });
+
+    // remove all tracks
+    this.attachedElements = [];
+    return detached;
+  }
+
+  stop() {
+    this.mediaStreamTrack.stop();
+  }
+
+  protected enable() {
+    this.mediaStreamTrack.enabled = true;
+  }
+
+  protected disable() {
+    this.mediaStreamTrack.enabled = false;
+  }
+
+  private recycleElement(element: HTMLMediaElement) {
+    if (element instanceof HTMLAudioElement) {
+    // we only need to re-use a single element
+      let shouldCache = true;
+      element.pause();
+      recycledElements.forEach((e) => {
+        if (!e.parentElement) {
+          shouldCache = false;
+        }
+      });
+      if (shouldCache) {
+        recycledElements.push(element);
+      }
+    }
+  }
+}
+
+/** @internal */
+export function attachToElement(track: MediaStreamTrack, element: HTMLMediaElement) {
+  let mediaStream: MediaStream;
+  if (element.srcObject instanceof MediaStream) {
+    mediaStream = element.srcObject;
+  } else {
+    mediaStream = new MediaStream();
+    element.srcObject = mediaStream;
+  }
+
+  // remove existing tracks of same type from stream
+  let existingTracks: MediaStreamTrack[];
+  if (track.kind === 'audio') {
+    existingTracks = mediaStream.getAudioTracks();
+  } else {
+    existingTracks = mediaStream.getVideoTracks();
+  }
+
+  existingTracks.forEach((et) => {
+    mediaStream.removeTrack(et);
+  });
+
+  mediaStream.addTrack(track);
+  if (isFireFox()) {
+    // sometimes firefox doesn't render local video on the first try.
+    // It needs to be re-attached after a timeout.
+    setTimeout(() => {
+      element.srcObject = mediaStream;
+    }, 1);
+  }
+}
+
+/** @internal */
+export function detachTrack(
+  track: MediaStreamTrack,
+  element: HTMLMediaElement,
+) {
+  if (element.srcObject instanceof MediaStream) {
+    const mediaStream = element.srcObject;
+    mediaStream.removeTrack(track);
+    element.srcObject = null;
+  }
+}
+
+export namespace Track {
+  export enum Kind {
+    Audio = 'audio',
+    Video = 'video',
+    Unknown = 'unknown',
+  }
+  export type SID = string;
+  export enum Source {
+    Camera = 'camera',
+    Microphone = 'microphone',
+    ScreenShare = 'screen_share',
+    ScreenShareAudio = 'screen_share_audio',
+    Unknown = 'unknown',
+  }
+
+  export enum StreamState {
+    Active = 'active',
+    Paused = 'paused',
+    Unknown = 'unknown',
+  }
+
+  export interface Dimensions {
+    width: number;
+    height: number;
+  }
+
+  /** @internal */
+  export function kindToProto(k: Kind): TrackType {
+    switch (k) {
+      case Kind.Audio:
+        return TrackType.AUDIO;
+      case Kind.Video:
+        return TrackType.VIDEO;
+      default:
+        return TrackType.UNRECOGNIZED;
+    }
+  }
+
+  /** @internal */
+  export function kindFromProto(t: TrackType): Kind | undefined {
+    switch (t) {
+      case TrackType.AUDIO:
+        return Kind.Audio;
+      case TrackType.VIDEO:
+        return Kind.Video;
+      default:
+        return Kind.Unknown;
+    }
+  }
+
+  /** @internal */
+  export function sourceToProto(s: Source): TrackSource {
+    switch (s) {
+      case Source.Camera:
+        return TrackSource.CAMERA;
+      case Source.Microphone:
+        return TrackSource.MICROPHONE;
+      case Source.ScreenShare:
+        return TrackSource.SCREEN_SHARE;
+      case Source.ScreenShareAudio:
+        return TrackSource.SCREEN_SHARE_AUDIO;
+      default:
+        return TrackSource.UNRECOGNIZED;
+    }
+  }
+
+  /** @internal */
+  export function sourceFromProto(s: TrackSource): Source {
+    switch (s) {
+      case TrackSource.CAMERA:
+        return Source.Camera;
+      case TrackSource.MICROPHONE:
+        return Source.Microphone;
+      case TrackSource.SCREEN_SHARE:
+        return Source.ScreenShare;
+      case TrackSource.SCREEN_SHARE_AUDIO:
+        return Source.ScreenShareAudio;
+      default:
+        return Source.Unknown;
+    }
+  }
+
+  /** @internal */
+  export function streamStateFromProto(s: ProtoStreamState): StreamState {
+    switch (s) {
+      case ProtoStreamState.ACTIVE:
+        return StreamState.Active;
+      case ProtoStreamState.PAUSED:
+        return StreamState.Paused;
+      default:
+        return StreamState.Unknown;
+    }
+  }
+}