livekit-client 0.17.5 → 0.17.6-rc1

package/src/room/track/RemoteVideoTrack.ts

@@ -1,235 +0,0 @@
-import { debounce } from 'ts-debounce';
-import { TrackEvent } from '../events';
-import { computeBitrate, monitorFrequency, VideoReceiverStats } from '../stats';
-import {
-  getIntersectionObserver, getResizeObserver, isMobile, ObservableMediaElement,
-} from '../utils';
-import RemoteTrack from './RemoteTrack';
-import { attachToElement, detachTrack, Track } from './Track';
-import { AdaptiveStreamSettings } from './types';
-
-const REACTION_DELAY = 100;
-
-export default class RemoteVideoTrack extends RemoteTrack {
-  /** @internal */
-  receiver?: RTCRtpReceiver;
-
-  private prevStats?: VideoReceiverStats;
-
-  private elementInfos: ElementInfo[] = [];
-
-  private adaptiveStreamSettings?: AdaptiveStreamSettings;
-
-  private lastVisible?: boolean;
-
-  private lastDimensions?: Track.Dimensions;
-
-  constructor(
-    mediaTrack: MediaStreamTrack,
-    sid: string,
-    receiver?: RTCRtpReceiver,
-    adaptiveStreamSettings?: AdaptiveStreamSettings,
-  ) {
-    super(mediaTrack, sid, Track.Kind.Video, receiver);
-    this.adaptiveStreamSettings = adaptiveStreamSettings;
-  }
-
-  get isAdaptiveStream(): boolean {
-    return this.adaptiveStreamSettings !== undefined;
-  }
-
-  /** @internal */
-  setMuted(muted: boolean) {
-    super.setMuted(muted);
-
-    this.attachedElements.forEach((element) => {
-      // detach or attach
-      if (muted) {
-        detachTrack(this.mediaStreamTrack, element);
-      } else {
-        attachToElement(this.mediaStreamTrack, element);
-      }
-    });
-  }
-
-  attach(): HTMLMediaElement;
-  attach(element: HTMLMediaElement): HTMLMediaElement;
-  attach(element?: HTMLMediaElement): HTMLMediaElement {
-    if (!element) {
-      element = super.attach();
-    } else {
-      super.attach(element);
-    }
-
-    // It's possible attach is called multiple times on an element. When that's
-    // the case, we'd want to avoid adding duplicate elementInfos
-    if (this.adaptiveStreamSettings
-      && this.elementInfos.find((info) => info.element === element) === undefined
-    ) {
-      this.elementInfos.push({
-        element,
-        visible: true, // default visible
-      });
-
-      (element as ObservableMediaElement)
-        .handleResize = this.debouncedHandleResize;
-      (element as ObservableMediaElement)
-        .handleVisibilityChanged = this.handleVisibilityChanged;
-
-      getIntersectionObserver().observe(element);
-      getResizeObserver().observe(element);
-
-      // trigger the first resize update cycle
-      // if the tab is backgrounded, the initial resize event does not fire until
-      // the tab comes into focus for the first time.
-      this.debouncedHandleResize();
-    }
-    return element;
-  }
-
-  detach(): HTMLMediaElement[];
-  detach(element: HTMLMediaElement): HTMLMediaElement;
-  detach(element?: HTMLMediaElement): HTMLMediaElement | HTMLMediaElement[] {
-    let detachedElements: HTMLMediaElement[] = [];
-    if (element) {
-      this.stopObservingElement(element);
-      return super.detach(element);
-    }
-    detachedElements = super.detach();
-
-    for (const e of detachedElements) {
-      this.stopObservingElement(e);
-    }
-
-    return detachedElements;
-  }
-
-  protected monitorReceiver = async () => {
-    if (!this.receiver) {
-      this._currentBitrate = 0;
-      return;
-    }
-    const stats = await this.getReceiverStats();
-
-    if (stats && this.prevStats && this.receiver) {
-      this._currentBitrate = computeBitrate(stats, this.prevStats);
-    }
-
-    this.prevStats = stats;
-    setTimeout(() => {
-      this.monitorReceiver();
-    }, monitorFrequency);
-  };
-
-  private async getReceiverStats(): Promise<VideoReceiverStats | undefined> {
-    if (!this.receiver) {
-      return;
-    }
-
-    const stats = await this.receiver.getStats();
-    let receiverStats: VideoReceiverStats | undefined;
-    stats.forEach((v) => {
-      if (v.type === 'inbound-rtp') {
-        receiverStats = {
-          type: 'video',
-          framesDecoded: v.framesDecoded,
-          framesDropped: v.framesDropped,
-          framesReceived: v.framesReceived,
-          packetsReceived: v.packetsReceived,
-          packetsLost: v.packetsLost,
-          frameWidth: v.frameWidth,
-          frameHeight: v.frameHeight,
-          pliCount: v.pliCount,
-          firCount: v.firCount,
-          nackCount: v.nackCount,
-          jitter: v.jitter,
-          timestamp: v.timestamp,
-          bytesReceived: v.bytesReceived,
-        };
-      }
-    });
-    return receiverStats;
-  }
-
-  private stopObservingElement(element: HTMLMediaElement) {
-    getIntersectionObserver()?.unobserve(element);
-    getResizeObserver()?.unobserve(element);
-    this.elementInfos = this.elementInfos.filter((info) => info.element !== element);
-  }
-
-  private handleVisibilityChanged = (entry: IntersectionObserverEntry) => {
-    const { target, isIntersecting } = entry;
-    const elementInfo = this.elementInfos.find((info) => info.element === target);
-    if (elementInfo) {
-      elementInfo.visible = isIntersecting;
-      elementInfo.visibilityChangedAt = Date.now();
-    }
-    this.updateVisibility();
-  };
-
-  protected async handleAppVisibilityChanged() {
-    await super.handleAppVisibilityChanged();
-    if (!this.isAdaptiveStream) return;
-    // on desktop don't pause when tab is backgrounded
-    if (!isMobile()) return;
-    this.updateVisibility();
-  }
-
-  private readonly debouncedHandleResize = debounce(() => {
-    this.updateDimensions();
-  }, REACTION_DELAY);
-
-  private updateVisibility() {
-    const lastVisibilityChange = this.elementInfos.reduce(
-      (prev, info) => Math.max(prev, info.visibilityChangedAt || 0),
-      0,
-    );
-    const isVisible = this.elementInfos.some((info) => info.visible) && !this.isInBackground;
-
-    if (this.lastVisible === isVisible) {
-      return;
-    }
-
-    if (!isVisible && Date.now() - lastVisibilityChange < REACTION_DELAY) {
-      // delay hidden events
-      setTimeout(() => {
-        this.updateVisibility();
-      }, REACTION_DELAY);
-      return;
-    }
-
-    this.lastVisible = isVisible;
-    this.emit(TrackEvent.VisibilityChanged, isVisible, this);
-  }
-
-  private updateDimensions() {
-    let maxWidth = 0;
-    let maxHeight = 0;
-    for (const info of this.elementInfos) {
-      const pixelDensity = this.adaptiveStreamSettings?.pixelDensity ?? 1;
-      const pixelDensityValue = pixelDensity === 'screen' ? window.devicePixelRatio : pixelDensity;
-      const currentElementWidth = info.element.clientWidth * pixelDensityValue;
-      const currentElementHeight = info.element.clientHeight * pixelDensityValue;
-      if (currentElementWidth + currentElementHeight > maxWidth + maxHeight) {
-        maxWidth = currentElementWidth;
-        maxHeight = currentElementHeight;
-      }
-    }
-
-    if (this.lastDimensions?.width === maxWidth && this.lastDimensions?.height === maxHeight) {
-      return;
-    }
-
-    this.lastDimensions = {
-      width: maxWidth,
-      height: maxHeight,
-    };
-    this.emit(TrackEvent.VideoDimensionsChanged, this.lastDimensions, this);
-  }
-}
-
-interface ElementInfo {
-  element: HTMLMediaElement;
-  visible: boolean;
-  visibilityChangedAt?: number;
-}

package/src/room/track/Track.ts

@@ -1,343 +0,0 @@
-import { EventEmitter } from 'events';
-import type TypedEventEmitter from 'typed-emitter';
-import { TrackSource, TrackType } from '../../proto/livekit_models';
-import { StreamState as ProtoStreamState } from '../../proto/livekit_rtc';
-import { TrackEvent } from '../events';
-import { isFireFox, isSafari, isWeb } from '../utils';
-
-// keep old audio elements when detached, we would re-use them since on iOS
-// Safari tracks which audio elements have been "blessed" by the user.
-const recycledElements: Array<HTMLAudioElement> = [];
-
-export class Track extends (EventEmitter as new () => TypedEventEmitter<TrackEventCallbacks>) {
-  kind: Track.Kind;
-
-  mediaStream?: MediaStream;
-
-  mediaStreamTrack: MediaStreamTrack;
-
-  attachedElements: HTMLMediaElement[] = [];
-
-  isMuted: boolean = false;
-
-  source: Track.Source;
-
-  protected isInBackground: boolean;
-
-  /**
-   * sid is set after track is published to server, or if it's a remote track
-   */
-  sid?: Track.SID;
-
-  protected _currentBitrate: number = 0;
-
-  protected constructor(mediaTrack: MediaStreamTrack, kind: Track.Kind) {
-    super();
-    this.kind = kind;
-    this.mediaStreamTrack = mediaTrack;
-    this.source = Track.Source.Unknown;
-    if (isWeb()) {
-      this.isInBackground = document.visibilityState === 'hidden';
-      document.addEventListener('visibilitychange', this.appVisibilityChangedListener);
-    } else {
-      this.isInBackground = false;
-    }
-  }
-
-  /** current receive bits per second */
-  get currentBitrate(): number {
-    return this._currentBitrate;
-  }
-
-  /**
-   * creates a new HTMLAudioElement or HTMLVideoElement, attaches to it, and returns it
-   */
-  attach(): HTMLMediaElement;
-
-  /**
-   * attaches track to an existing HTMLAudioElement or HTMLVideoElement
-   */
-  attach(element: HTMLMediaElement): HTMLMediaElement;
-  attach(element?: HTMLMediaElement): HTMLMediaElement {
-    let elementType = 'audio';
-    if (this.kind === Track.Kind.Video) {
-      elementType = 'video';
-    }
-    if (!element) {
-      if (elementType === 'audio') {
-        recycledElements.forEach((e) => {
-          if (e.parentElement === null && !element) {
-            element = e;
-          }
-        });
-        if (element) {
-          // remove it from pool
-          recycledElements.splice(recycledElements.indexOf(element), 1);
-        }
-      }
-      if (!element) {
-        element = <HTMLMediaElement>document.createElement(elementType);
-      }
-    }
-
-    if (!this.attachedElements.includes(element)) {
-      this.attachedElements.push(element);
-    }
-
-    // even if we believe it's already attached to the element, it's possible
-    // the element's srcObject was set to something else out of band.
-    // we'll want to re-attach it in that case
-    attachToElement(this.mediaStreamTrack, element);
-
-    if (element instanceof HTMLAudioElement) {
-      // manually play audio to detect audio playback status
-      element.play()
-        .then(() => {
-          this.emit(TrackEvent.AudioPlaybackStarted);
-        })
-        .catch((e) => {
-          this.emit(TrackEvent.AudioPlaybackFailed, e);
-        });
-    }
-
-    return element;
-  }
-
-  /**
-   * Detaches from all attached elements
-   */
-  detach(): HTMLMediaElement[];
-
-  /**
-   * Detach from a single element
-   * @param element
-   */
-  detach(element: HTMLMediaElement): HTMLMediaElement;
-  detach(element?: HTMLMediaElement): HTMLMediaElement | HTMLMediaElement[] {
-    // detach from a single element
-    if (element) {
-      detachTrack(this.mediaStreamTrack, element);
-      const idx = this.attachedElements.indexOf(element);
-      if (idx >= 0) {
-        this.attachedElements.splice(idx, 1);
-        this.recycleElement(element);
-      }
-      return element;
-    }
-
-    const detached: HTMLMediaElement[] = [];
-    this.attachedElements.forEach((elm) => {
-      detachTrack(this.mediaStreamTrack, elm);
-      detached.push(elm);
-      this.recycleElement(elm);
-    });
-
-    // remove all tracks
-    this.attachedElements = [];
-    return detached;
-  }
-
-  stop() {
-    this.mediaStreamTrack.stop();
-    if (isWeb()) {
-      document.removeEventListener('visibilitychange', this.appVisibilityChangedListener);
-    }
-  }
-
-  protected enable() {
-    this.mediaStreamTrack.enabled = true;
-  }
-
-  protected disable() {
-    this.mediaStreamTrack.enabled = false;
-  }
-
-  private recycleElement(element: HTMLMediaElement) {
-    if (element instanceof HTMLAudioElement) {
-    // we only need to re-use a single element
-      let shouldCache = true;
-      element.pause();
-      recycledElements.forEach((e) => {
-        if (!e.parentElement) {
-          shouldCache = false;
-        }
-      });
-      if (shouldCache) {
-        recycledElements.push(element);
-      }
-    }
-  }
-
-  appVisibilityChangedListener = () => {
-    this.handleAppVisibilityChanged();
-  };
-
-  protected async handleAppVisibilityChanged() {
-    this.isInBackground = document.visibilityState === 'hidden';
-  }
-}
-
-/** @internal */
-export function attachToElement(track: MediaStreamTrack, element: HTMLMediaElement) {
-  let mediaStream: MediaStream;
-  if (element.srcObject instanceof MediaStream) {
-    mediaStream = element.srcObject;
-  } else {
-    mediaStream = new MediaStream();
-  }
-
-  // check if track matches existing track
-  let existingTracks: MediaStreamTrack[];
-  if (track.kind === 'audio') {
-    existingTracks = mediaStream.getAudioTracks();
-  } else {
-    existingTracks = mediaStream.getVideoTracks();
-  }
-  if (!existingTracks.includes(track)) {
-    existingTracks.forEach((et) => {
-      mediaStream.removeTrack(et);
-    });
-    mediaStream.addTrack(track);
-  }
-
-  // avoid flicker
-  if (element.srcObject !== mediaStream) {
-    element.srcObject = mediaStream;
-    if ((isSafari() || isFireFox()) && element instanceof HTMLVideoElement) {
-      // Firefox also has a timing issue where video doesn't actually get attached unless
-      // performed out-of-band
-      // Safari 15 has a bug where in certain layouts, video element renders
-      // black until the page is resized or other changes take place.
-      // Resetting the src triggers it to render.
-      // https://developer.apple.com/forums/thread/690523
-      setTimeout(() => {
-        element.srcObject = mediaStream;
-      }, 0);
-    }
-  }
-  element.autoplay = true;
-  if (element instanceof HTMLVideoElement) {
-    element.playsInline = true;
-  }
-}
-
-/** @internal */
-export function detachTrack(
-  track: MediaStreamTrack,
-  element: HTMLMediaElement,
-) {
-  if (element.srcObject instanceof MediaStream) {
-    const mediaStream = element.srcObject;
-    mediaStream.removeTrack(track);
-    element.srcObject = null;
-  }
-}
-
-export namespace Track {
-  export enum Kind {
-    Audio = 'audio',
-    Video = 'video',
-    Unknown = 'unknown',
-  }
-  export type SID = string;
-  export enum Source {
-    Camera = 'camera',
-    Microphone = 'microphone',
-    ScreenShare = 'screen_share',
-    ScreenShareAudio = 'screen_share_audio',
-    Unknown = 'unknown',
-  }
-
-  export enum StreamState {
-    Active = 'active',
-    Paused = 'paused',
-    Unknown = 'unknown',
-  }
-
-  export interface Dimensions {
-    width: number;
-    height: number;
-  }
-
-  /** @internal */
-  export function kindToProto(k: Kind): TrackType {
-    switch (k) {
-      case Kind.Audio:
-        return TrackType.AUDIO;
-      case Kind.Video:
-        return TrackType.VIDEO;
-      default:
-        return TrackType.UNRECOGNIZED;
-    }
-  }
-
-  /** @internal */
-  export function kindFromProto(t: TrackType): Kind | undefined {
-    switch (t) {
-      case TrackType.AUDIO:
-        return Kind.Audio;
-      case TrackType.VIDEO:
-        return Kind.Video;
-      default:
-        return Kind.Unknown;
-    }
-  }
-
-  /** @internal */
-  export function sourceToProto(s: Source): TrackSource {
-    switch (s) {
-      case Source.Camera:
-        return TrackSource.CAMERA;
-      case Source.Microphone:
-        return TrackSource.MICROPHONE;
-      case Source.ScreenShare:
-        return TrackSource.SCREEN_SHARE;
-      case Source.ScreenShareAudio:
-        return TrackSource.SCREEN_SHARE_AUDIO;
-      default:
-        return TrackSource.UNRECOGNIZED;
-    }
-  }
-
-  /** @internal */
-  export function sourceFromProto(s: TrackSource): Source {
-    switch (s) {
-      case TrackSource.CAMERA:
-        return Source.Camera;
-      case TrackSource.MICROPHONE:
-        return Source.Microphone;
-      case TrackSource.SCREEN_SHARE:
-        return Source.ScreenShare;
-      case TrackSource.SCREEN_SHARE_AUDIO:
-        return Source.ScreenShareAudio;
-      default:
-        return Source.Unknown;
-    }
-  }
-
-  /** @internal */
-  export function streamStateFromProto(s: ProtoStreamState): StreamState {
-    switch (s) {
-      case ProtoStreamState.ACTIVE:
-        return StreamState.Active;
-      case ProtoStreamState.PAUSED:
-        return StreamState.Paused;
-      default:
-        return StreamState.Unknown;
-    }
-  }
-}
-
-export type TrackEventCallbacks = {
-  message: () => void,
-  muted: (track?: any) => void,
-  unmuted: (track?: any) => void,
-  ended: (track?: any) => void,
-  updateSettings: () => void,
-  updateSubscription: () => void,
-  audioPlaybackStarted: () => void,
-  audioPlaybackFailed: (error: Error) => void,
-  audioSilenceDetected: () => void,
-  visibilityChanged: (visible: boolean, track?: any) => void,
-  videoDimensionsChanged: (dimensions: Track.Dimensions, track?: any) => void,
-};

package/src/room/track/TrackPublication.ts

@@ -1,120 +0,0 @@
-import { EventEmitter } from 'events';
-import { TrackInfo } from '../../proto/livekit_models';
-import { TrackEvent } from '../events';
-import LocalAudioTrack from './LocalAudioTrack';
-import LocalVideoTrack from './LocalVideoTrack';
-import RemoteAudioTrack from './RemoteAudioTrack';
-import RemoteVideoTrack from './RemoteVideoTrack';
-import { Track } from './Track';
-
-export class TrackPublication extends EventEmitter {
-  kind: Track.Kind;
-
-  trackName: string;
-
-  trackSid: Track.SID;
-
-  track?: Track;
-
-  source: Track.Source;
-
-  /** MimeType of the published track */
-  mimeType?: string;
-
-  /** dimension of the original published stream, video-only */
-  dimensions?: Track.Dimensions;
-
-  /** true if track was simulcasted to server, video-only */
-  simulcasted?: boolean;
-
-  /** @internal */
-  trackInfo?: TrackInfo;
-
-  protected metadataMuted: boolean = false;
-
-  constructor(kind: Track.Kind, id: string, name: string) {
-    super();
-    this.kind = kind;
-    this.trackSid = id;
-    this.trackName = name;
-    this.source = Track.Source.Unknown;
-  }
-
-  /** @internal */
-  setTrack(track?: Track) {
-    if (this.track) {
-      this.track.off(TrackEvent.Muted, this.handleMuted);
-      this.track.off(TrackEvent.Unmuted, this.handleUnmuted);
-    }
-
-    this.track = track;
-
-    if (track) {
-      // forward events
-      track.on(TrackEvent.Muted, this.handleMuted);
-      track.on(TrackEvent.Unmuted, this.handleUnmuted);
-    }
-  }
-
-  get isMuted(): boolean {
-    return this.metadataMuted;
-  }
-
-  get isEnabled(): boolean {
-    return true;
-  }
-
-  get isSubscribed(): boolean {
-    return this.track !== undefined;
-  }
-
-  /**
-   * an [AudioTrack] if this publication holds an audio track
-   */
-  get audioTrack(): LocalAudioTrack | RemoteAudioTrack | undefined {
-    if (this.track instanceof LocalAudioTrack || this.track instanceof RemoteAudioTrack) {
-      return this.track;
-    }
-  }
-
-  /**
-   * an [VideoTrack] if this publication holds a video track
-   */
-  get videoTrack(): LocalVideoTrack | RemoteVideoTrack | undefined {
-    if (this.track instanceof LocalVideoTrack || this.track instanceof RemoteVideoTrack) {
-      return this.track;
-    }
-  }
-
-  handleMuted = () => {
-    this.emit(TrackEvent.Muted);
-  };
-
-  handleUnmuted = () => {
-    this.emit(TrackEvent.Unmuted);
-  };
-
-  /** @internal */
-  updateInfo(info: TrackInfo) {
-    this.trackSid = info.sid;
-    this.trackName = info.name;
-    this.source = Track.sourceFromProto(info.source);
-    this.mimeType = info.mimeType;
-    if (this.kind === Track.Kind.Video && info.width > 0) {
-      this.dimensions = {
-        width: info.width,
-        height: info.height,
-      };
-      this.simulcasted = info.simulcast;
-    }
-    this.trackInfo = info;
-  }
-}
-
-export namespace TrackPublication {
-  export enum SubscriptionStatus {
-    Subscribed = 'subscribed',
-    NotAllowed = 'not_allowed',
-    Unsubscribed = 'unsubscribed',
-  }
-}