livekit-client 0.18.4-RC7 → 0.18.5

package/src/room/track/Track.ts

@@ -0,0 +1,365 @@
+import { EventEmitter } from 'events';
+import type TypedEventEmitter from 'typed-emitter';
+import { TrackSource, TrackType } from '../../proto/livekit_models';
+import { StreamState as ProtoStreamState } from '../../proto/livekit_rtc';
+import { TrackEvent } from '../events';
+import { isFireFox, isSafari, isWeb } from '../utils';
+
+// keep old audio elements when detached, we would re-use them since on iOS
+// Safari tracks which audio elements have been "blessed" by the user.
+const recycledElements: Array<HTMLAudioElement> = [];
+
+export class Track extends (EventEmitter as new () => TypedEventEmitter<TrackEventCallbacks>) {
+  kind: Track.Kind;
+
+  attachedElements: HTMLMediaElement[] = [];
+
+  isMuted: boolean = false;
+
+  source: Track.Source;
+
+  /**
+   * sid is set after track is published to server, or if it's a remote track
+   */
+  sid?: Track.SID;
+
+  /**
+   * @internal
+   */
+  mediaStream?: MediaStream;
+
+  protected _mediaStreamTrack: MediaStreamTrack;
+
+  protected isInBackground: boolean;
+
+  protected _currentBitrate: number = 0;
+
+  protected constructor(mediaTrack: MediaStreamTrack, kind: Track.Kind) {
+    super();
+    this.kind = kind;
+    this._mediaStreamTrack = mediaTrack;
+    this.source = Track.Source.Unknown;
+    if (isWeb()) {
+      this.isInBackground = document.visibilityState === 'hidden';
+      document.addEventListener('visibilitychange', this.appVisibilityChangedListener);
+    } else {
+      this.isInBackground = false;
+    }
+  }
+
+  /** current receive bits per second */
+  get currentBitrate(): number {
+    return this._currentBitrate;
+  }
+
+  get mediaStreamTrack() {
+    return this._mediaStreamTrack;
+  }
+
+  /**
+   * creates a new HTMLAudioElement or HTMLVideoElement, attaches to it, and returns it
+   */
+  attach(): HTMLMediaElement;
+
+  /**
+   * attaches track to an existing HTMLAudioElement or HTMLVideoElement
+   */
+  attach(element: HTMLMediaElement): HTMLMediaElement;
+  attach(element?: HTMLMediaElement): HTMLMediaElement {
+    let elementType = 'audio';
+    if (this.kind === Track.Kind.Video) {
+      elementType = 'video';
+    }
+    if (!element) {
+      if (elementType === 'audio') {
+        recycledElements.forEach((e) => {
+          if (e.parentElement === null && !element) {
+            element = e;
+          }
+        });
+        if (element) {
+          // remove it from pool
+          recycledElements.splice(recycledElements.indexOf(element), 1);
+        }
+      }
+      if (!element) {
+        element = <HTMLMediaElement>document.createElement(elementType);
+      }
+    }
+
+    if (!this.attachedElements.includes(element)) {
+      this.attachedElements.push(element);
+    }
+
+    // even if we believe it's already attached to the element, it's possible
+    // the element's srcObject was set to something else out of band.
+    // we'll want to re-attach it in that case
+    attachToElement(this._mediaStreamTrack, element);
+
+    if (element instanceof HTMLAudioElement) {
+      // manually play audio to detect audio playback status
+      element
+        .play()
+        .then(() => {
+          this.emit(TrackEvent.AudioPlaybackStarted);
+        })
+        .catch((e) => {
+          this.emit(TrackEvent.AudioPlaybackFailed, e);
+        });
+    }
+
+    this.emit(TrackEvent.ElementAttached, element);
+    return element;
+  }
+
+  /**
+   * Detaches from all attached elements
+   */
+  detach(): HTMLMediaElement[];
+
+  /**
+   * Detach from a single element
+   * @param element
+   */
+  detach(element: HTMLMediaElement): HTMLMediaElement;
+  detach(element?: HTMLMediaElement): HTMLMediaElement | HTMLMediaElement[] {
+    // detach from a single element
+    if (element) {
+      detachTrack(this._mediaStreamTrack, element);
+      const idx = this.attachedElements.indexOf(element);
+      if (idx >= 0) {
+        this.attachedElements.splice(idx, 1);
+        this.recycleElement(element);
+        this.emit(TrackEvent.ElementDetached, element);
+      }
+      return element;
+    }
+
+    const detached: HTMLMediaElement[] = [];
+    this.attachedElements.forEach((elm) => {
+      detachTrack(this._mediaStreamTrack, elm);
+      detached.push(elm);
+      this.recycleElement(elm);
+      this.emit(TrackEvent.ElementDetached, elm);
+    });
+
+    // remove all tracks
+    this.attachedElements = [];
+    return detached;
+  }
+
+  stop() {
+    this._mediaStreamTrack.stop();
+    if (isWeb()) {
+      document.removeEventListener('visibilitychange', this.appVisibilityChangedListener);
+    }
+  }
+
+  protected enable() {
+    this._mediaStreamTrack.enabled = true;
+  }
+
+  protected disable() {
+    this._mediaStreamTrack.enabled = false;
+  }
+
+  private recycleElement(element: HTMLMediaElement) {
+    if (element instanceof HTMLAudioElement) {
+      // we only need to re-use a single element
+      let shouldCache = true;
+      element.pause();
+      recycledElements.forEach((e) => {
+        if (!e.parentElement) {
+          shouldCache = false;
+        }
+      });
+      if (shouldCache) {
+        recycledElements.push(element);
+      }
+    }
+  }
+
+  appVisibilityChangedListener = () => {
+    this.handleAppVisibilityChanged();
+  };
+
+  protected async handleAppVisibilityChanged() {
+    this.isInBackground = document.visibilityState === 'hidden';
+  }
+}
+
+/** @internal */
+export function attachToElement(track: MediaStreamTrack, element: HTMLMediaElement) {
+  let mediaStream: MediaStream;
+  if (element.srcObject instanceof MediaStream) {
+    mediaStream = element.srcObject;
+  } else {
+    mediaStream = new MediaStream();
+  }
+
+  // check if track matches existing track
+  let existingTracks: MediaStreamTrack[];
+  if (track.kind === 'audio') {
+    existingTracks = mediaStream.getAudioTracks();
+  } else {
+    existingTracks = mediaStream.getVideoTracks();
+  }
+  if (!existingTracks.includes(track)) {
+    existingTracks.forEach((et) => {
+      mediaStream.removeTrack(et);
+    });
+    mediaStream.addTrack(track);
+  }
+
+  // avoid flicker
+  if (element.srcObject !== mediaStream) {
+    element.srcObject = mediaStream;
+    if ((isSafari() || isFireFox()) && element instanceof HTMLVideoElement) {
+      // Firefox also has a timing issue where video doesn't actually get attached unless
+      // performed out-of-band
+      // Safari 15 has a bug where in certain layouts, video element renders
+      // black until the page is resized or other changes take place.
+      // Resetting the src triggers it to render.
+      // https://developer.apple.com/forums/thread/690523
+      setTimeout(() => {
+        element.srcObject = mediaStream;
+        // Safari 15 sometimes fails to start a video
+        // when the window is backgrounded before the first frame is drawn
+        // manually calling play here seems to fix that
+        element.play().catch(() => {
+          /* do nothing */
+        });
+      }, 0);
+    }
+  }
+  element.autoplay = true;
+  if (element instanceof HTMLVideoElement) {
+    element.playsInline = true;
+  }
+}
+
+/** @internal */
+export function detachTrack(track: MediaStreamTrack, element: HTMLMediaElement) {
+  if (element.srcObject instanceof MediaStream) {
+    const mediaStream = element.srcObject;
+    mediaStream.removeTrack(track);
+    if (mediaStream.getTracks().length > 0) {
+      element.srcObject = mediaStream;
+    } else {
+      element.srcObject = null;
+    }
+  }
+}
+
+export namespace Track {
+  export enum Kind {
+    Audio = 'audio',
+    Video = 'video',
+    Unknown = 'unknown',
+  }
+  export type SID = string;
+  export enum Source {
+    Camera = 'camera',
+    Microphone = 'microphone',
+    ScreenShare = 'screen_share',
+    ScreenShareAudio = 'screen_share_audio',
+    Unknown = 'unknown',
+  }
+
+  export enum StreamState {
+    Active = 'active',
+    Paused = 'paused',
+    Unknown = 'unknown',
+  }
+
+  export interface Dimensions {
+    width: number;
+    height: number;
+  }
+
+  /** @internal */
+  export function kindToProto(k: Kind): TrackType {
+    switch (k) {
+      case Kind.Audio:
+        return TrackType.AUDIO;
+      case Kind.Video:
+        return TrackType.VIDEO;
+      default:
+        return TrackType.UNRECOGNIZED;
+    }
+  }
+
+  /** @internal */
+  export function kindFromProto(t: TrackType): Kind | undefined {
+    switch (t) {
+      case TrackType.AUDIO:
+        return Kind.Audio;
+      case TrackType.VIDEO:
+        return Kind.Video;
+      default:
+        return Kind.Unknown;
+    }
+  }
+
+  /** @internal */
+  export function sourceToProto(s: Source): TrackSource {
+    switch (s) {
+      case Source.Camera:
+        return TrackSource.CAMERA;
+      case Source.Microphone:
+        return TrackSource.MICROPHONE;
+      case Source.ScreenShare:
+        return TrackSource.SCREEN_SHARE;
+      case Source.ScreenShareAudio:
+        return TrackSource.SCREEN_SHARE_AUDIO;
+      default:
+        return TrackSource.UNRECOGNIZED;
+    }
+  }
+
+  /** @internal */
+  export function sourceFromProto(s: TrackSource): Source {
+    switch (s) {
+      case TrackSource.CAMERA:
+        return Source.Camera;
+      case TrackSource.MICROPHONE:
+        return Source.Microphone;
+      case TrackSource.SCREEN_SHARE:
+        return Source.ScreenShare;
+      case TrackSource.SCREEN_SHARE_AUDIO:
+        return Source.ScreenShareAudio;
+      default:
+        return Source.Unknown;
+    }
+  }
+
+  /** @internal */
+  export function streamStateFromProto(s: ProtoStreamState): StreamState {
+    switch (s) {
+      case ProtoStreamState.ACTIVE:
+        return StreamState.Active;
+      case ProtoStreamState.PAUSED:
+        return StreamState.Paused;
+      default:
+        return StreamState.Unknown;
+    }
+  }
+}
+
+export type TrackEventCallbacks = {
+  message: () => void;
+  muted: (track?: any) => void;
+  unmuted: (track?: any) => void;
+  ended: (track?: any) => void;
+  updateSettings: () => void;
+  updateSubscription: () => void;
+  audioPlaybackStarted: () => void;
+  audioPlaybackFailed: (error: Error) => void;
+  audioSilenceDetected: () => void;
+  visibilityChanged: (visible: boolean, track?: any) => void;
+  videoDimensionsChanged: (dimensions: Track.Dimensions, track?: any) => void;
+  elementAttached: (element: HTMLMediaElement) => void;
+  elementDetached: (element: HTMLMediaElement) => void;
+  upstreamPaused: (track: any) => void;
+  upstreamResumed: (track: any) => void;
+};

package/src/room/track/TrackPublication.ts

@@ -0,0 +1,120 @@
+import { EventEmitter } from 'events';
+import { TrackInfo } from '../../proto/livekit_models';
+import { TrackEvent } from '../events';
+import LocalAudioTrack from './LocalAudioTrack';
+import LocalVideoTrack from './LocalVideoTrack';
+import RemoteAudioTrack from './RemoteAudioTrack';
+import RemoteVideoTrack from './RemoteVideoTrack';
+import { Track } from './Track';
+
+export class TrackPublication extends EventEmitter {
+  kind: Track.Kind;
+
+  trackName: string;
+
+  trackSid: Track.SID;
+
+  track?: Track;
+
+  source: Track.Source;
+
+  /** MimeType of the published track */
+  mimeType?: string;
+
+  /** dimension of the original published stream, video-only */
+  dimensions?: Track.Dimensions;
+
+  /** true if track was simulcasted to server, video-only */
+  simulcasted?: boolean;
+
+  /** @internal */
+  trackInfo?: TrackInfo;
+
+  protected metadataMuted: boolean = false;
+
+  constructor(kind: Track.Kind, id: string, name: string) {
+    super();
+    this.kind = kind;
+    this.trackSid = id;
+    this.trackName = name;
+    this.source = Track.Source.Unknown;
+  }
+
+  /** @internal */
+  setTrack(track?: Track) {
+    if (this.track) {
+      this.track.off(TrackEvent.Muted, this.handleMuted);
+      this.track.off(TrackEvent.Unmuted, this.handleUnmuted);
+    }
+
+    this.track = track;
+
+    if (track) {
+      // forward events
+      track.on(TrackEvent.Muted, this.handleMuted);
+      track.on(TrackEvent.Unmuted, this.handleUnmuted);
+    }
+  }
+
+  get isMuted(): boolean {
+    return this.metadataMuted;
+  }
+
+  get isEnabled(): boolean {
+    return true;
+  }
+
+  get isSubscribed(): boolean {
+    return this.track !== undefined;
+  }
+
+  /**
+   * an [AudioTrack] if this publication holds an audio track
+   */
+  get audioTrack(): LocalAudioTrack | RemoteAudioTrack | undefined {
+    if (this.track instanceof LocalAudioTrack || this.track instanceof RemoteAudioTrack) {
+      return this.track;
+    }
+  }
+
+  /**
+   * an [VideoTrack] if this publication holds a video track
+   */
+  get videoTrack(): LocalVideoTrack | RemoteVideoTrack | undefined {
+    if (this.track instanceof LocalVideoTrack || this.track instanceof RemoteVideoTrack) {
+      return this.track;
+    }
+  }
+
+  handleMuted = () => {
+    this.emit(TrackEvent.Muted);
+  };
+
+  handleUnmuted = () => {
+    this.emit(TrackEvent.Unmuted);
+  };
+
+  /** @internal */
+  updateInfo(info: TrackInfo) {
+    this.trackSid = info.sid;
+    this.trackName = info.name;
+    this.source = Track.sourceFromProto(info.source);
+    this.mimeType = info.mimeType;
+    if (this.kind === Track.Kind.Video && info.width > 0) {
+      this.dimensions = {
+        width: info.width,
+        height: info.height,
+      };
+      this.simulcasted = info.simulcast;
+    }
+    this.trackInfo = info;
+  }
+}
+
+export namespace TrackPublication {
+  export enum SubscriptionStatus {
+    Subscribed = 'subscribed',
+    NotAllowed = 'not_allowed',
+    Unsubscribed = 'unsubscribed',
+  }
+}

package/src/room/track/create.ts

@@ -0,0 +1,122 @@
+import { TrackInvalidError } from '../errors';
+import { mediaTrackToLocalTrack } from '../participant/publishUtils';
+import { audioDefaults, videoDefaults } from './defaults';
+import LocalAudioTrack from './LocalAudioTrack';
+import LocalTrack from './LocalTrack';
+import LocalVideoTrack from './LocalVideoTrack';
+import {
+  AudioCaptureOptions,
+  CreateLocalTracksOptions,
+  ScreenShareCaptureOptions,
+  VideoCaptureOptions,
+  VideoPresets,
+} from './options';
+import { Track } from './Track';
+import { constraintsForOptions, mergeDefaultOptions } from './utils';
+
+/**
+ * Creates a local video and audio track at the same time. When acquiring both
+ * audio and video tracks together, it'll display a single permission prompt to
+ * the user instead of two separate ones.
+ * @param options
+ */
+export async function createLocalTracks(
+  options?: CreateLocalTracksOptions,
+): Promise<Array<LocalTrack>> {
+  // set default options to true
+  options ??= {};
+  options.audio ??= true;
+  options.video ??= true;
+
+  const opts = mergeDefaultOptions(options, audioDefaults, videoDefaults);
+  const constraints = constraintsForOptions(opts);
+  const stream = await navigator.mediaDevices.getUserMedia(constraints);
+  return stream.getTracks().map((mediaStreamTrack) => {
+    const isAudio = mediaStreamTrack.kind === 'audio';
+    let trackOptions = isAudio ? options!.audio : options!.video;
+    if (typeof trackOptions === 'boolean' || !trackOptions) {
+      trackOptions = {};
+    }
+    let trackConstraints: MediaTrackConstraints | undefined;
+    const conOrBool = isAudio ? constraints.audio : constraints.video;
+    if (typeof conOrBool !== 'boolean') {
+      trackConstraints = conOrBool;
+    }
+    const track = mediaTrackToLocalTrack(mediaStreamTrack, trackConstraints);
+    if (track.kind === Track.Kind.Video) {
+      track.source = Track.Source.Camera;
+    } else if (track.kind === Track.Kind.Audio) {
+      track.source = Track.Source.Microphone;
+    }
+    track.mediaStream = stream;
+    return track;
+  });
+}
+
+/**
+ * Creates a [[LocalVideoTrack]] with getUserMedia()
+ * @param options
+ */
+export async function createLocalVideoTrack(
+  options?: VideoCaptureOptions,
+): Promise<LocalVideoTrack> {
+  const tracks = await createLocalTracks({
+    audio: false,
+    video: options,
+  });
+  return <LocalVideoTrack>tracks[0];
+}
+
+export async function createLocalAudioTrack(
+  options?: AudioCaptureOptions,
+): Promise<LocalAudioTrack> {
+  const tracks = await createLocalTracks({
+    audio: options,
+    video: false,
+  });
+  return <LocalAudioTrack>tracks[0];
+}
+
+/**
+ * Creates a screen capture tracks with getDisplayMedia().
+ * A LocalVideoTrack is always created and returned.
+ * If { audio: true }, and the browser supports audio capture, a LocalAudioTrack is also created.
+ */
+export async function createLocalScreenTracks(
+  options?: ScreenShareCaptureOptions,
+): Promise<Array<LocalTrack>> {
+  if (options === undefined) {
+    options = {};
+  }
+  if (options.resolution === undefined) {
+    options.resolution = VideoPresets.fhd.resolution;
+  }
+
+  let videoConstraints: MediaTrackConstraints | boolean = true;
+  if (options.resolution) {
+    videoConstraints = {
+      width: options.resolution.width,
+      height: options.resolution.height,
+    };
+  }
+  // typescript definition is missing getDisplayMedia: https://github.com/microsoft/TypeScript/issues/33232
+  // @ts-ignore
+  const stream: MediaStream = await navigator.mediaDevices.getDisplayMedia({
+    audio: options.audio ?? false,
+    video: videoConstraints,
+  });
+
+  const tracks = stream.getVideoTracks();
+  if (tracks.length === 0) {
+    throw new TrackInvalidError('no video track found');
+  }
+  const screenVideo = new LocalVideoTrack(tracks[0]);
+  screenVideo.source = Track.Source.ScreenShare;
+  const localTracks: Array<LocalTrack> = [screenVideo];
+  if (stream.getAudioTracks().length > 0) {
+    const screenAudio = new LocalAudioTrack(stream.getAudioTracks()[0]);
+    screenAudio.source = Track.Source.ScreenShareAudio;
+    localTracks.push(screenAudio);
+  }
+  return localTracks;
+}

package/src/room/track/defaults.ts

@@ -0,0 +1,26 @@
+import {
+  AudioCaptureOptions,
+  AudioPresets,
+  ScreenSharePresets,
+  TrackPublishDefaults,
+  VideoCaptureOptions,
+  VideoPresets,
+} from './options';
+
+export const publishDefaults: TrackPublishDefaults = {
+  audioBitrate: AudioPresets.speech.maxBitrate,
+  dtx: true,
+  simulcast: true,
+  screenShareEncoding: ScreenSharePresets.h1080fps15.encoding,
+  stopMicTrackOnMute: false,
+};
+
+export const audioDefaults: AudioCaptureOptions = {
+  autoGainControl: true,
+  echoCancellation: true,
+  noiseSuppression: true,
+};
+
+export const videoDefaults: VideoCaptureOptions = {
+  resolution: VideoPresets.h720.resolution,
+};