livekit-client 0.15.3 → 0.16.2

package/src/room/participant/LocalParticipant.ts

@@ -0,0 +1,685 @@
+import log from '../../logger';
+import { RoomOptions } from '../../options';
+import {
+  DataPacket, DataPacket_Kind,
+} from '../../proto/livekit_models';
+import { AddTrackRequest, SubscribedQualityUpdate, TrackPublishedResponse } from '../../proto/livekit_rtc';
+import {
+  TrackInvalidError,
+  UnexpectedConnectionState,
+} from '../errors';
+import { ParticipantEvent, TrackEvent } from '../events';
+import RTCEngine from '../RTCEngine';
+import LocalAudioTrack from '../track/LocalAudioTrack';
+import LocalTrack from '../track/LocalTrack';
+import LocalTrackPublication from '../track/LocalTrackPublication';
+import LocalVideoTrack, { videoLayersFromEncodings } from '../track/LocalVideoTrack';
+import {
+  CreateLocalTracksOptions,
+  ScreenShareCaptureOptions,
+  TrackPublishOptions, VideoCodec, VideoPresets,
+} from '../track/options';
+import { Track } from '../track/Track';
+import { constraintsForOptions, mergeDefaultOptions } from '../track/utils';
+import Participant from './Participant';
+import { ParticipantTrackPermission, trackPermissionToProto } from './ParticipantTrackPermission';
+import { computeVideoEncodings, mediaTrackToLocalTrack } from './publishUtils';
+import RemoteParticipant from './RemoteParticipant';
+
+export default class LocalParticipant extends Participant {
+  audioTracks: Map<string, LocalTrackPublication>;
+
+  videoTracks: Map<string, LocalTrackPublication>;
+
+  /** map of track sid => all published tracks */
+  tracks: Map<string, LocalTrackPublication>;
+
+  private pendingPublishing = new Set<Track.Source>();
+
+  private cameraError: Error | undefined;
+
+  private microphoneError: Error | undefined;
+
+  private engine: RTCEngine;
+
+  // keep a pointer to room options
+  private roomOptions?: RoomOptions;
+
+  /** @internal */
+  constructor(sid: string, identity: string, engine: RTCEngine, options: RoomOptions) {
+    super(sid, identity);
+    this.audioTracks = new Map();
+    this.videoTracks = new Map();
+    this.tracks = new Map();
+    this.engine = engine;
+    this.roomOptions = options;
+
+    this.engine.client.onRemoteMuteChanged = (trackSid: string, muted: boolean) => {
+      const pub = this.tracks.get(trackSid);
+      if (!pub || !pub.track) {
+        return;
+      }
+      if (muted) {
+        pub.mute();
+      } else {
+        pub.unmute();
+      }
+    };
+
+    this.engine.client.onSubscribedQualityUpdate = this.handleSubscribedQualityUpdate;
+  }
+
+  get lastCameraError(): Error | undefined {
+    return this.cameraError;
+  }
+
+  get lastMicrophoneError(): Error | undefined {
+    return this.microphoneError;
+  }
+
+  getTrack(source: Track.Source): LocalTrackPublication | undefined {
+    const track = super.getTrack(source);
+    if (track) {
+      return track as LocalTrackPublication;
+    }
+  }
+
+  getTrackByName(name: string): LocalTrackPublication | undefined {
+    const track = super.getTrackByName(name);
+    if (track) {
+      return track as LocalTrackPublication;
+    }
+  }
+
+  /**
+   * Enable or disable a participant's camera track.
+   *
+   * If a track has already published, it'll mute or unmute the track.
+   */
+  setCameraEnabled(enabled: boolean): Promise<void> {
+    return this.setTrackEnabled(Track.Source.Camera, enabled);
+  }
+
+  /**
+   * Enable or disable a participant's microphone track.
+   *
+   * If a track has already published, it'll mute or unmute the track.
+   */
+  setMicrophoneEnabled(enabled: boolean): Promise<void> {
+    return this.setTrackEnabled(Track.Source.Microphone, enabled);
+  }
+
+  /**
+   * Start or stop sharing a participant's screen
+   */
+  setScreenShareEnabled(enabled: boolean): Promise<void> {
+    return this.setTrackEnabled(Track.Source.ScreenShare, enabled);
+  }
+
+  /**
+   * Enable or disable publishing for a track by source. This serves as a simple
+   * way to manage the common tracks (camera, mic, or screen share)
+   */
+  private async setTrackEnabled(source: Track.Source, enabled: boolean): Promise<void> {
+    log.debug('setTrackEnabled', source, enabled);
+    const track = this.getTrack(source);
+    if (enabled) {
+      if (track) {
+        await track.unmute();
+      } else {
+        let localTrack: LocalTrack | undefined;
+        if (this.pendingPublishing.has(source)) {
+          log.info('skipping duplicate published source', source);
+          // no-op it's already been requested
+          return;
+        }
+        this.pendingPublishing.add(source);
+        try {
+          switch (source) {
+            case Track.Source.Camera:
+              [localTrack] = await this.createTracks({
+                video: true,
+              });
+              break;
+            case Track.Source.Microphone:
+              [localTrack] = await this.createTracks({
+                audio: true,
+              });
+              break;
+            case Track.Source.ScreenShare:
+              [localTrack] = await this.createScreenTracks({ audio: false });
+              break;
+            default:
+              throw new TrackInvalidError(source);
+          }
+
+          await this.publishTrack(localTrack);
+        } catch (e) {
+          if (e instanceof Error && !(e instanceof TrackInvalidError)) {
+            this.emit(ParticipantEvent.MediaDevicesError, e);
+          }
+          throw e;
+        } finally {
+          this.pendingPublishing.delete(source);
+        }
+      }
+    } else if (track && track.track) {
+      // screenshare cannot be muted, unpublish instead
+      if (source === Track.Source.ScreenShare) {
+        this.unpublishTrack(track.track);
+      } else {
+        await track.mute();
+      }
+    }
+  }
+
+  /**
+   * Publish both camera and microphone at the same time. This is useful for
+   * displaying a single Permission Dialog box to the end user.
+   */
+  async enableCameraAndMicrophone() {
+    if (this.pendingPublishing.has(Track.Source.Camera)
+        || this.pendingPublishing.has(Track.Source.Microphone)) {
+      // no-op it's already been requested
+      return;
+    }
+
+    this.pendingPublishing.add(Track.Source.Camera);
+    this.pendingPublishing.add(Track.Source.Microphone);
+    try {
+      const tracks: LocalTrack[] = await this.createTracks({
+        audio: true,
+        video: true,
+      });
+
+      await Promise.all(tracks.map((track) => this.publishTrack(track)));
+    } finally {
+      this.pendingPublishing.delete(Track.Source.Camera);
+      this.pendingPublishing.delete(Track.Source.Microphone);
+    }
+  }
+
+  /**
+   * Create local camera and/or microphone tracks
+   * @param options
+   * @returns
+   */
+  async createTracks(
+    options?: CreateLocalTracksOptions,
+  ): Promise<LocalTrack[]> {
+    const opts = mergeDefaultOptions(
+      options,
+      this.roomOptions?.audioCaptureDefaults,
+      this.roomOptions?.videoCaptureDefaults,
+    );
+
+    const constraints = constraintsForOptions(opts);
+    let stream: MediaStream | undefined;
+    try {
+      stream = await navigator.mediaDevices.getUserMedia(
+        constraints,
+      );
+    } catch (err) {
+      if (err instanceof Error) {
+        if (constraints.audio) {
+          this.microphoneError = err;
+        }
+        if (constraints.video) {
+          this.cameraError = err;
+        }
+      }
+
+      throw err;
+    }
+
+    if (constraints.audio) {
+      this.microphoneError = undefined;
+    }
+    if (constraints.video) {
+      this.cameraError = undefined;
+    }
+
+    return stream.getTracks().map((mediaStreamTrack) => {
+      const isAudio = mediaStreamTrack.kind === 'audio';
+      let trackOptions = isAudio ? options!.audio : options!.video;
+      if (typeof trackOptions === 'boolean' || !trackOptions) {
+        trackOptions = {};
+      }
+      let trackConstraints: MediaTrackConstraints | undefined;
+      const conOrBool = isAudio ? constraints.audio : constraints.video;
+      if (typeof conOrBool !== 'boolean') {
+        trackConstraints = conOrBool;
+      }
+      const track = mediaTrackToLocalTrack(mediaStreamTrack, trackConstraints);
+      if (track.kind === Track.Kind.Video) {
+        track.source = Track.Source.Camera;
+      } else if (track.kind === Track.Kind.Audio) {
+        track.source = Track.Source.Microphone;
+      }
+      return track;
+    });
+  }
+
+  /**
+   * Creates a screen capture tracks with getDisplayMedia().
+   * A LocalVideoTrack is always created and returned.
+   * If { audio: true }, and the browser supports audio capture, a LocalAudioTrack is also created.
+   */
+  async createScreenTracks(
+    options?: ScreenShareCaptureOptions,
+  ): Promise<Array<LocalTrack>> {
+    if (options === undefined) {
+      options = {};
+    }
+    if (options.resolution === undefined) {
+      options.resolution = VideoPresets.fhd.resolution;
+    }
+
+    let videoConstraints: MediaTrackConstraints | boolean = true;
+    if (options.resolution) {
+      videoConstraints = {
+        width: options.resolution.width,
+        height: options.resolution.height,
+        frameRate: options.resolution.frameRate,
+      };
+    }
+    // typescript definition is missing getDisplayMedia: https://github.com/microsoft/TypeScript/issues/33232
+    // @ts-ignore
+    const stream: MediaStream = await navigator.mediaDevices.getDisplayMedia({
+      audio: options.audio ?? false,
+      video: videoConstraints,
+    });
+
+    const tracks = stream.getVideoTracks();
+    if (tracks.length === 0) {
+      throw new TrackInvalidError('no video track found');
+    }
+    const screenVideo = new LocalVideoTrack(tracks[0]);
+    screenVideo.source = Track.Source.ScreenShare;
+    const localTracks: Array<LocalTrack> = [screenVideo];
+    if (stream.getAudioTracks().length > 0) {
+      const screenAudio = new LocalAudioTrack(stream.getAudioTracks()[0]);
+      screenAudio.source = Track.Source.ScreenShareAudio;
+      localTracks.push(screenAudio);
+    }
+    return localTracks;
+  }
+
+  /**
+   * Publish a new track to the room
+   * @param track
+   * @param options
+   */
+  async publishTrack(
+    track: LocalTrack | MediaStreamTrack,
+    options?: TrackPublishOptions,
+  ): Promise<LocalTrackPublication> {
+    const opts: TrackPublishOptions = {
+      ...this.roomOptions?.publishDefaults,
+      ...options,
+    };
+
+    // convert raw media track into audio or video track
+    if (track instanceof MediaStreamTrack) {
+      switch (track.kind) {
+        case 'audio':
+          track = new LocalAudioTrack(track);
+          break;
+        case 'video':
+          track = new LocalVideoTrack(track);
+          break;
+        default:
+          throw new TrackInvalidError(
+            `unsupported MediaStreamTrack kind ${track.kind}`,
+          );
+      }
+    }
+
+    // is it already published? if so skip
+    let existingPublication: LocalTrackPublication | undefined;
+    this.tracks.forEach((publication) => {
+      if (!publication.track) {
+        return;
+      }
+      if (publication.track === track) {
+        existingPublication = <LocalTrackPublication>publication;
+      }
+    });
+
+    if (existingPublication) return existingPublication;
+
+    if (opts.source) {
+      track.source = opts.source;
+    }
+    if (opts.stopMicTrackOnMute && track instanceof LocalAudioTrack) {
+      track.stopOnMute = true;
+    }
+
+    // handle track actions
+    track.on(TrackEvent.Muted, this.onTrackMuted);
+    track.on(TrackEvent.Unmuted, this.onTrackUnmuted);
+    track.on(TrackEvent.Ended, this.onTrackUnpublish);
+
+    // create track publication from track
+    const req = AddTrackRequest.fromPartial({
+      // get local track id for use during publishing
+      cid: track.mediaStreamTrack.id,
+      name: options?.name,
+      type: Track.kindToProto(track.kind),
+      muted: track.isMuted,
+      source: Track.sourceToProto(track.source),
+      disableDtx: !(opts?.dtx ?? true),
+    });
+
+    // compute encodings and layers for video
+    let encodings: RTCRtpEncodingParameters[] | undefined;
+    if (track.kind === Track.Kind.Video) {
+      // TODO: support react native, which doesn't expose getSettings
+      const settings = track.mediaStreamTrack.getSettings();
+      const width = settings.width ?? track.dimensions?.width;
+      const height = settings.height ?? track.dimensions?.height;
+      // width and height should be defined for video
+      req.width = width ?? 0;
+      req.height = height ?? 0;
+      encodings = computeVideoEncodings(
+        track.source === Track.Source.ScreenShare,
+        width,
+        height,
+        opts,
+      );
+      req.layers = videoLayersFromEncodings(req.width, req.height, encodings);
+    } else if (track.kind === Track.Kind.Audio && opts.audioBitrate) {
+      encodings = [
+        {
+          maxBitrate: opts.audioBitrate,
+        },
+      ];
+    }
+
+    const ti = await this.engine.addTrack(req);
+    const publication = new LocalTrackPublication(track.kind, ti, track);
+    track.sid = ti.sid;
+
+    if (!this.engine.publisher) {
+      throw new UnexpectedConnectionState('publisher is closed');
+    }
+    log.debug(`publishing ${track.kind} with encodings`, encodings, ti);
+    const transceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly' };
+    if (encodings) {
+      transceiverInit.sendEncodings = encodings;
+    }
+    const transceiver = this.engine.publisher.pc.addTransceiver(
+      track.mediaStreamTrack, transceiverInit,
+    );
+    this.engine.negotiate();
+
+    // store RTPSender
+    track.sender = transceiver.sender;
+    if (track instanceof LocalVideoTrack) {
+      const disableLayerPause = this.roomOptions?.expDisableLayerPause ?? false;
+      track.startMonitor(this.engine.client, disableLayerPause);
+    } else if (track instanceof LocalAudioTrack) {
+      track.startMonitor();
+    }
+
+    if (opts.videoCodec) {
+      this.setPreferredCodec(transceiver, track.kind, opts.videoCodec);
+    }
+    this.addTrackPublication(publication);
+
+    // send event for publication
+    this.emit(ParticipantEvent.LocalTrackPublished, publication);
+    return publication;
+  }
+
+  unpublishTrack(
+    track: LocalTrack | MediaStreamTrack,
+    stopOnUnpublish?: boolean,
+  ): LocalTrackPublication | null {
+    // look through all published tracks to find the right ones
+    const publication = this.getPublicationForTrack(track);
+
+    log.debug('unpublishTrack', 'unpublishing track', track);
+
+    if (!publication || !publication.track) {
+      log.warn(
+        'unpublishTrack',
+        'track was not unpublished because no publication was found',
+        track,
+      );
+      return null;
+    }
+
+    track = publication.track;
+
+    track.off(TrackEvent.Muted, this.onTrackMuted);
+    track.off(TrackEvent.Unmuted, this.onTrackUnmuted);
+    track.off(TrackEvent.Ended, this.onTrackUnpublish);
+
+    if (stopOnUnpublish === undefined) {
+      stopOnUnpublish = this.roomOptions?.stopLocalTrackOnUnpublish ?? true;
+    }
+    if (stopOnUnpublish) {
+      track.stop();
+    }
+
+    const { mediaStreamTrack } = track;
+
+    if (this.engine.publisher) {
+      const senders = this.engine.publisher.pc.getSenders();
+      senders.forEach((sender) => {
+        if (sender.track === mediaStreamTrack) {
+          try {
+            this.engine.publisher?.pc.removeTrack(sender);
+            this.engine.negotiate();
+          } catch (e) {
+            log.warn('unpublishTrack', 'failed to remove track', e);
+          }
+        }
+      });
+    }
+
+    // remove from our maps
+    this.tracks.delete(publication.trackSid);
+    switch (publication.kind) {
+      case Track.Kind.Audio:
+        this.audioTracks.delete(publication.trackSid);
+        break;
+      case Track.Kind.Video:
+        this.videoTracks.delete(publication.trackSid);
+        break;
+      default:
+        break;
+    }
+
+    publication.setTrack(undefined);
+    this.emit(ParticipantEvent.LocalTrackUnpublished, publication);
+
+    return publication;
+  }
+
+  unpublishTracks(
+    tracks: LocalTrack[] | MediaStreamTrack[],
+  ): LocalTrackPublication[] {
+    const publications: LocalTrackPublication[] = [];
+    tracks.forEach((track: LocalTrack | MediaStreamTrack) => {
+      const pub = this.unpublishTrack(track);
+      if (pub) {
+        publications.push(pub);
+      }
+    });
+    return publications;
+  }
+
+  /**
+   * Publish a new data payload to the room. Data will be forwarded to each
+   * participant in the room if the destination argument is empty
+   *
+   * @param data Uint8Array of the payload. To send string data, use TextEncoder.encode
+   * @param kind whether to send this as reliable or lossy.
+   * For data that you need delivery guarantee (such as chat messages), use Reliable.
+   * For data that should arrive as quickly as possible, but you are ok with dropped
+   * packets, use Lossy.
+   * @param destination the participants who will receive the message
+   */
+  async publishData(data: Uint8Array, kind: DataPacket_Kind,
+    destination?: RemoteParticipant[] | string[]) {
+    const dest: string[] = [];
+    if (destination !== undefined) {
+      destination.forEach((val : any) => {
+        if (val instanceof RemoteParticipant) {
+          dest.push(val.sid);
+        } else {
+          dest.push(val);
+        }
+      });
+    }
+
+    const packet: DataPacket = {
+      kind,
+      user: {
+        participantSid: this.sid,
+        payload: data,
+        destinationSids: dest,
+      },
+    };
+
+    await this.engine.sendDataPacket(packet, kind);
+  }
+
+  /**
+   * Control who can subscribe to LocalParticipant's published tracks.
+   *
+   * By default, all participants can subscribe. This allows fine-grained control over
+   * who is able to subscribe at a participant and track level.
+   *
+   * Note: if access is given at a track-level (i.e. both [allParticipantsAllowed] and
+   * [ParticipantTrackPermission.allTracksAllowed] are false), any newer published tracks
+   * will not grant permissions to any participants and will require a subsequent
+   * permissions update to allow subscription.
+   *
+   * @param allParticipantsAllowed Allows all participants to subscribe all tracks.
+   *  Takes precedence over [[participantTrackPermissions]] if set to true.
+   *  By default this is set to true.
+   * @param participantTrackPermissions Full list of individual permissions per
+   *  participant/track. Any omitted participants will not receive any permissions.
+   */
+  setTrackSubscriptionPermissions(
+    allParticipantsAllowed: boolean,
+    participantTrackPermissions: ParticipantTrackPermission[] = [],
+  ) {
+    this.engine.client.sendUpdateSubscriptionPermissions(
+      allParticipantsAllowed,
+      participantTrackPermissions.map((p) => trackPermissionToProto(p)),
+    );
+  }
+
+  /** @internal */
+  private onTrackUnmuted = (track: LocalTrack) => {
+    this.onTrackMuted(track, false);
+  };
+
+  // when the local track changes in mute status, we'll notify server as such
+  /** @internal */
+  private onTrackMuted = (
+    track: LocalTrack,
+    muted?: boolean,
+  ) => {
+    if (muted === undefined) {
+      muted = true;
+    }
+
+    if (!track.sid) {
+      log.error('could not update mute status for unpublished track', track);
+      return;
+    }
+
+    this.engine.updateMuteStatus(track.sid, muted);
+  };
+
+  private handleSubscribedQualityUpdate = (update: SubscribedQualityUpdate) => {
+    if (!this.roomOptions?.dynacast) {
+      return;
+    }
+    const pub = this.videoTracks.get(update.trackSid);
+    if (!pub) {
+      log.warn('handleSubscribedQualityUpdate',
+        'received subscribed quality update for unknown track', update.trackSid);
+      return;
+    }
+    pub.videoTrack?.setPublishingLayers(update.subscribedQualities);
+  };
+
+  private onTrackUnpublish = (track: LocalTrack) => {
+    this.unpublishTrack(track);
+  };
+
+  private getPublicationForTrack(
+    track: LocalTrack | MediaStreamTrack,
+  ): LocalTrackPublication | undefined {
+    let publication: LocalTrackPublication | undefined;
+    this.tracks.forEach((pub) => {
+      const localTrack = pub.track;
+      if (!localTrack) {
+        return;
+      }
+
+      // this looks overly complicated due to this object tree
+      if (track instanceof MediaStreamTrack) {
+        if (
+          localTrack instanceof LocalAudioTrack
+          || localTrack instanceof LocalVideoTrack
+        ) {
+          if (localTrack.mediaStreamTrack === track) {
+            publication = <LocalTrackPublication>pub;
+          }
+        }
+      } else if (track === localTrack) {
+        publication = <LocalTrackPublication>pub;
+      }
+    });
+    return publication;
+  }
+
+  private setPreferredCodec(
+    transceiver: RTCRtpTransceiver,
+    kind: Track.Kind,
+    videoCodec: VideoCodec,
+  ) {
+    if (!('getCapabilities' in RTCRtpSender)) {
+      return;
+    }
+    const cap = RTCRtpSender.getCapabilities(kind);
+    if (!cap) return;
+    const selected = cap.codecs.find((c) => {
+      const codec = c.mimeType.toLowerCase();
+      const matchesVideoCodec = codec === `video/${videoCodec}`;
+
+      // for h264 codecs that have sdpFmtpLine available, use only if the
+      // profile-level-id is 42e01f for cross-browser compatibility
+      if (videoCodec === 'h264' && c.sdpFmtpLine) {
+        return matchesVideoCodec && c.sdpFmtpLine.includes('profile-level-id=42e01f');
+      }
+
+      return matchesVideoCodec || codec === 'audio/opus';
+    });
+    if (selected && 'setCodecPreferences' in transceiver) {
+      // @ts-ignore
+      transceiver.setCodecPreferences([selected]);
+    }
+  }
+
+  /** @internal */
+  publishedTracksInfo(): TrackPublishedResponse[] {
+    const infos: TrackPublishedResponse[] = [];
+    this.tracks.forEach((track: LocalTrackPublication) => {
+      if (track.track !== undefined) {
+        infos.push({
+          cid: track.track.mediaStreamTrack.id,
+          track: track.trackInfo,
+        });
+      }
+    });
+    return infos;
+  }
+}