livekit-client 0.15.2 → 0.16.1

package/src/room/participant/Participant.ts

@@ -0,0 +1,214 @@
+import { EventEmitter } from 'events';
+import { ConnectionQuality as ProtoQuality, ParticipantInfo } from '../../proto/livekit_models';
+import { ParticipantEvent, TrackEvent } from '../events';
+import { Track } from '../track/Track';
+import { TrackPublication } from '../track/TrackPublication';
+
+export enum ConnectionQuality {
+  Excellent = 'excellent',
+  Good = 'good',
+  Poor = 'poor',
+  Unknown = 'unknown',
+}
+
+function qualityFromProto(q: ProtoQuality): ConnectionQuality {
+  switch (q) {
+    case ProtoQuality.EXCELLENT:
+      return ConnectionQuality.Excellent;
+    case ProtoQuality.GOOD:
+      return ConnectionQuality.Good;
+    case ProtoQuality.POOR:
+      return ConnectionQuality.Poor;
+    default:
+      return ConnectionQuality.Unknown;
+  }
+}
+
+export default class Participant extends EventEmitter {
+  protected participantInfo?: ParticipantInfo;
+
+  audioTracks: Map<string, TrackPublication>;
+
+  videoTracks: Map<string, TrackPublication>;
+
+  /** map of track sid => all published tracks */
+  tracks: Map<string, TrackPublication>;
+
+  /** audio level between 0-1.0, 1 being loudest, 0 being softest */
+  audioLevel: number = 0;
+
+  /** if participant is currently speaking */
+  isSpeaking: boolean = false;
+
+  /** server assigned unique id */
+  sid: string;
+
+  /** client assigned identity, encoded in JWT token */
+  identity: string;
+
+  /** client assigned display name, encoded in JWT token */
+  name?: string;
+
+  /** client metadata, opaque to livekit */
+  metadata?: string;
+
+  lastSpokeAt?: Date | undefined;
+
+  private _connectionQuality: ConnectionQuality = ConnectionQuality.Unknown;
+
+  /** @internal */
+  constructor(sid: string, identity: string) {
+    super();
+    this.sid = sid;
+    this.identity = identity;
+    this.audioTracks = new Map();
+    this.videoTracks = new Map();
+    this.tracks = new Map();
+  }
+
+  getTracks(): TrackPublication[] {
+    return Array.from(this.tracks.values());
+  }
+
+  /**
+   * Finds the first track that matches the source filter, for example, getting
+   * the user's camera track with getTrackBySource(Track.Source.Camera).
+   * @param source
+   * @returns
+   */
+  getTrack(source: Track.Source): TrackPublication | undefined {
+    if (source === Track.Source.Unknown) {
+      return;
+    }
+    for (const [, pub] of this.tracks) {
+      if (pub.source === source) {
+        return pub;
+      }
+      if (pub.source === Track.Source.Unknown) {
+        if (source === Track.Source.Microphone && pub.kind === Track.Kind.Audio && pub.trackName !== 'screen') {
+          return pub;
+        }
+        if (source === Track.Source.Camera && pub.kind === Track.Kind.Video && pub.trackName !== 'screen') {
+          return pub;
+        }
+        if (source === Track.Source.ScreenShare && pub.kind === Track.Kind.Video && pub.trackName === 'screen') {
+          return pub;
+        }
+        if (source === Track.Source.ScreenShareAudio && pub.kind === Track.Kind.Audio && pub.trackName === 'screen') {
+          return pub;
+        }
+      }
+    }
+  }
+
+  /**
+   * Finds the first track that matches the track's name.
+   * @param name
+   * @returns
+   */
+  getTrackByName(name: string): TrackPublication | undefined {
+    for (const [, pub] of this.tracks) {
+      if (pub.trackName === name) {
+        return pub;
+      }
+    }
+  }
+
+  get connectionQuality(): ConnectionQuality {
+    return this._connectionQuality;
+  }
+
+  get isCameraEnabled(): boolean {
+    const track = this.getTrack(Track.Source.Camera);
+    return !(track?.isMuted ?? true);
+  }
+
+  get isMicrophoneEnabled(): boolean {
+    const track = this.getTrack(Track.Source.Microphone);
+    return !(track?.isMuted ?? true);
+  }
+
+  get isScreenShareEnabled(): boolean {
+    const track = this.getTrack(Track.Source.ScreenShare);
+    return !!track;
+  }
+
+  /** when participant joined the room */
+  get joinedAt(): Date | undefined {
+    if (this.participantInfo) {
+      return new Date(this.participantInfo.joinedAt * 1000);
+    }
+    return new Date();
+  }
+
+  /** @internal */
+  updateInfo(info: ParticipantInfo) {
+    this.identity = info.identity;
+    this.sid = info.sid;
+    this.name = info.name;
+    this.setMetadata(info.metadata);
+    // set this last so setMetadata can detect changes
+    this.participantInfo = info;
+  }
+
+  /** @internal */
+  setMetadata(md: string) {
+    const changed = !this.participantInfo || this.participantInfo.metadata !== md;
+    const prevMetadata = this.metadata;
+    this.metadata = md;
+
+    if (changed) {
+      this.emit(ParticipantEvent.MetadataChanged, prevMetadata, this);
+      this.emit(ParticipantEvent.ParticipantMetadataChanged, prevMetadata, this);
+    }
+  }
+
+  /** @internal */
+  setIsSpeaking(speaking: boolean) {
+    if (speaking === this.isSpeaking) {
+      return;
+    }
+    this.isSpeaking = speaking;
+    if (speaking) {
+      this.lastSpokeAt = new Date();
+    }
+    this.emit(ParticipantEvent.IsSpeakingChanged, speaking);
+  }
+
+  /** @internal */
+  setConnectionQuality(q: ProtoQuality) {
+    const prevQuality = this._connectionQuality;
+    this._connectionQuality = qualityFromProto(q);
+    if (prevQuality !== this._connectionQuality) {
+      this.emit(ParticipantEvent.ConnectionQualityChanged, this._connectionQuality);
+    }
+  }
+
+  protected addTrackPublication(publication: TrackPublication) {
+    // forward publication driven events
+    publication.on(TrackEvent.Muted, () => {
+      this.emit(ParticipantEvent.TrackMuted, publication);
+    });
+
+    publication.on(TrackEvent.Unmuted, () => {
+      this.emit(ParticipantEvent.TrackUnmuted, publication);
+    });
+
+    const pub = publication;
+    if (pub.track) {
+      pub.track.sid = publication.trackSid;
+    }
+
+    this.tracks.set(publication.trackSid, publication);
+    switch (publication.kind) {
+      case Track.Kind.Audio:
+        this.audioTracks.set(publication.trackSid, publication);
+        break;
+      case Track.Kind.Video:
+        this.videoTracks.set(publication.trackSid, publication);
+        break;
+      default:
+        break;
+    }
+  }
+}

package/src/room/participant/ParticipantTrackPermission.ts

@@ -0,0 +1,32 @@
+import { TrackPermission } from '../../proto/livekit_rtc';
+
+export interface ParticipantTrackPermission {
+  /**
+   * The participant id this permission applies to.
+   */
+  participantSid: string;
+
+  /**
+   * Grant permission to all all tracks. Takes precedence over allowedTrackSids.
+   * false if unset.
+   */
+  allowAll?: boolean;
+
+  /**
+   * The list of track ids that the target participant can subscribe to.
+   * When unset, it'll allow all tracks to be subscribed by the participant.
+   * When empty, this participant is disallowed from subscribing to any tracks.
+   */
+  allowedTrackSids?: string[];
+}
+
+export function trackPermissionToProto(perms: ParticipantTrackPermission): TrackPermission {
+  if (!perms.participantSid) {
+    throw new Error('Invalid track permission, missing participantSid');
+  }
+  return {
+    participantSid: perms.participantSid,
+    allTracks: perms.allowAll ?? false,
+    trackSids: perms.allowedTrackSids || [],
+  };
+}

package/src/room/participant/RemoteParticipant.ts

@@ -0,0 +1,241 @@
+import { SignalClient } from '../../api/SignalClient';
+import log from '../../logger';
+import { ParticipantInfo } from '../../proto/livekit_models';
+import {
+  UpdateSubscription,
+  UpdateTrackSettings,
+} from '../../proto/livekit_rtc';
+import { ParticipantEvent, TrackEvent } from '../events';
+import RemoteAudioTrack from '../track/RemoteAudioTrack';
+import RemoteTrackPublication from '../track/RemoteTrackPublication';
+import RemoteVideoTrack from '../track/RemoteVideoTrack';
+import { Track } from '../track/Track';
+import { TrackPublication } from '../track/TrackPublication';
+import { RemoteTrack } from '../track/types';
+import Participant from './Participant';
+
+export default class RemoteParticipant extends Participant {
+  audioTracks: Map<string, RemoteTrackPublication>;
+
+  videoTracks: Map<string, RemoteTrackPublication>;
+
+  tracks: Map<string, RemoteTrackPublication>;
+
+  signalClient: SignalClient;
+
+  /** @internal */
+  static fromParticipantInfo(
+    signalClient: SignalClient,
+    pi: ParticipantInfo,
+  ): RemoteParticipant {
+    const rp = new RemoteParticipant(signalClient, pi.sid, pi.identity);
+    rp.updateInfo(pi);
+    return rp;
+  }
+
+  /** @internal */
+  constructor(signalClient: SignalClient, id: string, name?: string) {
+    super(id, name || '');
+    this.signalClient = signalClient;
+    this.tracks = new Map();
+    this.audioTracks = new Map();
+    this.videoTracks = new Map();
+  }
+
+  protected addTrackPublication(publication: TrackPublication) {
+    super.addTrackPublication(publication);
+
+    // register action events
+    publication.on(
+      TrackEvent.UpdateSettings,
+      (settings: UpdateTrackSettings) => {
+        this.signalClient.sendUpdateTrackSettings(settings);
+      },
+    );
+    publication.on(TrackEvent.UpdateSubscription, (sub: UpdateSubscription) => {
+      sub.participantTracks.forEach((pt) => {
+        pt.participantSid = this.sid;
+      });
+      this.signalClient.sendUpdateSubscription(sub);
+    });
+    publication.on(TrackEvent.Ended, (track: RemoteTrack) => {
+      this.emit(ParticipantEvent.TrackUnsubscribed, track, publication);
+    });
+  }
+
+  getTrack(source: Track.Source): RemoteTrackPublication | undefined {
+    const track = super.getTrack(source);
+    if (track) {
+      return track as RemoteTrackPublication;
+    }
+  }
+
+  getTrackByName(name: string): RemoteTrackPublication | undefined {
+    const track = super.getTrackByName(name);
+    if (track) {
+      return track as RemoteTrackPublication;
+    }
+  }
+
+  /** @internal */
+  addSubscribedMediaTrack(
+    mediaTrack: MediaStreamTrack,
+    sid: Track.SID,
+    mediaStream: MediaStream,
+    receiver?: RTCRtpReceiver,
+    adaptiveStream?: boolean,
+    triesLeft?: number,
+  ) {
+    // find the track publication
+    // it's possible for the media track to arrive before participant info
+    let publication = this.getTrackPublication(sid);
+
+    // it's also possible that the browser didn't honor our original track id
+    // FireFox would use its own local uuid instead of server track id
+    if (!publication) {
+      if (!sid.startsWith('TR')) {
+        // find the first track that matches type
+        this.tracks.forEach((p) => {
+          if (!publication && mediaTrack.kind === p.kind.toString()) {
+            publication = p;
+          }
+        });
+      }
+    }
+
+    // when we couldn't locate the track, it's possible that the metadata hasn't
+    // yet arrived. Wait a bit longer for it to arrive, or fire an error
+    if (!publication) {
+      if (triesLeft === 0) {
+        log.error('could not find published track', this.sid, sid);
+        this.emit(ParticipantEvent.TrackSubscriptionFailed, sid);
+        return;
+      }
+
+      if (triesLeft === undefined) triesLeft = 20;
+      setTimeout(() => {
+        this.addSubscribedMediaTrack(mediaTrack, sid, mediaStream,
+          receiver, adaptiveStream, triesLeft! - 1);
+      }, 150);
+      return;
+    }
+
+    const isVideo = mediaTrack.kind === 'video';
+    let track: RemoteTrack;
+    if (isVideo) {
+      track = new RemoteVideoTrack(mediaTrack, sid, receiver, adaptiveStream);
+    } else {
+      track = new RemoteAudioTrack(mediaTrack, sid, receiver);
+    }
+
+    // set track info
+    track.source = publication.source;
+    // keep publication's muted status
+    track.isMuted = publication.isMuted;
+    track.setMediaStream(mediaStream);
+    track.start();
+
+    publication.setTrack(track);
+
+    this.emit(ParticipantEvent.TrackSubscribed, track, publication);
+
+    return publication;
+  }
+
+  /** @internal */
+  get hasMetadata(): boolean {
+    return !!this.participantInfo;
+  }
+
+  getTrackPublication(sid: Track.SID): RemoteTrackPublication | undefined {
+    return this.tracks.get(sid);
+  }
+
+  /** @internal */
+  updateInfo(info: ParticipantInfo) {
+    const alreadyHasMetadata = this.hasMetadata;
+
+    super.updateInfo(info);
+
+    // we are getting a list of all available tracks, reconcile in here
+    // and send out events for changes
+
+    // reconcile track publications, publish events only if metadata is already there
+    // i.e. changes since the local participant has joined
+    const validTracks = new Map<string, RemoteTrackPublication>();
+    const newTracks = new Map<string, RemoteTrackPublication>();
+
+    info.tracks.forEach((ti) => {
+      let publication = this.getTrackPublication(ti.sid);
+      if (!publication) {
+        // new publication
+        const kind = Track.kindFromProto(ti.type);
+        if (!kind) {
+          return;
+        }
+        publication = new RemoteTrackPublication(kind, ti.sid, ti.name);
+        publication.updateInfo(ti);
+        newTracks.set(ti.sid, publication);
+        this.addTrackPublication(publication);
+      } else {
+        publication.updateInfo(ti);
+      }
+      validTracks.set(ti.sid, publication);
+    });
+
+    // send new tracks
+    if (alreadyHasMetadata) {
+      newTracks.forEach((publication) => {
+        this.emit(ParticipantEvent.TrackPublished, publication);
+      });
+    }
+
+    // detect removed tracks
+    this.tracks.forEach((publication) => {
+      if (!validTracks.has(publication.trackSid)) {
+        this.unpublishTrack(publication.trackSid, true);
+      }
+    });
+  }
+
+  /** @internal */
+  unpublishTrack(sid: Track.SID, sendUnpublish?: boolean) {
+    const publication = <RemoteTrackPublication> this.tracks.get(sid);
+    if (!publication) {
+      return;
+    }
+
+    this.tracks.delete(sid);
+
+    // remove from the right type map
+    switch (publication.kind) {
+      case Track.Kind.Audio:
+        this.audioTracks.delete(sid);
+        break;
+      case Track.Kind.Video:
+        this.videoTracks.delete(sid);
+        break;
+      default:
+        break;
+    }
+
+    // also send unsubscribe, if track is actively subscribed
+    const { track } = publication;
+    if (track) {
+      const { isSubscribed } = publication;
+      track.stop();
+      publication.setTrack(undefined);
+      // always send unsubscribed, since apps may rely on this
+      if (isSubscribed) {
+        this.emit(ParticipantEvent.TrackUnsubscribed, track, publication);
+      }
+    }
+    if (sendUnpublish) { this.emit(ParticipantEvent.TrackUnpublished, publication); }
+  }
+
+  /** @internal */
+  emit(event: string | symbol, ...args: any[]): boolean {
+    log.trace('participant event', this.sid, event, ...args);
+    return super.emit(event, ...args);
+  }
+}

package/src/room/participant/publishUtils.test.ts

@@ -0,0 +1,105 @@
+import { VideoPresets, VideoPresets43 } from '../track/options';
+import {
+  computeVideoEncodings,
+  determineAppropriateEncoding,
+  presets169,
+  presets43,
+  presetsForResolution,
+  presetsScreenShare,
+} from './publishUtils';
+
+describe('presetsForResolution', () => {
+  it('handles screenshare', () => {
+    expect(presetsForResolution(true, 600, 300)).toEqual(presetsScreenShare);
+  });
+
+  it('handles landscape', () => {
+    expect(presetsForResolution(false, 600, 300)).toEqual(presets169);
+    expect(presetsForResolution(false, 500, 500)).toEqual(presets43);
+  });
+
+  it('handles portrait', () => {
+    expect(presetsForResolution(false, 300, 600)).toEqual(presets169);
+    expect(presetsForResolution(false, 500, 500)).toEqual(presets43);
+  });
+});
+
+describe('determineAppropriateEncoding', () => {
+  it('uses higher encoding', () => {
+    expect(determineAppropriateEncoding(false, 600, 300))
+      .toEqual(VideoPresets.vga.encoding);
+  });
+
+  it('handles portrait', () => {
+    expect(determineAppropriateEncoding(false, 300, 600))
+      .toEqual(VideoPresets.vga.encoding);
+  });
+});
+
+describe('computeVideoEncodings', () => {
+  it('handles non-simulcast', () => {
+    const encodings = computeVideoEncodings(false, 640, 480, {
+      simulcast: false,
+    });
+    expect(encodings).toEqual([{}]);
+  });
+
+  it('respects client defined bitrate', () => {
+    const encodings = computeVideoEncodings(false, 640, 480, {
+      simulcast: false,
+      videoEncoding: {
+        maxBitrate: 1024,
+      },
+    });
+    expect(encodings).toHaveLength(1);
+    expect(encodings![0].maxBitrate).toBe(1024);
+  });
+
+  it('returns three encodings for high-res simulcast', () => {
+    const encodings = computeVideoEncodings(false, 960, 540, {
+      simulcast: true,
+    });
+    expect(encodings).toHaveLength(3);
+
+    // ensure they are what we expect
+    expect(encodings![0].rid).toBe('q');
+    expect(encodings![0].maxBitrate).toBe(VideoPresets.qvga.encoding.maxBitrate);
+    expect(encodings![0].scaleResolutionDownBy).toBe(3);
+    expect(encodings![1].rid).toBe('h');
+    expect(encodings![1].scaleResolutionDownBy).toBe(1.5);
+    expect(encodings![2].rid).toBe('f');
+  });
+
+  it('handles portrait simulcast', () => {
+    const encodings = computeVideoEncodings(false, 540, 960, {
+      simulcast: true,
+    });
+    expect(encodings).toHaveLength(3);
+    expect(encodings![0].scaleResolutionDownBy).toBe(3);
+    expect(encodings![1].scaleResolutionDownBy).toBe(1.5);
+    expect(encodings![2].maxBitrate).toBe(VideoPresets.qhd.encoding.maxBitrate);
+  });
+
+  it('returns two encodings for lower-res simulcast', () => {
+    const encodings = computeVideoEncodings(false, 640, 360, {
+      simulcast: true,
+    });
+    expect(encodings).toHaveLength(2);
+
+    // ensure they are what we expect
+    expect(encodings![0].rid).toBe('q');
+    expect(encodings![0].maxBitrate).toBe(VideoPresets.qvga.encoding.maxBitrate);
+    expect(encodings![1].rid).toBe('h');
+    expect(encodings![1].maxBitrate).toBe(VideoPresets.vga.encoding.maxBitrate);
+  });
+
+  it('respects provided min resolution', () => {
+    const encodings = computeVideoEncodings(false, 100, 120, {
+      simulcast: true,
+    });
+    expect(encodings).toHaveLength(1);
+    expect(encodings![0].rid).toBe('q');
+    expect(encodings![0].maxBitrate).toBe(VideoPresets43.qvga.encoding.maxBitrate);
+    expect(encodings![0].scaleResolutionDownBy).toBe(1);
+  });
+});