livekit-client 0.18.6 → 1.0.2

package/src/room/participant/LocalParticipant.ts

@@ -10,12 +10,15 @@ import {
   TrackUnpublishedResponse,
 } from '../../proto/livekit_rtc';
 import { TrackInvalidError, UnexpectedConnectionState } from '../errors';
-import { ParticipantEvent, TrackEvent } from '../events';
+import { EngineEvent, ParticipantEvent, TrackEvent } from '../events';
 import RTCEngine from '../RTCEngine';
 import LocalAudioTrack from '../track/LocalAudioTrack';
 import LocalTrack from '../track/LocalTrack';
 import LocalTrackPublication from '../track/LocalTrackPublication';
-import LocalVideoTrack, { videoLayersFromEncodings } from '../track/LocalVideoTrack';
+import LocalVideoTrack, {
+  SimulcastTrackInfo,
+  videoLayersFromEncodings,
+} from '../track/LocalVideoTrack';
 import {
   CreateLocalTracksOptions,
   ScreenShareCaptureOptions,
@@ -31,6 +34,7 @@ import { ParticipantTrackPermission, trackPermissionToProto } from './Participan
 import { computeVideoEncodings, mediaTrackToLocalTrack } from './publishUtils';
 import RemoteParticipant from './RemoteParticipant';
 
+const compatibleCodecForSVC = 'vp8';
 export default class LocalParticipant extends Participant {
   audioTracks: Map<string, LocalTrackPublication>;
 
@@ -47,6 +51,10 @@ export default class LocalParticipant extends Participant {
 
   private engine: RTCEngine;
 
+  private participantTrackPermissions: Array<ParticipantTrackPermission> = [];
+
+  private allParticipantsAllowedToSubscribe: boolean = true;
+
   // keep a pointer to room options
   private roomOptions?: RoomOptions;
 
@@ -74,6 +82,11 @@ export default class LocalParticipant extends Participant {
     this.engine.client.onSubscribedQualityUpdate = this.handleSubscribedQualityUpdate;
 
     this.engine.client.onLocalTrackUnpublished = this.handleLocalTrackUnpublished;
+
+    this.engine
+      .on(EngineEvent.Connected, this.updateTrackSubscriptionPermissions)
+      .on(EngineEvent.Restarted, this.updateTrackSubscriptionPermissions)
+      .on(EngineEvent.Resumed, this.updateTrackSubscriptionPermissions);
   }
 
   get lastCameraError(): Error | undefined {
@@ -384,7 +397,7 @@ export default class LocalParticipant extends Participant {
     // handle track actions
     track.on(TrackEvent.Muted, this.onTrackMuted);
     track.on(TrackEvent.Unmuted, this.onTrackUnmuted);
-    track.on(TrackEvent.Ended, this.onTrackUnpublish);
+    track.on(TrackEvent.Ended, this.handleTrackEnded);
     track.on(TrackEvent.UpstreamPaused, this.onTrackUpstreamPaused);
     track.on(TrackEvent.UpstreamResumed, this.onTrackUpstreamResumed);
 
@@ -401,6 +414,8 @@ export default class LocalParticipant extends Participant {
 
     // compute encodings and layers for video
     let encodings: RTCRtpEncodingParameters[] | undefined;
+    let simEncodings: RTCRtpEncodingParameters[] | undefined;
+    let simulcastTracks: SimulcastTrackInfo[] | undefined;
     if (track.kind === Track.Kind.Video) {
       // TODO: support react native, which doesn't expose getSettings
       const settings = track.mediaStreamTrack.getSettings();
@@ -409,16 +424,38 @@ export default class LocalParticipant extends Participant {
       // width and height should be defined for video
       req.width = width ?? 0;
       req.height = height ?? 0;
-      // for svc codecs, disable simulcast and enable scalability L3T3
-      // by default
-      if (track instanceof LocalVideoTrack) {
-        if (opts?.videoCodec === 'vp9' || opts?.videoCodec === 'av1') {
-          opts.simulcast = false;
-          opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3';
-        } else {
-          // other codecs, unset scalability
-          opts.scalabilityMode = undefined;
-        }
+      // for svc codecs, disable simulcast and use vp8 for backup codec
+      if (
+        track instanceof LocalVideoTrack &&
+        (opts?.videoCodec === 'vp9' || opts?.videoCodec === 'av1')
+      ) {
+        // set scalabilityMode to 'L3T3' by default
+        opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3';
+
+        // add backup codec track
+        const simOpts = { ...opts };
+        simOpts.simulcast = true;
+        simOpts.scalabilityMode = undefined;
+        simEncodings = computeVideoEncodings(
+          track.source === Track.Source.ScreenShare,
+          width,
+          height,
+          simOpts,
+        );
+        const simulcastTrack = track.addSimulcastTrack(compatibleCodecForSVC, simEncodings);
+        simulcastTracks = [simulcastTrack];
+        req.simulcastCodecs = [
+          {
+            codec: opts.videoCodec,
+            cid: track.mediaStreamTrack.id,
+            enableSimulcastLayers: true,
+          },
+          {
+            codec: simulcastTrack.codec,
+            cid: simulcastTrack.mediaStreamTrack.id,
+            enableSimulcastLayers: true,
+          },
+        ];
       }
 
       encodings = computeVideoEncodings(
@@ -427,7 +464,7 @@ export default class LocalParticipant extends Participant {
         height,
         opts,
       );
-      req.layers = videoLayersFromEncodings(req.width, req.height, encodings);
+      req.layers = videoLayersFromEncodings(req.width, req.height, simEncodings ?? encodings);
     } else if (track.kind === Track.Kind.Audio && opts.audioBitrate) {
       encodings = [
         {
@@ -436,8 +473,14 @@ export default class LocalParticipant extends Participant {
       ];
     }
 
+    if (!this.engine || this.engine.isClosed) {
+      throw new UnexpectedConnectionState('cannot publish track when not connected');
+    }
+
     const ti = await this.engine.addTrack(req);
     const publication = new LocalTrackPublication(track.kind, ti, track);
+    // save options for when it needs to be republished again
+    publication.options = opts;
     track.sid = ti.sid;
 
     if (!this.engine.publisher) {
@@ -448,13 +491,32 @@ export default class LocalParticipant extends Participant {
     if (encodings) {
       transceiverInit.sendEncodings = encodings;
     }
-    const transceiver = this.engine.publisher.pc.addTransceiver(
+    // addTransceiver for react-native is async. web is synchronous, but await won't effect it.
+    const transceiver = await this.engine.publisher.pc.addTransceiver(
       track.mediaStreamTrack,
       transceiverInit,
     );
-    if (opts.videoCodec) {
+    if (track.kind === Track.Kind.Video && opts.videoCodec) {
       this.setPreferredCodec(transceiver, track.kind, opts.videoCodec);
+      track.codec = opts.videoCodec;
+    }
+
+    const localTrack = track as LocalVideoTrack;
+    if (simulcastTracks) {
+      for await (const simulcastTrack of simulcastTracks) {
+        const simTransceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly' };
+        if (simulcastTrack.encodings) {
+          simTransceiverInit.sendEncodings = simulcastTrack.encodings;
+        }
+        const simTransceiver = await this.engine.publisher!.pc.addTransceiver(
+          simulcastTrack.mediaStreamTrack,
+          simTransceiverInit,
+        );
+        this.setPreferredCodec(simTransceiver, localTrack.kind, simulcastTrack.codec);
+        localTrack.setSimulcastTrackSender(simulcastTrack.codec, simTransceiver.sender);
+      }
     }
+
     this.engine.negotiate();
 
     // store RTPSender
@@ -464,6 +526,7 @@ export default class LocalParticipant extends Participant {
     } else if (track instanceof LocalAudioTrack) {
       track.startMonitor();
     }
+
     this.addTrackPublication(publication);
 
     // send event for publication
@@ -490,9 +553,10 @@ export default class LocalParticipant extends Participant {
 
     track = publication.track;
 
+    track.sender = undefined;
     track.off(TrackEvent.Muted, this.onTrackMuted);
     track.off(TrackEvent.Unmuted, this.onTrackUnmuted);
-    track.off(TrackEvent.Ended, this.onTrackUnpublish);
+    track.off(TrackEvent.Ended, this.handleTrackEnded);
     track.off(TrackEvent.UpstreamPaused, this.onTrackUpstreamPaused);
     track.off(TrackEvent.UpstreamResumed, this.onTrackUpstreamResumed);
 
@@ -505,7 +569,7 @@ export default class LocalParticipant extends Participant {
 
     const { mediaStreamTrack } = track;
 
-    if (this.engine.publisher) {
+    if (this.engine.publisher && this.engine.publisher.pc.connectionState !== 'closed') {
       const senders = this.engine.publisher.pc.getSenders();
       senders.forEach((sender) => {
         if (sender.track === mediaStreamTrack) {
@@ -609,11 +673,23 @@ export default class LocalParticipant extends Participant {
     allParticipantsAllowed: boolean,
     participantTrackPermissions: ParticipantTrackPermission[] = [],
   ) {
+    this.participantTrackPermissions = participantTrackPermissions;
+    this.allParticipantsAllowedToSubscribe = allParticipantsAllowed;
+    if (this.engine.client.isConnected) {
+      this.updateTrackSubscriptionPermissions();
+    }
+  }
+
+  private updateTrackSubscriptionPermissions = () => {
+    log.debug('updating track subscription permissions', {
+      allParticipantsAllowed: this.allParticipantsAllowedToSubscribe,
+      participantTrackPermissions: this.participantTrackPermissions,
+    });
     this.engine.client.sendUpdateSubscriptionPermissions(
-      allParticipantsAllowed,
-      participantTrackPermissions.map((p) => trackPermissionToProto(p)),
+      this.allParticipantsAllowedToSubscribe,
+      this.participantTrackPermissions.map((p) => trackPermissionToProto(p)),
     );
-  }
+  };
 
   /** @internal */
   private onTrackUnmuted = (track: LocalTrack) => {
@@ -657,7 +733,11 @@ export default class LocalParticipant extends Participant {
       });
       return;
     }
-    pub.videoTrack?.setPublishingLayers(update.subscribedQualities);
+    if (update.subscribedCodecs.length > 0) {
+      pub.videoTrack?.setPublishingCodecs(update.subscribedCodecs);
+    } else if (update.subscribedQualities.length > 0) {
+      pub.videoTrack?.setPublishingLayers(update.subscribedQualities);
+    }
   };
 
   private handleLocalTrackUnpublished = (unpublished: TrackUnpublishedResponse) => {
@@ -672,7 +752,10 @@ export default class LocalParticipant extends Participant {
     this.unpublishTrack(track.track!);
   };
 
-  private onTrackUnpublish = (track: LocalTrack) => {
+  private handleTrackEnded = (track: LocalTrack) => {
+    log.debug('unpublishing local track due to TrackEnded', {
+      track: track.sid,
+    });
     this.unpublishTrack(track);
   };
 
@@ -710,6 +793,7 @@ export default class LocalParticipant extends Participant {
     }
     const cap = RTCRtpSender.getCapabilities(kind);
     if (!cap) return;
+    log.debug('get capabilities', cap);
     let selected: RTCRtpCodecCapability | undefined;
     const codecs: RTCRtpCodecCapability[] = [];
     cap.codecs.forEach((c) => {
@@ -734,6 +818,7 @@ export default class LocalParticipant extends Participant {
       }
       codecs.push(c);
     });
+
     if (selected && 'setCodecPreferences' in transceiver) {
       // @ts-ignore
       codecs.unshift(selected);

package/src/room/participant/Participant.ts

@@ -188,7 +188,6 @@ export default class Participant extends (EventEmitter as new () => TypedEmitter
     this.metadata = md;
 
     if (changed) {
-      this.emit(ParticipantEvent.MetadataChanged, prevMetadata);
       this.emit(ParticipantEvent.ParticipantMetadataChanged, prevMetadata);
     }
   }
@@ -266,10 +265,6 @@ export type ParticipantEventCallbacks = {
   trackUnmuted: (publication: TrackPublication) => void;
   localTrackPublished: (publication: LocalTrackPublication) => void;
   localTrackUnpublished: (publication: LocalTrackPublication) => void;
-  /**
-   * @deprecated use [[participantMetadataChanged]] instead
-   */
-  metadataChanged: (prevMetadata: string | undefined, participant?: any) => void;
   participantMetadataChanged: (prevMetadata: string | undefined, participant?: any) => void;
   dataReceived: (payload: Uint8Array, kind: DataPacket_Kind) => void;
   isSpeakingChanged: (speaking: boolean) => void;

package/src/room/participant/RemoteParticipant.ts

@@ -19,6 +19,8 @@ export default class RemoteParticipant extends Participant {
 
   signalClient: SignalClient;
 
+  private volume?: number;
+
   /** @internal */
   static fromParticipantInfo(signalClient: SignalClient, pi: ParticipantInfo): RemoteParticipant {
     const rp = new RemoteParticipant(signalClient, pi.sid, pi.identity);
@@ -40,6 +42,7 @@ export default class RemoteParticipant extends Participant {
 
     // register action events
     publication.on(TrackEvent.UpdateSettings, (settings: UpdateTrackSettings) => {
+      log.debug('send update settings', settings);
       this.signalClient.sendUpdateTrackSettings(settings);
     });
     publication.on(TrackEvent.UpdateSubscription, (sub: UpdateSubscription) => {
@@ -68,24 +71,26 @@ export default class RemoteParticipant extends Participant {
   }
 
   /**
-   * sets the volume on the participant's microphone track if it exists.
+   * sets the volume on the participant's microphone track
+   * if no track exists the volume will be applied when the microphone track is added
    */
   setVolume(volume: number) {
+    this.volume = volume;
     const audioPublication = this.getTrack(Track.Source.Microphone);
-    if (audioPublication) {
+    if (audioPublication && audioPublication.track) {
       (audioPublication.track as RemoteAudioTrack).setVolume(volume);
     }
   }
 
   /**
    * gets the volume on the participant's microphone track
-   * returns undefined if no microphone track exists
    */
   getVolume() {
     const audioPublication = this.getTrack(Track.Source.Microphone);
-    if (audioPublication) {
+    if (audioPublication && audioPublication.track) {
       return (audioPublication.track as RemoteAudioTrack).getVolume();
     }
+    return this.volume;
   }
 
   /** @internal */
@@ -153,7 +158,14 @@ export default class RemoteParticipant extends Participant {
     track.start();
 
     publication.setTrack(track);
-
+    // set participant volume on new microphone tracks
+    if (
+      this.volume !== undefined &&
+      track instanceof RemoteAudioTrack &&
+      track.source === Track.Source.Microphone
+    ) {
+      track.setVolume(this.volume);
+    }
     this.emit(ParticipantEvent.TrackSubscribed, track, publication);
 
     return publication;

package/src/room/participant/publishUtils.test.ts

@@ -28,11 +28,11 @@ describe('presetsForResolution', () => {
 
 describe('determineAppropriateEncoding', () => {
   it('uses higher encoding', () => {
-    expect(determineAppropriateEncoding(false, 600, 300)).toEqual(VideoPresets.vga.encoding);
+    expect(determineAppropriateEncoding(false, 600, 300)).toEqual(VideoPresets.h360.encoding);
   });
 
   it('handles portrait', () => {
-    expect(determineAppropriateEncoding(false, 300, 600)).toEqual(VideoPresets.vga.encoding);
+    expect(determineAppropriateEncoding(false, 300, 600)).toEqual(VideoPresets.h360.encoding);
   });
 });
 

package/src/room/stats.ts

@@ -104,6 +104,8 @@ export interface VideoReceiverStats extends ReceiverStats {
   pliCount?: number;
 
   nackCount?: number;
+
+  decoderImplementation?: string;
 }
 
 export function computeBitrate<T extends ReceiverStats | SenderStats>(

package/src/room/track/LocalAudioTrack.ts

@@ -1,6 +1,7 @@
 import log from '../../logger';
 import { TrackEvent } from '../events';
 import { AudioSenderStats, computeBitrate, monitorFrequency } from '../stats';
+import { isWeb } from '../utils';
 import LocalTrack from './LocalTrack';
 import { AudioCaptureOptions } from './options';
 import { Track } from './Track';
@@ -68,6 +69,9 @@ export default class LocalAudioTrack extends LocalTrack {
 
   /* @internal */
   startMonitor() {
+    if (!isWeb()) {
+      return;
+    }
     setTimeout(() => {
       this.monitorSender();
     }, monitorFrequency);

package/src/room/track/LocalTrack.ts

@@ -2,6 +2,7 @@ import log from '../../logger';
 import DeviceManager from '../DeviceManager';
 import { TrackInvalidError } from '../errors';
 import { TrackEvent } from '../events';
+import { VideoCodec } from './options';
 import { getEmptyAudioStreamTrack, getEmptyVideoStreamTrack, isMobile } from '../utils';
 import { attachToElement, detachTrack, Track } from './Track';
 
@@ -9,6 +10,9 @@ export default class LocalTrack extends Track {
   /** @internal */
   sender?: RTCRtpSender;
 
+  /** @internal */
+  codec?: VideoCodec;
+
   protected constraints: MediaTrackConstraints;
 
   protected wasMuted: boolean;
@@ -94,7 +98,9 @@ export default class LocalTrack extends Track {
     track.addEventListener('ended', this.handleEnded);
     log.debug('replace MediaStreamTrack');
 
-    await this.sender.replaceTrack(track);
+    if (this.sender) {
+      await this.sender.replaceTrack(track);
+    }
     this._mediaStreamTrack = track;
 
     this.attachedElements.forEach((el) => {
@@ -106,9 +112,6 @@ export default class LocalTrack extends Track {
   }
 
   protected async restart(constraints?: MediaTrackConstraints): Promise<LocalTrack> {
-    if (!this.sender) {
-      throw new TrackInvalidError('unable to restart an unpublished track');
-    }
     if (!constraints) {
       constraints = this.constraints;
     }
@@ -141,7 +144,11 @@ export default class LocalTrack extends Track {
     newTrack.addEventListener('ended', this.handleEnded);
     log.debug('re-acquired MediaStreamTrack');
 
-    await this.sender.replaceTrack(newTrack);
+    if (this.sender) {
+      // Track can be restarted after it's unpublished
+      await this.sender.replaceTrack(newTrack);
+    }
+
     this._mediaStreamTrack = newTrack;
 
     this.attachedElements.forEach((el) => {