livekit-client 1.0.1 → 1.0.2

package/src/room/Room.ts

@@ -27,7 +27,9 @@ import Participant, { ConnectionQuality } from './participant/Participant';
 import RemoteParticipant from './participant/RemoteParticipant';
 import RTCEngine, { maxICEConnectTimeout } from './RTCEngine';
 import { audioDefaults, publishDefaults, videoDefaults } from './track/defaults';
+import LocalAudioTrack from './track/LocalAudioTrack';
 import LocalTrackPublication from './track/LocalTrackPublication';
+import LocalVideoTrack from './track/LocalVideoTrack';
 import RemoteTrackPublication from './track/RemoteTrackPublication';
 import { Track } from './track/Track';
 import { TrackPublication } from './track/TrackPublication';
@@ -84,6 +86,8 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
   /** options of room */
   options: RoomOptions;
 
+  private identityToSid: Map<string, string>;
+
   /** connect options of room */
   private connOptions?: RoomConnectOptions;
 
@@ -101,6 +105,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
   constructor(options?: RoomOptions) {
     super();
     this.participants = new Map();
+    this.identityToSid = new Map();
     this.options = options || {};
 
     switch (this.options?.publishDefaults?.videoCodec) {
@@ -157,8 +162,9 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       .on(EngineEvent.ActiveSpeakersUpdate, this.handleActiveSpeakersUpdate)
       .on(EngineEvent.DataPacketReceived, this.handleDataPacket)
       .on(EngineEvent.Resuming, () => {
-        this.setAndEmitConnectionState(ConnectionState.Reconnecting);
-        this.emit(RoomEvent.Reconnecting);
+        if (this.setAndEmitConnectionState(ConnectionState.Reconnecting)) {
+          this.emit(RoomEvent.Reconnecting);
+        }
       })
       .on(EngineEvent.Resumed, () => {
         this.setAndEmitConnectionState(ConnectionState.Connected);
@@ -287,6 +293,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       this.name = joinResponse.room!.name;
       this.sid = joinResponse.room!.sid;
       this.metadata = joinResponse.room!.metadata;
+      this.emit(RoomEvent.SignalConnected);
     } catch (err) {
       this.engine.close();
       this.setAndEmitConnectionState(ConnectionState.Disconnected);
@@ -338,10 +345,14 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       return;
     }
     // send leave
-    if (this.engine) {
+    if (this.engine?.client.isConnected) {
       this.engine.client.sendLeave();
+    }
+    // close engine (also closes client)
+    if (this.engine) {
       this.engine.close();
     }
+
     this.handleDisconnect(stopTracks);
     /* @ts-ignore */
     this.engine = undefined;
@@ -353,20 +364,20 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
    * @returns
    */
   getParticipantByIdentity(identity: string): Participant | undefined {
-    for (const [, p] of this.participants) {
-      if (p.identity === identity) {
-        return p;
-      }
-    }
     if (this.localParticipant.identity === identity) {
       return this.localParticipant;
     }
+    const sid = this.identityToSid.get(identity);
+    if (sid) {
+      return this.participants.get(sid);
+    }
   }
 
   /**
    * @internal for testing
    */
   simulateScenario(scenario: string) {
+    let postAction = () => {};
     let req: SimulateScenario | undefined;
     switch (scenario) {
       case 'speaker':
@@ -389,10 +400,19 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
           migration: true,
         });
         break;
+      case 'switch-candidate':
+        req = SimulateScenario.fromPartial({
+          switchCandidateProtocol: 1,
+        });
+        postAction = () => {
+          this.engine.publisher?.createAndSendOffer({ iceRestart: true });
+        };
+        break;
       default:
     }
     if (req) {
       this.engine.client.sendSimulateScenario(req);
+      postAction();
     }
   }
 
@@ -512,8 +532,9 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
   }
 
   private handleRestarting = () => {
-    this.setAndEmitConnectionState(ConnectionState.Reconnecting);
-    this.emit(RoomEvent.Reconnecting);
+    if (this.setAndEmitConnectionState(ConnectionState.Reconnecting)) {
+      this.emit(RoomEvent.Reconnecting);
+    }
 
     // also unwind existing participants & existing subscriptions
     for (const p of this.participants.values()) {
@@ -522,7 +543,9 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
   };
 
   private handleRestarted = async (joinResponse: JoinResponse) => {
-    log.debug(`reconnected to server region ${joinResponse.serverRegion}`);
+    log.debug(`reconnected to server`, {
+      region: joinResponse.serverRegion,
+    });
     this.setAndEmitConnectionState(ConnectionState.Connected);
     this.emit(RoomEvent.Reconnected);
 
@@ -546,7 +569,17 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       localPubs.map(async (pub) => {
         const track = pub.track!;
         this.localParticipant.unpublishTrack(track, false);
-        this.localParticipant.publishTrack(track, pub.options);
+        if (!track.isMuted) {
+          if (track instanceof LocalAudioTrack || track instanceof LocalVideoTrack) {
+            // we need to restart the track before publishing, often a full reconnect
+            // is necessary because computer had gone to sleep.
+            log.debug('restarting existing track', {
+              track: pub.trackSid,
+            });
+            await track.restartTrack();
+          }
+          await this.localParticipant.publishTrack(track, pub.options);
+        }
       }),
     );
   };
@@ -593,6 +626,13 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
         return;
       }
 
+      // ensure identity <=> sid mapping
+      const sid = this.identityToSid.get(info.identity);
+      if (sid && sid !== info.sid) {
+        // sid had changed, need to remove previous participant
+        this.handleParticipantDisconnected(sid, this.participants.get(sid));
+      }
+
       let remoteParticipant = this.participants.get(info.sid);
       const isNewParticipant = !remoteParticipant;
 
@@ -603,6 +643,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       if (info.state === ParticipantInfo_State.DISCONNECTED) {
         this.handleParticipantDisconnected(info.sid, remoteParticipant);
       } else if (isNewParticipant) {
+        this.identityToSid.set(info.identity, info.sid);
         // fire connected event
         this.emit(RoomEvent.ParticipantConnected, remoteParticipant);
       } else {
@@ -619,8 +660,9 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       return;
     }
 
+    this.identityToSid.delete(participant.identity);
     participant.tracks.forEach((publication) => {
-      participant.unpublishTrack(publication.trackSid);
+      participant.unpublishTrack(publication.trackSid, true);
     });
     this.emit(RoomEvent.ParticipantDisconnected, participant);
   }
@@ -913,12 +955,14 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     }
   }
 
-  private setAndEmitConnectionState(state: ConnectionState) {
+  private setAndEmitConnectionState(state: ConnectionState): boolean {
     if (state === this.state) {
-      return;
+      // unchanged
+      return false;
     }
     this.state = state;
     this.emit(RoomEvent.ConnectionStateChanged, this.state);
+    return true;
   }
 
   // /** @internal */
@@ -991,4 +1035,5 @@ export type RoomEventCallbacks = {
     participant: RemoteParticipant,
   ) => void;
   audioPlaybackChanged: (playing: boolean) => void;
+  signalConnected: () => void;
 };

package/src/room/events.ts

@@ -230,6 +230,11 @@ export enum RoomEvent {
    * args: (prevPermissions: [[ParticipantPermission]], participant: [[Participant]])
    */
   ParticipantPermissionsChanged = 'participantPermissionsChanged',
+
+  /**
+   * Signal connected, can publish tracks.
+   */
+  SignalConnected = 'signalConnected',
 }
 
 export enum ParticipantEvent {

package/src/room/participant/LocalParticipant.ts

@@ -10,12 +10,15 @@ import {
   TrackUnpublishedResponse,
 } from '../../proto/livekit_rtc';
 import { TrackInvalidError, UnexpectedConnectionState } from '../errors';
-import { ParticipantEvent, TrackEvent } from '../events';
+import { EngineEvent, ParticipantEvent, TrackEvent } from '../events';
 import RTCEngine from '../RTCEngine';
 import LocalAudioTrack from '../track/LocalAudioTrack';
 import LocalTrack from '../track/LocalTrack';
 import LocalTrackPublication from '../track/LocalTrackPublication';
-import LocalVideoTrack, { videoLayersFromEncodings } from '../track/LocalVideoTrack';
+import LocalVideoTrack, {
+  SimulcastTrackInfo,
+  videoLayersFromEncodings,
+} from '../track/LocalVideoTrack';
 import {
   CreateLocalTracksOptions,
   ScreenShareCaptureOptions,
@@ -31,6 +34,7 @@ import { ParticipantTrackPermission, trackPermissionToProto } from './Participan
 import { computeVideoEncodings, mediaTrackToLocalTrack } from './publishUtils';
 import RemoteParticipant from './RemoteParticipant';
 
+const compatibleCodecForSVC = 'vp8';
 export default class LocalParticipant extends Participant {
   audioTracks: Map<string, LocalTrackPublication>;
 
@@ -47,6 +51,10 @@ export default class LocalParticipant extends Participant {
 
   private engine: RTCEngine;
 
+  private participantTrackPermissions: Array<ParticipantTrackPermission> = [];
+
+  private allParticipantsAllowedToSubscribe: boolean = true;
+
   // keep a pointer to room options
   private roomOptions?: RoomOptions;
 
@@ -74,6 +82,11 @@ export default class LocalParticipant extends Participant {
     this.engine.client.onSubscribedQualityUpdate = this.handleSubscribedQualityUpdate;
 
     this.engine.client.onLocalTrackUnpublished = this.handleLocalTrackUnpublished;
+
+    this.engine
+      .on(EngineEvent.Connected, this.updateTrackSubscriptionPermissions)
+      .on(EngineEvent.Restarted, this.updateTrackSubscriptionPermissions)
+      .on(EngineEvent.Resumed, this.updateTrackSubscriptionPermissions);
   }
 
   get lastCameraError(): Error | undefined {
@@ -384,7 +397,7 @@ export default class LocalParticipant extends Participant {
     // handle track actions
     track.on(TrackEvent.Muted, this.onTrackMuted);
     track.on(TrackEvent.Unmuted, this.onTrackUnmuted);
-    track.on(TrackEvent.Ended, this.onTrackUnpublish);
+    track.on(TrackEvent.Ended, this.handleTrackEnded);
     track.on(TrackEvent.UpstreamPaused, this.onTrackUpstreamPaused);
     track.on(TrackEvent.UpstreamResumed, this.onTrackUpstreamResumed);
 
@@ -401,6 +414,8 @@ export default class LocalParticipant extends Participant {
 
     // compute encodings and layers for video
     let encodings: RTCRtpEncodingParameters[] | undefined;
+    let simEncodings: RTCRtpEncodingParameters[] | undefined;
+    let simulcastTracks: SimulcastTrackInfo[] | undefined;
     if (track.kind === Track.Kind.Video) {
       // TODO: support react native, which doesn't expose getSettings
       const settings = track.mediaStreamTrack.getSettings();
@@ -409,16 +424,38 @@ export default class LocalParticipant extends Participant {
       // width and height should be defined for video
       req.width = width ?? 0;
       req.height = height ?? 0;
-      // for svc codecs, disable simulcast and enable scalability L3T3
-      // by default
-      if (track instanceof LocalVideoTrack) {
-        if (opts?.videoCodec === 'vp9' || opts?.videoCodec === 'av1') {
-          opts.simulcast = false;
-          opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3';
-        } else {
-          // other codecs, unset scalability
-          opts.scalabilityMode = undefined;
-        }
+      // for svc codecs, disable simulcast and use vp8 for backup codec
+      if (
+        track instanceof LocalVideoTrack &&
+        (opts?.videoCodec === 'vp9' || opts?.videoCodec === 'av1')
+      ) {
+        // set scalabilityMode to 'L3T3' by default
+        opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3';
+
+        // add backup codec track
+        const simOpts = { ...opts };
+        simOpts.simulcast = true;
+        simOpts.scalabilityMode = undefined;
+        simEncodings = computeVideoEncodings(
+          track.source === Track.Source.ScreenShare,
+          width,
+          height,
+          simOpts,
+        );
+        const simulcastTrack = track.addSimulcastTrack(compatibleCodecForSVC, simEncodings);
+        simulcastTracks = [simulcastTrack];
+        req.simulcastCodecs = [
+          {
+            codec: opts.videoCodec,
+            cid: track.mediaStreamTrack.id,
+            enableSimulcastLayers: true,
+          },
+          {
+            codec: simulcastTrack.codec,
+            cid: simulcastTrack.mediaStreamTrack.id,
+            enableSimulcastLayers: true,
+          },
+        ];
       }
 
       encodings = computeVideoEncodings(
@@ -427,7 +464,7 @@ export default class LocalParticipant extends Participant {
         height,
         opts,
       );
-      req.layers = videoLayersFromEncodings(req.width, req.height, encodings);
+      req.layers = videoLayersFromEncodings(req.width, req.height, simEncodings ?? encodings);
     } else if (track.kind === Track.Kind.Audio && opts.audioBitrate) {
       encodings = [
         {
@@ -442,6 +479,8 @@ export default class LocalParticipant extends Participant {
 
     const ti = await this.engine.addTrack(req);
     const publication = new LocalTrackPublication(track.kind, ti, track);
+    // save options for when it needs to be republished again
+    publication.options = opts;
     track.sid = ti.sid;
 
     if (!this.engine.publisher) {
@@ -452,13 +491,32 @@ export default class LocalParticipant extends Participant {
     if (encodings) {
       transceiverInit.sendEncodings = encodings;
     }
-    const transceiver = this.engine.publisher.pc.addTransceiver(
+    // addTransceiver for react-native is async. web is synchronous, but await won't effect it.
+    const transceiver = await this.engine.publisher.pc.addTransceiver(
       track.mediaStreamTrack,
       transceiverInit,
     );
-    if (opts.videoCodec) {
+    if (track.kind === Track.Kind.Video && opts.videoCodec) {
       this.setPreferredCodec(transceiver, track.kind, opts.videoCodec);
+      track.codec = opts.videoCodec;
     }
+
+    const localTrack = track as LocalVideoTrack;
+    if (simulcastTracks) {
+      for await (const simulcastTrack of simulcastTracks) {
+        const simTransceiverInit: RTCRtpTransceiverInit = { direction: 'sendonly' };
+        if (simulcastTrack.encodings) {
+          simTransceiverInit.sendEncodings = simulcastTrack.encodings;
+        }
+        const simTransceiver = await this.engine.publisher!.pc.addTransceiver(
+          simulcastTrack.mediaStreamTrack,
+          simTransceiverInit,
+        );
+        this.setPreferredCodec(simTransceiver, localTrack.kind, simulcastTrack.codec);
+        localTrack.setSimulcastTrackSender(simulcastTrack.codec, simTransceiver.sender);
+      }
+    }
+
     this.engine.negotiate();
 
     // store RTPSender
@@ -468,6 +526,7 @@ export default class LocalParticipant extends Participant {
     } else if (track instanceof LocalAudioTrack) {
       track.startMonitor();
     }
+
     this.addTrackPublication(publication);
 
     // send event for publication
@@ -494,9 +553,10 @@ export default class LocalParticipant extends Participant {
 
     track = publication.track;
 
+    track.sender = undefined;
     track.off(TrackEvent.Muted, this.onTrackMuted);
     track.off(TrackEvent.Unmuted, this.onTrackUnmuted);
-    track.off(TrackEvent.Ended, this.onTrackUnpublish);
+    track.off(TrackEvent.Ended, this.handleTrackEnded);
     track.off(TrackEvent.UpstreamPaused, this.onTrackUpstreamPaused);
     track.off(TrackEvent.UpstreamResumed, this.onTrackUpstreamResumed);
 
@@ -509,7 +569,7 @@ export default class LocalParticipant extends Participant {
 
     const { mediaStreamTrack } = track;
 
-    if (this.engine.publisher) {
+    if (this.engine.publisher && this.engine.publisher.pc.connectionState !== 'closed') {
       const senders = this.engine.publisher.pc.getSenders();
       senders.forEach((sender) => {
         if (sender.track === mediaStreamTrack) {
@@ -613,11 +673,23 @@ export default class LocalParticipant extends Participant {
     allParticipantsAllowed: boolean,
     participantTrackPermissions: ParticipantTrackPermission[] = [],
   ) {
+    this.participantTrackPermissions = participantTrackPermissions;
+    this.allParticipantsAllowedToSubscribe = allParticipantsAllowed;
+    if (this.engine.client.isConnected) {
+      this.updateTrackSubscriptionPermissions();
+    }
+  }
+
+  private updateTrackSubscriptionPermissions = () => {
+    log.debug('updating track subscription permissions', {
+      allParticipantsAllowed: this.allParticipantsAllowedToSubscribe,
+      participantTrackPermissions: this.participantTrackPermissions,
+    });
     this.engine.client.sendUpdateSubscriptionPermissions(
-      allParticipantsAllowed,
-      participantTrackPermissions.map((p) => trackPermissionToProto(p)),
+      this.allParticipantsAllowedToSubscribe,
+      this.participantTrackPermissions.map((p) => trackPermissionToProto(p)),
     );
-  }
+  };
 
   /** @internal */
   private onTrackUnmuted = (track: LocalTrack) => {
@@ -661,7 +733,11 @@ export default class LocalParticipant extends Participant {
       });
       return;
     }
-    pub.videoTrack?.setPublishingLayers(update.subscribedQualities);
+    if (update.subscribedCodecs.length > 0) {
+      pub.videoTrack?.setPublishingCodecs(update.subscribedCodecs);
+    } else if (update.subscribedQualities.length > 0) {
+      pub.videoTrack?.setPublishingLayers(update.subscribedQualities);
+    }
   };
 
   private handleLocalTrackUnpublished = (unpublished: TrackUnpublishedResponse) => {
@@ -676,7 +752,10 @@ export default class LocalParticipant extends Participant {
     this.unpublishTrack(track.track!);
   };
 
-  private onTrackUnpublish = (track: LocalTrack) => {
+  private handleTrackEnded = (track: LocalTrack) => {
+    log.debug('unpublishing local track due to TrackEnded', {
+      track: track.sid,
+    });
     this.unpublishTrack(track);
   };
 
@@ -714,6 +793,7 @@ export default class LocalParticipant extends Participant {
     }
     const cap = RTCRtpSender.getCapabilities(kind);
     if (!cap) return;
+    log.debug('get capabilities', cap);
     let selected: RTCRtpCodecCapability | undefined;
     const codecs: RTCRtpCodecCapability[] = [];
     cap.codecs.forEach((c) => {
@@ -738,6 +818,7 @@ export default class LocalParticipant extends Participant {
       }
       codecs.push(c);
     });
+
     if (selected && 'setCodecPreferences' in transceiver) {
       // @ts-ignore
       codecs.unshift(selected);

package/src/room/participant/RemoteParticipant.ts

@@ -42,6 +42,7 @@ export default class RemoteParticipant extends Participant {
 
     // register action events
     publication.on(TrackEvent.UpdateSettings, (settings: UpdateTrackSettings) => {
+      log.debug('send update settings', settings);
       this.signalClient.sendUpdateTrackSettings(settings);
     });
     publication.on(TrackEvent.UpdateSubscription, (sub: UpdateSubscription) => {

package/src/room/stats.ts

@@ -104,6 +104,8 @@ export interface VideoReceiverStats extends ReceiverStats {
   pliCount?: number;
 
   nackCount?: number;
+
+  decoderImplementation?: string;
 }
 
 export function computeBitrate<T extends ReceiverStats | SenderStats>(

package/src/room/track/LocalAudioTrack.ts

@@ -1,6 +1,7 @@
 import log from '../../logger';
 import { TrackEvent } from '../events';
 import { AudioSenderStats, computeBitrate, monitorFrequency } from '../stats';
+import { isWeb } from '../utils';
 import LocalTrack from './LocalTrack';
 import { AudioCaptureOptions } from './options';
 import { Track } from './Track';
@@ -68,6 +69,9 @@ export default class LocalAudioTrack extends LocalTrack {
 
   /* @internal */
   startMonitor() {
+    if (!isWeb()) {
+      return;
+    }
     setTimeout(() => {
       this.monitorSender();
     }, monitorFrequency);

package/src/room/track/LocalTrack.ts

@@ -2,6 +2,7 @@ import log from '../../logger';
 import DeviceManager from '../DeviceManager';
 import { TrackInvalidError } from '../errors';
 import { TrackEvent } from '../events';
+import { VideoCodec } from './options';
 import { getEmptyAudioStreamTrack, getEmptyVideoStreamTrack, isMobile } from '../utils';
 import { attachToElement, detachTrack, Track } from './Track';
 
@@ -9,6 +10,9 @@ export default class LocalTrack extends Track {
   /** @internal */
   sender?: RTCRtpSender;
 
+  /** @internal */
+  codec?: VideoCodec;
+
   protected constraints: MediaTrackConstraints;
 
   protected wasMuted: boolean;
@@ -94,7 +98,9 @@ export default class LocalTrack extends Track {
     track.addEventListener('ended', this.handleEnded);
     log.debug('replace MediaStreamTrack');
 
-    await this.sender.replaceTrack(track);
+    if (this.sender) {
+      await this.sender.replaceTrack(track);
+    }
     this._mediaStreamTrack = track;
 
     this.attachedElements.forEach((el) => {
@@ -106,9 +112,6 @@ export default class LocalTrack extends Track {
   }
 
   protected async restart(constraints?: MediaTrackConstraints): Promise<LocalTrack> {
-    if (!this.sender) {
-      throw new TrackInvalidError('unable to restart an unpublished track');
-    }
     if (!constraints) {
       constraints = this.constraints;
     }
@@ -141,7 +144,11 @@ export default class LocalTrack extends Track {
     newTrack.addEventListener('ended', this.handleEnded);
     log.debug('re-acquired MediaStreamTrack');
 
-    await this.sender.replaceTrack(newTrack);
+    if (this.sender) {
+      // Track can be restarted after it's unpublished
+      await this.sender.replaceTrack(newTrack);
+    }
+
     this._mediaStreamTrack = newTrack;
 
     this.attachedElements.forEach((el) => {