livekit-client 1.2.0 → 1.2.3

package/src/room/RTCEngine.ts

@@ -2,6 +2,7 @@ import { EventEmitter } from 'events';
 import type TypedEventEmitter from 'typed-emitter';
 import { SignalClient, SignalOptions } from '../api/SignalClient';
 import log from '../logger';
+import { RoomOptions } from '../options';
 import {
   ClientConfigSetting,
   ClientConfiguration,
@@ -19,16 +20,16 @@ import {
   SignalTarget,
   TrackPublishedResponse,
 } from '../proto/livekit_rtc';
+import DefaultReconnectPolicy from './DefaultReconnectPolicy';
 import { ConnectionError, TrackInvalidError, UnexpectedConnectionState } from './errors';
 import { EngineEvent } from './events';
 import PCTransport from './PCTransport';
+import { ReconnectContext, ReconnectPolicy } from './ReconnectPolicy';
 import { isFireFox, isWeb, sleep } from './utils';
 
 const lossyDataChannel = '_lossy';
 const reliableDataChannel = '_reliable';
-const maxReconnectRetries = 10;
 const minReconnectWait = 2 * 1000;
-const maxReconnectDuration = 60 * 1000;
 export const maxICEConnectTimeout = 15 * 1000;
 
 enum PCState {
@@ -96,9 +97,15 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
 
   private attemptingReconnect: boolean = false;
 
-  constructor() {
+  private reconnectPolicy: ReconnectPolicy;
+
+  private reconnectTimeout?: ReturnType<typeof setTimeout>;
+
+  constructor(private options: RoomOptions) {
     super();
     this.client = new SignalClient();
+    this.client.signalLatency = this.options.expSignalLatency;
+    this.reconnectPolicy = this.options.reconnectPolicy ?? new DefaultReconnectPolicy();
   }
 
   async join(
@@ -157,8 +164,16 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
     if (this.pendingTrackResolvers[req.cid]) {
       throw new TrackInvalidError('a track with the same ID has already been published');
     }
-    return new Promise<TrackInfo>((resolve) => {
-      this.pendingTrackResolvers[req.cid] = resolve;
+    return new Promise<TrackInfo>((resolve, reject) => {
+      const publicationTimeout = setTimeout(() => {
+        reject(
+          new ConnectionError('publication of local track timed out, no response from server'),
+        );
+      }, 15_000);
+      this.pendingTrackResolvers[req.cid] = (info: TrackInfo) => {
+        clearTimeout(publicationTimeout);
+        resolve(info);
+      };
       this.client.sendAddTrack(req);
     });
   }
@@ -317,8 +332,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
       await this.subscriber.setRemoteDescription(sd);
 
       // answer the offer
-      const answer = await this.subscriber.pc.createAnswer();
-      await this.subscriber.pc.setLocalDescription(answer);
+      const answer = await this.subscriber.createAndSetAnswer();
       this.client.sendAnswer(answer);
     };
 
@@ -437,18 +451,42 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
   // websocket reconnect behavior. if websocket is interrupted, and the PeerConnection
   // continues to work, we can reconnect to websocket to continue the session
   // after a number of retries, we'll close and give up permanently
-  private handleDisconnect = (connection: string) => {
+  private handleDisconnect = (connection: string, signalEvents: boolean = false) => {
     if (this._isClosed) {
       return;
     }
+
     log.debug(`${connection} disconnected`);
     if (this.reconnectAttempts === 0) {
       // only reset start time on the first try
       this.reconnectStart = Date.now();
     }
 
-    const delay = this.reconnectAttempts * this.reconnectAttempts * 300;
-    setTimeout(async () => {
+    const disconnect = (duration: number) => {
+      log.info(
+        `could not recover connection after ${this.reconnectAttempts} attempts, ${duration}ms. giving up`,
+      );
+      this.emit(EngineEvent.Disconnected);
+      this.close();
+    };
+
+    const duration = Date.now() - this.reconnectStart;
+    const delay = this.getNextRetryDelay({
+      elapsedMs: duration,
+      retryCount: this.reconnectAttempts,
+    });
+
+    if (delay === null) {
+      disconnect(duration);
+      return;
+    }
+
+    log.debug(`reconnecting in ${delay}ms`);
+
+    if (this.reconnectTimeout) {
+      clearTimeout(this.reconnectTimeout);
+    }
+    this.reconnectTimeout = setTimeout(async () => {
       if (this._isClosed) {
         return;
       }
@@ -469,16 +507,20 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
       try {
         this.attemptingReconnect = true;
         if (this.fullReconnectOnNext) {
-          await this.restartConnection();
+          await this.restartConnection(signalEvents);
         } else {
-          await this.resumeConnection();
+          await this.resumeConnection(signalEvents);
         }
         this.reconnectAttempts = 0;
         this.fullReconnectOnNext = false;
+        if (this.reconnectTimeout) {
+          clearTimeout(this.reconnectTimeout);
+        }
       } catch (e) {
         this.reconnectAttempts += 1;
         let reconnectRequired = false;
         let recoverable = true;
+        let requireSignalEvents = false;
         if (e instanceof UnexpectedConnectionState) {
           log.debug('received unrecoverable error', { error: e });
           // unrecoverable
@@ -488,26 +530,17 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
           reconnectRequired = true;
         }
 
-        // when we flip from resume to reconnect, we need to reset reconnectAttempts
-        // this is needed to fire the right reconnecting events
+        // when we flip from resume to reconnect
+        // we need to fire the right reconnecting events
         if (reconnectRequired && !this.fullReconnectOnNext) {
           this.fullReconnectOnNext = true;
-          this.reconnectAttempts = 0;
-        }
-
-        const duration = Date.now() - this.reconnectStart;
-        if (this.reconnectAttempts >= maxReconnectRetries || duration > maxReconnectDuration) {
-          recoverable = false;
+          requireSignalEvents = true;
         }
 
         if (recoverable) {
-          this.handleDisconnect('reconnect');
+          this.handleDisconnect('reconnect', requireSignalEvents);
         } else {
-          log.info(
-            `could not recover connection after ${maxReconnectRetries} attempts, ${duration}ms. giving up`,
-          );
-          this.emit(EngineEvent.Disconnected);
-          this.close();
+          disconnect(Date.now() - this.reconnectStart);
         }
       } finally {
         this.attemptingReconnect = false;
@@ -515,14 +548,25 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
     }, delay);
   };
 
-  private async restartConnection() {
+  private getNextRetryDelay(context: ReconnectContext) {
+    try {
+      return this.reconnectPolicy.nextRetryDelayInMs(context);
+    } catch (e) {
+      log.warn('encountered error in reconnect policy', { error: e });
+    }
+
+    // error in user code with provided reconnect policy, stop reconnecting
+    return null;
+  }
+
+  private async restartConnection(emitRestarting: boolean = false) {
     if (!this.url || !this.token) {
       // permanent failure, don't attempt reconnection
       throw new UnexpectedConnectionState('could not reconnect, url or token not saved');
     }
 
     log.info(`reconnecting, attempt: ${this.reconnectAttempts}`);
-    if (this.reconnectAttempts === 0) {
+    if (emitRestarting || this.reconnectAttempts === 0) {
       this.emit(EngineEvent.Restarting);
     }
 
@@ -550,7 +594,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
     this.emit(EngineEvent.Restarted, joinResponse);
   }
 
-  private async resumeConnection(): Promise<void> {
+  private async resumeConnection(emitResuming: boolean = false): Promise<void> {
     if (!this.url || !this.token) {
       // permanent failure, don't attempt reconnection
       throw new UnexpectedConnectionState('could not reconnect, url or token not saved');
@@ -561,14 +605,18 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
     }
 
     log.info(`resuming signal connection, attempt ${this.reconnectAttempts}`);
-    if (this.reconnectAttempts === 0) {
+    if (emitResuming || this.reconnectAttempts === 0) {
       this.emit(EngineEvent.Resuming);
     }
 
     try {
       await this.client.reconnect(this.url, this.token);
     } catch (e) {
-      throw new SignalReconnectError();
+      let message = '';
+      if (e instanceof Error) {
+        message = e.message;
+      }
+      throw new SignalReconnectError(message);
     }
     this.emit(EngineEvent.SignalResumed);
 

package/src/room/ReconnectPolicy.ts

@@ -0,0 +1,25 @@
+/** Controls reconnecting of the client */
+export interface ReconnectPolicy {
+  /** Called after disconnect was detected
+   *
+   * @returns {number | null} Amount of time in milliseconds to delay the next reconnect attempt, `null` signals to stop retrying.
+   */
+  nextRetryDelayInMs(context: ReconnectContext): number | null;
+}
+
+export interface ReconnectContext {
+  /**
+   * Number of failed reconnect attempts
+   */
+  readonly retryCount: number;
+
+  /**
+   * Elapsed amount of time in milliseconds since the disconnect.
+   */
+  readonly elapsedMs: number;
+
+  /**
+   * Reason for retrying
+   */
+  readonly retryReason?: Error;
+}

package/src/room/Room.ts

@@ -134,9 +134,8 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       return;
     }
 
-    this.engine = new RTCEngine();
+    this.engine = new RTCEngine(this.options);
 
-    this.engine.client.signalLatency = this.options.expSignalLatency;
     this.engine.client.onParticipantUpdate = this.handleParticipantUpdates;
     this.engine.client.onRoomUpdate = this.handleRoomUpdate;
     this.engine.client.onSpeakersChanged = this.handleSpeakersChanged;
@@ -255,36 +254,16 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       this.localParticipant.updateInfo(pi);
       // forward metadata changed for the local participant
       this.localParticipant
-        .on(ParticipantEvent.ParticipantMetadataChanged, (metadata: string | undefined) => {
-          this.emit(RoomEvent.ParticipantMetadataChanged, metadata, this.localParticipant);
-        })
-        .on(ParticipantEvent.TrackMuted, (pub: TrackPublication) => {
-          this.emit(RoomEvent.TrackMuted, pub, this.localParticipant);
-        })
-        .on(ParticipantEvent.TrackUnmuted, (pub: TrackPublication) => {
-          this.emit(RoomEvent.TrackUnmuted, pub, this.localParticipant);
-        })
-        .on(ParticipantEvent.LocalTrackPublished, (pub: LocalTrackPublication) => {
-          this.emit(RoomEvent.LocalTrackPublished, pub, this.localParticipant);
-        })
-        .on(ParticipantEvent.LocalTrackUnpublished, (pub: LocalTrackPublication) => {
-          this.emit(RoomEvent.LocalTrackUnpublished, pub, this.localParticipant);
-        })
-        .on(ParticipantEvent.ConnectionQualityChanged, (quality: ConnectionQuality) => {
-          this.emit(RoomEvent.ConnectionQualityChanged, quality, this.localParticipant);
-        })
-        .on(ParticipantEvent.MediaDevicesError, (e: Error) => {
-          this.emit(RoomEvent.MediaDevicesError, e);
-        })
+        .on(ParticipantEvent.ParticipantMetadataChanged, this.onLocalParticipantMetadataChanged)
+        .on(ParticipantEvent.TrackMuted, this.onLocalTrackMuted)
+        .on(ParticipantEvent.TrackUnmuted, this.onLocalTrackUnmuted)
+        .on(ParticipantEvent.LocalTrackPublished, this.onLocalTrackPublished)
+        .on(ParticipantEvent.LocalTrackUnpublished, this.onLocalTrackUnpublished)
+        .on(ParticipantEvent.ConnectionQualityChanged, this.onLocalConnectionQualityChanged)
+        .on(ParticipantEvent.MediaDevicesError, this.onMediaDevicesError)
         .on(
           ParticipantEvent.ParticipantPermissionsChanged,
-          (prevPermissions: ParticipantPermission) => {
-            this.emit(
-              RoomEvent.ParticipantPermissionsChanged,
-              prevPermissions,
-              this.localParticipant,
-            );
-          },
+          this.onLocalParticipantPermissionsChanged,
         );
 
       // populate remote participants, these should not trigger new events
@@ -657,6 +636,19 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       });
     });
 
+    this.localParticipant
+      .off(ParticipantEvent.ParticipantMetadataChanged, this.onLocalParticipantMetadataChanged)
+      .off(ParticipantEvent.TrackMuted, this.onLocalTrackMuted)
+      .off(ParticipantEvent.TrackUnmuted, this.onLocalTrackUnmuted)
+      .off(ParticipantEvent.LocalTrackPublished, this.onLocalTrackPublished)
+      .off(ParticipantEvent.LocalTrackUnpublished, this.onLocalTrackUnpublished)
+      .off(ParticipantEvent.ConnectionQualityChanged, this.onLocalConnectionQualityChanged)
+      .off(ParticipantEvent.MediaDevicesError, this.onMediaDevicesError)
+      .off(
+        ParticipantEvent.ParticipantPermissionsChanged,
+        this.onLocalParticipantPermissionsChanged,
+      );
+
     this.localParticipant.tracks.forEach((pub) => {
       if (pub.track) {
         this.localParticipant.unpublishTrack(pub.track, shouldStopTracks);
@@ -666,6 +658,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
         pub.track?.stop();
       }
     });
+
     this.localParticipant.tracks.clear();
     this.localParticipant.videoTracks.clear();
     this.localParticipant.audioTracks.clear();
@@ -1081,6 +1074,38 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     return false;
   }
 
+  private onLocalParticipantMetadataChanged = (metadata: string | undefined) => {
+    this.emit(RoomEvent.ParticipantMetadataChanged, metadata, this.localParticipant);
+  };
+
+  private onLocalTrackMuted = (pub: TrackPublication) => {
+    this.emit(RoomEvent.TrackMuted, pub, this.localParticipant);
+  };
+
+  private onLocalTrackUnmuted = (pub: TrackPublication) => {
+    this.emit(RoomEvent.TrackUnmuted, pub, this.localParticipant);
+  };
+
+  private onLocalTrackPublished = (pub: LocalTrackPublication) => {
+    this.emit(RoomEvent.LocalTrackPublished, pub, this.localParticipant);
+  };
+
+  private onLocalTrackUnpublished = (pub: LocalTrackPublication) => {
+    this.emit(RoomEvent.LocalTrackUnpublished, pub, this.localParticipant);
+  };
+
+  private onLocalConnectionQualityChanged = (quality: ConnectionQuality) => {
+    this.emit(RoomEvent.ConnectionQualityChanged, quality, this.localParticipant);
+  };
+
+  private onMediaDevicesError = (e: Error) => {
+    this.emit(RoomEvent.MediaDevicesError, e);
+  };
+
+  private onLocalParticipantPermissionsChanged = (prevPermissions: ParticipantPermission) => {
+    this.emit(RoomEvent.ParticipantPermissionsChanged, prevPermissions, this.localParticipant);
+  };
+
   // /** @internal */
   emit<E extends keyof RoomEventCallbacks>(
     event: E,

package/src/room/events.ts

@@ -218,8 +218,8 @@ export enum RoomEvent {
    * When we have encountered an error while attempting to create a track.
    * The errors take place in getUserMedia().
    * Use MediaDeviceFailure.getFailure(error) to get the reason of failure.
-   * [[getAudioCreateError]] and [[getVideoCreateError]] will indicate if it had
-   * an error while creating the audio or video track respectively.
+   * [[LocalParticipant.lastCameraError]] and [[LocalParticipant.lastMicrophoneError]]
+   * will indicate if it had an error while creating the audio or video track respectively.
    *
    * args: (error: Error)
    */

package/src/room/participant/LocalParticipant.ts

@@ -125,8 +125,9 @@ export default class LocalParticipant extends Participant {
   setCameraEnabled(
     enabled: boolean,
     options?: VideoCaptureOptions,
+    publishOptions?: TrackPublishOptions,
   ): Promise<LocalTrackPublication | undefined> {
-    return this.setTrackEnabled(Track.Source.Camera, enabled, options);
+    return this.setTrackEnabled(Track.Source.Camera, enabled, options, publishOptions);
   }
 
   /**
@@ -138,8 +139,9 @@ export default class LocalParticipant extends Participant {
   setMicrophoneEnabled(
     enabled: boolean,
     options?: AudioCaptureOptions,
+    publishOptions?: TrackPublishOptions,
   ): Promise<LocalTrackPublication | undefined> {
-    return this.setTrackEnabled(Track.Source.Microphone, enabled, options);
+    return this.setTrackEnabled(Track.Source.Microphone, enabled, options, publishOptions);
   }
 
   /**
@@ -149,8 +151,9 @@ export default class LocalParticipant extends Participant {
   setScreenShareEnabled(
     enabled: boolean,
     options?: ScreenShareCaptureOptions,
+    publishOptions?: TrackPublishOptions,
   ): Promise<LocalTrackPublication | undefined> {
-    return this.setTrackEnabled(Track.Source.ScreenShare, enabled, options);
+    return this.setTrackEnabled(Track.Source.ScreenShare, enabled, options, publishOptions);
   }
 
   /** @internal */
@@ -172,21 +175,25 @@ export default class LocalParticipant extends Participant {
     source: Extract<Track.Source, Track.Source.Camera>,
     enabled: boolean,
     options?: VideoCaptureOptions,
+    publishOptions?: TrackPublishOptions,
   ): Promise<LocalTrackPublication | undefined>;
   private async setTrackEnabled(
     source: Extract<Track.Source, Track.Source.Microphone>,
     enabled: boolean,
     options?: AudioCaptureOptions,
+    publishOptions?: TrackPublishOptions,
   ): Promise<LocalTrackPublication | undefined>;
   private async setTrackEnabled(
     source: Extract<Track.Source, Track.Source.ScreenShare>,
     enabled: boolean,
     options?: ScreenShareCaptureOptions,
+    publishOptions?: TrackPublishOptions,
   ): Promise<LocalTrackPublication | undefined>;
   private async setTrackEnabled(
     source: Track.Source,
     enabled: true,
     options?: VideoCaptureOptions | AudioCaptureOptions | ScreenShareCaptureOptions,
+    publishOptions?: TrackPublishOptions,
   ) {
     log.debug('setTrackEnabled', { source, enabled });
     let track = this.getTrack(source);
@@ -224,7 +231,7 @@ export default class LocalParticipant extends Participant {
           }
           const publishPromises: Array<Promise<LocalTrackPublication>> = [];
           for (const localTrack of localTracks) {
-            publishPromises.push(this.publishTrack(localTrack));
+            publishPromises.push(this.publishTrack(localTrack, publishOptions));
           }
           const publishedTracks = await Promise.all(publishPromises);
           // for screen share publications including audio, this will only return the screen share publication, not the screen share audio one
@@ -427,6 +434,23 @@ export default class LocalParticipant extends Participant {
     if (opts.source) {
       track.source = opts.source;
     }
+    const existingTrackOfSource = Array.from(this.tracks.values()).find(
+      (publishedTrack) => track instanceof LocalTrack && publishedTrack.source === track.source,
+    );
+    if (existingTrackOfSource) {
+      try {
+        // throw an Error in order to capture the stack trace
+        throw Error(`publishing a second track with the same source: ${track.source}`);
+      } catch (e: unknown) {
+        if (e instanceof Error) {
+          log.warn(e.message, {
+            oldTrack: existingTrackOfSource,
+            newTrack: track,
+            trace: e.stack,
+          });
+        }
+      }
+    }
     if (opts.stopMicTrackOnMute && track instanceof LocalAudioTrack) {
       track.stopOnMute = true;
     }
@@ -544,6 +568,14 @@ export default class LocalParticipant extends Participant {
       track.codec = opts.videoCodec;
     }
 
+    if (track.codec === 'av1' && encodings && encodings[0]?.maxBitrate) {
+      this.engine.publisher.setTrackCodecBitrate(
+        req.cid,
+        track.codec,
+        encodings[0].maxBitrate / 1000,
+      );
+    }
+
     this.engine.negotiate();
 
     // store RTPSender
@@ -642,6 +674,13 @@ export default class LocalParticipant extends Participant {
     this.setPreferredCodec(transceiver, track.kind, opts.videoCodec);
     track.setSimulcastTrackSender(opts.videoCodec, transceiver.sender);
 
+    if (videoCodec === 'av1' && encodings[0]?.maxBitrate) {
+      this.engine.publisher.setTrackCodecBitrate(
+        req.cid,
+        videoCodec,
+        encodings[0].maxBitrate / 1000,
+      );
+    }
     this.engine.negotiate();
     log.debug(`published ${opts.videoCodec} for track ${track.sid}`, { encodings, trackInfo: ti });
   }
@@ -664,8 +703,6 @@ export default class LocalParticipant extends Participant {
     }
 
     track = publication.track;
-
-    track.sender = undefined;
     track.off(TrackEvent.Muted, this.onTrackMuted);
     track.off(TrackEvent.Unmuted, this.onTrackUnmuted);
     track.off(TrackEvent.Ended, this.handleTrackEnded);
@@ -679,22 +716,21 @@ export default class LocalParticipant extends Participant {
       track.stop();
     }
 
-    const { mediaStreamTrack } = track;
-
-    if (this.engine.publisher && this.engine.publisher.pc.connectionState !== 'closed') {
-      const senders = this.engine.publisher.pc.getSenders();
-      senders.forEach((sender) => {
-        if (sender.track === mediaStreamTrack) {
-          try {
-            this.engine.publisher?.pc.removeTrack(sender);
-            this.engine.negotiate();
-          } catch (e) {
-            log.warn('failed to remove track', { error: e, method: 'unpublishTrack' });
-          }
-        }
-      });
+    if (
+      this.engine.publisher &&
+      this.engine.publisher.pc.connectionState !== 'closed' &&
+      track.sender
+    ) {
+      try {
+        this.engine.publisher.pc.removeTrack(track.sender);
+        this.engine.negotiate();
+      } catch (e) {
+        log.warn('failed to remove track', { error: e, method: 'unpublishTrack' });
+      }
     }
 
+    track.sender = undefined;
+
     // remove from our maps
     this.tracks.delete(publication.trackSid);
     switch (publication.kind) {

package/src/room/participant/RemoteParticipant.ts

@@ -220,6 +220,19 @@ export default class RemoteParticipant extends Participant {
     // always emit events for new publications, Room will not forward them unless it's ready
     newTracks.forEach((publication) => {
       this.emit(ParticipantEvent.TrackPublished, publication);
+      const existingTrackOfSource = Array.from(this.tracks.values()).find(
+        (publishedTrack) => publishedTrack.source === publication.source,
+      );
+      if (existingTrackOfSource) {
+        log.warn(
+          `received a second track publication for ${this.identity} with the same source: ${publication.source}`,
+          {
+            oldTrack: existingTrackOfSource,
+            newTrack: publication,
+            participant: this,
+          },
+        );
+      }
     });
 
     // detect removed tracks

package/src/room/participant/publishUtils.ts

@@ -106,7 +106,7 @@ export function computeVideoEncodings(
           encodings.push({
             rid: videoRids[2 - i],
             scaleResolutionDownBy: 2 ** i,
-            maxBitrate: videoEncoding ? videoEncoding.maxBitrate / 2 ** i : 0,
+            maxBitrate: videoEncoding ? videoEncoding.maxBitrate / 3 ** i : 0,
             /* @ts-ignore */
             maxFramerate: original.encoding.maxFramerate,
             /* @ts-ignore */

package/src/room/track/LocalVideoTrack.ts

@@ -75,7 +75,6 @@ export default class LocalVideoTrack extends LocalTrack {
   }
 
   stop() {
-    this.sender = undefined;
     this._mediaStreamTrack.getConstraints();
     this.simulcastCodecs.forEach((trackInfo) => {
       trackInfo.mediaStreamTrack.stop();