livekit-client 1.2.2 → 1.2.5

package/src/room/DeviceManager.ts

@@ -1,3 +1,6 @@
+import log from '../logger';
+import { isSafari } from './utils';
+
 const defaultId = 'default';
 
 export default class DeviceManager {
@@ -12,13 +15,32 @@ export default class DeviceManager {
     return this.instance;
   }
 
+  static userMediaPromiseMap: Map<MediaDeviceKind, Promise<MediaStream>> = new Map();
+
   async getDevices(
     kind?: MediaDeviceKind,
     requestPermissions: boolean = true,
   ): Promise<MediaDeviceInfo[]> {
+    if (DeviceManager.userMediaPromiseMap?.size > 0) {
+      log.debug('awaiting getUserMedia promise');
+      try {
+        if (kind) {
+          await DeviceManager.userMediaPromiseMap.get(kind);
+        } else {
+          await Promise.all(DeviceManager.userMediaPromiseMap.values());
+        }
+      } catch (e: any) {
+        log.warn('error waiting for media permissons');
+      }
+    }
     let devices = await navigator.mediaDevices.enumerateDevices();
 
-    if (requestPermissions) {
+    if (
+      requestPermissions &&
+      kind &&
+      // for safari we need to skip this check, as otherwise it will re-acquire user media and fail on iOS https://bugs.webkit.org/show_bug.cgi?id=179363
+      (!DeviceManager.userMediaPromiseMap.get(kind) || !isSafari())
+    ) {
       const isDummyDeviceOrEmpty =
         devices.length === 0 ||
         devices.some((device) => {

package/src/room/PCTransport.ts

@@ -1,4 +1,5 @@
 import { debounce } from 'ts-debounce';
+import { MediaDescription, parse, write } from 'sdp-transform';
 import log from '../logger';
 
 /** @internal */
@@ -88,31 +89,71 @@ export default class PCTransport {
     log.debug('starting to negotiate');
     const offer = await this.pc.createOffer(options);
 
-    // mung sdp for codec bitrate setting that can't apply by sendEncoding
-    this.trackBitrates.forEach((trackbr) => {
-      let sdp = offer.sdp ?? '';
-      const sidIndex = sdp.search(new RegExp(`msid.* ${trackbr.sid}`));
-      if (sidIndex < 0) {
-        return;
+    const sdpParsed = parse(offer.sdp ?? '');
+    sdpParsed.media.forEach((media) => {
+      if (media.type === 'audio') {
+        ensureAudioNack(media);
+      } else if (media.type === 'video') {
+        // mung sdp for codec bitrate setting that can't apply by sendEncoding
+        this.trackBitrates.some((trackbr): boolean => {
+          if (!media.msid || !media.msid.includes(trackbr.sid)) {
+            return false;
+          }
+
+          let codecPayload = 0;
+          media.rtp.some((rtp): boolean => {
+            if (rtp.codec.toUpperCase() === trackbr.codec.toUpperCase()) {
+              codecPayload = rtp.payload;
+              return true;
+            }
+            return false;
+          });
+
+          // add x-google-max-bitrate to fmtp line if not exist
+          if (codecPayload > 0) {
+            if (
+              !media.fmtp.some((fmtp): boolean => {
+                if (fmtp.payload === codecPayload) {
+                  if (!fmtp.config.includes('x-google-max-bitrate')) {
+                    fmtp.config += `;x-google-max-bitrate=${trackbr.maxbr}`;
+                  }
+                  return true;
+                }
+                return false;
+              })
+            ) {
+              media.fmtp.push({
+                payload: codecPayload,
+                config: `x-google-max-bitrate=${trackbr.maxbr}`,
+              });
+            }
+          }
+
+          return true;
+        });
       }
-
-      const mlineStart = sdp.substring(0, sidIndex).lastIndexOf('m=');
-      const mlineEnd = sdp.indexOf('m=', sidIndex);
-      const mediaSection = sdp.substring(mlineStart, mlineEnd);
-
-      const mungedMediaSection = mediaSection.replace(
-        new RegExp(`a=rtpmap:(\\d+) ${trackbr.codec}/\\d+`, 'i'),
-        '$'.concat(`&\r\na=fmtp:$1 x-google-max-bitrate=${trackbr.maxbr}`), // Unity replaces '$&' by some random values when building ( Using concat as a workaround )
-      );
-      sdp = sdp.substring(0, mlineStart) + mungedMediaSection + sdp.substring(mlineEnd);
-      offer.sdp = sdp;
     });
+
+    offer.sdp = write(sdpParsed);
     this.trackBitrates = [];
 
     await this.pc.setLocalDescription(offer);
     this.onOffer(offer);
   }
 
+  async createAndSetAnswer(): Promise<RTCSessionDescriptionInit> {
+    const answer = await this.pc.createAnswer();
+    const sdpParsed = parse(answer.sdp ?? '');
+    sdpParsed.media.forEach((media) => {
+      if (media.type === 'audio') {
+        ensureAudioNack(media);
+      }
+    });
+    answer.sdp = write(sdpParsed);
+    await this.pc.setLocalDescription(answer);
+    return answer;
+  }
+
   setTrackCodecBitrate(sid: string, codec: string, maxbr: number) {
     this.trackBitrates.push({
       sid,
@@ -125,3 +166,36 @@ export default class PCTransport {
     this.pc.close();
   }
 }
+
+function ensureAudioNack(
+  media: {
+    type: string;
+    port: number;
+    protocol: string;
+    payloads?: string | undefined;
+  } & MediaDescription,
+) {
+  // found opus codec to add nack fb
+  let opusPayload = 0;
+  media.rtp.some((rtp): boolean => {
+    if (rtp.codec === 'opus') {
+      opusPayload = rtp.payload;
+      return true;
+    }
+    return false;
+  });
+
+  // add nack rtcpfb if not exist
+  if (opusPayload > 0) {
+    if (!media.rtcpFb) {
+      media.rtcpFb = [];
+    }
+
+    if (!media.rtcpFb.some((fb) => fb.payload === opusPayload && fb.type === 'nack')) {
+      media.rtcpFb.push({
+        payload: opusPayload,
+        type: 'nack',
+      });
+    }
+  }
+}

package/src/room/RTCEngine.ts

@@ -2,6 +2,7 @@ import { EventEmitter } from 'events';
 import type TypedEventEmitter from 'typed-emitter';
 import { SignalClient, SignalOptions } from '../api/SignalClient';
 import log from '../logger';
+import { RoomOptions } from '../options';
 import {
   ClientConfigSetting,
   ClientConfiguration,
@@ -19,16 +20,16 @@ import {
   SignalTarget,
   TrackPublishedResponse,
 } from '../proto/livekit_rtc';
+import DefaultReconnectPolicy from './DefaultReconnectPolicy';
 import { ConnectionError, TrackInvalidError, UnexpectedConnectionState } from './errors';
 import { EngineEvent } from './events';
 import PCTransport from './PCTransport';
+import { ReconnectContext, ReconnectPolicy } from './ReconnectPolicy';
 import { isFireFox, isWeb, sleep } from './utils';
 
 const lossyDataChannel = '_lossy';
 const reliableDataChannel = '_reliable';
-const maxReconnectRetries = 10;
 const minReconnectWait = 2 * 1000;
-const maxReconnectDuration = 60 * 1000;
 export const maxICEConnectTimeout = 15 * 1000;
 
 enum PCState {
@@ -71,7 +72,9 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
 
   private _isClosed: boolean = true;
 
-  private pendingTrackResolvers: { [key: string]: (info: TrackInfo) => void } = {};
+  private pendingTrackResolvers: {
+    [key: string]: { resolve: (info: TrackInfo) => void; reject: () => void };
+  } = {};
 
   // true if publisher connection has already been established.
   // this is helpful to know if we need to restart ICE on the publisher connection
@@ -96,9 +99,15 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
 
   private attemptingReconnect: boolean = false;
 
-  constructor() {
+  private reconnectPolicy: ReconnectPolicy;
+
+  private reconnectTimeout?: ReturnType<typeof setTimeout>;
+
+  constructor(private options: RoomOptions) {
     super();
     this.client = new SignalClient();
+    this.client.signalLatency = this.options.expSignalLatency;
+    this.reconnectPolicy = this.options.reconnectPolicy ?? new DefaultReconnectPolicy();
   }
 
   async join(
@@ -157,12 +166,42 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
     if (this.pendingTrackResolvers[req.cid]) {
       throw new TrackInvalidError('a track with the same ID has already been published');
     }
-    return new Promise<TrackInfo>((resolve) => {
-      this.pendingTrackResolvers[req.cid] = resolve;
+    return new Promise<TrackInfo>((resolve, reject) => {
+      const publicationTimeout = setTimeout(() => {
+        delete this.pendingTrackResolvers[req.cid];
+        reject(
+          new ConnectionError('publication of local track timed out, no response from server'),
+        );
+      }, 10_000);
+      this.pendingTrackResolvers[req.cid] = {
+        resolve: (info: TrackInfo) => {
+          clearTimeout(publicationTimeout);
+          resolve(info);
+        },
+        reject: () => {
+          clearTimeout(publicationTimeout);
+          reject('Cancelled publication by calling unpublish');
+        },
+      };
       this.client.sendAddTrack(req);
     });
   }
 
+  removeTrack(sender: RTCRtpSender) {
+    if (sender.track && this.pendingTrackResolvers[sender.track.id]) {
+      const { reject } = this.pendingTrackResolvers[sender.track.id];
+      if (reject) {
+        reject();
+      }
+      delete this.pendingTrackResolvers[sender.track.id];
+    }
+    try {
+      this.publisher?.pc.removeTrack(sender);
+    } catch (e: unknown) {
+      log.warn('failed to remove track', { error: e, method: 'removeTrack' });
+    }
+  }
+
   updateMuteStatus(trackSid: string, muted: boolean) {
     this.client.sendMuteTrack(trackSid, muted);
   }
@@ -317,14 +356,13 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
       await this.subscriber.setRemoteDescription(sd);
 
       // answer the offer
-      const answer = await this.subscriber.pc.createAnswer();
-      await this.subscriber.pc.setLocalDescription(answer);
+      const answer = await this.subscriber.createAndSetAnswer();
       this.client.sendAnswer(answer);
     };
 
     this.client.onLocalTrackPublished = (res: TrackPublishedResponse) => {
       log.debug('received trackPublishedResponse', res);
-      const resolve = this.pendingTrackResolvers[res.cid];
+      const { resolve } = this.pendingTrackResolvers[res.cid];
       if (!resolve) {
         log.error(`missing track resolver for ${res.cid}`);
         return;
@@ -437,18 +475,42 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
   // websocket reconnect behavior. if websocket is interrupted, and the PeerConnection
   // continues to work, we can reconnect to websocket to continue the session
   // after a number of retries, we'll close and give up permanently
-  private handleDisconnect = (connection: string) => {
+  private handleDisconnect = (connection: string, signalEvents: boolean = false) => {
     if (this._isClosed) {
       return;
     }
+
     log.debug(`${connection} disconnected`);
     if (this.reconnectAttempts === 0) {
       // only reset start time on the first try
       this.reconnectStart = Date.now();
     }
 
-    const delay = this.reconnectAttempts * this.reconnectAttempts * 300;
-    setTimeout(async () => {
+    const disconnect = (duration: number) => {
+      log.info(
+        `could not recover connection after ${this.reconnectAttempts} attempts, ${duration}ms. giving up`,
+      );
+      this.emit(EngineEvent.Disconnected);
+      this.close();
+    };
+
+    const duration = Date.now() - this.reconnectStart;
+    const delay = this.getNextRetryDelay({
+      elapsedMs: duration,
+      retryCount: this.reconnectAttempts,
+    });
+
+    if (delay === null) {
+      disconnect(duration);
+      return;
+    }
+
+    log.debug(`reconnecting in ${delay}ms`);
+
+    if (this.reconnectTimeout) {
+      clearTimeout(this.reconnectTimeout);
+    }
+    this.reconnectTimeout = setTimeout(async () => {
       if (this._isClosed) {
         return;
       }
@@ -469,16 +531,20 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
       try {
         this.attemptingReconnect = true;
         if (this.fullReconnectOnNext) {
-          await this.restartConnection();
+          await this.restartConnection(signalEvents);
         } else {
-          await this.resumeConnection();
+          await this.resumeConnection(signalEvents);
         }
         this.reconnectAttempts = 0;
         this.fullReconnectOnNext = false;
+        if (this.reconnectTimeout) {
+          clearTimeout(this.reconnectTimeout);
+        }
       } catch (e) {
         this.reconnectAttempts += 1;
         let reconnectRequired = false;
         let recoverable = true;
+        let requireSignalEvents = false;
         if (e instanceof UnexpectedConnectionState) {
           log.debug('received unrecoverable error', { error: e });
           // unrecoverable
@@ -488,26 +554,17 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
           reconnectRequired = true;
         }
 
-        // when we flip from resume to reconnect, we need to reset reconnectAttempts
-        // this is needed to fire the right reconnecting events
+        // when we flip from resume to reconnect
+        // we need to fire the right reconnecting events
         if (reconnectRequired && !this.fullReconnectOnNext) {
           this.fullReconnectOnNext = true;
-          this.reconnectAttempts = 0;
-        }
-
-        const duration = Date.now() - this.reconnectStart;
-        if (this.reconnectAttempts >= maxReconnectRetries || duration > maxReconnectDuration) {
-          recoverable = false;
+          requireSignalEvents = true;
         }
 
         if (recoverable) {
-          this.handleDisconnect('reconnect');
+          this.handleDisconnect('reconnect', requireSignalEvents);
         } else {
-          log.info(
-            `could not recover connection after ${maxReconnectRetries} attempts, ${duration}ms. giving up`,
-          );
-          this.emit(EngineEvent.Disconnected);
-          this.close();
+          disconnect(Date.now() - this.reconnectStart);
         }
       } finally {
         this.attemptingReconnect = false;
@@ -515,14 +572,25 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
     }, delay);
   };
 
-  private async restartConnection() {
+  private getNextRetryDelay(context: ReconnectContext) {
+    try {
+      return this.reconnectPolicy.nextRetryDelayInMs(context);
+    } catch (e) {
+      log.warn('encountered error in reconnect policy', { error: e });
+    }
+
+    // error in user code with provided reconnect policy, stop reconnecting
+    return null;
+  }
+
+  private async restartConnection(emitRestarting: boolean = false) {
     if (!this.url || !this.token) {
       // permanent failure, don't attempt reconnection
       throw new UnexpectedConnectionState('could not reconnect, url or token not saved');
     }
 
     log.info(`reconnecting, attempt: ${this.reconnectAttempts}`);
-    if (this.reconnectAttempts === 0) {
+    if (emitRestarting || this.reconnectAttempts === 0) {
       this.emit(EngineEvent.Restarting);
     }
 
@@ -550,7 +618,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
     this.emit(EngineEvent.Restarted, joinResponse);
   }
 
-  private async resumeConnection(): Promise<void> {
+  private async resumeConnection(emitResuming: boolean = false): Promise<void> {
     if (!this.url || !this.token) {
       // permanent failure, don't attempt reconnection
       throw new UnexpectedConnectionState('could not reconnect, url or token not saved');
@@ -561,14 +629,18 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
     }
 
     log.info(`resuming signal connection, attempt ${this.reconnectAttempts}`);
-    if (this.reconnectAttempts === 0) {
+    if (emitResuming || this.reconnectAttempts === 0) {
       this.emit(EngineEvent.Resuming);
     }
 
     try {
       await this.client.reconnect(this.url, this.token);
     } catch (e) {
-      throw new SignalReconnectError();
+      let message = '';
+      if (e instanceof Error) {
+        message = e.message;
+      }
+      throw new SignalReconnectError(message);
     }
     this.emit(EngineEvent.SignalResumed);
 

package/src/room/ReconnectPolicy.ts

@@ -0,0 +1,25 @@
+/** Controls reconnecting of the client */
+export interface ReconnectPolicy {
+  /** Called after disconnect was detected
+   *
+   * @returns {number | null} Amount of time in milliseconds to delay the next reconnect attempt, `null` signals to stop retrying.
+   */
+  nextRetryDelayInMs(context: ReconnectContext): number | null;
+}
+
+export interface ReconnectContext {
+  /**
+   * Number of failed reconnect attempts
+   */
+  readonly retryCount: number;
+
+  /**
+   * Elapsed amount of time in milliseconds since the disconnect.
+   */
+  readonly elapsedMs: number;
+
+  /**
+   * Reason for retrying
+   */
+  readonly retryReason?: Error;
+}