livekit-client 1.6.1 → 1.6.3

package/src/room/PCTransport.ts

@@ -30,6 +30,8 @@ export default class PCTransport extends EventEmitter {
 
   remoteStereoMids: string[] = [];
 
+  remoteNackMids: string[] = [];
+
   onOffer?: (offer: RTCSessionDescriptionInit) => void;
 
   constructor(config?: RTCConfiguration) {
@@ -50,7 +52,9 @@ export default class PCTransport extends EventEmitter {
 
   async setRemoteDescription(sd: RTCSessionDescriptionInit): Promise<void> {
     if (sd.type === 'offer') {
-      this.remoteStereoMids = extractStereoTracksFromOffer(sd);
+      let { stereoMids, nackMids } = extractStereoAndNackAudioFromOffer(sd);
+      this.remoteStereoMids = stereoMids;
+      this.remoteNackMids = nackMids;
     }
     await this.pc.setRemoteDescription(sd);
 
@@ -116,7 +120,7 @@ export default class PCTransport extends EventEmitter {
     const sdpParsed = parse(offer.sdp ?? '');
     sdpParsed.media.forEach((media) => {
       if (media.type === 'audio') {
-        ensureAudioNackAndStereo(media, []);
+        ensureAudioNackAndStereo(media, [], []);
       } else if (media.type === 'video') {
         // mung sdp for codec bitrate setting that can't apply by sendEncoding
         this.trackBitrates.some((trackbr): boolean => {
@@ -169,7 +173,7 @@ export default class PCTransport extends EventEmitter {
     const sdpParsed = parse(answer.sdp ?? '');
     sdpParsed.media.forEach((media) => {
       if (media.type === 'audio') {
-        ensureAudioNackAndStereo(media, this.remoteStereoMids);
+        ensureAudioNackAndStereo(media, this.remoteStereoMids, this.remoteNackMids);
       }
     });
     await this.setMungedLocalDescription(answer, write(sdpParsed));
@@ -226,6 +230,7 @@ function ensureAudioNackAndStereo(
     payloads?: string | undefined;
   } & MediaDescription,
   stereoMids: string[],
+  nackMids: string[],
 ) {
   // found opus codec to add nack fb
   let opusPayload = 0;
@@ -243,7 +248,10 @@ function ensureAudioNackAndStereo(
       media.rtcpFb = [];
     }
 
-    if (!media.rtcpFb.some((fb) => fb.payload === opusPayload && fb.type === 'nack')) {
+    if (
+      nackMids.includes(media.mid!) &&
+      !media.rtcpFb.some((fb) => fb.payload === opusPayload && fb.type === 'nack')
+    ) {
       media.rtcpFb.push({
         payload: opusPayload,
         type: 'nack',
@@ -264,8 +272,12 @@ function ensureAudioNackAndStereo(
   }
 }
 
-function extractStereoTracksFromOffer(offer: RTCSessionDescriptionInit): string[] {
+function extractStereoAndNackAudioFromOffer(offer: RTCSessionDescriptionInit): {
+  stereoMids: string[];
+  nackMids: string[];
+} {
   const stereoMids: string[] = [];
+  const nackMids: string[] = [];
   const sdpParsed = parse(offer.sdp ?? '');
   let opusPayload = 0;
   sdpParsed.media.forEach((media) => {
@@ -278,6 +290,10 @@ function extractStereoTracksFromOffer(offer: RTCSessionDescriptionInit): string[
         return false;
       });
 
+      if (media.rtcpFb?.some((fb) => fb.payload === opusPayload && fb.type === 'nack')) {
+        nackMids.push(media.mid!);
+      }
+
       media.fmtp.some((fmtp): boolean => {
         if (fmtp.payload === opusPayload) {
           if (fmtp.config.includes('sprop-stereo=1')) {
@@ -289,5 +305,5 @@ function extractStereoTracksFromOffer(offer: RTCSessionDescriptionInit): string[
       });
     }
   });
-  return stereoMids;
+  return { stereoMids, nackMids };
 }

package/src/room/RTCEngine.ts

@@ -17,6 +17,7 @@ import {
   AddTrackRequest,
   JoinResponse,
   LeaveRequest,
+  ReconnectResponse,
   SignalTarget,
   TrackPublishedResponse,
 } from '../proto/livekit_rtc';
@@ -31,6 +32,7 @@ import {
 import { EngineEvent } from './events';
 import PCTransport, { PCEvents } from './PCTransport';
 import type { ReconnectContext, ReconnectPolicy } from './ReconnectPolicy';
+import CriticalTimers from './timers';
 import type LocalTrack from './track/LocalTrack';
 import type LocalVideoTrack from './track/LocalVideoTrack';
 import type { SimulcastTrackInfo } from './track/LocalVideoTrack';
@@ -279,35 +281,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
 
     this.participantSid = joinResponse.participant?.sid;
 
-    const rtcConfig = { ...this.rtcConfig };
-
-    // update ICE servers before creating PeerConnection
-    if (joinResponse.iceServers && !rtcConfig.iceServers) {
-      const rtcIceServers: RTCIceServer[] = [];
-      joinResponse.iceServers.forEach((iceServer) => {
-        const rtcIceServer: RTCIceServer = {
-          urls: iceServer.urls,
-        };
-        if (iceServer.username) rtcIceServer.username = iceServer.username;
-        if (iceServer.credential) {
-          rtcIceServer.credential = iceServer.credential;
-        }
-        rtcIceServers.push(rtcIceServer);
-      });
-      rtcConfig.iceServers = rtcIceServers;
-    }
-
-    if (
-      joinResponse.clientConfiguration &&
-      joinResponse.clientConfiguration.forceRelay === ClientConfigSetting.ENABLED
-    ) {
-      rtcConfig.iceTransportPolicy = 'relay';
-    }
-
-    // @ts-ignore
-    rtcConfig.sdpSemantics = 'unified-plan';
-    // @ts-ignore
-    rtcConfig.continualGatheringPolicy = 'gather_continually';
+    const rtcConfig = this.makeRTCConfiguration(joinResponse);
 
     this.publisher = new PCTransport(rtcConfig);
     this.subscriber = new PCTransport(rtcConfig);
@@ -448,6 +422,40 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
     };
   }
 
+  private makeRTCConfiguration(serverResponse: JoinResponse | ReconnectResponse): RTCConfiguration {
+    const rtcConfig = { ...this.rtcConfig };
+
+    // update ICE servers before creating PeerConnection
+    if (serverResponse.iceServers && !rtcConfig.iceServers) {
+      const rtcIceServers: RTCIceServer[] = [];
+      serverResponse.iceServers.forEach((iceServer) => {
+        const rtcIceServer: RTCIceServer = {
+          urls: iceServer.urls,
+        };
+        if (iceServer.username) rtcIceServer.username = iceServer.username;
+        if (iceServer.credential) {
+          rtcIceServer.credential = iceServer.credential;
+        }
+        rtcIceServers.push(rtcIceServer);
+      });
+      rtcConfig.iceServers = rtcIceServers;
+    }
+
+    if (
+      serverResponse.clientConfiguration &&
+      serverResponse.clientConfiguration.forceRelay === ClientConfigSetting.ENABLED
+    ) {
+      rtcConfig.iceTransportPolicy = 'relay';
+    }
+
+    // @ts-ignore
+    rtcConfig.sdpSemantics = 'unified-plan';
+    // @ts-ignore
+    rtcConfig.continualGatheringPolicy = 'gather_continually';
+
+    return rtcConfig;
+  }
+
   private createDataChannels() {
     if (!this.publisher) {
       return;
@@ -703,10 +711,11 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
 
     log.debug(`reconnecting in ${delay}ms`);
 
-    if (this.reconnectTimeout) {
-      clearTimeout(this.reconnectTimeout);
-    }
-    this.reconnectTimeout = setTimeout(() => this.attemptReconnect(signalEvents), delay);
+    this.clearReconnectTimeout();
+    this.reconnectTimeout = CriticalTimers.setTimeout(
+      () => this.attemptReconnect(signalEvents),
+      delay,
+    );
   };
 
   private async attemptReconnect(signalEvents: boolean = false) {
@@ -733,11 +742,8 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
       } else {
         await this.resumeConnection(signalEvents);
       }
-      this.reconnectAttempts = 0;
+      this.clearPendingReconnect();
       this.fullReconnectOnNext = false;
-      if (this.reconnectTimeout) {
-        clearTimeout(this.reconnectTimeout);
-      }
     } catch (e) {
       this.reconnectAttempts += 1;
       let reconnectRequired = false;
@@ -841,7 +847,12 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
     }
 
     try {
-      await this.client.reconnect(this.url, this.token, this.participantSid);
+      const res = await this.client.reconnect(this.url, this.token, this.participantSid);
+      if (res) {
+        const rtcConfig = this.makeRTCConfiguration(res);
+        this.publisher.pc.setConfiguration(rtcConfig);
+        this.subscriber.pc.setConfiguration(rtcConfig);
+      }
     } catch (e) {
       let message = '';
       if (e instanceof Error) {
@@ -1034,19 +1045,21 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
     }
   }
 
-  private clearPendingReconnect() {
+  private clearReconnectTimeout() {
     if (this.reconnectTimeout) {
-      clearTimeout(this.reconnectTimeout);
+      CriticalTimers.clearTimeout(this.reconnectTimeout);
     }
+  }
+
+  private clearPendingReconnect() {
+    this.clearReconnectTimeout();
     this.reconnectAttempts = 0;
   }
 
   private handleBrowserOnLine = () => {
     // in case the engine is currently reconnecting, attempt a reconnect immediately after the browser state has changed to 'onLine'
     if (this.client.isReconnecting) {
-      if (this.reconnectTimeout) {
-        clearTimeout(this.reconnectTimeout);
-      }
+      this.clearReconnectTimeout();
       this.attemptReconnect(true);
     }
   };

package/src/room/Room.ts

@@ -44,6 +44,7 @@ import type Participant from './participant/Participant';
 import type { ConnectionQuality } from './participant/Participant';
 import RemoteParticipant from './participant/RemoteParticipant';
 import RTCEngine from './RTCEngine';
+import CriticalTimers from './timers';
 import LocalAudioTrack from './track/LocalAudioTrack';
 import LocalTrackPublication from './track/LocalTrackPublication';
 import LocalVideoTrack from './track/LocalVideoTrack';
@@ -364,7 +365,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       }
 
       // don't return until ICE connected
-      const connectTimeout = setTimeout(() => {
+      const connectTimeout = CriticalTimers.setTimeout(() => {
         // timeout
         this.recreateEngine();
         this.handleDisconnect(this.options.stopLocalTrackOnUnpublish);
@@ -372,7 +373,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       }, this.connOptions.peerConnectionTimeout);
       const abortHandler = () => {
         log.warn('closing engine');
-        clearTimeout(connectTimeout);
+        CriticalTimers.clearTimeout(connectTimeout);
         this.recreateEngine();
         this.handleDisconnect(this.options.stopLocalTrackOnUnpublish);
         reject(new ConnectionError('room connection has been cancelled'));
@@ -383,7 +384,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       this.abortController?.signal.addEventListener('abort', abortHandler);
 
       this.engine.once(EngineEvent.Connected, () => {
-        clearTimeout(connectTimeout);
+        CriticalTimers.clearTimeout(connectTimeout);
         this.abortController?.signal.removeEventListener('abort', abortHandler);
         // also hook unload event
         if (isWeb()) {
@@ -1281,10 +1282,11 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
    * No actual connection to a server will be established, all state is
    * @experimental
    */
-  simulateParticipants(options: SimulationOptions) {
+  async simulateParticipants(options: SimulationOptions) {
     const publishOptions = {
       audio: true,
       video: true,
+      useRealTracks: false,
       ...options.publish,
     };
     const participantOptions = {
@@ -1317,12 +1319,14 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
           name: 'video-dummy',
         }),
         new LocalVideoTrack(
-          createDummyVideoStreamTrack(
-            160 * participantOptions.aspectRatios[0] ?? 1,
-            160,
-            true,
-            true,
-          ),
+          publishOptions.useRealTracks
+            ? (await navigator.mediaDevices.getUserMedia({ video: true })).getVideoTracks()[0]
+            : createDummyVideoStreamTrack(
+                160 * participantOptions.aspectRatios[0] ?? 1,
+                160,
+                true,
+                true,
+              ),
         ),
       );
       // @ts-ignore
@@ -1337,7 +1341,11 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
           sid: Math.floor(Math.random() * 10_000).toString(),
           type: TrackType.AUDIO,
         }),
-        new LocalAudioTrack(getEmptyAudioStreamTrack()),
+        new LocalAudioTrack(
+          publishOptions.useRealTracks
+            ? (await navigator.mediaDevices.getUserMedia({ audio: true })).getAudioTracks()[0]
+            : getEmptyAudioStreamTrack(),
+        ),
       );
       // @ts-ignore
       this.localParticipant.addTrackPublication(audioPub);

package/src/room/timers.ts

@@ -0,0 +1,16 @@
+/**
+ * Timers that can be overridden with platform specific implementations
+ * that ensure that they are fired. These should be used when it is critical
+ * that the timer fires on time.
+ */
+export default class CriticalTimers {
+  // eslint-disable-next-line @typescript-eslint/no-implied-eval
+  static setTimeout = (...args: Parameters<typeof setTimeout>) => setTimeout(...args);
+
+  // eslint-disable-next-line @typescript-eslint/no-implied-eval
+  static setInterval = (...args: Parameters<typeof setInterval>) => setInterval(...args);
+
+  static clearTimeout = (...args: Parameters<typeof clearTimeout>) => clearTimeout(...args);
+
+  static clearInterval = (...args: Parameters<typeof clearInterval>) => clearInterval(...args);
+}

package/src/room/track/RemoteVideoTrack.ts

@@ -2,6 +2,7 @@ import { debounce } from 'ts-debounce';
 import log from '../../logger';
 import { TrackEvent } from '../events';
 import { computeBitrate, VideoReceiverStats } from '../stats';
+import CriticalTimers from '../timers';
 import { getIntersectionObserver, getResizeObserver, ObservableMediaElement } from '../utils';
 import RemoteTrack from './RemoteTrack';
 import { attachToElement, detachTrack, Track } from './Track';
@@ -233,7 +234,7 @@ export default class RemoteVideoTrack extends RemoteTrack {
 
     if (!isVisible && Date.now() - lastVisibilityChange < REACTION_DELAY) {
       // delay hidden events
-      setTimeout(() => {
+      CriticalTimers.setTimeout(() => {
         this.updateVisibility();
       }, REACTION_DELAY);
       return;

package/src/room/types.ts

@@ -2,6 +2,7 @@ export type SimulationOptions = {
   publish?: {
     audio?: boolean;
     video?: boolean;
+    useRealTracks?: boolean;
   };
   participants?: {
     count?: number;