livekit-client 1.7.1 → 1.8.0

package/src/room/Room.ts

@@ -37,14 +37,13 @@ import {
   videoDefaults,
 } from './defaults';
 import DeviceManager from './DeviceManager';
-import { ConnectionError, UnsupportedServer } from './errors';
+import { ConnectionError, ConnectionErrorReason, UnsupportedServer } from './errors';
 import { EngineEvent, ParticipantEvent, RoomEvent, TrackEvent } from './events';
 import LocalParticipant from './participant/LocalParticipant';
 import type Participant from './participant/Participant';
 import type { ConnectionQuality } from './participant/Participant';
 import RemoteParticipant from './participant/RemoteParticipant';
 import RTCEngine from './RTCEngine';
-import CriticalTimers from './timers';
 import LocalAudioTrack from './track/LocalAudioTrack';
 import LocalTrackPublication from './track/LocalTrackPublication';
 import LocalVideoTrack from './track/LocalVideoTrack';
@@ -59,11 +58,13 @@ import {
   createDummyVideoStreamTrack,
   Future,
   getEmptyAudioStreamTrack,
+  isCloud,
   isWeb,
   Mutex,
   supportsSetSinkId,
   unpackStreamId,
 } from './utils';
+import { RegionUrlProvider } from './RegionUrlProvider';
 
 export enum ConnectionState {
   Disconnected = 'disconnected',
@@ -206,7 +207,10 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
         }
       })
       .on(EngineEvent.Restarting, this.handleRestarting)
-      .on(EngineEvent.Restarted, this.handleRestarted);
+      .on(EngineEvent.Restarted, this.handleRestarted)
+      .on(EngineEvent.DCBufferStatusChanged, (status, kind) => {
+        this.emit(RoomEvent.DCBufferStatusChanged, status, kind);
+      });
 
     if (this.localParticipant) {
       this.localParticipant.setupEngine(this.engine);
@@ -258,155 +262,219 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
 
     this.setAndEmitConnectionState(ConnectionState.Connecting);
 
-    const connectFn = async (resolve: () => void, reject: (reason: any) => void) => {
-      if (!this.abortController || this.abortController.signal.aborted) {
-        this.abortController = new AbortController();
+    const urlProvider = new RegionUrlProvider(url, token);
+
+    const connectFn = async (
+      resolve: () => void,
+      reject: (reason: any) => void,
+      regionUrl?: string,
+    ) => {
+      if (this.abortController) {
+        this.abortController.abort();
       }
+      this.abortController = new AbortController();
+
       // at this point the intention to connect has been signalled so we can allow cancelling of the connection via disconnect() again
-      unlockDisconnect();
+      unlockDisconnect?.();
 
-      if (this.state === ConnectionState.Reconnecting) {
-        log.info('Reconnection attempt replaced by new connection attempt');
-        // make sure we close and recreate the existing engine in order to get rid of any potentially ongoing reconnection attempts
-        this.recreateEngine();
-      } else {
-        // create engine if previously disconnected
-        this.maybeCreateEngine();
+      try {
+        await this.attemptConnection(regionUrl ?? url, token, opts, this.abortController);
+        this.abortController = undefined;
+        resolve();
+      } catch (e) {
+        if (
+          isCloud(new URL(url)) &&
+          e instanceof ConnectionError &&
+          e.reason !== ConnectionErrorReason.Cancelled
+        ) {
+          let nextUrl: string | null = null;
+          try {
+            nextUrl = await urlProvider.getNextBestRegionUrl(this.abortController?.signal);
+          } catch (error) {
+            if (
+              error instanceof ConnectionError &&
+              (error.status === 401 || error.reason === ConnectionErrorReason.Cancelled)
+            ) {
+              reject(error);
+              return;
+            }
+          }
+          if (nextUrl) {
+            log.debug('initial connection failed, retrying with another region');
+            await connectFn(resolve, reject, nextUrl);
+          } else {
+            reject(e);
+          }
+        } else {
+          reject(e);
+        }
       }
+    };
+    this.connectFuture = new Future(connectFn, () => {
+      this.clearConnectionFutures();
+    });
 
-      this.acquireAudioContext();
-
-      this.connOptions = { ...roomConnectOptionDefaults, ...opts } as InternalRoomConnectOptions;
+    return this.connectFuture.promise;
+  };
 
-      if (this.connOptions.rtcConfig) {
-        this.engine.rtcConfig = this.connOptions.rtcConfig;
-      }
-      if (this.connOptions.peerConnectionTimeout) {
-        this.engine.peerConnectionTimeout = this.connOptions.peerConnectionTimeout;
-      }
+  private connectSignal = async (
+    url: string,
+    token: string,
+    engine: RTCEngine,
+    connectOptions: InternalRoomConnectOptions,
+    roomOptions: InternalRoomOptions,
+    abortController: AbortController,
+  ): Promise<JoinResponse> => {
+    const joinResponse = await engine.join(
+      url,
+      token,
+      {
+        autoSubscribe: connectOptions.autoSubscribe,
+        publishOnly: connectOptions.publishOnly,
+        adaptiveStream:
+          typeof roomOptions.adaptiveStream === 'object' ? true : roomOptions.adaptiveStream,
+        maxRetries: connectOptions.maxRetries,
+      },
+      abortController.signal,
+    );
 
-      try {
-        const joinResponse = await this.engine.join(
-          url,
-          token,
-          {
-            autoSubscribe: this.connOptions.autoSubscribe,
-            publishOnly: this.connOptions.publishOnly,
-            adaptiveStream:
-              typeof this.options.adaptiveStream === 'object' ? true : this.options.adaptiveStream,
-            maxRetries: this.connOptions.maxRetries,
-          },
-          this.abortController.signal,
-        );
+    let serverInfo: Partial<ServerInfo> | undefined = joinResponse.serverInfo;
+    if (!serverInfo) {
+      serverInfo = { version: joinResponse.serverVersion, region: joinResponse.serverRegion };
+    }
 
-        let serverInfo: Partial<ServerInfo> | undefined = joinResponse.serverInfo;
-        if (!serverInfo) {
-          serverInfo = { version: joinResponse.serverVersion, region: joinResponse.serverRegion };
-        }
+    log.debug(
+      `connected to Livekit Server ${Object.entries(serverInfo)
+        .map(([key, value]) => `${key}: ${value}`)
+        .join(', ')}`,
+    );
 
-        log.debug(
-          `connected to Livekit Server ${Object.entries(serverInfo)
-            .map(([key, value]) => `${key}: ${value}`)
-            .join(', ')}`,
-        );
+    if (!joinResponse.serverVersion) {
+      throw new UnsupportedServer('unknown server version');
+    }
 
-        if (!joinResponse.serverVersion) {
-          throw new UnsupportedServer('unknown server version');
-        }
+    if (joinResponse.serverVersion === '0.15.1' && this.options.dynacast) {
+      log.debug('disabling dynacast due to server version');
+      // dynacast has a bug in 0.15.1, so we cannot use it then
+      roomOptions.dynacast = false;
+    }
 
-        if (joinResponse.serverVersion === '0.15.1' && this.options.dynacast) {
-          log.debug('disabling dynacast due to server version');
-          // dynacast has a bug in 0.15.1, so we cannot use it then
-          this.options.dynacast = false;
-        }
+    return joinResponse;
+  };
 
-        const pi = joinResponse.participant!;
+  private applyJoinResponse = (joinResponse: JoinResponse) => {
+    const pi = joinResponse.participant!;
 
-        this.localParticipant.sid = pi.sid;
-        this.localParticipant.identity = pi.identity;
+    this.localParticipant.sid = pi.sid;
+    this.localParticipant.identity = pi.identity;
 
-        this.localParticipant.updateInfo(pi);
-        // forward metadata changed for the local participant
-        this.setupLocalParticipantEvents();
+    this.localParticipant.updateInfo(pi);
+    // forward metadata changed for the local participant
+    this.setupLocalParticipantEvents();
 
-        // populate remote participants, these should not trigger new events
-        joinResponse.otherParticipants.forEach((info) => {
-          if (
-            info.sid !== this.localParticipant.sid &&
-            info.identity !== this.localParticipant.identity
-          ) {
-            this.getOrCreateParticipant(info.sid, info);
-          } else {
-            log.warn('received info to create local participant as remote participant', {
-              info,
-              localParticipant: this.localParticipant,
-            });
-          }
+    // populate remote participants, these should not trigger new events
+    joinResponse.otherParticipants.forEach((info) => {
+      if (
+        info.sid !== this.localParticipant.sid &&
+        info.identity !== this.localParticipant.identity
+      ) {
+        this.getOrCreateParticipant(info.sid, info);
+      } else {
+        log.warn('received info to create local participant as remote participant', {
+          info,
+          localParticipant: this.localParticipant,
         });
-
-        this.name = joinResponse.room!.name;
-        this.sid = joinResponse.room!.sid;
-        this.metadata = joinResponse.room!.metadata;
-        if (this._isRecording !== joinResponse.room!.activeRecording) {
-          this._isRecording = joinResponse.room!.activeRecording;
-          this.emit(RoomEvent.RecordingStatusChanged, joinResponse.room!.activeRecording);
-        }
-        this.emit(RoomEvent.SignalConnected);
-      } catch (err) {
-        this.recreateEngine();
-        this.handleDisconnect(this.options.stopLocalTrackOnUnpublish);
-        const resultingError = new ConnectionError(`could not establish signal connection`);
-        if (err instanceof Error) {
-          resultingError.message = `${resultingError.message}: ${err.message}`;
-        }
-        if (err instanceof ConnectionError) {
-          resultingError.reason = err.reason;
-          resultingError.status = err.status;
-        }
-        log.debug(`error trying to establish signal connection`, { error: err });
-        reject(resultingError);
-        return;
       }
+    });
 
-      // don't return until ICE connected
-      const connectTimeout = CriticalTimers.setTimeout(() => {
-        // timeout
-        this.recreateEngine();
-        this.handleDisconnect(this.options.stopLocalTrackOnUnpublish);
-        reject(new ConnectionError('could not connect PeerConnection after timeout'));
-      }, this.connOptions.peerConnectionTimeout);
-      const abortHandler = () => {
-        log.warn('closing engine');
-        CriticalTimers.clearTimeout(connectTimeout);
-        this.recreateEngine();
-        this.handleDisconnect(this.options.stopLocalTrackOnUnpublish);
-        reject(new ConnectionError('room connection has been cancelled'));
-      };
-      if (this.abortController?.signal.aborted) {
-        abortHandler();
+    this.name = joinResponse.room!.name;
+    this.sid = joinResponse.room!.sid;
+    this.metadata = joinResponse.room!.metadata;
+    if (this._isRecording !== joinResponse.room!.activeRecording) {
+      this._isRecording = joinResponse.room!.activeRecording;
+      this.emit(RoomEvent.RecordingStatusChanged, joinResponse.room!.activeRecording);
+    }
+  };
+
+  private attemptConnection = async (
+    url: string,
+    token: string,
+    opts: RoomConnectOptions | undefined,
+    abortController: AbortController,
+  ) => {
+    if (this.state === ConnectionState.Reconnecting) {
+      log.info('Reconnection attempt replaced by new connection attempt');
+      // make sure we close and recreate the existing engine in order to get rid of any potentially ongoing reconnection attempts
+      this.recreateEngine();
+    } else {
+      // create engine if previously disconnected
+      this.maybeCreateEngine();
+    }
+
+    this.acquireAudioContext();
+
+    this.connOptions = { ...roomConnectOptionDefaults, ...opts } as InternalRoomConnectOptions;
+
+    if (this.connOptions.rtcConfig) {
+      this.engine.rtcConfig = this.connOptions.rtcConfig;
+    }
+    if (this.connOptions.peerConnectionTimeout) {
+      this.engine.peerConnectionTimeout = this.connOptions.peerConnectionTimeout;
+    }
+
+    try {
+      const joinResponse = await this.connectSignal(
+        url,
+        token,
+        this.engine,
+        this.connOptions,
+        this.options,
+        abortController,
+      );
+
+      this.applyJoinResponse(joinResponse);
+      this.emit(RoomEvent.SignalConnected);
+    } catch (err) {
+      this.recreateEngine();
+      this.handleDisconnect(this.options.stopLocalTrackOnUnpublish);
+      const resultingError = new ConnectionError(`could not establish signal connection`);
+      if (err instanceof Error) {
+        resultingError.message = `${resultingError.message}: ${err.message}`;
       }
-      this.abortController?.signal.addEventListener('abort', abortHandler);
-
-      this.engine.once(EngineEvent.Connected, () => {
-        CriticalTimers.clearTimeout(connectTimeout);
-        this.abortController?.signal.removeEventListener('abort', abortHandler);
-        // also hook unload event
-        if (isWeb() && this.options.disconnectOnPageLeave) {
-          // capturing both 'pagehide' and 'beforeunload' to capture broadest set of browser behaviors
-          window.addEventListener('pagehide', this.onPageLeave);
-          window.addEventListener('beforeunload', this.onPageLeave);
-          navigator.mediaDevices?.addEventListener('devicechange', this.handleDeviceChange);
-        }
-        this.setAndEmitConnectionState(ConnectionState.Connected);
-        this.emit(RoomEvent.Connected);
-        resolve();
-      });
-    };
-    this.connectFuture = new Future(connectFn, () => {
-      this.clearConnectionFutures();
-    });
+      if (err instanceof ConnectionError) {
+        resultingError.reason = err.reason;
+        resultingError.status = err.status;
+      }
+      log.debug(`error trying to establish signal connection`, { error: err });
+      throw resultingError;
+    }
 
-    return this.connectFuture.promise;
+    if (abortController.signal.aborted) {
+      this.recreateEngine();
+      this.handleDisconnect(this.options.stopLocalTrackOnUnpublish);
+      throw new ConnectionError(`Connection attempt aborted`);
+    }
+
+    try {
+      await this.engine.waitForPCInitialConnection(
+        this.connOptions.peerConnectionTimeout,
+        abortController,
+      );
+    } catch (e) {
+      this.recreateEngine();
+      this.handleDisconnect(this.options.stopLocalTrackOnUnpublish);
+      throw e;
+    }
+
+    // also hook unload event
+    if (isWeb() && this.options.disconnectOnPageLeave) {
+      // capturing both 'pagehide' and 'beforeunload' to capture broadest set of browser behaviors
+      window.addEventListener('pagehide', this.onPageLeave);
+      window.addEventListener('beforeunload', this.onPageLeave);
+      navigator.mediaDevices?.addEventListener('devicechange', this.handleDeviceChange);
+    }
+    this.setAndEmitConnectionState(ConnectionState.Connected);
+    this.emit(RoomEvent.Connected);
   };
 
   /**
@@ -1501,4 +1569,5 @@ export type RoomEventCallbacks = {
   audioPlaybackChanged: (playing: boolean) => void;
   signalConnected: () => void;
   recordingStatusChanged: (recording: boolean) => void;
+  dcBufferStatusChanged: (isLow: boolean, kind: DataPacket_Kind) => void;
 };

package/src/room/errors.ts

@@ -11,6 +11,7 @@ export const enum ConnectionErrorReason {
   NotAllowed,
   ServerUnreachable,
   InternalError,
+  Cancelled,
 }
 
 export class ConnectionError extends LivekitError {

package/src/room/events.ts

@@ -264,6 +264,12 @@ export enum RoomEvent {
    * args: (isRecording: boolean)
    */
   RecordingStatusChanged = 'recordingStatusChanged',
+
+  /**
+   * Emits whenever the current buffer status of a data channel changes
+   * args: (isLow: boolean, kind: [[DataPacket_Kind]])
+   */
+  DCBufferStatusChanged = 'dcBufferStatusChanged',
 }
 
 export enum ParticipantEvent {
@@ -431,6 +437,7 @@ export enum EngineEvent {
   MediaTrackAdded = 'mediaTrackAdded',
   ActiveSpeakersUpdate = 'activeSpeakersUpdate',
   DataPacketReceived = 'dataPacketReceived',
+  DCBufferStatusChanged = 'dcBufferStatusChanged',
 }
 
 export enum TrackEvent {

package/src/room/track/LocalAudioTrack.ts

@@ -39,7 +39,8 @@ export default class LocalAudioTrack extends LocalTrack {
   }
 
   async mute(): Promise<LocalAudioTrack> {
-    await this.muteQueue.run(async () => {
+    const unlock = await this.muteLock.lock();
+    try {
       // disabled special handling as it will cause BT headsets to switch communication modes
       if (this.source === Track.Source.Microphone && this.stopOnMute && !this.isUserProvided) {
         log.debug('stopping mic track');
@@ -47,12 +48,15 @@ export default class LocalAudioTrack extends LocalTrack {
         this._mediaStreamTrack.stop();
       }
       await super.mute();
-    });
-    return this;
+      return this;
+    } finally {
+      unlock();
+    }
   }
 
   async unmute(): Promise<LocalAudioTrack> {
-    await this.muteQueue.run(async () => {
+    const unlock = await this.muteLock.lock();
+    try {
       if (
         this.source === Track.Source.Microphone &&
         (this.stopOnMute || this._mediaStreamTrack.readyState === 'ended') &&
@@ -62,8 +66,11 @@ export default class LocalAudioTrack extends LocalTrack {
         await this.restartTrack();
       }
       await super.unmute();
-    });
-    return this;
+
+      return this;
+    } finally {
+      unlock();
+    }
   }
 
   async restartTrack(options?: AudioCaptureOptions) {

package/src/room/track/LocalTrack.ts

@@ -1,9 +1,14 @@
-import Queue from 'async-await-queue';
 import log from '../../logger';
 import DeviceManager from '../DeviceManager';
 import { TrackInvalidError } from '../errors';
 import { TrackEvent } from '../events';
-import { getEmptyAudioStreamTrack, getEmptyVideoStreamTrack, isMobile, sleep } from '../utils';
+import {
+  getEmptyAudioStreamTrack,
+  getEmptyVideoStreamTrack,
+  isMobile,
+  Mutex,
+  sleep,
+} from '../utils';
 import type { VideoCodec } from './options';
 import { attachToElement, detachTrack, Track } from './Track';
 
@@ -22,7 +27,9 @@ export default abstract class LocalTrack extends Track {
 
   protected providedByUser: boolean;
 
-  protected muteQueue: Queue;
+  protected muteLock: Mutex;
+
+  protected pauseUpstreamLock: Mutex;
 
   /**
    *
@@ -42,7 +49,8 @@ export default abstract class LocalTrack extends Track {
     this.constraints = constraints ?? mediaTrack.getConstraints();
     this.reacquireTrack = false;
     this.providedByUser = userProvidedTrack;
-    this.muteQueue = new Queue();
+    this.muteLock = new Mutex();
+    this.pauseUpstreamLock = new Mutex();
   }
 
   get id(): string {
@@ -246,7 +254,8 @@ export default abstract class LocalTrack extends Track {
   };
 
   async pauseUpstream() {
-    this.muteQueue.run(async () => {
+    const unlock = await this.pauseUpstreamLock.lock();
+    try {
       if (this._isUpstreamPaused === true) {
         return;
       }
@@ -260,11 +269,14 @@ export default abstract class LocalTrack extends Track {
       const emptyTrack =
         this.kind === Track.Kind.Audio ? getEmptyAudioStreamTrack() : getEmptyVideoStreamTrack();
       await this.sender.replaceTrack(emptyTrack);
-    });
+    } finally {
+      unlock();
+    }
   }
 
   async resumeUpstream() {
-    this.muteQueue.run(async () => {
+    const unlock = await this.pauseUpstreamLock.lock();
+    try {
       if (this._isUpstreamPaused === false) {
         return;
       }
@@ -276,7 +288,9 @@ export default abstract class LocalTrack extends Track {
       this.emit(TrackEvent.UpstreamResumed, this);
 
       await this.sender.replaceTrack(this._mediaStreamTrack);
-    });
+    } finally {
+      unlock();
+    }
   }
 
   protected abstract monitorSender(): void;

package/src/room/track/LocalVideoTrack.ts

@@ -97,26 +97,32 @@ export default class LocalVideoTrack extends LocalTrack {
   }
 
   async mute(): Promise<LocalVideoTrack> {
-    await this.muteQueue.run(async () => {
+    const unlock = await this.muteLock.lock();
+    try {
       if (this.source === Track.Source.Camera && !this.isUserProvided) {
         log.debug('stopping camera track');
         // also stop the track, so that camera indicator is turned off
         this._mediaStreamTrack.stop();
       }
       await super.mute();
-    });
-    return this;
+      return this;
+    } finally {
+      unlock();
+    }
   }
 
   async unmute(): Promise<LocalVideoTrack> {
-    await this.muteQueue.run(async () => {
+    const unlock = await this.muteLock.lock();
+    try {
       if (this.source === Track.Source.Camera && !this.isUserProvided) {
         log.debug('reacquiring camera track');
         await this.restartTrack();
       }
       await super.unmute();
-    });
-    return this;
+      return this;
+    } finally {
+      unlock();
+    }
   }
 
   async getSenderStats(): Promise<VideoSenderStats[]> {

package/src/room/track/RemoteTrackPublication.ts

@@ -4,7 +4,7 @@ import { UpdateSubscription, UpdateTrackSettings } from '../../proto/livekit_rtc
 import { TrackEvent } from '../events';
 import type RemoteTrack from './RemoteTrack';
 import RemoteVideoTrack from './RemoteVideoTrack';
-import type { Track } from './Track';
+import { Track } from './Track';
 import { TrackPublication } from './TrackPublication';
 
 export default class RemoteTrackPublication extends TrackPublication {
@@ -181,6 +181,7 @@ export default class RemoteTrackPublication extends TrackPublication {
       prevTrack.off(TrackEvent.VisibilityChanged, this.handleVisibilityChange);
       prevTrack.off(TrackEvent.Ended, this.handleEnded);
       prevTrack.detach();
+      prevTrack.stopMonitor();
       this.emit(TrackEvent.Unsubscribed, prevTrack);
     }
     super.setTrack(track);
@@ -238,8 +239,8 @@ export default class RemoteTrackPublication extends TrackPublication {
   }
 
   private isManualOperationAllowed(): boolean {
-    if (this.isAdaptiveStream) {
-      log.warn('adaptive stream is enabled, cannot change track settings', {
+    if (this.kind === Track.Kind.Video && this.isAdaptiveStream) {
+      log.warn('adaptive stream is enabled, cannot change video track settings', {
         trackSid: this.trackSid,
       });
       return false;

package/src/room/track/RemoteVideoTrack.ts

@@ -4,6 +4,7 @@ import { TrackEvent } from '../events';
 import { computeBitrate, VideoReceiverStats } from '../stats';
 import CriticalTimers from '../timers';
 import {
+  getDevicePixelRatio,
   getIntersectionObserver,
   getResizeObserver,
   isWeb,
@@ -26,7 +27,7 @@ export default class RemoteVideoTrack extends RemoteTrack {
 
   private lastDimensions?: Track.Dimensions;
 
-  private hasUsedAttach: boolean = false;
+  private isObserved: boolean = false;
 
   constructor(
     mediaTrack: MediaStreamTrack,
@@ -43,7 +44,7 @@ export default class RemoteVideoTrack extends RemoteTrack {
   }
 
   get mediaStreamTrack() {
-    if (this.isAdaptiveStream && !this.hasUsedAttach) {
+    if (this.isAdaptiveStream && !this.isObserved) {
       log.warn(
         'When using adaptiveStream, you need to use remoteVideoTrack.attach() to add the track to a HTMLVideoElement, otherwise your video tracks might never start',
       );
@@ -83,7 +84,6 @@ export default class RemoteVideoTrack extends RemoteTrack {
       const elementInfo = new HTMLElementInfo(element);
       this.observeElementInfo(elementInfo);
     }
-    this.hasUsedAttach = true;
     return element;
   }
 
@@ -110,6 +110,7 @@ export default class RemoteVideoTrack extends RemoteTrack {
       // the tab comes into focus for the first time.
       this.debouncedHandleResize();
       this.updateVisibility();
+      this.isObserved = true;
     } else {
       log.warn('visibility resize observer not triggered');
     }
@@ -253,7 +254,7 @@ export default class RemoteVideoTrack extends RemoteTrack {
     let maxHeight = 0;
     for (const info of this.elementInfos) {
       const pixelDensity = this.adaptiveStreamSettings?.pixelDensity ?? 1;
-      const pixelDensityValue = pixelDensity === 'screen' ? window.devicePixelRatio : pixelDensity;
+      const pixelDensityValue = pixelDensity === 'screen' ? getDevicePixelRatio() : pixelDensity;
       const currentElementWidth = info.width() * pixelDensityValue;
       const currentElementHeight = info.height() * pixelDensityValue;
       if (currentElementWidth + currentElementHeight > maxWidth + maxHeight) {