livekit-client 1.12.3 → 1.13.1

package/src/room/Room.ts

@@ -209,10 +209,10 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
    */
   async setE2EEEnabled(enabled: boolean) {
     if (this.e2eeManager) {
-      await Promise.all([
-        this.localParticipant.setE2EEEnabled(enabled),
-        this.e2eeManager.setParticipantCryptorEnabled(enabled),
-      ]);
+      await Promise.all([this.localParticipant.setE2EEEnabled(enabled)]);
+      if (this.localParticipant.identity !== '') {
+        this.e2eeManager.setParticipantCryptorEnabled(enabled, this.localParticipant.identity);
+      }
     } else {
       throw Error('e2ee not configured, please set e2ee settings within the room options');
     }
@@ -230,7 +230,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
           this.emit(RoomEvent.ParticipantEncryptionStatusChanged, enabled, participant);
         },
       );
-      this.e2eeManager.on(EncryptionEvent.Error, (error) =>
+      this.e2eeManager.on(EncryptionEvent.EncryptionError, (error) =>
         this.emit(RoomEvent.EncryptionError, error),
       );
       this.e2eeManager?.setup(this);
@@ -424,13 +424,15 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
         this.abortController.abort();
       }
 
-      this.abortController = new AbortController();
+      // explicit creation as local var needed to satisfy TS compiler when passing it to `attemptConnection` further down
+      const abortController = new AbortController();
+      this.abortController = abortController;
 
       // at this point the intention to connect has been signalled so we can allow cancelling of the connection via disconnect() again
       unlockDisconnect?.();
 
       try {
-        await this.attemptConnection(regionUrl ?? url, token, opts, this.abortController);
+        await this.attemptConnection(regionUrl ?? url, token, opts, abortController);
         this.abortController = undefined;
         resolve();
       } catch (e) {
@@ -500,6 +502,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
           typeof roomOptions.adaptiveStream === 'object' ? true : roomOptions.adaptiveStream,
         maxRetries: connectOptions.maxRetries,
         e2eeEnabled: !!this.e2eeManager,
+        websocketTimeout: connectOptions.websocketTimeout,
       },
       abortController.signal,
     );
@@ -540,6 +543,10 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     if (joinResponse.room) {
       this.handleRoomUpdate(joinResponse.room);
     }
+
+    if (this.options.e2ee && this.e2eeManager) {
+      this.e2eeManager.setSifTrailer(joinResponse.sifTrailer);
+    }
   };
 
   private attemptConnection = async (
@@ -692,7 +699,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
   /**
    * @internal for testing
    */
-  async simulateScenario(scenario: SimulationScenario) {
+  async simulateScenario(scenario: SimulationScenario, arg?: any) {
     let postAction = () => {};
     let req: SimulateScenario | undefined;
     switch (scenario) {
@@ -762,6 +769,17 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
           }
         };
         break;
+      case 'subscriber-bandwidth':
+        if (arg === undefined || typeof arg !== 'number') {
+          throw new Error('subscriber-bandwidth requires a number as argument');
+        }
+        req = new SimulateScenario({
+          scenario: {
+            case: 'subscriberBandwidth',
+            value: BigInt(arg),
+          },
+        });
+        break;
       default:
     }
     if (req) {
@@ -782,7 +800,6 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
    * - `getUserMedia`
    */
   async startAudio() {
-    await this.acquireAudioContext();
     const elements: Array<HTMLMediaElement> = [];
     const browser = getBrowser();
     if (browser && browser.os === 'iOS') {
@@ -831,12 +848,13 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     });
 
     try {
-      await Promise.all(
-        elements.map((e) => {
+      await Promise.all([
+        this.acquireAudioContext(),
+        ...elements.map((e) => {
           e.muted = false;
           return e.play();
         }),
-      );
+      ]);
       this.handleAudioPlaybackStarted();
     } catch (err) {
       this.handleAudioPlaybackFailed(err);
@@ -1007,7 +1025,15 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       log.warn('tried to create RemoteParticipant for local participant');
       return;
     }
-    const participant = this.getOrCreateParticipant(participantId);
+
+    const participant = this.participants.get(participantId) as RemoteParticipant | undefined;
+
+    if (!participant) {
+      log.error(
+        `Tried to add a track for a participant, that's not present. Sid: ${participantId}`,
+      );
+      return;
+    }
 
     let adaptiveStreamSettings: AdaptiveStreamSettings | undefined;
     if (this.options.adaptiveStream) {
@@ -1405,6 +1431,8 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       this.participants.forEach((participant) => participant.setAudioContext(this.audioContext));
     }
 
+    this.localParticipant.setAudioContext(this.audioContext);
+
     const newContextIsRunning = this.audioContext?.state === 'running';
     if (newContextIsRunning !== this.canPlaybackAudio) {
       this.audioEnabled = newContextIsRunning;
@@ -1425,20 +1453,17 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     return participant;
   }
 
-  private getOrCreateParticipant(id: string, info?: ParticipantInfo): RemoteParticipant {
+  private getOrCreateParticipant(id: string, info: ParticipantInfo): RemoteParticipant {
     if (this.participants.has(id)) {
       return this.participants.get(id) as RemoteParticipant;
     }
-    // it's possible for the RTC track to arrive before signaling data
-    // when this happens, we'll create the participant and make the track work
     const participant = this.createParticipant(id, info);
     this.participants.set(id, participant);
-    if (info) {
-      this.identityToSid.set(info.identity, info.sid);
-      // if we have valid info and the participant wasn't in the map before, we can assume the participant is new
-      // firing here to make sure that `ParticipantConnected` fires before the initial track events
-      this.emitWhenConnected(RoomEvent.ParticipantConnected, participant);
-    }
+
+    this.identityToSid.set(info.identity, info.sid);
+    // if we have valid info and the participant wasn't in the map before, we can assume the participant is new
+    // firing here to make sure that `ParticipantConnected` fires before the initial track events
+    this.emitWhenConnected(RoomEvent.ParticipantConnected, participant);
 
     // also forward events
     // trackPublished is only fired for tracks added after both local participant

package/src/room/defaults.ts

@@ -46,4 +46,5 @@ export const roomConnectOptionDefaults: InternalRoomConnectOptions = {
   autoSubscribe: true,
   maxRetries: 1,
   peerConnectionTimeout: 15_000,
+  websocketTimeout: 15_000,
 } as const;

package/src/room/participant/LocalParticipant.ts

@@ -104,6 +104,10 @@ export default class LocalParticipant extends Participant {
     return this.microphoneError;
   }
 
+  get isE2EEEnabled(): boolean {
+    return this.encryptionType !== Encryption_Type.NONE;
+  }
+
   getTrack(source: Track.Source): LocalTrackPublication | undefined {
     const track = super.getTrack(source);
     if (track) {
@@ -457,7 +461,12 @@ export default class LocalParticipant extends Participant {
     screenVideo.source = Track.Source.ScreenShare;
     const localTracks: Array<LocalTrack> = [screenVideo];
     if (stream.getAudioTracks().length > 0) {
-      const screenAudio = new LocalAudioTrack(stream.getAudioTracks()[0], undefined, false);
+      const screenAudio = new LocalAudioTrack(
+        stream.getAudioTracks()[0],
+        undefined,
+        false,
+        this.audioContext,
+      );
       screenAudio.source = Track.Source.ScreenShareAudio;
       localTracks.push(screenAudio);
     }
@@ -505,7 +514,7 @@ export default class LocalParticipant extends Participant {
     if (track instanceof MediaStreamTrack) {
       switch (track.kind) {
         case 'audio':
-          track = new LocalAudioTrack(track, defaultConstraints, true);
+          track = new LocalAudioTrack(track, defaultConstraints, true, this.audioContext);
           break;
         case 'video':
           track = new LocalVideoTrack(track, defaultConstraints, true);
@@ -515,6 +524,10 @@ export default class LocalParticipant extends Participant {
       }
     }
 
+    if (track instanceof LocalAudioTrack) {
+      track.setAudioContext(this.audioContext);
+    }
+
     // is it already published? if so skip
     let existingPublication: LocalTrackPublication | undefined;
     this.tracks.forEach((publication) => {
@@ -673,6 +686,9 @@ export default class LocalParticipant extends Participant {
 
         // set up backup
         if (opts.videoCodec && opts.backupCodec && opts.videoCodec !== opts.backupCodec.codec) {
+          if (!this.roomOptions.dynacast) {
+            this.roomOptions.dynacast = true;
+          }
           const simOpts = { ...opts };
           simOpts.simulcast = true;
           simEncodings = computeTrackBackupEncodings(track, opts.backupCodec.codec, simOpts);

package/src/room/participant/Participant.ts

@@ -9,7 +9,9 @@ import {
   SubscriptionError,
 } from '../../proto/livekit_models_pb';
 import { ParticipantEvent, TrackEvent } from '../events';
+import LocalAudioTrack from '../track/LocalAudioTrack';
 import type LocalTrackPublication from '../track/LocalTrackPublication';
+import RemoteAudioTrack from '../track/RemoteAudioTrack';
 import type RemoteTrack from '../track/RemoteTrack';
 import type RemoteTrackPublication from '../track/RemoteTrackPublication';
 import { Track } from '../track/Track';
@@ -69,6 +71,8 @@ export default class Participant extends (EventEmitter as new () => TypedEmitter
 
   private _connectionQuality: ConnectionQuality = ConnectionQuality.Unknown;
 
+  protected audioContext?: AudioContext;
+
   get isEncrypted() {
     return this.tracks.size > 0 && Array.from(this.tracks.values()).every((tr) => tr.isEncrypted);
   }
@@ -238,6 +242,18 @@ export default class Participant extends (EventEmitter as new () => TypedEmitter
     }
   }
 
+  /**
+   * @internal
+   */
+  setAudioContext(ctx: AudioContext | undefined) {
+    this.audioContext = ctx;
+    this.audioTracks.forEach(
+      (track) =>
+        (track.track instanceof RemoteAudioTrack || track.track instanceof LocalAudioTrack) &&
+        track.track.setAudioContext(ctx),
+    );
+  }
+
   protected addTrackPublication(publication: TrackPublication) {
     // forward publication driven events
     publication.on(TrackEvent.Muted, () => {

package/src/room/participant/RemoteParticipant.ts

@@ -25,8 +25,6 @@ export default class RemoteParticipant extends Participant {
 
   private volumeMap: Map<Track.Source, number>;
 
-  private audioContext?: AudioContext;
-
   private audioOutput?: AudioOutputOptions;
 
   /** @internal */
@@ -324,16 +322,6 @@ export default class RemoteParticipant extends Participant {
     }
   }
 
-  /**
-   * @internal
-   */
-  setAudioContext(ctx: AudioContext | undefined) {
-    this.audioContext = ctx;
-    this.audioTracks.forEach(
-      (track) => track.track instanceof RemoteAudioTrack && track.track.setAudioContext(ctx),
-    );
-  }
-
   /**
    * @internal
    */

package/src/room/track/LocalAudioTrack.ts

@@ -6,12 +6,15 @@ import { isWeb, unwrapConstraint } from '../utils';
 import LocalTrack from './LocalTrack';
 import { Track } from './Track';
 import type { AudioCaptureOptions } from './options';
+import type { TrackProcessor } from './processor/types';
 import { constraintsForOptions, detectSilence } from './utils';
 
 export default class LocalAudioTrack extends LocalTrack {
   /** @internal */
   stopOnMute: boolean = false;
 
+  private audioContext?: AudioContext;
+
   private prevStats?: AudioSenderStats;
 
   /**
@@ -24,8 +27,10 @@ export default class LocalAudioTrack extends LocalTrack {
     mediaTrack: MediaStreamTrack,
     constraints?: MediaTrackConstraints,
     userProvidedTrack = true,
+    audioContext?: AudioContext,
   ) {
     super(mediaTrack, Track.Kind.Audio, constraints, userProvidedTrack);
+    this.audioContext = audioContext;
     this.checkForSilence();
   }
 
@@ -133,6 +138,46 @@ export default class LocalAudioTrack extends LocalTrack {
     this.prevStats = stats;
   };
 
+  async setProcessor(processor: TrackProcessor<typeof this.kind>) {
+    const unlock = await this.processorLock.lock();
+    try {
+      if (!this.audioContext) {
+        throw Error(
+          'Audio context needs to be set on LocalAudioTrack in order to enable processors',
+        );
+      }
+      if (this.processor) {
+        await this.stopProcessor();
+      }
+      if (this.kind === 'unknown') {
+        throw TypeError('cannot set processor on track of unknown kind');
+      }
+
+      const processorOptions = {
+        kind: this.kind,
+        track: this._mediaStreamTrack,
+        audioContext: this.audioContext,
+      };
+      log.debug(`setting up audio processor ${processor.name}`);
+
+      await processor.init(processorOptions);
+      this.processor = processor;
+      if (this.processor.processedTrack) {
+        await this.sender?.replaceTrack(this.processor.processedTrack);
+      }
+    } finally {
+      unlock();
+    }
+  }
+
+  /**
+   * @internal
+   * @experimental
+   */
+  setAudioContext(audioContext: AudioContext | undefined) {
+    this.audioContext = audioContext;
+  }
+
   async getSenderStats(): Promise<AudioSenderStats | undefined> {
     if (!this.sender?.getStats) {
       return undefined;

package/src/room/track/LocalTrack.ts

@@ -128,8 +128,23 @@ export default abstract class LocalTrack extends Track {
       newTrack.addEventListener('unmute', this.handleTrackUnmuteEvent);
       this._constraints = newTrack.getConstraints();
     }
+    let processedTrack: MediaStreamTrack | undefined;
+    if (this.processor && newTrack && this.processorElement) {
+      log.debug('restarting processor');
+      if (this.kind === 'unknown') {
+        throw TypeError('cannot set processor on track of unknown kind');
+      }
+
+      attachToElement(newTrack, this.processorElement);
+      await this.processor.restart({
+        track: newTrack,
+        kind: this.kind,
+        element: this.processorElement,
+      });
+      processedTrack = this.processor.processedTrack;
+    }
     if (this.sender) {
-      await this.sender.replaceTrack(newTrack);
+      await this.sender.replaceTrack(processedTrack ?? newTrack);
     }
     this._mediaStreamTrack = newTrack;
     if (newTrack) {
@@ -138,7 +153,7 @@ export default abstract class LocalTrack extends Track {
       // when a valid track is replace, we'd want to start producing
       await this.resumeUpstream();
       this.attachedElements.forEach((el) => {
-        attachToElement(newTrack, el);
+        attachToElement(processedTrack ?? newTrack, el);
       });
     }
   }
@@ -236,14 +251,7 @@ export default abstract class LocalTrack extends Track {
 
     await this.setMediaStreamTrack(newTrack);
     this._constraints = constraints;
-    if (this.processor) {
-      const processor = this.processor;
-      await this.setProcessor(processor);
-    } else {
-      this.attachedElements.forEach((el) => {
-        attachToElement(this._mediaStreamTrack, el);
-      });
-    }
+
     this.emit(TrackEvent.Restarted, this);
     return this;
   }
@@ -320,7 +328,7 @@ export default abstract class LocalTrack extends Track {
    * the server.
    * this API is unsupported on Safari < 12 due to a bug
    **/
-  pauseUpstream = async () => {
+  async pauseUpstream() {
     const unlock = await this.pauseUpstreamLock.lock();
     try {
       if (this._isUpstreamPaused === true) {
@@ -342,9 +350,9 @@ export default abstract class LocalTrack extends Track {
     } finally {
       unlock();
     }
-  };
+  }
 
-  resumeUpstream = async () => {
+  async resumeUpstream() {
     const unlock = await this.pauseUpstreamLock.lock();
     try {
       if (this._isUpstreamPaused === false) {
@@ -362,7 +370,7 @@ export default abstract class LocalTrack extends Track {
     } finally {
       unlock();
     }
-  };
+  }
 
   /**
    * Sets a processor on this track.

package/src/room/track/LocalVideoTrack.ts

@@ -9,6 +9,7 @@ import { Mutex, isFireFox, isMobile, isWeb, unwrapConstraint } from '../utils';
 import LocalTrack from './LocalTrack';
 import { Track } from './Track';
 import type { VideoCaptureOptions, VideoCodec } from './options';
+import type { TrackProcessor } from './processor/types';
 import { constraintsForOptions } from './utils';
 
 export class SimulcastTrackInfo {
@@ -98,6 +99,20 @@ export default class LocalVideoTrack extends LocalTrack {
     super.stop();
   }
 
+  async pauseUpstream() {
+    await super.pauseUpstream();
+    for await (const sc of this.simulcastCodecs.values()) {
+      await sc.sender?.replaceTrack(null);
+    }
+  }
+
+  async resumeUpstream() {
+    await super.resumeUpstream();
+    for await (const sc of this.simulcastCodecs.values()) {
+      await sc.sender?.replaceTrack(sc.mediaStreamTrack);
+    }
+  }
+
   async mute(): Promise<LocalVideoTrack> {
     const unlock = await this.muteLock.lock();
     try {
@@ -127,6 +142,13 @@ export default class LocalVideoTrack extends LocalTrack {
     }
   }
 
+  protected setTrackMuted(muted: boolean) {
+    super.setTrackMuted(muted);
+    for (const sc of this.simulcastCodecs.values()) {
+      sc.mediaStreamTrack.enabled = !muted;
+    }
+  }
+
   async getSenderStats(): Promise<VideoSenderStats[]> {
     if (!this.sender?.getStats) {
       return [];
@@ -211,6 +233,23 @@ export default class LocalVideoTrack extends LocalTrack {
       }
     }
     await this.restart(constraints);
+
+    for await (const sc of this.simulcastCodecs.values()) {
+      if (sc.sender) {
+        sc.mediaStreamTrack = this.mediaStreamTrack.clone();
+        await sc.sender.replaceTrack(sc.mediaStreamTrack);
+      }
+    }
+  }
+
+  async setProcessor(processor: TrackProcessor<Track.Kind>, showProcessedStreamLocally = true) {
+    await super.setProcessor(processor, showProcessedStreamLocally);
+
+    if (this.processor?.processedTrack) {
+      for await (const sc of this.simulcastCodecs.values()) {
+        await sc.sender?.replaceTrack(this.processor.processedTrack);
+      }
+    }
   }
 
   addSimulcastTrack(codec: VideoCodec, encodings?: RTCRtpEncodingParameters[]): SimulcastTrackInfo {

package/src/room/track/RemoteAudioTrack.ts

@@ -2,7 +2,7 @@ import log from '../../logger';
 import { TrackEvent } from '../events';
 import { computeBitrate } from '../stats';
 import type { AudioReceiverStats } from '../stats';
-import { supportsSetSinkId } from '../utils';
+import { isReactNative, supportsSetSinkId } from '../utils';
 import RemoteTrack from './RemoteTrack';
 import { Track } from './Track';
 import type { AudioOutputOptions } from './options';
@@ -48,6 +48,10 @@ export default class RemoteAudioTrack extends RemoteTrack {
         el.volume = volume;
       }
     }
+    if (isReactNative()) {
+      // @ts-ignore
+      this._mediaStreamTrack._setVolume(volume);
+    }
     this.elementVolume = volume;
   }
 
@@ -58,6 +62,10 @@ export default class RemoteAudioTrack extends RemoteTrack {
     if (this.elementVolume) {
       return this.elementVolume;
     }
+    if (isReactNative()) {
+      // RN volume value defaults to 1.0 if hasn't been changed.
+      return 1.0;
+    }
     let highestVolume = 0;
     this.attachedElements.forEach((element) => {
       if (element.volume > highestVolume) {

package/src/room/track/RemoteTrackPublication.ts

@@ -297,8 +297,8 @@ export default class RemoteTrackPublication extends TrackPublication {
       fps: this.fps,
     });
     if (this.videoDimensions) {
-      settings.width = this.videoDimensions.width;
-      settings.height = this.videoDimensions.height;
+      settings.width = Math.ceil(this.videoDimensions.width);
+      settings.height = Math.ceil(this.videoDimensions.height);
     } else if (this.currentVideoQuality !== undefined) {
       settings.quality = this.currentVideoQuality;
     } else {

package/src/room/track/facingMode.ts

@@ -1,4 +1,4 @@
-import log from 'loglevel';
+import log from '../../logger';
 import LocalTrack from './LocalTrack';
 import type { VideoCaptureOptions } from './options';
 

package/src/room/track/processor/types.ts

@@ -12,9 +12,25 @@ export type ProcessorOptions<T extends Track.Kind> = {
 /**
  * @experimental
  */
-export interface TrackProcessor<T extends Track.Kind> {
+export interface AudioProcessorOptions extends ProcessorOptions<Track.Kind.Audio> {
+  audioContext: AudioContext;
+}
+
+/**
+ * @experimental
+ */
+export interface VideoProcessorOptions extends ProcessorOptions<Track.Kind.Video> {}
+
+/**
+ * @experimental
+ */
+export interface TrackProcessor<
+  T extends Track.Kind,
+  U extends ProcessorOptions<T> = ProcessorOptions<T>,
+> {
   name: string;
-  init: (opts: ProcessorOptions<T>) => void;
+  init: (opts: U) => Promise<void>;
+  restart: (opts: U) => Promise<void>;
   destroy: () => Promise<void>;
   processedTrack?: MediaStreamTrack;
 }

package/src/room/types.ts

@@ -36,4 +36,8 @@ export type SimulationScenario =
   | 'resume-reconnect'
   | 'force-tcp'
   | 'force-tls'
-  | 'full-reconnect';
+  | 'full-reconnect'
+  // overrides server-side bandwidth estimator with set bandwidth
+  // this can be used to test application behavior when congested or
+  // to disable congestion control entirely (by setting bandwidth to 100Mbps)
+  | 'subscriber-bandwidth';