livekit-client 1.12.3 → 1.13.0

package/src/proto/livekit_rtc_pb.ts

@@ -1,3 +1,17 @@
+// Copyright 2023 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // @generated by protoc-gen-es v1.3.0 with parameter "target=ts"
 // @generated from file livekit_rtc.proto (package livekit, syntax proto3)
 /* eslint-disable */

package/src/room/DeviceManager.ts

@@ -37,9 +37,8 @@ export default class DeviceManager {
 
     if (
       requestPermissions &&
-      kind &&
       // for safari we need to skip this check, as otherwise it will re-acquire user media and fail on iOS https://bugs.webkit.org/show_bug.cgi?id=179363
-      (!DeviceManager.userMediaPromiseMap.get(kind) || !isSafari())
+      !(isSafari() && this.hasDeviceInUse(kind))
     ) {
       const isDummyDeviceOrEmpty =
         devices.length === 0 ||
@@ -85,4 +84,10 @@ export default class DeviceManager {
 
     return device?.deviceId;
   }
+
+  private hasDeviceInUse(kind?: MediaDeviceKind): boolean {
+    return kind
+      ? DeviceManager.userMediaPromiseMap.has(kind)
+      : DeviceManager.userMediaPromiseMap.size > 0;
+  }
 }

package/src/room/RTCEngine.ts

@@ -191,8 +191,11 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
     this.url = url;
     this.token = token;
     this.signalOpts = opts;
+    this.maxJoinAttempts = opts.maxRetries;
     try {
       this.joinAttempts += 1;
+
+      this.setupSignalClientCallbacks();
       const joinResponse = await this.client.join(url, token, opts, abortSignal);
       this._isClosed = false;
       this.latestJoinResponse = joinResponse;
@@ -206,7 +209,6 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
       if (!this.subscriberPrimary) {
         this.negotiate();
       }
-      this.setupSignalClientCallbacks();
 
       this.clientConfiguration = joinResponse.clientConfiguration;
 

package/src/room/Room.ts

@@ -424,13 +424,15 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
         this.abortController.abort();
       }
 
-      this.abortController = new AbortController();
+      // explicit creation as local var needed to satisfy TS compiler when passing it to `attemptConnection` further down
+      const abortController = new AbortController();
+      this.abortController = abortController;
 
       // at this point the intention to connect has been signalled so we can allow cancelling of the connection via disconnect() again
       unlockDisconnect?.();
 
       try {
-        await this.attemptConnection(regionUrl ?? url, token, opts, this.abortController);
+        await this.attemptConnection(regionUrl ?? url, token, opts, abortController);
         this.abortController = undefined;
         resolve();
       } catch (e) {
@@ -500,6 +502,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
           typeof roomOptions.adaptiveStream === 'object' ? true : roomOptions.adaptiveStream,
         maxRetries: connectOptions.maxRetries,
         e2eeEnabled: !!this.e2eeManager,
+        websocketTimeout: connectOptions.websocketTimeout,
       },
       abortController.signal,
     );
@@ -540,6 +543,10 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     if (joinResponse.room) {
       this.handleRoomUpdate(joinResponse.room);
     }
+
+    if (this.options.e2ee && this.e2eeManager) {
+      this.e2eeManager.setSifTrailer(joinResponse.sifTrailer);
+    }
   };
 
   private attemptConnection = async (
@@ -692,7 +699,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
   /**
    * @internal for testing
    */
-  async simulateScenario(scenario: SimulationScenario) {
+  async simulateScenario(scenario: SimulationScenario, arg?: any) {
     let postAction = () => {};
     let req: SimulateScenario | undefined;
     switch (scenario) {
@@ -762,6 +769,17 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
           }
         };
         break;
+      case 'subscriber-bandwidth':
+        if (arg === undefined || typeof arg !== 'number') {
+          throw new Error('subscriber-bandwidth requires a number as argument');
+        }
+        req = new SimulateScenario({
+          scenario: {
+            case: 'subscriberBandwidth',
+            value: BigInt(arg),
+          },
+        });
+        break;
       default:
     }
     if (req) {
@@ -782,7 +800,6 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
    * - `getUserMedia`
    */
   async startAudio() {
-    await this.acquireAudioContext();
     const elements: Array<HTMLMediaElement> = [];
     const browser = getBrowser();
     if (browser && browser.os === 'iOS') {
@@ -831,12 +848,13 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     });
 
     try {
-      await Promise.all(
-        elements.map((e) => {
+      await Promise.all([
+        this.acquireAudioContext(),
+        ...elements.map((e) => {
           e.muted = false;
           return e.play();
         }),
-      );
+      ]);
       this.handleAudioPlaybackStarted();
     } catch (err) {
       this.handleAudioPlaybackFailed(err);
@@ -1405,6 +1423,8 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       this.participants.forEach((participant) => participant.setAudioContext(this.audioContext));
     }
 
+    this.localParticipant.setAudioContext(this.audioContext);
+
     const newContextIsRunning = this.audioContext?.state === 'running';
     if (newContextIsRunning !== this.canPlaybackAudio) {
       this.audioEnabled = newContextIsRunning;

package/src/room/defaults.ts

@@ -46,4 +46,5 @@ export const roomConnectOptionDefaults: InternalRoomConnectOptions = {
   autoSubscribe: true,
   maxRetries: 1,
   peerConnectionTimeout: 15_000,
+  websocketTimeout: 15_000,
 } as const;

package/src/room/participant/LocalParticipant.ts

@@ -457,7 +457,12 @@ export default class LocalParticipant extends Participant {
     screenVideo.source = Track.Source.ScreenShare;
     const localTracks: Array<LocalTrack> = [screenVideo];
     if (stream.getAudioTracks().length > 0) {
-      const screenAudio = new LocalAudioTrack(stream.getAudioTracks()[0], undefined, false);
+      const screenAudio = new LocalAudioTrack(
+        stream.getAudioTracks()[0],
+        undefined,
+        false,
+        this.audioContext,
+      );
       screenAudio.source = Track.Source.ScreenShareAudio;
       localTracks.push(screenAudio);
     }
@@ -505,7 +510,7 @@ export default class LocalParticipant extends Participant {
     if (track instanceof MediaStreamTrack) {
       switch (track.kind) {
         case 'audio':
-          track = new LocalAudioTrack(track, defaultConstraints, true);
+          track = new LocalAudioTrack(track, defaultConstraints, true, this.audioContext);
           break;
         case 'video':
           track = new LocalVideoTrack(track, defaultConstraints, true);
@@ -515,6 +520,10 @@ export default class LocalParticipant extends Participant {
       }
     }
 
+    if (track instanceof LocalAudioTrack) {
+      track.setAudioContext(this.audioContext);
+    }
+
     // is it already published? if so skip
     let existingPublication: LocalTrackPublication | undefined;
     this.tracks.forEach((publication) => {
@@ -673,6 +682,9 @@ export default class LocalParticipant extends Participant {
 
         // set up backup
         if (opts.videoCodec && opts.backupCodec && opts.videoCodec !== opts.backupCodec.codec) {
+          if (!this.roomOptions.dynacast) {
+            this.roomOptions.dynacast = true;
+          }
           const simOpts = { ...opts };
           simOpts.simulcast = true;
           simEncodings = computeTrackBackupEncodings(track, opts.backupCodec.codec, simOpts);

package/src/room/participant/Participant.ts

@@ -9,7 +9,9 @@ import {
   SubscriptionError,
 } from '../../proto/livekit_models_pb';
 import { ParticipantEvent, TrackEvent } from '../events';
+import LocalAudioTrack from '../track/LocalAudioTrack';
 import type LocalTrackPublication from '../track/LocalTrackPublication';
+import RemoteAudioTrack from '../track/RemoteAudioTrack';
 import type RemoteTrack from '../track/RemoteTrack';
 import type RemoteTrackPublication from '../track/RemoteTrackPublication';
 import { Track } from '../track/Track';
@@ -69,6 +71,8 @@ export default class Participant extends (EventEmitter as new () => TypedEmitter
 
   private _connectionQuality: ConnectionQuality = ConnectionQuality.Unknown;
 
+  protected audioContext?: AudioContext;
+
   get isEncrypted() {
     return this.tracks.size > 0 && Array.from(this.tracks.values()).every((tr) => tr.isEncrypted);
   }
@@ -238,6 +242,18 @@ export default class Participant extends (EventEmitter as new () => TypedEmitter
     }
   }
 
+  /**
+   * @internal
+   */
+  setAudioContext(ctx: AudioContext | undefined) {
+    this.audioContext = ctx;
+    this.audioTracks.forEach(
+      (track) =>
+        (track.track instanceof RemoteAudioTrack || track.track instanceof LocalAudioTrack) &&
+        track.track.setAudioContext(ctx),
+    );
+  }
+
   protected addTrackPublication(publication: TrackPublication) {
     // forward publication driven events
     publication.on(TrackEvent.Muted, () => {

package/src/room/participant/RemoteParticipant.ts

@@ -25,8 +25,6 @@ export default class RemoteParticipant extends Participant {
 
   private volumeMap: Map<Track.Source, number>;
 
-  private audioContext?: AudioContext;
-
   private audioOutput?: AudioOutputOptions;
 
   /** @internal */
@@ -324,16 +322,6 @@ export default class RemoteParticipant extends Participant {
     }
   }
 
-  /**
-   * @internal
-   */
-  setAudioContext(ctx: AudioContext | undefined) {
-    this.audioContext = ctx;
-    this.audioTracks.forEach(
-      (track) => track.track instanceof RemoteAudioTrack && track.track.setAudioContext(ctx),
-    );
-  }
-
   /**
    * @internal
    */

package/src/room/track/LocalAudioTrack.ts

@@ -6,12 +6,15 @@ import { isWeb, unwrapConstraint } from '../utils';
 import LocalTrack from './LocalTrack';
 import { Track } from './Track';
 import type { AudioCaptureOptions } from './options';
+import type { TrackProcessor } from './processor/types';
 import { constraintsForOptions, detectSilence } from './utils';
 
 export default class LocalAudioTrack extends LocalTrack {
   /** @internal */
   stopOnMute: boolean = false;
 
+  private audioContext?: AudioContext;
+
   private prevStats?: AudioSenderStats;
 
   /**
@@ -24,8 +27,10 @@ export default class LocalAudioTrack extends LocalTrack {
     mediaTrack: MediaStreamTrack,
     constraints?: MediaTrackConstraints,
     userProvidedTrack = true,
+    audioContext?: AudioContext,
   ) {
     super(mediaTrack, Track.Kind.Audio, constraints, userProvidedTrack);
+    this.audioContext = audioContext;
     this.checkForSilence();
   }
 
@@ -133,6 +138,46 @@ export default class LocalAudioTrack extends LocalTrack {
     this.prevStats = stats;
   };
 
+  async setProcessor(processor: TrackProcessor<typeof this.kind>) {
+    const unlock = await this.processorLock.lock();
+    try {
+      if (!this.audioContext) {
+        throw Error(
+          'Audio context needs to be set on LocalAudioTrack in order to enable processors',
+        );
+      }
+      if (this.processor) {
+        await this.stopProcessor();
+      }
+      if (this.kind === 'unknown') {
+        throw TypeError('cannot set processor on track of unknown kind');
+      }
+
+      const processorOptions = {
+        kind: this.kind,
+        track: this._mediaStreamTrack,
+        audioContext: this.audioContext,
+      };
+      log.debug(`setting up audio processor ${processor.name}`);
+
+      await processor.init(processorOptions);
+      this.processor = processor;
+      if (this.processor.processedTrack) {
+        await this.sender?.replaceTrack(this.processor.processedTrack);
+      }
+    } finally {
+      unlock();
+    }
+  }
+
+  /**
+   * @internal
+   * @experimental
+   */
+  setAudioContext(audioContext: AudioContext | undefined) {
+    this.audioContext = audioContext;
+  }
+
   async getSenderStats(): Promise<AudioSenderStats | undefined> {
     if (!this.sender?.getStats) {
       return undefined;

package/src/room/track/LocalTrack.ts

@@ -320,7 +320,7 @@ export default abstract class LocalTrack extends Track {
    * the server.
    * this API is unsupported on Safari < 12 due to a bug
    **/
-  pauseUpstream = async () => {
+  async pauseUpstream() {
     const unlock = await this.pauseUpstreamLock.lock();
     try {
       if (this._isUpstreamPaused === true) {
@@ -342,9 +342,9 @@ export default abstract class LocalTrack extends Track {
     } finally {
       unlock();
     }
-  };
+  }
 
-  resumeUpstream = async () => {
+  async resumeUpstream() {
     const unlock = await this.pauseUpstreamLock.lock();
     try {
       if (this._isUpstreamPaused === false) {
@@ -362,7 +362,7 @@ export default abstract class LocalTrack extends Track {
     } finally {
       unlock();
     }
-  };
+  }
 
   /**
    * Sets a processor on this track.

package/src/room/track/LocalVideoTrack.ts

@@ -9,6 +9,7 @@ import { Mutex, isFireFox, isMobile, isWeb, unwrapConstraint } from '../utils';
 import LocalTrack from './LocalTrack';
 import { Track } from './Track';
 import type { VideoCaptureOptions, VideoCodec } from './options';
+import type { TrackProcessor } from './processor/types';
 import { constraintsForOptions } from './utils';
 
 export class SimulcastTrackInfo {
@@ -98,6 +99,20 @@ export default class LocalVideoTrack extends LocalTrack {
     super.stop();
   }
 
+  async pauseUpstream() {
+    await super.pauseUpstream();
+    for await (const sc of this.simulcastCodecs.values()) {
+      await sc.sender?.replaceTrack(null);
+    }
+  }
+
+  async resumeUpstream() {
+    await super.resumeUpstream();
+    for await (const sc of this.simulcastCodecs.values()) {
+      await sc.sender?.replaceTrack(sc.mediaStreamTrack);
+    }
+  }
+
   async mute(): Promise<LocalVideoTrack> {
     const unlock = await this.muteLock.lock();
     try {
@@ -127,6 +142,13 @@ export default class LocalVideoTrack extends LocalTrack {
     }
   }
 
+  protected setTrackMuted(muted: boolean) {
+    super.setTrackMuted(muted);
+    for (const sc of this.simulcastCodecs.values()) {
+      sc.mediaStreamTrack.enabled = !muted;
+    }
+  }
+
   async getSenderStats(): Promise<VideoSenderStats[]> {
     if (!this.sender?.getStats) {
       return [];
@@ -211,6 +233,23 @@ export default class LocalVideoTrack extends LocalTrack {
       }
     }
     await this.restart(constraints);
+
+    for await (const sc of this.simulcastCodecs.values()) {
+      if (sc.sender) {
+        sc.mediaStreamTrack = this.mediaStreamTrack.clone();
+        await sc.sender.replaceTrack(sc.mediaStreamTrack);
+      }
+    }
+  }
+
+  async setProcessor(processor: TrackProcessor<Track.Kind>, showProcessedStreamLocally = true) {
+    await super.setProcessor(processor, showProcessedStreamLocally);
+
+    if (this.processor?.processedTrack) {
+      for await (const sc of this.simulcastCodecs.values()) {
+        await sc.sender?.replaceTrack(this.processor.processedTrack);
+      }
+    }
   }
 
   addSimulcastTrack(codec: VideoCodec, encodings?: RTCRtpEncodingParameters[]): SimulcastTrackInfo {

package/src/room/track/RemoteAudioTrack.ts

@@ -2,7 +2,7 @@ import log from '../../logger';
 import { TrackEvent } from '../events';
 import { computeBitrate } from '../stats';
 import type { AudioReceiverStats } from '../stats';
-import { supportsSetSinkId } from '../utils';
+import { isReactNative, supportsSetSinkId } from '../utils';
 import RemoteTrack from './RemoteTrack';
 import { Track } from './Track';
 import type { AudioOutputOptions } from './options';
@@ -48,6 +48,10 @@ export default class RemoteAudioTrack extends RemoteTrack {
         el.volume = volume;
       }
     }
+    if (isReactNative()) {
+      // @ts-ignore
+      this._mediaStreamTrack._setVolume(volume);
+    }
     this.elementVolume = volume;
   }
 
@@ -58,6 +62,10 @@ export default class RemoteAudioTrack extends RemoteTrack {
     if (this.elementVolume) {
       return this.elementVolume;
     }
+    if (isReactNative()) {
+      // RN volume value defaults to 1.0 if hasn't been changed.
+      return 1.0;
+    }
     let highestVolume = 0;
     this.attachedElements.forEach((element) => {
       if (element.volume > highestVolume) {

package/src/room/track/RemoteTrackPublication.ts

@@ -297,8 +297,8 @@ export default class RemoteTrackPublication extends TrackPublication {
       fps: this.fps,
     });
     if (this.videoDimensions) {
-      settings.width = this.videoDimensions.width;
-      settings.height = this.videoDimensions.height;
+      settings.width = Math.ceil(this.videoDimensions.width);
+      settings.height = Math.ceil(this.videoDimensions.height);
     } else if (this.currentVideoQuality !== undefined) {
       settings.quality = this.currentVideoQuality;
     } else {

package/src/room/track/processor/types.ts

@@ -12,9 +12,24 @@ export type ProcessorOptions<T extends Track.Kind> = {
 /**
  * @experimental
  */
-export interface TrackProcessor<T extends Track.Kind> {
+export interface AudioProcessorOptions extends ProcessorOptions<Track.Kind.Audio> {
+  audioContext: AudioContext;
+}
+
+/**
+ * @experimental
+ */
+export interface VideoProcessorOptions extends ProcessorOptions<Track.Kind.Video> {}
+
+/**
+ * @experimental
+ */
+export interface TrackProcessor<
+  T extends Track.Kind,
+  U extends ProcessorOptions<T> = ProcessorOptions<T>,
+> {
   name: string;
-  init: (opts: ProcessorOptions<T>) => void;
+  init: (opts: U) => void;
   destroy: () => Promise<void>;
   processedTrack?: MediaStreamTrack;
 }

package/src/room/types.ts

@@ -36,4 +36,8 @@ export type SimulationScenario =
   | 'resume-reconnect'
   | 'force-tcp'
   | 'force-tls'
-  | 'full-reconnect';
+  | 'full-reconnect'
+  // overrides server-side bandwidth estimator with set bandwidth
+  // this can be used to test application behavior when congested or
+  // to disable congestion control entirely (by setting bandwidth to 100Mbps)
+  | 'subscriber-bandwidth';