livekit-client 2.0.9 → 2.1.0

package/dist/ts4.2/src/room/track/Track.d.ts

@@ -1,4 +1,4 @@
-import { StreamState as ProtoStreamState, TrackSource, TrackType } from '@livekit/protocol';
+import { AudioTrackFeature, StreamState as ProtoStreamState, TrackSource, TrackType } from '@livekit/protocol';
 import type TypedEventEmitter from 'typed-emitter';
 import type { SignalClient } from '../../api/SignalClient';
 import { StructuredLogger } from '../../logger';
@@ -136,6 +136,7 @@ export type TrackEventCallbacks = {
     upstreamPaused: (track: any) => void;
     upstreamResumed: (track: any) => void;
     trackProcessorUpdate: (processor?: TrackProcessor<Track.Kind, any>) => void;
+    audioTrackFeatureUpdate: (track: any, feature: AudioTrackFeature, enabled: boolean) => void;
 };
 export {};
 //# sourceMappingURL=Track.d.ts.map

package/dist/ts4.2/src/room/track/options.d.ts

@@ -263,7 +263,7 @@ export declare function isBackupCodec(codec: string): codec is BackupVideoCodec;
 /**
  * scalability modes for svc.
  */
-export type ScalabilityMode = 'L1T3' | 'L2T3' | 'L2T3_KEY' | 'L3T3' | 'L3T3_KEY';
+export type ScalabilityMode = 'L1T1' | 'L1T2' | 'L1T3' | 'L2T1' | 'L2T1h' | 'L2T1_KEY' | 'L2T2' | 'L2T2h' | 'L2T2_KEY' | 'L2T3' | 'L2T3h' | 'L2T3_KEY' | 'L3T1' | 'L3T1h' | 'L3T1_KEY' | 'L3T2' | 'L3T2h' | 'L3T2_KEY' | 'L3T3' | 'L3T3h' | 'L3T3_KEY';
 export declare namespace AudioPresets {
     const telephone: AudioPreset;
     const speech: AudioPreset;

package/dist/ts4.2/src/room/utils.d.ts

@@ -79,6 +79,9 @@ export declare function createAudioAnalyser(track: LocalAudioTrack | RemoteAudio
     analyser: AnalyserNode;
     cleanup: () => Promise<void>;
 };
+/**
+ * @internal
+ */
 export declare class Mutex {
     private _locking;
     private _locks;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "livekit-client",
-  "version": "2.0.9",
+  "version": "2.1.0",
   "description": "JavaScript/TypeScript client SDK for LiveKit",
   "main": "./dist/livekit-client.umd.js",
   "unpkg": "./dist/livekit-client.umd.js",
@@ -36,7 +36,7 @@
   "author": "David Zhao <david@davidzhao.com>",
   "license": "Apache-2.0",
   "dependencies": {
-    "@livekit/protocol": "1.10.4",
+    "@livekit/protocol": "1.13.0",
     "events": "^3.3.0",
     "loglevel": "^1.8.0",
     "sdp-transform": "^2.14.1",
@@ -46,8 +46,8 @@
     "webrtc-adapter": "^8.1.1"
   },
   "devDependencies": {
-    "@babel/core": "7.23.9",
-    "@babel/preset-env": "7.23.9",
+    "@babel/core": "7.24.4",
+    "@babel/preset-env": "7.24.4",
     "@bufbuild/protoc-gen-es": "^1.3.0",
     "@changesets/cli": "2.27.1",
     "@livekit/changesets-changelog-github": "^0.0.4",
@@ -65,23 +65,23 @@
     "@typescript-eslint/eslint-plugin": "5.62.0",
     "@typescript-eslint/parser": "5.62.0",
     "downlevel-dts": "^0.11.0",
-    "eslint": "8.56.0",
-    "eslint-config-airbnb-typescript": "17.1.0",
+    "eslint": "8.57.0",
+    "eslint-config-airbnb-typescript": "18.0.0",
     "eslint-config-prettier": "9.1.0",
     "eslint-plugin-ecmascript-compat": "^3.0.0",
     "eslint-plugin-import": "2.29.1",
     "gh-pages": "6.1.1",
     "jsdom": "^24.0.0",
     "prettier": "^3.0.0",
-    "rollup": "4.9.6",
+    "rollup": "4.14.0",
     "rollup-plugin-delete": "^2.0.0",
     "rollup-plugin-re": "1.0.7",
     "rollup-plugin-typescript2": "0.36.0",
     "size-limit": "^8.2.4",
-    "typedoc": "0.25.7",
+    "typedoc": "0.25.12",
     "typedoc-plugin-no-inherit": "1.4.0",
-    "typescript": "5.3.3",
-    "vite": "5.0.12",
+    "typescript": "5.4.3",
+    "vite": "5.0.13",
     "vitest": "^1.0.0"
   },
   "scripts": {

package/src/api/SignalClient.ts

@@ -1,5 +1,6 @@
 import {
   AddTrackRequest,
+  AudioTrackFeature,
   ClientInfo,
   ConnectionQualityUpdate,
   DisconnectReason,
@@ -27,6 +28,7 @@ import {
   TrackPublishedResponse,
   TrackUnpublishedResponse,
   TrickleRequest,
+  UpdateLocalAudioTrack,
   UpdateParticipantMetadata,
   UpdateSubscription,
   UpdateTrackSettings,
@@ -583,6 +585,13 @@ export class SignalClient {
     ]);
   }
 
+  sendUpdateLocalAudioTrack(trackSid: string, features: AudioTrackFeature[]) {
+    return this.sendRequest({
+      case: 'updateAudioTrack',
+      value: new UpdateLocalAudioTrack({ trackSid, features }),
+    });
+  }
+
   sendLeave() {
     return this.sendRequest({
       case: 'leave',

package/src/connectionHelper/ConnectionCheck.ts

@@ -1,7 +1,7 @@
 import { EventEmitter } from 'events';
 import type TypedEmitter from 'typed-emitter';
+import type { CheckInfo, CheckerOptions, InstantiableCheck } from './checks/Checker';
 import { CheckStatus, Checker } from './checks/Checker';
-import type { CheckInfo, InstantiableCheck } from './checks/Checker';
 import { PublishAudioCheck } from './checks/publishAudio';
 import { PublishVideoCheck } from './checks/publishVideo';
 import { ReconnectCheck } from './checks/reconnect';
@@ -16,12 +16,15 @@ export class ConnectionCheck extends (EventEmitter as new () => TypedEmitter<Con
 
   url: string;
 
+  options: CheckerOptions = {};
+
   private checkResults: Map<number, CheckInfo> = new Map();
 
-  constructor(url: string, token: string) {
+  constructor(url: string, token: string, options: CheckerOptions = {}) {
     super();
     this.url = url;
     this.token = token;
+    this.options = options;
   }
 
   private getNextCheckId() {
@@ -50,7 +53,7 @@ export class ConnectionCheck extends (EventEmitter as new () => TypedEmitter<Con
 
   async createAndRunCheck<T extends Checker>(check: InstantiableCheck<T>) {
     const checkId = this.getNextCheckId();
-    const test = new check(this.url, this.token);
+    const test = new check(this.url, this.token, this.options);
     const handleUpdate = (info: CheckInfo) => {
       this.updateCheck(checkId, info);
     };

package/src/e2ee/worker/FrameCryptor.ts

@@ -607,7 +607,6 @@ export class FrameCryptor extends BaseFrameCryptor {
     if (this.rtpMap.size === 0) {
       return undefined;
     }
-    // @ts-expect-error payloadType is not yet part of the typescript definition and currently not supported in Safari
     const payloadType = frame.getMetadata().payloadType;
     const codec = payloadType ? this.rtpMap.get(payloadType) : undefined;
     return codec;

package/src/e2ee/worker/e2ee.worker.ts

@@ -245,9 +245,11 @@ function handleSifTrailer(trailer: Uint8Array) {
 if (self.RTCTransformEvent) {
   workerLogger.debug('setup transform event');
   // @ts-ignore
-  self.onrtctransform = (event) => {
+  self.onrtctransform = (event: RTCTransformEvent) => {
+    // @ts-ignore .transformer property is part of RTCTransformEvent
     const transformer = event.transformer;
     workerLogger.debug('transformer', transformer);
+    // @ts-ignore monkey patching non standard flag
     transformer.handled = true;
     const { kind, participantIdentity, trackId, codec } = transformer.options;
     const cryptor = getTrackCryptor(participantIdentity, trackId);

package/src/index.ts

@@ -20,6 +20,7 @@ import { TrackPublication } from './room/track/TrackPublication';
 import type { LiveKitReactNativeInfo } from './room/types';
 import type { AudioAnalyserOptions } from './room/utils';
 import {
+  Mutex,
   createAudioAnalyser,
   getEmptyAudioStreamTrack,
   getEmptyVideoStreamTrack,
@@ -32,6 +33,7 @@ import {
 import { getBrowser } from './utils/browserParser';
 
 export * from './connectionHelper/ConnectionCheck';
+export * from './connectionHelper/checks/Checker';
 export * from './e2ee';
 export * from './options';
 export * from './room/errors';
@@ -79,6 +81,7 @@ export {
   supportsAdaptiveStream,
   supportsDynacast,
   supportsVP9,
+  Mutex,
 };
 export type {
   AudioAnalyserOptions,

package/src/room/RTCEngine.ts

@@ -169,6 +169,8 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
 
   private loggerOptions: LoggerOptions;
 
+  private publisherConnectionPromise: Promise<void> | undefined;
+
   constructor(private options: InternalRoomOptions) {
     super();
     this.log = getLogger(options.loggerName ?? LoggerNames.Engine);
@@ -398,6 +400,11 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
     this.pcManager.onDataChannel = this.handleDataChannel;
     this.pcManager.onStateChange = async (connectionState, publisherState, subscriberState) => {
       this.log.debug(`primary PC state changed ${connectionState}`, this.logContext);
+
+      if (['closed', 'disconnected', 'failed'].includes(publisherState)) {
+        // reset publisher connection promise
+        this.publisherConnectionPromise = undefined;
+      }
       if (connectionState === PCTransportState.CONNECTED) {
         const shouldEmit = this.pcState === PCState.New;
         this.pcState = PCState.Connected;
@@ -1179,7 +1186,10 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
   }
 
   private async ensurePublisherConnected(kind: DataPacket_Kind) {
-    await this.ensureDataTransportConnected(kind, false);
+    if (!this.publisherConnectionPromise) {
+      this.publisherConnectionPromise = this.ensureDataTransportConnected(kind, false);
+    }
+    await this.publisherConnectionPromise;
   }
 
   /* @internal */

package/src/room/events.ts

@@ -557,4 +557,9 @@ export enum TrackEvent {
    * @internal
    */
   TrackProcessorUpdate = 'trackProcessorUpdate',
+
+  /**
+   * @internal
+   */
+  AudioTrackFeatureUpdate = 'audioTrackFeatureUpdate',
 }

package/src/room/participant/LocalParticipant.ts

@@ -685,6 +685,7 @@ export default class LocalParticipant extends Participant {
     track.on(TrackEvent.Ended, this.handleTrackEnded);
     track.on(TrackEvent.UpstreamPaused, this.onTrackUpstreamPaused);
     track.on(TrackEvent.UpstreamResumed, this.onTrackUpstreamResumed);
+    track.on(TrackEvent.AudioTrackFeatureUpdate, this.onTrackFeatureUpdate);
 
     // create track publication from track
     const req = new AddTrackRequest({
@@ -1037,6 +1038,7 @@ export default class LocalParticipant extends Participant {
     track.off(TrackEvent.Ended, this.handleTrackEnded);
     track.off(TrackEvent.UpstreamPaused, this.onTrackUpstreamPaused);
     track.off(TrackEvent.UpstreamResumed, this.onTrackUpstreamResumed);
+    track.off(TrackEvent.AudioTrackFeatureUpdate, this.onTrackFeatureUpdate);
 
     if (stopOnUnpublish === undefined) {
       stopOnUnpublish = this.roomOptions?.stopLocalTrackOnUnpublish ?? true;
@@ -1293,6 +1295,18 @@ export default class LocalParticipant extends Participant {
     this.onTrackMuted(track, track.isMuted);
   };
 
+  private onTrackFeatureUpdate = (track: LocalAudioTrack) => {
+    const pub = this.audioTrackPublications.get(track.sid!);
+    if (!pub) {
+      this.log.warn(
+        `Could not update local audio track settings, missing publication for track ${track.sid}`,
+        this.logContext,
+      );
+      return;
+    }
+    this.engine.client.sendUpdateLocalAudioTrack(pub.trackSid, pub.getTrackFeatures());
+  };
+
   private handleSubscribedQualityUpdate = async (update: SubscribedQualityUpdate) => {
     if (!this.roomOptions?.dynacast) {
       return;

package/src/room/participant/publishUtils.ts

@@ -150,11 +150,12 @@ export function computeVideoEncodings(
       isSafari() ||
       (browser?.name === 'Chrome' && compareVersions(browser?.version, '113') < 0)
     ) {
+      const bitratesRatio = sm.suffix == 'h' ? 2 : 3;
       for (let i = 0; i < sm.spatial; i += 1) {
         // in legacy SVC, scaleResolutionDownBy cannot be set
         encodings.push({
           rid: videoRids[2 - i],
-          maxBitrate: videoEncoding.maxBitrate / 3 ** i,
+          maxBitrate: videoEncoding.maxBitrate / bitratesRatio ** i,
           maxFramerate: original.encoding.maxFramerate,
         });
       }

package/src/room/stats.ts

@@ -42,14 +42,20 @@ export interface VideoSenderStats extends SenderStats {
 
   frameHeight: number;
 
+  framesPerSecond: number;
+
   framesSent: number;
 
   // bandwidth, cpu, other, none
-  qualityLimitationReason: string;
+  qualityLimitationReason?: string;
+
+  qualityLimitationDurations?: Record<string, number>;
+
+  qualityLimitationResolutionChanges?: number;
 
-  qualityLimitationResolutionChanges: number;
+  retransmittedPacketsSent?: number;
 
-  retransmittedPacketsSent: number;
+  targetBitrate: number;
 }
 
 interface ReceiverStats {

package/src/room/track/LocalAudioTrack.ts

@@ -1,3 +1,4 @@
+import { AudioTrackFeature } from '@livekit/protocol';
 import { TrackEvent } from '../events';
 import { computeBitrate, monitorFrequency } from '../stats';
 import type { AudioSenderStats } from '../stats';
@@ -15,8 +16,17 @@ export default class LocalAudioTrack extends LocalTrack<Track.Kind.Audio> {
 
   private prevStats?: AudioSenderStats;
 
+  private isKrispNoiseFilterEnabled = false;
+
   protected processor?: TrackProcessor<Track.Kind.Audio, AudioProcessorOptions> | undefined;
 
+  /**
+   * boolean indicating whether enhanced noise cancellation is currently being used on this track
+   */
+  get enhancedNoiseCancellation() {
+    return this.isKrispNoiseFilterEnabled;
+  }
+
   /**
    *
    * @param mediaTrack
@@ -152,6 +162,28 @@ export default class LocalAudioTrack extends LocalTrack<Track.Kind.Audio> {
     this.prevStats = stats;
   };
 
+  private handleKrispNoiseFilterEnable = () => {
+    this.isKrispNoiseFilterEnabled = true;
+    this.log.debug(`Krisp noise filter enabled`, this.logContext);
+    this.emit(
+      TrackEvent.AudioTrackFeatureUpdate,
+      this,
+      AudioTrackFeature.TF_ENHANCED_NOISE_CANCELLATION,
+      true,
+    );
+  };
+
+  private handleKrispNoiseFilterDisable = () => {
+    this.isKrispNoiseFilterEnabled = false;
+    this.log.debug(`Krisp noise filter disabled`, this.logContext);
+    this.emit(
+      TrackEvent.AudioTrackFeatureUpdate,
+      this,
+      AudioTrackFeature.TF_ENHANCED_NOISE_CANCELLATION,
+      false,
+    );
+  };
+
   async setProcessor(processor: TrackProcessor<Track.Kind.Audio, AudioProcessorOptions>) {
     const unlock = await this.processorLock.lock();
     try {
@@ -175,6 +207,14 @@ export default class LocalAudioTrack extends LocalTrack<Track.Kind.Audio> {
       this.processor = processor;
       if (this.processor.processedTrack) {
         await this.sender?.replaceTrack(this.processor.processedTrack);
+        this.processor.processedTrack.addEventListener(
+          'enable-lk-krisp-noise-filter',
+          this.handleKrispNoiseFilterEnable,
+        );
+        this.processor.processedTrack.addEventListener(
+          'disable-lk-krisp-noise-filter',
+          this.handleKrispNoiseFilterDisable,
+        );
       }
       this.emit(TrackEvent.TrackProcessorUpdate, this.processor);
     } finally {

package/src/room/track/LocalTrack.ts

@@ -448,6 +448,10 @@ export default abstract class LocalTrack<
     const unlock = await this.processorLock.lock();
     try {
       this.log.debug('setting up processor', this.logContext);
+
+      this.processorElement =
+        this.processorElement ?? (document.createElement(this.kind) as HTMLMediaElement);
+
       const processorOptions = {
         kind: this.kind,
         track: this._mediaStreamTrack,
@@ -461,8 +465,6 @@ export default abstract class LocalTrack<
       if (this.kind === 'unknown') {
         throw TypeError('cannot set processor on track of unknown kind');
       }
-      this.processorElement =
-        this.processorElement ?? (document.createElement(this.kind) as HTMLMediaElement);
 
       attachToElement(this._mediaStreamTrack, this.processorElement);
       this.processorElement.muted = true;

package/src/room/track/LocalTrackPublication.ts

@@ -1,7 +1,7 @@
-import type { TrackInfo } from '@livekit/protocol';
+import { AudioTrackFeature, TrackInfo } from '@livekit/protocol';
 import { TrackEvent } from '../events';
 import type { LoggerOptions } from '../types';
-import type LocalAudioTrack from './LocalAudioTrack';
+import LocalAudioTrack from './LocalAudioTrack';
 import type LocalTrack from './LocalTrack';
 import type LocalVideoTrack from './LocalVideoTrack';
 import type { Track } from './Track';
@@ -82,6 +82,32 @@ export default class LocalTrackPublication extends TrackPublication {
     await this.track?.resumeUpstream();
   }
 
+  getTrackFeatures() {
+    if (this.track instanceof LocalAudioTrack) {
+      const settings = this.track!.mediaStreamTrack.getSettings();
+      const features: Set<AudioTrackFeature> = new Set();
+      if (settings.autoGainControl) {
+        features.add(AudioTrackFeature.TF_AUTO_GAIN_CONTROL);
+      }
+      if (settings.echoCancellation) {
+        features.add(AudioTrackFeature.TF_ECHO_CANCELLATION);
+      }
+      if (settings.noiseSuppression) {
+        features.add(AudioTrackFeature.TF_NOISE_SUPPRESSION);
+      }
+      if (settings.channelCount && settings.channelCount > 1) {
+        features.add(AudioTrackFeature.TF_STEREO);
+      }
+      if (!this.options?.dtx) {
+        features.add(AudioTrackFeature.TF_STEREO);
+      }
+      if (this.track.enhancedNoiseCancellation) {
+        features.add(AudioTrackFeature.TF_ENHANCED_NOISE_CANCELLATION);
+      }
+      return Array.from(features.values());
+    } else return [];
+  }
+
   handleTrackEnded = () => {
     this.emit(TrackEvent.Ended);
   };

package/src/room/track/LocalVideoTrack.ts

@@ -180,17 +180,20 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
           streamId: v.id,
           frameHeight: v.frameHeight,
           frameWidth: v.frameWidth,
+          framesPerSecond: v.framesPerSecond,
+          framesSent: v.framesSent,
           firCount: v.firCount,
           pliCount: v.pliCount,
           nackCount: v.nackCount,
           packetsSent: v.packetsSent,
           bytesSent: v.bytesSent,
-          framesSent: v.framesSent,
-          timestamp: v.timestamp,
-          rid: v.rid ?? v.id,
-          retransmittedPacketsSent: v.retransmittedPacketsSent,
           qualityLimitationReason: v.qualityLimitationReason,
+          qualityLimitationDurations: v.qualityLimitationDurations,
           qualityLimitationResolutionChanges: v.qualityLimitationResolutionChanges,
+          rid: v.rid ?? v.id,
+          retransmittedPacketsSent: v.retransmittedPacketsSent,
+          targetBitrate: v.targetBitrate,
+          timestamp: v.timestamp,
         };
 
         // locate the appropriate remote-inbound-rtp item
@@ -205,6 +208,8 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
       }
     });
 
+    // make sure highest res layer is always first
+    items.sort((a, b) => (b.frameWidth ?? 0) - (a.frameWidth ?? 0));
     return items;
   }
 
@@ -567,13 +572,17 @@ export function videoLayersFromEncodings(
     const encodingSM = encodings[0].scalabilityMode as string;
     const sm = new ScalabilityMode(encodingSM);
     const layers = [];
+    const resRatio = sm.suffix == 'h' ? 1.5 : 2;
+    const bitratesRatio = sm.suffix == 'h' ? 2 : 3;
     for (let i = 0; i < sm.spatial; i += 1) {
       layers.push(
         new VideoLayer({
           quality: VideoQuality.HIGH - i,
-          width: Math.ceil(width / 2 ** i),
-          height: Math.ceil(height / 2 ** i),
-          bitrate: encodings[0].maxBitrate ? Math.ceil(encodings[0].maxBitrate / 3 ** i) : 0,
+          width: Math.ceil(width / resRatio ** i),
+          height: Math.ceil(height / resRatio ** i),
+          bitrate: encodings[0].maxBitrate
+            ? Math.ceil(encodings[0].maxBitrate / bitratesRatio ** i)
+            : 0,
           ssrc: 0,
         }),
       );

package/src/room/track/Track.ts

@@ -1,4 +1,5 @@
 import {
+  AudioTrackFeature,
   VideoQuality as ProtoQuality,
   StreamState as ProtoStreamState,
   TrackSource,
@@ -300,6 +301,17 @@ export abstract class Track<
 
   protected async handleAppVisibilityChanged() {
     this.isInBackground = document.visibilityState === 'hidden';
+    if (!this.isInBackground && this.kind === Track.Kind.Video) {
+      setTimeout(
+        () =>
+          this.attachedElements.forEach((el) =>
+            el.play().catch(() => {
+              /** catch clause necessary for Safari */
+            }),
+          ),
+        0,
+      );
+    }
   }
 
   protected addAppVisibilityListener() {
@@ -504,4 +516,5 @@ export type TrackEventCallbacks = {
   upstreamPaused: (track: any) => void;
   upstreamResumed: (track: any) => void;
   trackProcessorUpdate: (processor?: TrackProcessor<Track.Kind, any>) => void;
+  audioTrackFeatureUpdate: (track: any, feature: AudioTrackFeature, enabled: boolean) => void;
 };

package/src/room/track/options.ts

@@ -350,7 +350,28 @@ export function isBackupCodec(codec: string): codec is BackupVideoCodec {
 /**
  * scalability modes for svc.
  */
-export type ScalabilityMode = 'L1T3' | 'L2T3' | 'L2T3_KEY' | 'L3T3' | 'L3T3_KEY';
+export type ScalabilityMode =
+  | 'L1T1'
+  | 'L1T2'
+  | 'L1T3'
+  | 'L2T1'
+  | 'L2T1h'
+  | 'L2T1_KEY'
+  | 'L2T2'
+  | 'L2T2h'
+  | 'L2T2_KEY'
+  | 'L2T3'
+  | 'L2T3h'
+  | 'L2T3_KEY'
+  | 'L3T1'
+  | 'L3T1h'
+  | 'L3T1_KEY'
+  | 'L3T2'
+  | 'L3T2h'
+  | 'L3T2_KEY'
+  | 'L3T3'
+  | 'L3T3h'
+  | 'L3T3_KEY';
 
 export namespace AudioPresets {
   export const telephone: AudioPreset = {

package/src/room/utils.ts

@@ -447,6 +447,9 @@ export function createAudioAnalyser(
   return { calculateVolume, analyser, cleanup };
 }
 
+/**
+ * @internal
+ */
 export class Mutex {
   private _locking: Promise<void>;