livekit-client 1.9.0 → 1.9.2

package/src/room/Room.ts

@@ -45,6 +45,7 @@ import LocalParticipant from './participant/LocalParticipant';
 import type Participant from './participant/Participant';
 import type { ConnectionQuality } from './participant/Participant';
 import RemoteParticipant from './participant/RemoteParticipant';
+import CriticalTimers from './timers';
 import LocalAudioTrack from './track/LocalAudioTrack';
 import LocalTrackPublication from './track/LocalTrackPublication';
 import LocalVideoTrack from './track/LocalVideoTrack';
@@ -73,6 +74,8 @@ export enum ConnectionState {
   Reconnecting = 'reconnecting',
 }
 
+const connectionReconcileFrequency = 2 * 1000;
+
 /** @deprecated RoomState has been renamed to [[ConnectionState]] */
 export const RoomState = ConnectionState;
 
@@ -124,6 +127,10 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
 
   private disconnectLock: Mutex;
 
+  private cachedParticipantSids: Array<string>;
+
+  private connectionReconcileInterval?: ReturnType<typeof setInterval>;
+
   /**
    * Creates a new Room, the primary construct for a LiveKit session.
    * @param options
@@ -132,6 +139,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     super();
     this.setMaxListeners(100);
     this.participants = new Map();
+    this.cachedParticipantSids = [];
     this.identityToSid = new Map();
     this.options = { ...roomOptionDefaults, ...options };
 
@@ -186,7 +194,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
   }
 
   private maybeCreateEngine() {
-    if (this.engine) {
+    if (this.engine && !this.engine.isClosed) {
       return;
     }
 
@@ -212,14 +220,24 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       .on(EngineEvent.ActiveSpeakersUpdate, this.handleActiveSpeakersUpdate)
       .on(EngineEvent.DataPacketReceived, this.handleDataPacket)
       .on(EngineEvent.Resuming, () => {
+        this.clearConnectionReconcile();
         if (this.setAndEmitConnectionState(ConnectionState.Reconnecting)) {
           this.emit(RoomEvent.Reconnecting);
         }
+        this.cachedParticipantSids = Array.from(this.participants.keys());
       })
       .on(EngineEvent.Resumed, () => {
         this.setAndEmitConnectionState(ConnectionState.Connected);
         this.emit(RoomEvent.Reconnected);
+        this.registerConnectionReconcile();
         this.updateSubscriptions();
+
+        // once reconnected, figure out if any participants connected during reconnect and emit events for it
+        const diffParticipants = Array.from(this.participants.values()).filter(
+          (p) => !this.cachedParticipantSids.includes(p.sid),
+        );
+        diffParticipants.forEach((p) => this.emit(RoomEvent.ParticipantConnected, p));
+        this.cachedParticipantSids = [];
       })
       .on(EngineEvent.SignalResumed, () => {
         if (this.state === ConnectionState.Reconnecting) {
@@ -477,6 +495,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     }
     this.setAndEmitConnectionState(ConnectionState.Connected);
     this.emit(RoomEvent.Connected);
+    this.registerConnectionReconcile();
   };
 
   /**
@@ -818,6 +837,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
   }
 
   private handleRestarting = () => {
+    this.clearConnectionReconcile();
     // also unwind existing participants & existing subscriptions
     for (const p of this.participants.values()) {
       this.handleParticipantDisconnected(p.sid, p);
@@ -833,6 +853,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       region: joinResponse.serverRegion,
     });
 
+    this.cachedParticipantSids = [];
     this.applyJoinResponse(joinResponse);
 
     try {
@@ -882,6 +903,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     }
     this.setAndEmitConnectionState(ConnectionState.Connected);
     this.emit(RoomEvent.Reconnected);
+    this.registerConnectionReconcile();
 
     // emit participant connected events after connection has been re-established
     this.participants.forEach((participant) => {
@@ -890,57 +912,61 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
   };
 
   private handleDisconnect(shouldStopTracks = true, reason?: DisconnectReason) {
+    this.clearConnectionReconcile();
     if (this.state === ConnectionState.Disconnected) {
       return;
     }
 
-    this.participants.forEach((p) => {
-      p.tracks.forEach((pub) => {
-        p.unpublishTrack(pub.trackSid);
+    try {
+      this.participants.forEach((p) => {
+        p.tracks.forEach((pub) => {
+          p.unpublishTrack(pub.trackSid);
+        });
       });
-    });
 
-    this.localParticipant.tracks.forEach((pub) => {
-      if (pub.track) {
-        this.localParticipant.unpublishTrack(pub.track, shouldStopTracks);
-      }
-      if (shouldStopTracks) {
-        pub.track?.detach();
-        pub.track?.stop();
-      }
-    });
+      this.localParticipant.tracks.forEach((pub) => {
+        if (pub.track) {
+          this.localParticipant.unpublishTrack(pub.track, shouldStopTracks);
+        }
+        if (shouldStopTracks) {
+          pub.track?.detach();
+          pub.track?.stop();
+        }
+      });
 
-    this.localParticipant
-      .off(ParticipantEvent.ParticipantMetadataChanged, this.onLocalParticipantMetadataChanged)
-      .off(ParticipantEvent.ParticipantNameChanged, this.onLocalParticipantNameChanged)
-      .off(ParticipantEvent.TrackMuted, this.onLocalTrackMuted)
-      .off(ParticipantEvent.TrackUnmuted, this.onLocalTrackUnmuted)
-      .off(ParticipantEvent.LocalTrackPublished, this.onLocalTrackPublished)
-      .off(ParticipantEvent.LocalTrackUnpublished, this.onLocalTrackUnpublished)
-      .off(ParticipantEvent.ConnectionQualityChanged, this.onLocalConnectionQualityChanged)
-      .off(ParticipantEvent.MediaDevicesError, this.onMediaDevicesError)
-      .off(
-        ParticipantEvent.ParticipantPermissionsChanged,
-        this.onLocalParticipantPermissionsChanged,
-      );
+      this.localParticipant
+        .off(ParticipantEvent.ParticipantMetadataChanged, this.onLocalParticipantMetadataChanged)
+        .off(ParticipantEvent.ParticipantNameChanged, this.onLocalParticipantNameChanged)
+        .off(ParticipantEvent.TrackMuted, this.onLocalTrackMuted)
+        .off(ParticipantEvent.TrackUnmuted, this.onLocalTrackUnmuted)
+        .off(ParticipantEvent.LocalTrackPublished, this.onLocalTrackPublished)
+        .off(ParticipantEvent.LocalTrackUnpublished, this.onLocalTrackUnpublished)
+        .off(ParticipantEvent.ConnectionQualityChanged, this.onLocalConnectionQualityChanged)
+        .off(ParticipantEvent.MediaDevicesError, this.onMediaDevicesError)
+        .off(
+          ParticipantEvent.ParticipantPermissionsChanged,
+          this.onLocalParticipantPermissionsChanged,
+        );
 
-    this.localParticipant.tracks.clear();
-    this.localParticipant.videoTracks.clear();
-    this.localParticipant.audioTracks.clear();
+      this.localParticipant.tracks.clear();
+      this.localParticipant.videoTracks.clear();
+      this.localParticipant.audioTracks.clear();
 
-    this.participants.clear();
-    this.activeSpeakers = [];
-    if (this.audioContext && typeof this.options.expWebAudioMix === 'boolean') {
-      this.audioContext.close();
-      this.audioContext = undefined;
-    }
-    if (isWeb()) {
-      window.removeEventListener('beforeunload', this.onPageLeave);
-      window.removeEventListener('pagehide', this.onPageLeave);
-      navigator.mediaDevices?.removeEventListener('devicechange', this.handleDeviceChange);
+      this.participants.clear();
+      this.activeSpeakers = [];
+      if (this.audioContext && typeof this.options.expWebAudioMix === 'boolean') {
+        this.audioContext.close();
+        this.audioContext = undefined;
+      }
+      if (isWeb()) {
+        window.removeEventListener('beforeunload', this.onPageLeave);
+        window.removeEventListener('pagehide', this.onPageLeave);
+        navigator.mediaDevices?.removeEventListener('devicechange', this.handleDeviceChange);
+      }
+    } finally {
+      this.setAndEmitConnectionState(ConnectionState.Disconnected);
+      this.emit(RoomEvent.Disconnected, reason);
     }
-    this.setAndEmitConnectionState(ConnectionState.Disconnected);
-    this.emit(RoomEvent.Disconnected, reason);
   }
 
   private handleParticipantUpdates = (participantInfos: ParticipantInfo[]) => {
@@ -1329,6 +1355,37 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     }
   }
 
+  private registerConnectionReconcile() {
+    this.clearConnectionReconcile();
+    let consecutiveFailures = 0;
+    this.connectionReconcileInterval = CriticalTimers.setInterval(() => {
+      if (
+        // ensure we didn't tear it down
+        !this.engine ||
+        // engine detected close, but Room missed it
+        this.engine.isClosed ||
+        // transports failed without notifying engine
+        !this.engine.verifyTransport()
+      ) {
+        consecutiveFailures++;
+        log.warn('detected connection state mismatch', { numFailures: consecutiveFailures });
+        if (consecutiveFailures >= 3)
+          this.handleDisconnect(
+            this.options.stopLocalTrackOnUnpublish,
+            DisconnectReason.UNKNOWN_REASON,
+          );
+      } else {
+        consecutiveFailures = 0;
+      }
+    }, connectionReconcileFrequency);
+  }
+
+  private clearConnectionReconcile() {
+    if (this.connectionReconcileInterval) {
+      CriticalTimers.clearInterval(this.connectionReconcileInterval);
+    }
+  }
+
   private setAndEmitConnectionState(state: ConnectionState): boolean {
     if (state === this.state) {
       // unchanged

package/src/room/participant/LocalParticipant.ts

@@ -27,10 +27,11 @@ import {
   TrackPublishOptions,
   VideoCaptureOptions,
   isBackupCodec,
+  isCodecEqual,
 } from '../track/options';
 import { constraintsForOptions, mergeDefaultOptions } from '../track/utils';
 import type { DataPublishOptions } from '../types';
-import { Future, isFireFox, isSafari, isWeb, supportsAV1 } from '../utils';
+import { Future, isFireFox, isSVCCodec, isSafari, isWeb, supportsAV1, supportsVP9 } from '../utils';
 import Participant from './Participant';
 import { ParticipantTrackPermission, trackPermissionToProto } from './ParticipantTrackPermission';
 import RemoteParticipant from './RemoteParticipant';
@@ -570,10 +571,13 @@ export default class LocalParticipant extends Participant {
       opts.simulcast = false;
     }
 
-    // require full AV1 SVC support prior to using it
+    // require full AV1/VP9 SVC support prior to using it
     if (opts.videoCodec === 'av1' && !supportsAV1()) {
       opts.videoCodec = undefined;
     }
+    if (opts.videoCodec === 'vp9' && !supportsVP9()) {
+      opts.videoCodec = undefined;
+    }
 
     // handle track actions
     track.on(TrackEvent.Muted, this.onTrackMuted);
@@ -614,7 +618,7 @@ export default class LocalParticipant extends Participant {
       req.height = dims.height;
       // for svc codecs, disable simulcast and use vp8 for backup codec
       if (track instanceof LocalVideoTrack) {
-        if (opts?.videoCodec === 'av1') {
+        if (isSVCCodec(opts.videoCodec)) {
           // set scalabilityMode to 'L3T3' by default
           opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3';
         }
@@ -660,6 +664,33 @@ export default class LocalParticipant extends Participant {
     }
 
     const ti = await this.engine.addTrack(req);
+    let primaryCodecSupported = false;
+    let backupCodecSupported = false;
+    ti.codecs.forEach((c) => {
+      if (isCodecEqual(c.mimeType, opts.videoCodec)) {
+        primaryCodecSupported = true;
+      } else if (opts.backupCodec && isCodecEqual(c.mimeType, opts.backupCodec.codec)) {
+        backupCodecSupported = true;
+      }
+    });
+
+    if (req.simulcastCodecs.length > 0) {
+      if (!primaryCodecSupported && !backupCodecSupported) {
+        throw Error('cannot publish track, codec not supported by server');
+      }
+
+      if (!primaryCodecSupported && opts.backupCodec) {
+        const backupCodec = opts.backupCodec;
+        opts = { ...opts };
+        log.debug(
+          `primary codec ${opts.videoCodec} not supported, fallback to ${backupCodec.codec}`,
+        );
+        opts.videoCodec = backupCodec.codec;
+        opts.videoEncoding = backupCodec.encoding;
+        encodings = simEncodings;
+      }
+    }
+
     const publication = new LocalTrackPublication(track.kind, ti, track);
     // save options for when it needs to be republished again
     publication.options = opts;
@@ -673,7 +704,7 @@ export default class LocalParticipant extends Participant {
     // store RTPSender
     track.sender = await this.engine.createSender(track, opts, encodings);
 
-    if (track.codec === 'av1' && encodings && encodings[0]?.maxBitrate) {
+    if (track.codec && isSVCCodec(track.codec) && encodings && encodings[0]?.maxBitrate) {
       this.engine.publisher.setTrackCodecBitrate(
         req.cid,
         track.codec,

package/src/room/participant/publishUtils.ts

@@ -13,6 +13,7 @@ import {
   VideoPresets,
   VideoPresets43,
 } from '../track/options';
+import { isSVCCodec } from '../utils';
 
 /** @internal */
 export function mediaTrackToLocalTrack(
@@ -121,7 +122,7 @@ export function computeVideoEncodings(
     videoEncoding.maxFramerate,
   );
 
-  if (scalabilityMode && videoCodec === 'av1') {
+  if (scalabilityMode && isSVCCodec(videoCodec)) {
     log.debug(`using svc with scalabilityMode ${scalabilityMode}`);
 
     const encodings: RTCRtpEncodingParameters[] = [];
@@ -248,8 +249,13 @@ export function determineAppropriateEncoding(
   if (codec) {
     switch (codec) {
       case 'av1':
+        encoding = { ...encoding };
         encoding.maxBitrate = encoding.maxBitrate * 0.7;
         break;
+      case 'vp9':
+        encoding = { ...encoding };
+        encoding.maxBitrate = encoding.maxBitrate * 0.85;
+        break;
       default:
         break;
     }
@@ -303,13 +309,15 @@ function encodingsFromPresets(
     }
     const size = Math.min(width, height);
     const rid = videoRids[idx];
-    encodings.push({
+    const encoding: RTCRtpEncodingParameters = {
       rid,
       scaleResolutionDownBy: Math.max(1, size / Math.min(preset.width, preset.height)),
       maxBitrate: preset.encoding.maxBitrate,
-      /* @ts-ignore */
-      maxFramerate: preset.encoding.maxFramerate,
-    });
+    };
+    if (preset.encoding.maxFramerate) {
+      encoding.maxFramerate = preset.encoding.maxFramerate;
+    }
+    encodings.push(encoding);
   });
   return encodings;
 }

package/src/room/track/options.ts

@@ -145,6 +145,12 @@ export interface ScreenShareCaptureOptions {
 
   /** specifies whether the browser should include the system audio among the possible audio sources offered to the user */
   systemAudio?: 'include' | 'exclude';
+
+  /**
+   * Experimental option to control whether the audio playing in a tab will continue to be played out of a user's
+   * local speakers when the tab is captured.
+   */
+  suppressLocalAudioPlayback?: boolean;
 }
 
 export interface AudioCaptureOptions {
@@ -241,7 +247,7 @@ export interface AudioPreset {
   maxBitrate: number;
 }
 
-const codecs = ['vp8', 'h264', 'av1'] as const;
+const codecs = ['vp8', 'h264', 'vp9', 'av1'] as const;
 const backupCodecs = ['vp8', 'h264'] as const;
 
 export type VideoCodec = (typeof codecs)[number];
@@ -252,6 +258,13 @@ export function isBackupCodec(codec: string): codec is BackupVideoCodec {
   return !!backupCodecs.find((backup) => backup === codec);
 }
 
+export function isCodecEqual(c1: string | undefined, c2: string | undefined): boolean {
+  return (
+    c1?.toLowerCase().replace(/audio\/|video\//y, '') ===
+    c2?.toLowerCase().replace(/audio\/|video\//y, '')
+  );
+}
+
 /**
  * scalability modes for svc, only supprot l3t3 now.
  */

package/src/room/utils.ts

@@ -7,6 +7,8 @@ import { getNewAudioContext } from './track/utils';
 import type { LiveKitReactNativeInfo } from './types';
 
 const separator = '|';
+export const ddExtensionURI =
+  'https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension';
 
 export function unpackStreamId(packed: string): string[] {
   const parts = packed.split(separator);
@@ -41,7 +43,6 @@ export function supportsDynacast() {
 export function supportsAV1(): boolean {
   const capabilities = RTCRtpReceiver.getCapabilities('video');
   let hasAV1 = false;
-  let hasDDExt = false;
   if (capabilities) {
     for (const codec of capabilities.codecs) {
       if (codec.mimeType === 'video/AV1') {
@@ -49,17 +50,26 @@ export function supportsAV1(): boolean {
         break;
       }
     }
-    for (const ext of capabilities.headerExtensions) {
-      if (
-        ext.uri ===
-        'https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension'
-      ) {
-        hasDDExt = true;
+  }
+  return hasAV1;
+}
+
+export function supportsVP9(): boolean {
+  const capabilities = RTCRtpReceiver.getCapabilities('video');
+  let hasVP9 = false;
+  if (capabilities) {
+    for (const codec of capabilities.codecs) {
+      if (codec.mimeType === 'video/VP9') {
+        hasVP9 = true;
         break;
       }
     }
   }
-  return hasAV1 && hasDDExt;
+  return hasVP9;
+}
+
+export function isSVCCodec(codec?: string): boolean {
+  return codec === 'av1' || codec === 'vp9';
 }
 
 export function supportsSetSinkId(elm?: HTMLMediaElement): boolean {