livekit-client 1.9.7 → 1.11.0

package/src/room/PCTransport.ts

@@ -1,4 +1,4 @@
-import { EventEmitter } from 'events';
+import EventEmitter from 'eventemitter3';
 import { parse, write } from 'sdp-transform';
 import type { MediaDescription } from 'sdp-transform';
 import { debounce } from 'ts-debounce';

package/src/room/RTCEngine.ts

@@ -1,5 +1,4 @@
-import { EventEmitter } from 'events';
-import type TypedEventEmitter from 'typed-emitter';
+import EventEmitter from 'eventemitter3';
 import { SignalClient } from '../api/SignalClient';
 import type { SignalOptions } from '../api/SignalClient';
 import log from '../logger';
@@ -65,7 +64,7 @@ enum PCState {
 }
 
 /** @internal */
-export default class RTCEngine extends (EventEmitter as new () => TypedEventEmitter<EngineEventCallbacks>) {
+export default class RTCEngine extends EventEmitter<EngineEventCallbacks> {
   publisher?: PCTransport;
 
   subscriber?: PCTransport;
@@ -1046,6 +1045,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
       };
       this.once(EngineEvent.Restarted, onRestarted);
       this.once(EngineEvent.Disconnected, onDisconnected);
+      this.once(EngineEvent.Closing, onDisconnected);
     });
   };
 
@@ -1172,7 +1172,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
       this.hasPublished = true;
 
       const handleClosed = () => {
-        log.debug('engine disconnected while negotiation was ongoing');
+        log.warn('engine disconnected while negotiation was ongoing');
         cleanup();
         resolve();
         return;

package/src/room/Room.ts

@@ -1,5 +1,4 @@
-import { EventEmitter } from 'events';
-import type TypedEmitter from 'typed-emitter';
+import EventEmitter from 'eventemitter3';
 import 'webrtc-adapter';
 import { toProtoSessionDescription } from '../api/SignalClient';
 import log from '../logger';
@@ -18,6 +17,7 @@ import {
   Room as RoomModel,
   ServerInfo,
   SpeakerInfo,
+  SubscriptionError,
   TrackInfo,
   TrackSource,
   TrackType,
@@ -29,6 +29,7 @@ import {
   SimulateScenario,
   StreamStateUpdate,
   SubscriptionPermissionUpdate,
+  SubscriptionResponse,
 } from '../proto/livekit_rtc';
 import DeviceManager from './DeviceManager';
 import RTCEngine from './RTCEngine';
@@ -63,6 +64,7 @@ import {
   createDummyVideoStreamTrack,
   getEmptyAudioStreamTrack,
   isCloud,
+  isSafari,
   isWeb,
   supportsSetSinkId,
   unpackStreamId,
@@ -88,7 +90,7 @@ export const RoomState = ConnectionState;
  *
  * @noInheritDoc
  */
-class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>) {
+class Room extends EventEmitter<RoomEventCallbacks> {
   state: ConnectionState = ConnectionState.Disconnected;
 
   /** map of sid: [[RemoteParticipant]] */
@@ -138,7 +140,6 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
    */
   constructor(options?: RoomOptions) {
     super();
-    this.setMaxListeners(100);
     this.participants = new Map();
     this.cachedParticipantSids = [];
     this.identityToSid = new Map();
@@ -207,6 +208,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     this.engine.client.onStreamStateUpdate = this.handleStreamStateUpdate;
     this.engine.client.onSubscriptionPermissionUpdate = this.handleSubscriptionPermissionUpdate;
     this.engine.client.onConnectionQuality = this.handleConnectionQualityUpdate;
+    this.engine.client.onSubscriptionError = this.handleSubscriptionError;
 
     this.engine
       .on(
@@ -492,6 +494,9 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       // capturing both 'pagehide' and 'beforeunload' to capture broadest set of browser behaviors
       window.addEventListener('pagehide', this.onPageLeave);
       window.addEventListener('beforeunload', this.onPageLeave);
+    }
+    if (isWeb()) {
+      document.addEventListener('freeze', this.onPageLeave);
       navigator.mediaDevices?.addEventListener('devicechange', this.handleDeviceChange);
     }
     this.setAndEmitConnectionState(ConnectionState.Connected);
@@ -654,8 +659,31 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
    */
   async startAudio() {
     await this.acquireAudioContext();
-
     const elements: Array<HTMLMediaElement> = [];
+
+    if (isSafari()) {
+      /**
+       * iOS Safari blocks audio element playback if
+       * - user is not publishing audio themselves and
+       * - no other audio source is playing
+       *
+       * as a workaround, we create an audio element with an empty track, so that
+       * silent audio is always playing
+       */
+      const audioId = 'livekit-dummy-audio-el';
+      let dummyAudioEl = document.getElementById(audioId) as HTMLAudioElement | null;
+      if (!dummyAudioEl) {
+        dummyAudioEl = document.createElement('audio');
+        dummyAudioEl.autoplay = true;
+        dummyAudioEl.hidden = true;
+        const track = getEmptyAudioStreamTrack();
+        track.enabled = true;
+        dummyAudioEl.srcObject = new MediaStream([track]);
+        document.body.append(dummyAudioEl);
+      }
+      elements.push(dummyAudioEl);
+    }
+
     this.participants.forEach((p) => {
       p.audioTracks.forEach((t) => {
         if (t.track) {
@@ -962,6 +990,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       if (isWeb()) {
         window.removeEventListener('beforeunload', this.onPageLeave);
         window.removeEventListener('pagehide', this.onPageLeave);
+        window.removeEventListener('freeze', this.onPageLeave);
         navigator.mediaDevices?.removeEventListener('devicechange', this.handleDeviceChange);
       }
     } finally {
@@ -1114,6 +1143,21 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     pub.setAllowed(update.allowed);
   };
 
+  private handleSubscriptionError = (update: SubscriptionResponse) => {
+    const participant = Array.from(this.participants.values()).find((p) =>
+      p.tracks.has(update.trackSid),
+    );
+    if (!participant) {
+      return;
+    }
+    const pub = participant.getTrackPublication(update.trackSid);
+    if (!pub) {
+      return;
+    }
+
+    pub.setSubscriptionError(update.err);
+  };
+
   private handleDataPacket = (userPacket: UserPacket, kind: DataPacket_Kind) => {
     // find the participant
     const participant = this.participants.get(userPacket.participantSid);
@@ -1280,6 +1324,9 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       .on(ParticipantEvent.TrackSubscriptionStatusChanged, (pub, status) => {
         this.emitWhenConnected(RoomEvent.TrackSubscriptionStatusChanged, pub, status, participant);
       })
+      .on(ParticipantEvent.TrackSubscriptionFailed, (trackSid, error) => {
+        this.emit(RoomEvent.TrackSubscriptionFailed, trackSid, participant, error);
+      })
       .on(ParticipantEvent.TrackSubscriptionPermissionChanged, (pub, status) => {
         this.emitWhenConnected(
           RoomEvent.TrackSubscriptionPermissionChanged,
@@ -1399,9 +1446,9 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     return true;
   }
 
-  private emitWhenConnected<E extends keyof RoomEventCallbacks>(
-    event: E,
-    ...args: Parameters<RoomEventCallbacks[E]>
+  private emitWhenConnected<T extends EventEmitter.EventNames<RoomEventCallbacks>>(
+    event: T,
+    ...args: EventEmitter.EventArgs<RoomEventCallbacks, T>
   ): boolean {
     if (this.state === ConnectionState.Connected) {
       return this.emit(event, ...args);
@@ -1578,10 +1625,14 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
   }
 
   // /** @internal */
-  emit<E extends keyof RoomEventCallbacks>(
-    event: E,
-    ...args: Parameters<RoomEventCallbacks[E]>
+  emit<T extends EventEmitter.EventNames<RoomEventCallbacks>>(
+    event: T,
+    ...args: EventEmitter.EventArgs<RoomEventCallbacks, T>
   ): boolean {
+    // emit<E extends keyof RoomEventCallbacks>(
+    //   event: E,
+    //   ...args: Parameters<RoomEventCallbacks[E]>
+    // ): boolean {
     // active speaker updates are too spammy
     if (event !== RoomEvent.ActiveSpeakersChanged) {
       log.debug(`room event ${event}`, { event, args });
@@ -1609,7 +1660,11 @@ export type RoomEventCallbacks = {
     publication: RemoteTrackPublication,
     participant: RemoteParticipant,
   ) => void;
-  trackSubscriptionFailed: (trackSid: string, participant: RemoteParticipant) => void;
+  trackSubscriptionFailed: (
+    trackSid: string,
+    participant: RemoteParticipant,
+    reason?: SubscriptionError,
+  ) => void;
   trackUnpublished: (publication: RemoteTrackPublication, participant: RemoteParticipant) => void;
   trackUnsubscribed: (
     track: RemoteTrack,

package/src/room/events.ts

@@ -508,4 +508,8 @@ export enum TrackEvent {
    * Fires on RemoteTrackPublication
    */
   SubscriptionStatusChanged = 'subscriptionStatusChanged',
+  /**
+   * Fires on RemoteTrackPublication
+   */
+  SubscriptionFailed = 'subscriptionFailed',
 }

package/src/room/participant/LocalParticipant.ts

@@ -143,8 +143,11 @@ export default class LocalParticipant extends Participant {
   };
 
   private handleDisconnected = () => {
-    this.reconnectFuture?.reject?.('Got disconnected during publishing attempt');
-    this.reconnectFuture = undefined;
+    if (this.reconnectFuture) {
+      this.reconnectFuture.promise.catch((e) => log.warn(e));
+      this.reconnectFuture?.reject?.('Got disconnected during reconnection attempt');
+      this.reconnectFuture = undefined;
+    }
   };
 
   /**
@@ -625,8 +628,8 @@ export default class LocalParticipant extends Participant {
       // for svc codecs, disable simulcast and use vp8 for backup codec
       if (track instanceof LocalVideoTrack) {
         if (isSVCCodec(opts.videoCodec)) {
-          // set scalabilityMode to 'L3T3' by default
-          opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3';
+          // set scalabilityMode to 'L3T3_KEY' by default
+          opts.scalabilityMode = opts.scalabilityMode ?? 'L3T3_KEY';
         }
 
         // set up backup
@@ -647,6 +650,16 @@ export default class LocalParticipant extends Participant {
               enableSimulcastLayers: true,
             },
           ];
+        } else if (opts.videoCodec) {
+          // pass codec info to sfu so it can prefer codec for the client which don't support
+          // setCodecPreferences
+          req.simulcastCodecs = [
+            {
+              codec: opts.videoCodec,
+              cid: track.mediaStreamTrack.id,
+              enableSimulcastLayers: opts.simulcast ?? false,
+            },
+          ];
         }
       }
 
@@ -656,7 +669,12 @@ export default class LocalParticipant extends Participant {
         dims.height,
         opts,
       );
-      req.layers = videoLayersFromEncodings(req.width, req.height, simEncodings ?? encodings);
+      req.layers = videoLayersFromEncodings(
+        req.width,
+        req.height,
+        encodings,
+        isSVCCodec(opts.videoCodec),
+      );
     } else if (track.kind === Track.Kind.Audio) {
       encodings = [
         {
@@ -850,6 +868,16 @@ export default class LocalParticipant extends Participant {
       trackSender
     ) {
       try {
+        for (const transceiver of this.engine.publisher.pc.getTransceivers()) {
+          // if sender is not currently sending (after replaceTrack(null))
+          // removeTrack would have no effect.
+          // to ensure we end up successfully removing the track, manually set
+          // the transceiver to inactive
+          if (transceiver.sender === trackSender) {
+            transceiver.direction = 'inactive';
+            negotiationNeeded = true;
+          }
+        }
         if (this.engine.removeTrack(trackSender)) {
           negotiationNeeded = true;
         }

package/src/room/participant/Participant.ts

@@ -1,11 +1,11 @@
-import { EventEmitter } from 'events';
-import type TypedEmitter from 'typed-emitter';
+import EventEmitter from 'eventemitter3';
 import log from '../../logger';
 import {
   DataPacket_Kind,
   ParticipantInfo,
   ParticipantPermission,
   ConnectionQuality as ProtoQuality,
+  SubscriptionError,
 } from '../../proto/livekit_models';
 import { ParticipantEvent, TrackEvent } from '../events';
 import type LocalTrackPublication from '../track/LocalTrackPublication';
@@ -34,7 +34,7 @@ function qualityFromProto(q: ProtoQuality): ConnectionQuality {
   }
 }
 
-export default class Participant extends (EventEmitter as new () => TypedEmitter<ParticipantEventCallbacks>) {
+export default class Participant extends EventEmitter<ParticipantEventCallbacks> {
   protected participantInfo?: ParticipantInfo;
 
   audioTracks: Map<string, TrackPublication>;
@@ -71,7 +71,6 @@ export default class Participant extends (EventEmitter as new () => TypedEmitter
   /** @internal */
   constructor(sid: string, identity: string, name?: string, metadata?: string) {
     super();
-    this.setMaxListeners(100);
     this.sid = sid;
     this.identity = identity;
     this.name = name;
@@ -265,7 +264,7 @@ export default class Participant extends (EventEmitter as new () => TypedEmitter
 export type ParticipantEventCallbacks = {
   trackPublished: (publication: RemoteTrackPublication) => void;
   trackSubscribed: (track: RemoteTrack, publication: RemoteTrackPublication) => void;
-  trackSubscriptionFailed: (trackSid: string) => void;
+  trackSubscriptionFailed: (trackSid: string, reason?: SubscriptionError) => void;
   trackUnpublished: (publication: RemoteTrackPublication) => void;
   trackUnsubscribed: (track: RemoteTrack, publication: RemoteTrackPublication) => void;
   trackMuted: (publication: TrackPublication) => void;

package/src/room/participant/RemoteParticipant.ts

@@ -1,6 +1,7 @@
+import type EventEmitter from 'eventemitter3';
 import type { SignalClient } from '../../api/SignalClient';
 import log from '../../logger';
-import type { ParticipantInfo } from '../../proto/livekit_models';
+import type { ParticipantInfo, SubscriptionError } from '../../proto/livekit_models';
 import type { UpdateSubscription, UpdateTrackSettings } from '../../proto/livekit_rtc';
 import { ParticipantEvent, TrackEvent } from '../events';
 import RemoteAudioTrack from '../track/RemoteAudioTrack';
@@ -81,6 +82,9 @@ export default class RemoteParticipant extends Participant {
     publication.on(TrackEvent.Unsubscribed, (previousTrack: RemoteTrack) => {
       this.emit(ParticipantEvent.TrackUnsubscribed, previousTrack, publication);
     });
+    publication.on(TrackEvent.SubscriptionFailed, (error: SubscriptionError) => {
+      this.emit(ParticipantEvent.TrackSubscriptionFailed, publication.trackSid, error);
+    });
   }
 
   getTrack(source: Track.Source): RemoteTrackPublication | undefined {
@@ -342,9 +346,9 @@ export default class RemoteParticipant extends Participant {
   }
 
   /** @internal */
-  emit<E extends keyof ParticipantEventCallbacks>(
-    event: E,
-    ...args: Parameters<ParticipantEventCallbacks[E]>
+  emit<T extends EventEmitter.EventNames<ParticipantEventCallbacks>>(
+    event: T,
+    ...args: EventEmitter.EventArgs<ParticipantEventCallbacks, T>
   ): boolean {
     log.trace('participant event', { participant: this.sid, event, args });
     return super.emit(event, ...args);

package/src/room/participant/publishUtils.ts

@@ -123,29 +123,25 @@ export function computeVideoEncodings(
   if (scalabilityMode && isSVCCodec(videoCodec)) {
     log.debug(`using svc with scalabilityMode ${scalabilityMode}`);
 
-    const encodings: RTCRtpEncodingParameters[] = [];
+    const sm = new ScalabilityMode(scalabilityMode);
 
-    // svc use first encoding as the original, so we sort encoding from high to low
-    switch (scalabilityMode) {
-      case 'L3T3':
-        for (let i = 0; i < 3; i += 1) {
-          encodings.push({
-            rid: videoRids[2 - i],
-            scaleResolutionDownBy: 2 ** i,
-            maxBitrate: videoEncoding.maxBitrate / 3 ** i,
-            /* @ts-ignore */
-            maxFramerate: original.encoding.maxFramerate,
-            /* @ts-ignore */
-            scalabilityMode: 'L3T3',
-          });
-        }
-        log.debug('encodings', encodings);
-        return encodings;
+    const encodings: RTCRtpEncodingParameters[] = [];
 
-      default:
-        // TODO : support other scalability modes
-        throw new Error(`unsupported scalabilityMode: ${scalabilityMode}`);
+    if (sm.spatial > 3) {
+      throw new Error(`unsupported scalabilityMode: ${scalabilityMode}`);
+    }
+    for (let i = 0; i < sm.spatial; i += 1) {
+      encodings.push({
+        rid: videoRids[2 - i],
+        maxBitrate: videoEncoding.maxBitrate / 3 ** i,
+        /* @ts-ignore */
+        maxFramerate: original.encoding.maxFramerate,
+      });
     }
+    /* @ts-ignore */
+    encodings[0].scalabilityMode = scalabilityMode;
+    log.debug('encodings', encodings);
+    return encodings;
   }
 
   if (!useSimulcast) {
@@ -368,3 +364,34 @@ export function sortPresets(presets: Array<VideoPreset> | undefined) {
     return 0;
   });
 }
+
+/** @internal */
+export class ScalabilityMode {
+  spatial: number;
+
+  temporal: number;
+
+  suffix: undefined | 'h' | '_KEY' | '_KEY_SHIFT';
+
+  constructor(scalabilityMode: string) {
+    const results = scalabilityMode.match(/^L(\d)T(\d)(h|_KEY|_KEY_SHIFT){0,1}$/);
+    if (!results) {
+      throw new Error('invalid scalability mode');
+    }
+
+    this.spatial = parseInt(results[1]);
+    this.temporal = parseInt(results[2]);
+    if (results.length > 3) {
+      switch (results[3]) {
+        case 'h':
+        case '_KEY':
+        case '_KEY_SHIFT':
+          this.suffix = results[3];
+      }
+    }
+  }
+
+  toString(): string {
+    return `L${this.spatial}T${this.temporal}${this.suffix ?? ''}`;
+  }
+}