livekit-client 1.15.3 → 1.15.5

package/src/room/Room.ts

@@ -195,7 +195,10 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       );
     }
     if (this.options.audioOutput?.deviceId) {
-      this.switchActiveDevice('audiooutput', unwrapConstraint(this.options.audioOutput.deviceId));
+      this.switchActiveDevice(
+        'audiooutput',
+        unwrapConstraint(this.options.audioOutput.deviceId),
+      ).catch((e) => log.warn(`Could not set audio output: ${e.message}`));
     }
 
     if (this.options.e2ee) {
@@ -865,19 +868,29 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
   };
 
   startVideo = async () => {
+    const elements: HTMLMediaElement[] = [];
     for (const p of this.participants.values()) {
       p.videoTracks.forEach((tr) => {
         tr.track?.attachedElements.forEach((el) => {
-          el.play().catch((e) => {
-            if (e.name === 'NotAllowedError') {
-              log.warn(
-                'Resuming video playback failed, make sure you call `startVideo` directly in a user gesture handler',
-              );
-            }
-          });
+          if (!elements.includes(el)) {
+            elements.push(el);
+          }
         });
       });
     }
+    await Promise.all(elements.map((el) => el.play()))
+      .then(() => {
+        this.handleVideoPlaybackStarted();
+      })
+      .catch((e) => {
+        if (e.name === 'NotAllowedError') {
+          this.handleVideoPlaybackFailed();
+        } else {
+          log.warn(
+            'Resuming video playback failed, make sure you call `startVideo` directly in a user gesture handler',
+          );
+        }
+      });
   };
 
   /**
@@ -1494,6 +1507,11 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     if (this.options.expWebAudioMix) {
       participant.setAudioContext(this.audioContext);
     }
+    if (this.options.audioOutput?.deviceId) {
+      participant
+        .setAudioOutput(this.options.audioOutput)
+        .catch((e) => log.warn(`Could not set audio output: ${e.message}`));
+    }
     return participant;
   }
 

package/src/room/events.ts

@@ -486,6 +486,9 @@ export enum EngineEvent {
   ConnectionQualityUpdate = 'connectionQualityUpdate',
   SubscriptionError = 'subscriptionError',
   SubscriptionPermissionUpdate = 'subscriptionPermissionUpdate',
+  RemoteMute = 'remoteMute',
+  SubscribedQualityUpdate = 'subscribedQualityUpdate',
+  LocalTrackUnpublished = 'localTrackUnpublished',
 }
 
 export enum TrackEvent {

package/src/room/participant/LocalParticipant.ts

@@ -127,7 +127,7 @@ export default class LocalParticipant extends Participant {
    */
   setupEngine(engine: RTCEngine) {
     this.engine = engine;
-    this.engine.client.onRemoteMuteChanged = (trackSid: string, muted: boolean) => {
+    this.engine.on(EngineEvent.RemoteMute, (trackSid: string, muted: boolean) => {
       const pub = this.tracks.get(trackSid);
       if (!pub || !pub.track) {
         return;
@@ -137,11 +137,7 @@ export default class LocalParticipant extends Participant {
       } else {
         pub.unmute();
       }
-    };
-
-    this.engine.client.onSubscribedQualityUpdate = this.handleSubscribedQualityUpdate;
-
-    this.engine.client.onLocalTrackUnpublished = this.handleLocalTrackUnpublished;
+    });
 
     this.engine
       .on(EngineEvent.Connected, this.handleReconnected)
@@ -149,6 +145,8 @@ export default class LocalParticipant extends Participant {
       .on(EngineEvent.SignalResumed, this.handleReconnected)
       .on(EngineEvent.Restarting, this.handleReconnecting)
       .on(EngineEvent.Resuming, this.handleReconnecting)
+      .on(EngineEvent.LocalTrackUnpublished, this.handleLocalTrackUnpublished)
+      .on(EngineEvent.SubscribedQualityUpdate, this.handleSubscribedQualityUpdate)
       .on(EngineEvent.Disconnected, this.handleDisconnected);
   }
 

package/src/room/participant/Participant.ts

@@ -21,6 +21,11 @@ export enum ConnectionQuality {
   Excellent = 'excellent',
   Good = 'good',
   Poor = 'poor',
+  /**
+   * Indicates that a participant has temporarily (or permanently) lost connection to LiveKit.
+   * For permanent disconnection a `ParticipantDisconnected` event will be emitted after a timeout
+   */
+  Lost = 'lost',
   Unknown = 'unknown',
 }
 
@@ -32,6 +37,8 @@ function qualityFromProto(q: ProtoQuality): ConnectionQuality {
       return ConnectionQuality.Good;
     case ProtoQuality.POOR:
       return ConnectionQuality.Poor;
+    case ProtoQuality.LOST:
+      return ConnectionQuality.Lost;
     default:
       return ConnectionQuality.Unknown;
   }
@@ -77,6 +84,10 @@ export default class Participant extends (EventEmitter as new () => TypedEmitter
     return this.tracks.size > 0 && Array.from(this.tracks.values()).every((tr) => tr.isEncrypted);
   }
 
+  get isAgent() {
+    return this.permissions?.agent ?? false;
+  }
+
   /** @internal */
   constructor(sid: string, identity: string, name?: string, metadata?: string) {
     super();

package/src/room/stats.ts

@@ -106,6 +106,8 @@ export interface VideoReceiverStats extends ReceiverStats {
   nackCount?: number;
 
   decoderImplementation?: string;
+
+  mimeType?: string;
 }
 
 export function computeBitrate<T extends ReceiverStats | SenderStats>(

package/src/room/track/RemoteVideoTrack.ts

@@ -169,8 +169,11 @@ export default class RemoteVideoTrack extends RemoteTrack {
 
     const stats = await this.receiver.getStats();
     let receiverStats: VideoReceiverStats | undefined;
+    let codecID = '';
+    let codecs = new Map<string, any>();
     stats.forEach((v) => {
       if (v.type === 'inbound-rtp') {
+        codecID = v.codecId;
         receiverStats = {
           type: 'video',
           framesDecoded: v.framesDecoded,
@@ -188,8 +191,13 @@ export default class RemoteVideoTrack extends RemoteTrack {
           bytesReceived: v.bytesReceived,
           decoderImplementation: v.decoderImplementation,
         };
+      } else if (v.type === 'codec') {
+        codecs.set(v.id, v);
       }
     });
+    if (receiverStats && codecID !== '' && codecs.get(codecID)) {
+      receiverStats.mimeType = codecs.get(codecID).mimeType;
+    }
     return receiverStats;
   }
 

package/src/room/track/Track.ts

@@ -1,7 +1,7 @@
 import { EventEmitter } from 'events';
-import { debounce } from 'ts-debounce';
 import type TypedEventEmitter from 'typed-emitter';
 import type { SignalClient } from '../../api/SignalClient';
+import log from '../../logger';
 import { TrackSource, TrackType } from '../../proto/livekit_models_pb';
 import { StreamState as ProtoStreamState } from '../../proto/livekit_rtc_pb';
 import { TrackEvent } from '../events';
@@ -113,9 +113,6 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
 
     if (!this.attachedElements.includes(element)) {
       this.attachedElements.push(element);
-      // listen to suspend events in order to detect auto playback issues
-      element.addEventListener('suspend', this.handleElementSuspended);
-      element.addEventListener('playing', this.handleElementPlay);
     }
 
     // even if we believe it's already attached to the element, it's possible
@@ -125,27 +122,38 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
 
     // handle auto playback failures
     const allMediaStreamTracks = (element.srcObject as MediaStream).getTracks();
-    if (allMediaStreamTracks.some((tr) => tr.kind === 'audio')) {
-      // manually play audio to detect audio playback status
-      element
-        .play()
-        .then(() => {
-          this.emit(TrackEvent.AudioPlaybackStarted);
-        })
-        .catch((e) => {
-          // If audio playback isn't allowed make sure we still play back the video
-          if (
-            element &&
-            allMediaStreamTracks.some((tr) => tr.kind === 'video') &&
-            e.name === 'NotAllowedError'
-          ) {
-            element.muted = true;
-            element.play().catch(() => {
-              // catch for Safari, exceeded options at this point to automatically play the media element
-            });
-          }
-        });
-    }
+    const hasAudio = allMediaStreamTracks.some((tr) => tr.kind === 'audio');
+
+    // manually play media to detect auto playback status
+    element
+      .play()
+      .then(() => {
+        this.emit(hasAudio ? TrackEvent.AudioPlaybackStarted : TrackEvent.VideoPlaybackStarted);
+      })
+      .catch((e) => {
+        if (e.name === 'NotAllowedError') {
+          this.emit(hasAudio ? TrackEvent.AudioPlaybackFailed : TrackEvent.VideoPlaybackFailed, e);
+        } else if (e.name === 'AbortError') {
+          // commonly triggered by another `play` request, only log for debugging purposes
+          log.debug(
+            `${hasAudio ? 'audio' : 'video'} playback aborted, likely due to new play request`,
+          );
+        } else {
+          log.warn(`could not playback ${hasAudio ? 'audio' : 'video'}`, e);
+        }
+        // If audio playback isn't allowed make sure we still play back the video
+        if (
+          hasAudio &&
+          element &&
+          allMediaStreamTracks.some((tr) => tr.kind === 'video') &&
+          e.name === 'NotAllowedError'
+        ) {
+          element.muted = true;
+          element.play().catch(() => {
+            // catch for Safari, exceeded options at this point to automatically play the media element
+          });
+        }
+      });
 
     this.emit(TrackEvent.ElementAttached, element);
     return element;
@@ -170,8 +178,6 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
         if (idx >= 0) {
           this.attachedElements.splice(idx, 1);
           this.recycleElement(element);
-          element.removeEventListener('suspend', this.handleElementSuspended);
-          element.removeEventListener('playing', this.handleElementPlay);
           this.emit(TrackEvent.ElementDetached, element);
         }
         return element;
@@ -182,8 +188,6 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
         detachTrack(this.mediaStreamTrack, elm);
         detached.push(elm);
         this.recycleElement(elm);
-        elm.removeEventListener('suspend', this.handleElementSuspended);
-        elm.removeEventListener('playing', this.handleElementPlay);
         this.emit(TrackEvent.ElementDetached, elm);
       });
 
@@ -270,24 +274,6 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
       document.removeEventListener('visibilitychange', this.appVisibilityChangedListener);
     }
   }
-
-  private handleElementSuspended = () => {
-    this.debouncedPlaybackStateChange(false);
-  };
-
-  private handleElementPlay = () => {
-    this.debouncedPlaybackStateChange(true);
-  };
-
-  private debouncedPlaybackStateChange = debounce((allowed: boolean) => {
-    // we debounce this as Safari triggers both `playing` and `suspend` shortly after one another
-    // in order not to raise the wrong event, we debounce the call to make sure we only emit the correct status
-    if (this.kind === Track.Kind.Audio) {
-      this.emit(allowed ? TrackEvent.AudioPlaybackStarted : TrackEvent.AudioPlaybackFailed);
-    } else if (this.kind === Track.Kind.Video) {
-      this.emit(allowed ? TrackEvent.VideoPlaybackStarted : TrackEvent.VideoPlaybackFailed);
-    }
-  }, 300);
 }
 
 export function attachToElement(track: MediaStreamTrack, element: HTMLMediaElement) {
@@ -340,7 +326,7 @@ export function attachToElement(track: MediaStreamTrack, element: HTMLMediaEleme
         // when the window is backgrounded before the first frame is drawn
         // manually calling play here seems to fix that
         element.play().catch(() => {
-          /** do nothing, we watch the `suspended` event do deal with these failures */
+          /** do nothing */
         });
       }, 0);
     }

package/src/version.ts

@@ -1,4 +1,4 @@
 import { version as v } from '../package.json';
 
 export const version = v;
-export const protocolVersion = 10;
+export const protocolVersion = 11;