livekit-client 1.15.0 → 1.15.2

package/src/room/Room.ts

@@ -2,7 +2,6 @@ import { protoInt64 } from '@bufbuild/protobuf';
 import { EventEmitter } from 'events';
 import type TypedEmitter from 'typed-emitter';
 import 'webrtc-adapter';
-import { toProtoSessionDescription } from '../api/SignalClient';
 import { EncryptionEvent } from '../e2ee';
 import { E2EEManager } from '../e2ee/E2eeManager';
 import log from '../logger';
@@ -35,8 +34,6 @@ import {
   StreamStateUpdate,
   SubscriptionPermissionUpdate,
   SubscriptionResponse,
-  SyncState,
-  UpdateSubscription,
 } from '../proto/livekit_rtc_pb';
 import { getBrowser } from '../utils/browserParser';
 import DeviceManager from './DeviceManager';
@@ -152,6 +149,8 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
 
   private regionUrl?: string;
 
+  private isVideoPlaybackBlocked: boolean = false;
+
   /**
    * Creates a new Room, the primary construct for a LiveKit session.
    * @param options
@@ -865,6 +864,22 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     }
   };
 
+  startVideo = async () => {
+    for (const p of this.participants.values()) {
+      p.videoTracks.forEach((tr) => {
+        tr.track?.attachedElements.forEach((el) => {
+          el.play().catch((e) => {
+            if (e.name === 'NotAllowedError') {
+              log.warn(
+                'Resuming video playback failed, make sure you call `startVideo` directly in a user gesture handler',
+              );
+            }
+          });
+        });
+      });
+    }
+  };
+
   /**
    * Returns true if audio playback is enabled
    */
@@ -872,6 +887,13 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     return this.audioEnabled;
   }
 
+  /**
+   * Returns true if video playback is enabled
+   */
+  get canPlaybackVideo(): boolean {
+    return !this.isVideoPlaybackBlocked;
+  }
+
   /**
    * Returns the active audio output device used in this room.
    * @return the previously successfully set audio output device ID or an empty string if the default device is used.
@@ -1384,6 +1406,20 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     this.emit(RoomEvent.AudioPlaybackStatusChanged, false);
   };
 
+  private handleVideoPlaybackStarted = () => {
+    if (this.isVideoPlaybackBlocked) {
+      this.isVideoPlaybackBlocked = false;
+      this.emit(RoomEvent.VideoPlaybackStatusChanged, true);
+    }
+  };
+
+  private handleVideoPlaybackFailed = () => {
+    if (!this.isVideoPlaybackBlocked) {
+      this.isVideoPlaybackBlocked = true;
+      this.emit(RoomEvent.VideoPlaybackStatusChanged, false);
+    }
+  };
+
   private handleDeviceChange = async () => {
     this.emit(RoomEvent.MediaDevicesChanged);
   };
@@ -1487,6 +1523,9 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
           if (track.kind === Track.Kind.Audio) {
             track.on(TrackEvent.AudioPlaybackStarted, this.handleAudioPlaybackStarted);
             track.on(TrackEvent.AudioPlaybackFailed, this.handleAudioPlaybackFailed);
+          } else if (track.kind === Track.Kind.Video) {
+            track.on(TrackEvent.VideoPlaybackFailed, this.handleVideoPlaybackFailed);
+            track.on(TrackEvent.VideoPlaybackStarted, this.handleVideoPlaybackStarted);
           }
           this.emit(RoomEvent.TrackSubscribed, track, publication, participant);
         },
@@ -1551,49 +1590,12 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
   }
 
   private sendSyncState() {
-    const previousAnswer = this.engine.subscriber?.getLocalDescription();
-    const previousOffer = this.engine.subscriber?.getRemoteDescription();
-
-    if (!previousAnswer) {
-      return;
-    }
-
-    /* 1. autosubscribe on, so subscribed tracks = all tracks - unsub tracks,
-          in this case, we send unsub tracks, so server add all tracks to this
-          subscribe pc and unsub special tracks from it.
-       2. autosubscribe off, we send subscribed tracks.
-    */
-    const autoSubscribe = this.connOptions?.autoSubscribe ?? true;
-    const trackSids = new Array<string>();
-    this.participants.forEach((participant) => {
-      participant.tracks.forEach((track) => {
-        if (track.isDesired !== autoSubscribe) {
-          trackSids.push(track.trackSid);
-        }
-      });
-    });
-
-    this.engine.client.sendSyncState(
-      new SyncState({
-        answer: toProtoSessionDescription({
-          sdp: previousAnswer.sdp,
-          type: previousAnswer.type,
-        }),
-        offer: previousOffer
-          ? toProtoSessionDescription({
-              sdp: previousOffer.sdp,
-              type: previousOffer.type,
-            })
-          : undefined,
-        subscription: new UpdateSubscription({
-          trackSids,
-          subscribe: !autoSubscribe,
-          participantTracks: [],
-        }),
-        publishTracks: this.localParticipant.publishedTracksInfo(),
-        dataChannels: this.localParticipant.dataChannelsInfo(),
-      }),
-    );
+    const remoteTracks = Array.from(this.participants.values()).reduce((acc, participant) => {
+      acc.push(...(participant.getTracks() as RemoteTrackPublication[])); // FIXME would be nice to have this return RemoteTrackPublications directly instead of the type cast
+      return acc;
+    }, [] as RemoteTrackPublication[]);
+    const localTracks = this.localParticipant.getTracks() as LocalTrackPublication[]; // FIXME would be nice to have this return LocalTrackPublications directly instead of the type cast
+    this.engine.sendSyncState(remoteTracks, localTracks);
   }
 
   /**
@@ -1929,6 +1931,7 @@ export type RoomEventCallbacks = {
     participant: RemoteParticipant,
   ) => void;
   audioPlaybackChanged: (playing: boolean) => void;
+  videoPlaybackChanged: (playing: boolean) => void;
   signalConnected: () => void;
   recordingStatusChanged: (recording: boolean) => void;
   participantEncryptionStatusChanged: (encrypted: boolean, participant?: Participant) => void;

package/src/room/events.ts

@@ -245,6 +245,13 @@ export enum RoomEvent {
    */
   AudioPlaybackStatusChanged = 'audioPlaybackChanged',
 
+  /**
+   * LiveKit will attempt to autoplay all video tracks when you attach them to
+   * a video element. However, if that fails, we'll notify you via VideoPlaybackStatusChanged.
+   * Calling `room.startVideo()` in a user gesture event handler will resume the video playback.
+   */
+  VideoPlaybackStatusChanged = 'videoPlaybackChanged',
+
   /**
    * When we have encountered an error while attempting to create a track.
    * The errors take place in getUserMedia().
@@ -510,6 +517,10 @@ export enum TrackEvent {
   /** @internal */
   VideoDimensionsChanged = 'videoDimensionsChanged',
   /** @internal */
+  VideoPlaybackStarted = 'videoPlaybackStarted',
+  /** @internal */
+  VideoPlaybackFailed = 'videoPlaybackFailed',
+  /** @internal */
   ElementAttached = 'elementAttached',
   /** @internal */
   ElementDetached = 'elementDetached',

package/src/room/participant/LocalParticipant.ts

@@ -10,13 +10,11 @@ import {
 } from '../../proto/livekit_models_pb';
 import {
   AddTrackRequest,
-  DataChannelInfo,
-  SignalTarget,
   SimulcastCodec,
   SubscribedQualityUpdate,
-  TrackPublishedResponse,
   TrackUnpublishedResponse,
 } from '../../proto/livekit_rtc_pb';
+import { PCTransportState } from '../PCTransportManager';
 import type RTCEngine from '../RTCEngine';
 import { defaultVideoCodec } from '../defaults';
 import { DeviceUnsupportedError, TrackInvalidError, UnexpectedConnectionState } from '../errors';
@@ -695,7 +693,12 @@ export default class LocalParticipant extends Participant {
         if (opts.backupCodec === true) {
           opts.backupCodec = { codec: defaultVideoCodec };
         }
-        if (opts.backupCodec && videoCodec !== opts.backupCodec.codec) {
+        if (
+          opts.backupCodec &&
+          videoCodec !== opts.backupCodec.codec &&
+          // TODO remove this once e2ee is supported for backup codecs
+          req.encryption === Encryption_Type.NONE
+        ) {
           // multi-codec simulcast requires dynacast
           if (!this.roomOptions.dynacast) {
             this.roomOptions.dynacast = true;
@@ -766,8 +769,8 @@ export default class LocalParticipant extends Participant {
     publication.options = opts;
     track.sid = ti.sid;
 
-    if (!this.engine.publisher) {
-      throw new UnexpectedConnectionState('publisher is closed');
+    if (!this.engine.pcManager) {
+      throw new UnexpectedConnectionState('pcManager is not ready');
     }
     log.debug(`publishing ${track.kind} with encodings`, { encodings, trackInfo: ti });
 
@@ -783,21 +786,21 @@ export default class LocalParticipant extends Participant {
            fix the issue.
          */
         let trackTransceiver: RTCRtpTransceiver | undefined = undefined;
-        for (const transceiver of this.engine.publisher.getTransceivers()) {
+        for (const transceiver of this.engine.pcManager.publisher.getTransceivers()) {
           if (transceiver.sender === track.sender) {
             trackTransceiver = transceiver;
             break;
           }
         }
         if (trackTransceiver) {
-          this.engine.publisher.setTrackCodecBitrate({
+          this.engine.pcManager.publisher.setTrackCodecBitrate({
             transceiver: trackTransceiver,
             codec: 'opus',
             maxbr: encodings[0]?.maxBitrate ? encodings[0].maxBitrate / 1000 : 0,
           });
         }
       } else if (track.codec && isSVCCodec(track.codec) && encodings[0]?.maxBitrate) {
-        this.engine.publisher.setTrackCodecBitrate({
+        this.engine.pcManager.publisher.setTrackCodecBitrate({
           cid: req.cid,
           codec: track.codec,
           maxbr: encodings[0].maxBitrate / 1000,
@@ -929,12 +932,12 @@ export default class LocalParticipant extends Participant {
     const trackSender = track.sender;
     track.sender = undefined;
     if (
-      this.engine.publisher &&
-      this.engine.publisher.getConnectionState() !== 'closed' &&
+      this.engine.pcManager &&
+      this.engine.pcManager.currentState < PCTransportState.FAILED &&
       trackSender
     ) {
       try {
-        for (const transceiver of this.engine.publisher.getTransceivers()) {
+        for (const transceiver of this.engine.pcManager.publisher.getTransceivers()) {
           // if sender is not currently sending (after replaceTrack(null))
           // removeTrack would have no effect.
           // to ensure we end up successfully removing the track, manually set
@@ -1310,44 +1313,4 @@ export default class LocalParticipant extends Participant {
     });
     return publication;
   }
-
-  /** @internal */
-  publishedTracksInfo(): TrackPublishedResponse[] {
-    const infos: TrackPublishedResponse[] = [];
-    this.tracks.forEach((track: LocalTrackPublication) => {
-      if (track.track !== undefined) {
-        infos.push(
-          new TrackPublishedResponse({
-            cid: track.track.mediaStreamID,
-            track: track.trackInfo,
-          }),
-        );
-      }
-    });
-    return infos;
-  }
-
-  /** @internal */
-  dataChannelsInfo(): DataChannelInfo[] {
-    const infos: DataChannelInfo[] = [];
-    const getInfo = (dc: RTCDataChannel | undefined, target: SignalTarget) => {
-      if (dc?.id !== undefined && dc.id !== null) {
-        infos.push(
-          new DataChannelInfo({
-            label: dc.label,
-            id: dc.id,
-            target,
-          }),
-        );
-      }
-    };
-    getInfo(this.engine.dataChannelForKind(DataPacket_Kind.LOSSY), SignalTarget.PUBLISHER);
-    getInfo(this.engine.dataChannelForKind(DataPacket_Kind.RELIABLE), SignalTarget.PUBLISHER);
-    getInfo(this.engine.dataChannelForKind(DataPacket_Kind.LOSSY, true), SignalTarget.SUBSCRIBER);
-    getInfo(
-      this.engine.dataChannelForKind(DataPacket_Kind.RELIABLE, true),
-      SignalTarget.SUBSCRIBER,
-    );
-    return infos;
-  }
 }

package/src/room/track/Track.ts

@@ -1,7 +1,7 @@
 import { EventEmitter } from 'events';
+import { debounce } from 'ts-debounce';
 import type TypedEventEmitter from 'typed-emitter';
 import type { SignalClient } from '../../api/SignalClient';
-import log from '../../logger';
 import { TrackSource, TrackType } from '../../proto/livekit_models_pb';
 import { StreamState as ProtoStreamState } from '../../proto/livekit_rtc_pb';
 import { TrackEvent } from '../events';
@@ -113,6 +113,9 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
 
     if (!this.attachedElements.includes(element)) {
       this.attachedElements.push(element);
+      // listen to suspend events in order to detect auto playback issues
+      element.addEventListener('suspend', this.handleElementSuspended);
+      element.addEventListener('playing', this.handleElementPlay);
     }
 
     // even if we believe it's already attached to the element, it's possible
@@ -130,11 +133,6 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
           this.emit(TrackEvent.AudioPlaybackStarted);
         })
         .catch((e) => {
-          if (e.name === 'NotAllowedError') {
-            this.emit(TrackEvent.AudioPlaybackFailed, e);
-          } else {
-            log.warn('could not playback audio', e);
-          }
           // If audio playback isn't allowed make sure we still play back the video
           if (
             element &&
@@ -172,6 +170,8 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
         if (idx >= 0) {
           this.attachedElements.splice(idx, 1);
           this.recycleElement(element);
+          element.removeEventListener('suspend', this.handleElementSuspended);
+          element.removeEventListener('playing', this.handleElementPlay);
           this.emit(TrackEvent.ElementDetached, element);
         }
         return element;
@@ -182,6 +182,8 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
         detachTrack(this.mediaStreamTrack, elm);
         detached.push(elm);
         this.recycleElement(elm);
+        elm.removeEventListener('suspend', this.handleElementSuspended);
+        elm.removeEventListener('playing', this.handleElementPlay);
         this.emit(TrackEvent.ElementDetached, elm);
       });
 
@@ -268,9 +270,26 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
       document.removeEventListener('visibilitychange', this.appVisibilityChangedListener);
     }
   }
+
+  private handleElementSuspended = () => {
+    this.debouncedPlaybackStateChange(false);
+  };
+
+  private handleElementPlay = () => {
+    this.debouncedPlaybackStateChange(true);
+  };
+
+  private debouncedPlaybackStateChange = debounce((allowed: boolean) => {
+    // we debounce this as Safari triggers both `playing` and `suspend` shortly after one another
+    // in order not to raise the wrong event, we debounce the call to make sure we only emit the correct status
+    if (this.kind === Track.Kind.Audio) {
+      this.emit(allowed ? TrackEvent.AudioPlaybackStarted : TrackEvent.AudioPlaybackFailed);
+    } else if (this.kind === Track.Kind.Video) {
+      this.emit(allowed ? TrackEvent.VideoPlaybackStarted : TrackEvent.VideoPlaybackFailed);
+    }
+  }, 300);
 }
 
-/** @internal */
 export function attachToElement(track: MediaStreamTrack, element: HTMLMediaElement) {
   let mediaStream: MediaStream;
   if (element.srcObject instanceof MediaStream) {
@@ -321,7 +340,7 @@ export function attachToElement(track: MediaStreamTrack, element: HTMLMediaEleme
         // when the window is backgrounded before the first frame is drawn
         // manually calling play here seems to fix that
         element.play().catch(() => {
-          /* do nothing */
+          /** do nothing, we watch the `suspended` event do deal with these failures */
         });
       }, 0);
     }
@@ -446,10 +465,12 @@ export type TrackEventCallbacks = {
   updateSettings: () => void;
   updateSubscription: () => void;
   audioPlaybackStarted: () => void;
-  audioPlaybackFailed: (error: Error) => void;
+  audioPlaybackFailed: (error?: Error) => void;
   audioSilenceDetected: () => void;
   visibilityChanged: (visible: boolean, track?: any) => void;
   videoDimensionsChanged: (dimensions: Track.Dimensions, track?: any) => void;
+  videoPlaybackStarted: () => void;
+  videoPlaybackFailed: (error?: Error) => void;
   elementAttached: (element: HTMLMediaElement) => void;
   elementDetached: (element: HTMLMediaElement) => void;
   upstreamPaused: (track: any) => void;

package/src/room/track/utils.ts

@@ -1,6 +1,8 @@
+import { TrackPublishedResponse } from '../../proto/livekit_rtc_pb';
 import { cloneDeep } from '../../utils/cloneDeep';
 import { isSafari, sleep } from '../utils';
 import { Track } from './Track';
+import type { TrackPublication } from './TrackPublication';
 import {
   type AudioCaptureOptions,
   type CreateLocalTracksOptions,
@@ -190,3 +192,20 @@ export function mimeTypeToVideoCodecString(mimeType: string) {
   }
   return codec;
 }
+
+export function getTrackPublicationInfo<T extends TrackPublication>(
+  tracks: T[],
+): TrackPublishedResponse[] {
+  const infos: TrackPublishedResponse[] = [];
+  tracks.forEach((track: TrackPublication) => {
+    if (track.track !== undefined) {
+      infos.push(
+        new TrackPublishedResponse({
+          cid: track.track.mediaStreamID,
+          track: track.trackInfo,
+        }),
+      );
+    }
+  });
+  return infos;
+}