livekit-client 1.15.0 → 1.15.1

package/dist/ts4.2/src/room/events.d.ts

@@ -215,6 +215,12 @@ export declare enum RoomEvent {
      * `Room.canPlaybackAudio` will indicate if audio playback is permitted.
      */
     AudioPlaybackStatusChanged = "audioPlaybackChanged",
+    /**
+     * LiveKit will attempt to autoplay all video tracks when you attach them to
+     * a video element. However, if that fails, we'll notify you via VideoPlaybackStatusChanged.
+     * Calling `room.startVideo()` in a user gesture event handler will resume the video playback.
+     */
+    VideoPlaybackStatusChanged = "videoPlaybackChanged",
     /**
      * When we have encountered an error while attempting to create a track.
      * The errors take place in getUserMedia().
@@ -449,6 +455,10 @@ export declare enum TrackEvent {
     /** @internal */
     VideoDimensionsChanged = "videoDimensionsChanged",
     /** @internal */
+    VideoPlaybackStarted = "videoPlaybackStarted",
+    /** @internal */
+    VideoPlaybackFailed = "videoPlaybackFailed",
+    /** @internal */
     ElementAttached = "elementAttached",
     /** @internal */
     ElementDetached = "elementDetached",

package/dist/ts4.2/src/room/participant/LocalParticipant.d.ts

@@ -1,6 +1,5 @@
 import type { InternalRoomOptions } from '../../options';
 import { DataPacket_Kind, ParticipantInfo, ParticipantPermission } from '../../proto/livekit_models_pb';
-import { DataChannelInfo, TrackPublishedResponse } from '../../proto/livekit_rtc_pb';
 import type RTCEngine from '../RTCEngine';
 import LocalTrack from '../track/LocalTrack';
 import LocalTrackPublication from '../track/LocalTrackPublication';
@@ -174,9 +173,5 @@ export default class LocalParticipant extends Participant {
     private handleLocalTrackUnpublished;
     private handleTrackEnded;
     private getPublicationForTrack;
-    /** @internal */
-    publishedTracksInfo(): TrackPublishedResponse[];
-    /** @internal */
-    dataChannelsInfo(): DataChannelInfo[];
 }
 //# sourceMappingURL=LocalParticipant.d.ts.map

package/dist/ts4.2/src/room/track/Track.d.ts

@@ -64,8 +64,10 @@ export declare abstract class Track extends Track_base {
     protected handleAppVisibilityChanged(): Promise<void>;
     protected addAppVisibilityListener(): void;
     protected removeAppVisibilityListener(): void;
+    private handleElementSuspended;
+    private handleElementPlay;
+    private debouncedPlaybackStateChange;
 }
-/** @internal */
 export declare function attachToElement(track: MediaStreamTrack, element: HTMLMediaElement): void;
 /** @internal */
 export declare function detachTrack(track: MediaStreamTrack, element: HTMLMediaElement): void;
@@ -112,10 +114,12 @@ export type TrackEventCallbacks = {
     updateSettings: () => void;
     updateSubscription: () => void;
     audioPlaybackStarted: () => void;
-    audioPlaybackFailed: (error: Error) => void;
+    audioPlaybackFailed: (error?: Error) => void;
     audioSilenceDetected: () => void;
     visibilityChanged: (visible: boolean, track?: any) => void;
     videoDimensionsChanged: (dimensions: Track.Dimensions, track?: any) => void;
+    videoPlaybackStarted: () => void;
+    videoPlaybackFailed: (error?: Error) => void;
     elementAttached: (element: HTMLMediaElement) => void;
     elementDetached: (element: HTMLMediaElement) => void;
     upstreamPaused: (track: any) => void;

package/dist/ts4.2/src/room/track/utils.d.ts

@@ -1,4 +1,6 @@
+import { TrackPublishedResponse } from '../../proto/livekit_rtc_pb';
 import { Track } from './Track';
+import type { TrackPublication } from './TrackPublication';
 import type { AudioCaptureOptions, CreateLocalTracksOptions, ScreenShareCaptureOptions, VideoCaptureOptions } from './options';
 import type { AudioTrack } from './types';
 export declare function mergeDefaultOptions(options?: CreateLocalTracksOptions, audioDefaults?: AudioCaptureOptions, videoDefaults?: VideoCaptureOptions): CreateLocalTracksOptions;
@@ -25,4 +27,5 @@ export declare function sourceToKind(source: Track.Source): MediaDeviceKind | un
  */
 export declare function screenCaptureToDisplayMediaStreamOptions(options: ScreenShareCaptureOptions): DisplayMediaStreamOptions;
 export declare function mimeTypeToVideoCodecString(mimeType: string): "vp8" | "h264" | "vp9" | "av1";
+export declare function getTrackPublicationInfo<T extends TrackPublication>(tracks: T[]): TrackPublishedResponse[];
 //# sourceMappingURL=utils.d.ts.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "livekit-client",
-  "version": "1.15.0",
+  "version": "1.15.1",
   "description": "JavaScript/TypeScript client SDK for LiveKit",
   "main": "./dist/livekit-client.umd.js",
   "unpkg": "./dist/livekit-client.umd.js",

package/src/connectionHelper/checks/webrtc.ts

@@ -38,8 +38,8 @@ export class WebRTCCheck extends Checker {
         }
       };
 
-      if (this.room.engine.subscriber) {
-        this.room.engine.subscriber.onIceCandidateError = (ev) => {
+      if (this.room.engine.pcManager) {
+        this.room.engine.pcManager.subscriber.onIceCandidateError = (ev) => {
           if (ev instanceof RTCPeerConnectionIceErrorEvent) {
             this.appendWarning(
               `error with ICE candidate: ${ev.errorCode} ${ev.errorText} ${ev.url}`,

package/src/room/PCTransport.ts

@@ -33,10 +33,16 @@ export default class PCTransport extends EventEmitter {
   private _pc: RTCPeerConnection | null;
 
   private get pc() {
-    if (this._pc) return this._pc;
-    throw new UnexpectedConnectionState('Expected peer connection to be available');
+    if (!this._pc) {
+      this._pc = this.createPC();
+    }
+    return this._pc;
   }
 
+  private config?: RTCConfiguration;
+
+  private mediaConstraints: Record<string, unknown>;
+
   pendingCandidates: RTCIceCandidateInit[] = [];
 
   restartingIce: boolean = false;
@@ -57,32 +63,53 @@ export default class PCTransport extends EventEmitter {
 
   onConnectionStateChange?: (state: RTCPeerConnectionState) => void;
 
+  onIceConnectionStateChange?: (state: RTCIceConnectionState) => void;
+
+  onSignalingStatechange?: (state: RTCSignalingState) => void;
+
   onDataChannel?: (ev: RTCDataChannelEvent) => void;
 
   onTrack?: (ev: RTCTrackEvent) => void;
 
   constructor(config?: RTCConfiguration, mediaConstraints: Record<string, unknown> = {}) {
     super();
-    this._pc = isChromiumBased()
+    this.config = config;
+    this.mediaConstraints = mediaConstraints;
+    this._pc = this.createPC();
+  }
+
+  private createPC() {
+    const pc = isChromiumBased()
       ? // @ts-expect-error chrome allows additional media constraints to be passed into the RTCPeerConnection constructor
-        new RTCPeerConnection(config, mediaConstraints)
-      : new RTCPeerConnection(config);
-    this._pc.onicecandidate = (ev) => {
+        new RTCPeerConnection(this.config, this.mediaConstraints)
+      : new RTCPeerConnection(this.config);
+
+    pc.onicecandidate = (ev) => {
       if (!ev.candidate) return;
       this.onIceCandidate?.(ev.candidate);
     };
-    this._pc.onicecandidateerror = (ev) => {
+    pc.onicecandidateerror = (ev) => {
       this.onIceCandidateError?.(ev);
     };
-    this._pc.onconnectionstatechange = () => {
-      this.onConnectionStateChange?.(this._pc?.connectionState ?? 'closed');
+
+    pc.oniceconnectionstatechange = () => {
+      this.onIceConnectionStateChange?.(pc.iceConnectionState);
     };
-    this._pc.ondatachannel = (ev) => {
+
+    pc.onsignalingstatechange = () => {
+      this.onSignalingStatechange?.(pc.signalingState);
+    };
+
+    pc.onconnectionstatechange = () => {
+      this.onConnectionStateChange?.(pc.connectionState);
+    };
+    pc.ondatachannel = (ev) => {
       this.onDataChannel?.(ev);
     };
-    this._pc.ontrack = (ev) => {
+    pc.ontrack = (ev) => {
       this.onTrack?.(ev);
     };
+    return pc;
   }
 
   get isICEConnected(): boolean {
@@ -168,7 +195,7 @@ export default class PCTransport extends EventEmitter {
 
     if (this.renegotiate) {
       this.renegotiate = false;
-      this.createAndSendOffer();
+      await this.createAndSendOffer();
     } else if (sd.type === 'answer') {
       this.emit(PCEvents.NegotiationComplete);
       if (sd.sdp) {
@@ -183,10 +210,10 @@ export default class PCTransport extends EventEmitter {
   }
 
   // debounced negotiate interface
-  negotiate = debounce((onError?: (e: Error) => void) => {
+  negotiate = debounce(async (onError?: (e: Error) => void) => {
     this.emit(PCEvents.NegotiationStarted);
     try {
-      this.createAndSendOffer();
+      await this.createAndSendOffer();
     } catch (e) {
       if (onError) {
         onError(e as Error);
@@ -209,11 +236,11 @@ export default class PCTransport extends EventEmitter {
     if (this._pc && this._pc.signalingState === 'have-local-offer') {
       // we're waiting for the peer to accept our offer, so we'll just wait
       // the only exception to this is when ICE restart is needed
-      const currentSD = this.pc.remoteDescription;
+      const currentSD = this._pc.remoteDescription;
       if (options?.iceRestart && currentSD) {
         // TODO: handle when ICE restart is needed but we don't have a remote description
         // the best thing to do is to recreate the peerconnection
-        await this.pc.setRemoteDescription(currentSD);
+        await this._pc.setRemoteDescription(currentSD);
       } else {
         this.renegotiate = true;
         return;
@@ -307,7 +334,10 @@ export default class PCTransport extends EventEmitter {
   }
 
   addTrack(track: MediaStreamTrack) {
-    return this.pc.addTrack(track);
+    if (!this._pc) {
+      throw new UnexpectedConnectionState('PC closed, cannot add track');
+    }
+    return this._pc.addTrack(track);
   }
 
   setTrackCodecBitrate(info: TrackBitrateInfo) {
@@ -315,43 +345,46 @@ export default class PCTransport extends EventEmitter {
   }
 
   setConfiguration(rtcConfig: RTCConfiguration) {
-    return this.pc.setConfiguration(rtcConfig);
+    if (!this._pc) {
+      throw new UnexpectedConnectionState('PC closed, cannot configure');
+    }
+    return this._pc?.setConfiguration(rtcConfig);
   }
 
   canRemoveTrack(): boolean {
-    return !!this.pc.removeTrack;
+    return !!this._pc?.removeTrack;
   }
 
   removeTrack(sender: RTCRtpSender) {
-    return this.pc.removeTrack(sender);
+    return this._pc?.removeTrack(sender);
   }
 
   getConnectionState() {
-    return this.pc.connectionState;
+    return this._pc?.connectionState ?? 'closed';
   }
 
   getICEConnectionState() {
-    return this.pc.iceConnectionState;
+    return this._pc?.iceConnectionState ?? 'closed';
   }
 
   getSignallingState() {
-    return this.pc.signalingState;
+    return this._pc?.signalingState ?? 'closed';
   }
 
   getTransceivers() {
-    return this.pc.getTransceivers();
+    return this._pc?.getTransceivers() ?? [];
   }
 
   getSenders() {
-    return this.pc.getSenders();
+    return this._pc?.getSenders() ?? [];
   }
 
   getLocalDescription() {
-    return this.pc.localDescription;
+    return this._pc?.localDescription;
   }
 
   getRemoteDescription() {
-    return this.pc.remoteDescription;
+    return this.pc?.remoteDescription;
   }
 
   getStats() {
@@ -395,7 +428,7 @@ export default class PCTransport extends EventEmitter {
     return candidates.get(selectedID);
   }
 
-  close() {
+  close = () => {
     if (!this._pc) {
       return;
     }
@@ -412,7 +445,7 @@ export default class PCTransport extends EventEmitter {
     this._pc.onconnectionstatechange = null;
     this._pc.oniceconnectionstatechange = null;
     this._pc = null;
-  }
+  };
 
   private async setMungedSDP(sd: RTCSessionDescriptionInit, munged?: string, remote?: boolean) {
     if (munged) {

package/src/room/PCTransportManager.ts

@@ -0,0 +1,336 @@
+import log from '../logger';
+import { SignalTarget } from '../proto/livekit_rtc_pb';
+import PCTransport, { PCEvents } from './PCTransport';
+import { roomConnectOptionDefaults } from './defaults';
+import { ConnectionError, ConnectionErrorReason } from './errors';
+import CriticalTimers from './timers';
+import { Mutex, sleep } from './utils';
+
+export enum PCTransportState {
+  NEW,
+  CONNECTING,
+  CONNECTED,
+  FAILED,
+  CLOSING,
+  CLOSED,
+}
+
+export class PCTransportManager {
+  public publisher: PCTransport;
+
+  public subscriber: PCTransport;
+
+  public peerConnectionTimeout: number = roomConnectOptionDefaults.peerConnectionTimeout;
+
+  public get needsPublisher() {
+    return this.isPublisherConnectionRequired;
+  }
+
+  public get needsSubscriber() {
+    return this.isSubscriberConnectionRequired;
+  }
+
+  public get currentState() {
+    return this.state;
+  }
+
+  public onStateChange?: (
+    state: PCTransportState,
+    pubState: RTCPeerConnectionState,
+    subState: RTCPeerConnectionState,
+  ) => void;
+
+  public onIceCandidate?: (ev: RTCIceCandidate, target: SignalTarget) => void;
+
+  public onDataChannel?: (ev: RTCDataChannelEvent) => void;
+
+  public onTrack?: (ev: RTCTrackEvent) => void;
+
+  public onPublisherOffer?: (offer: RTCSessionDescriptionInit) => void;
+
+  private isPublisherConnectionRequired: boolean;
+
+  private isSubscriberConnectionRequired: boolean;
+
+  private state: PCTransportState;
+
+  private connectionLock: Mutex;
+
+  constructor(rtcConfig: RTCConfiguration, subscriberPrimary: boolean) {
+    this.isPublisherConnectionRequired = !subscriberPrimary;
+    this.isSubscriberConnectionRequired = subscriberPrimary;
+    const googConstraints = { optional: [{ googDscp: true }] };
+    this.publisher = new PCTransport(rtcConfig, googConstraints);
+    this.subscriber = new PCTransport(rtcConfig);
+
+    this.publisher.onConnectionStateChange = this.updateState;
+    this.subscriber.onConnectionStateChange = this.updateState;
+    this.publisher.onIceConnectionStateChange = this.updateState;
+    this.subscriber.onIceConnectionStateChange = this.updateState;
+    this.publisher.onSignalingStatechange = this.updateState;
+    this.subscriber.onSignalingStatechange = this.updateState;
+    this.publisher.onIceCandidate = (candidate) => {
+      this.onIceCandidate?.(candidate, SignalTarget.PUBLISHER);
+    };
+    this.subscriber.onIceCandidate = (candidate) => {
+      this.onIceCandidate?.(candidate, SignalTarget.SUBSCRIBER);
+    };
+    // in subscriber primary mode, server side opens sub data channels.
+    this.subscriber.onDataChannel = (ev) => {
+      this.onDataChannel?.(ev);
+    };
+    this.subscriber.onTrack = (ev) => {
+      this.onTrack?.(ev);
+    };
+    this.publisher.onOffer = (offer) => {
+      this.onPublisherOffer?.(offer);
+    };
+
+    this.state = PCTransportState.NEW;
+
+    this.connectionLock = new Mutex();
+  }
+
+  requirePublisher(require = true) {
+    this.isPublisherConnectionRequired = require;
+    this.updateState();
+  }
+
+  requireSubscriber(require = true) {
+    this.isSubscriberConnectionRequired = require;
+    this.updateState();
+  }
+
+  createAndSendPublisherOffer(options?: RTCOfferOptions) {
+    return this.publisher.createAndSendOffer(options);
+  }
+
+  setPublisherAnswer(sd: RTCSessionDescriptionInit) {
+    return this.publisher.setRemoteDescription(sd);
+  }
+
+  removeTrack(sender: RTCRtpSender) {
+    return this.publisher.removeTrack(sender);
+  }
+
+  async close() {
+    if (this.publisher && this.publisher.getSignallingState() !== 'closed') {
+      const publisher = this.publisher;
+      for (const sender of publisher.getSenders()) {
+        try {
+          // TODO: react-native-webrtc doesn't have removeTrack yet.
+          if (publisher.canRemoveTrack()) {
+            publisher.removeTrack(sender);
+          }
+        } catch (e) {
+          log.warn('could not removeTrack', { error: e });
+        }
+      }
+    }
+    await Promise.all([this.publisher.close(), this.subscriber.close()]);
+    this.updateState();
+  }
+
+  async triggerIceRestart() {
+    this.subscriber.restartingIce = true;
+    // only restart publisher if it's needed
+    if (this.needsPublisher) {
+      await this.createAndSendPublisherOffer({ iceRestart: true });
+    }
+  }
+
+  async addIceCandidate(candidate: RTCIceCandidateInit, target: SignalTarget) {
+    if (target === SignalTarget.PUBLISHER) {
+      await this.publisher.addIceCandidate(candidate);
+    } else {
+      await this.subscriber.addIceCandidate(candidate);
+    }
+  }
+
+  async createSubscriberAnswerFromOffer(sd: RTCSessionDescriptionInit) {
+    log.debug('received server offer', {
+      RTCSdpType: sd.type,
+      signalingState: this.subscriber.getSignallingState().toString(),
+    });
+    await this.subscriber.setRemoteDescription(sd);
+
+    // answer the offer
+    const answer = await this.subscriber.createAndSetAnswer();
+    return answer;
+  }
+
+  updateConfiguration(config: RTCConfiguration, iceRestart?: boolean) {
+    this.publisher.setConfiguration(config);
+    this.subscriber.setConfiguration(config);
+    if (iceRestart) {
+      this.triggerIceRestart();
+    }
+  }
+
+  async ensurePCTransportConnection(abortController?: AbortController, timeout?: number) {
+    const unlock = await this.connectionLock.lock();
+    try {
+      if (
+        this.isPublisherConnectionRequired &&
+        this.publisher.getConnectionState() !== 'connected' &&
+        this.publisher.getConnectionState() !== 'connecting'
+      ) {
+        log.debug('negotiation required, start negotiating');
+        this.publisher.negotiate();
+      }
+      await Promise.all(
+        this.requiredTransports?.map((transport) =>
+          this.ensureTransportConnected(transport, abortController, timeout),
+        ),
+      );
+    } finally {
+      unlock();
+    }
+  }
+
+  async negotiate(abortController: AbortController) {
+    return new Promise<void>(async (resolve, reject) => {
+      const negotiationTimeout = setTimeout(() => {
+        reject('negotiation timed out');
+      }, this.peerConnectionTimeout);
+
+      const abortHandler = () => {
+        clearTimeout(negotiationTimeout);
+        reject('negotiation aborted');
+      };
+
+      abortController.signal.addEventListener('abort', abortHandler);
+      this.publisher.once(PCEvents.NegotiationStarted, () => {
+        if (abortController.signal.aborted) {
+          return;
+        }
+        this.publisher.once(PCEvents.NegotiationComplete, () => {
+          clearTimeout(negotiationTimeout);
+          resolve();
+        });
+      });
+
+      await this.publisher.negotiate((e) => {
+        clearTimeout(negotiationTimeout);
+        reject(e);
+      });
+    });
+  }
+
+  addPublisherTransceiver(track: MediaStreamTrack, transceiverInit: RTCRtpTransceiverInit) {
+    return this.publisher.addTransceiver(track, transceiverInit);
+  }
+
+  addPublisherTrack(track: MediaStreamTrack) {
+    return this.publisher.addTrack(track);
+  }
+
+  createPublisherDataChannel(label: string, dataChannelDict: RTCDataChannelInit) {
+    return this.publisher.createDataChannel(label, dataChannelDict);
+  }
+
+  /**
+   * Returns the first required transport's address if no explicit target is specified
+   */
+  getConnectedAddress(target?: SignalTarget) {
+    if (target === SignalTarget.PUBLISHER) {
+      return this.publisher.getConnectedAddress();
+    } else if (target === SignalTarget.SUBSCRIBER) {
+      return this.publisher.getConnectedAddress();
+    }
+    return this.requiredTransports[0].getConnectedAddress();
+  }
+
+  private get requiredTransports() {
+    const transports: PCTransport[] = [];
+    if (this.isPublisherConnectionRequired) {
+      transports.push(this.publisher);
+    }
+    if (this.isSubscriberConnectionRequired) {
+      transports.push(this.subscriber);
+    }
+    return transports;
+  }
+
+  private updateState = () => {
+    const previousState = this.state;
+
+    const connectionStates = this.requiredTransports.map((tr) => tr.getConnectionState());
+    if (connectionStates.every((st) => st === 'connected')) {
+      this.state = PCTransportState.CONNECTED;
+    } else if (connectionStates.some((st) => st === 'failed')) {
+      this.state = PCTransportState.FAILED;
+    } else if (connectionStates.some((st) => st === 'connecting')) {
+      this.state = PCTransportState.CONNECTING;
+    } else if (connectionStates.every((st) => st === 'closed')) {
+      this.state = PCTransportState.CLOSED;
+    } else if (connectionStates.some((st) => st === 'closed')) {
+      this.state = PCTransportState.CLOSING;
+    } else if (connectionStates.every((st) => st === 'new')) {
+      this.state = PCTransportState.NEW;
+    }
+
+    if (previousState !== this.state) {
+      log.debug(
+        `pc state change: from ${PCTransportState[previousState]} to ${
+          PCTransportState[this.state]
+        }`,
+      );
+      this.onStateChange?.(
+        this.state,
+        this.publisher.getConnectionState(),
+        this.subscriber.getConnectionState(),
+      );
+    }
+  };
+
+  private async ensureTransportConnected(
+    pcTransport: PCTransport,
+    abortController?: AbortController,
+    timeout: number = this.peerConnectionTimeout,
+  ) {
+    const connectionState = pcTransport.getConnectionState();
+    if (connectionState === 'connected') {
+      return;
+    }
+
+    return new Promise<void>(async (resolve, reject) => {
+      const abortHandler = () => {
+        log.warn('abort transport connection');
+        CriticalTimers.clearTimeout(connectTimeout);
+
+        reject(
+          new ConnectionError(
+            'room connection has been cancelled',
+            ConnectionErrorReason.Cancelled,
+          ),
+        );
+      };
+      if (abortController?.signal.aborted) {
+        abortHandler();
+      }
+      abortController?.signal.addEventListener('abort', abortHandler);
+
+      const connectTimeout = CriticalTimers.setTimeout(() => {
+        abortController?.signal.removeEventListener('abort', abortHandler);
+        reject(new ConnectionError('could not establish pc connection'));
+      }, timeout);
+
+      while (this.state !== PCTransportState.CONNECTED) {
+        await sleep(50); // FIXME we shouldn't rely on `sleep` in the connection paths, as it invokes `setTimeout` which can be drastically throttled by browser implementations
+        if (abortController?.signal.aborted) {
+          reject(
+            new ConnectionError(
+              'room connection has been cancelled',
+              ConnectionErrorReason.Cancelled,
+            ),
+          );
+          return;
+        }
+      }
+      CriticalTimers.clearTimeout(connectTimeout);
+      abortController?.signal.removeEventListener('abort', abortHandler);
+      resolve();
+    });
+  }
+}