@stream-io/video-client 1.5.0-0 → 1.5.0

package/src/rtc/Publisher.ts

@@ -26,6 +26,8 @@ import { getOSInfo } from '../client-details';
 import { VideoLayerSetting } from '../gen/video/sfu/event/events';
 import { TargetResolutionResponse } from '../gen/shims';
 
+const logger: Logger = getLogger(['Publisher']);
+
 export type PublisherConstructorOpts = {
   sfuClient: StreamSfuClient;
   state: CallState;
@@ -33,17 +35,15 @@ export type PublisherConstructorOpts = {
   connectionConfig?: RTCConfiguration;
   isDtxEnabled: boolean;
   isRedEnabled: boolean;
+  iceRestartDelay?: number;
   onUnrecoverableError?: () => void;
-  logTag: string;
 };
 
 /**
  * The `Publisher` is responsible for publishing/unpublishing media streams to/from the SFU
- *
  * @internal
  */
 export class Publisher {
-  private readonly logger: Logger;
   private pc: RTCPeerConnection;
   private readonly state: CallState;
 
@@ -67,9 +67,9 @@ export class Publisher {
    * This is needed because some browsers (Firefox) don't reliably report
    * trackId and `mid` parameters.
    *
-   * @internal
+   * @private
    */
-  private readonly transceiverInitOrder: TrackType[] = [];
+  private transceiverInitOrder: TrackType[] = [];
 
   private readonly trackKindMapping: {
     [key in TrackType]: 'video' | 'audio' | undefined;
@@ -97,7 +97,9 @@ export class Publisher {
   private readonly unsubscribeOnIceRestart: () => void;
   private readonly onUnrecoverableError?: () => void;
 
+  private readonly iceRestartDelay: number;
   private isIceRestarting = false;
+  private iceRestartTimeout?: NodeJS.Timeout;
 
   // workaround for the lack of RTCPeerConnection.getConfiguration() method in react-native-webrtc
   private _connectionConfiguration: RTCConfiguration | undefined;
@@ -128,7 +130,6 @@ export class Publisher {
    * @param isRedEnabled whether RED is enabled.
    * @param iceRestartDelay the delay in milliseconds to wait before restarting ICE once connection goes to `disconnected` state.
    * @param onUnrecoverableError a callback to call when an unrecoverable error occurs.
-   * @param logTag the log tag to use.
    */
   constructor({
     connectionConfig,
@@ -137,21 +138,21 @@ export class Publisher {
     state,
     isDtxEnabled,
     isRedEnabled,
+    iceRestartDelay = 2500,
     onUnrecoverableError,
-    logTag,
   }: PublisherConstructorOpts) {
-    this.logger = getLogger(['Publisher', logTag]);
     this.pc = this.createPeerConnection(connectionConfig);
     this.sfuClient = sfuClient;
     this.state = state;
     this.isDtxEnabled = isDtxEnabled;
     this.isRedEnabled = isRedEnabled;
+    this.iceRestartDelay = iceRestartDelay;
     this.onUnrecoverableError = onUnrecoverableError;
 
     this.unsubscribeOnIceRestart = dispatcher.on('iceRestart', (iceRestart) => {
       if (iceRestart.peerType !== PeerType.PUBLISHER_UNSPECIFIED) return;
       this.restartIce().catch((err) => {
-        this.logger('warn', `ICERestart failed`, err);
+        logger('warn', `ICERestart failed`, err);
         this.onUnrecoverableError?.();
       });
     });
@@ -179,7 +180,7 @@ export class Publisher {
   /**
    * Closes the publisher PeerConnection and cleans up the resources.
    */
-  close = ({ stopTracks }: { stopTracks: boolean }) => {
+  close = ({ stopTracks = true } = {}) => {
     if (stopTracks) {
       this.stopPublishing();
       Object.keys(this.transceiverRegistry).forEach((trackType) => {
@@ -192,33 +193,10 @@ export class Publisher {
       });
     }
 
-    this.detachEventHandlers();
-    this.pc.close();
-  };
-
-  /**
-   * Detaches the event handlers from the `RTCPeerConnection`.
-   * This is useful when we want to replace the `RTCPeerConnection`
-   * instance with a new one (in case of migration).
-   */
-  detachEventHandlers = () => {
+    clearTimeout(this.iceRestartTimeout);
     this.unsubscribeOnIceRestart();
-
-    this.pc.removeEventListener('icecandidate', this.onIceCandidate);
     this.pc.removeEventListener('negotiationneeded', this.onNegotiationNeeded);
-    this.pc.removeEventListener('icecandidateerror', this.onIceCandidateError);
-    this.pc.removeEventListener(
-      'iceconnectionstatechange',
-      this.onIceConnectionStateChange,
-    );
-    this.pc.removeEventListener(
-      'icegatheringstatechange',
-      this.onIceGatheringStateChange,
-    );
-    this.pc.removeEventListener(
-      'signalingstatechange',
-      this.onSignalingStateChange,
-    );
+    this.pc.close();
   };
 
   /**
@@ -256,7 +234,7 @@ export class Publisher {
      * Once the track has ended, it will notify the SFU and update the state.
      */
     const handleTrackEnded = async () => {
-      this.logger(
+      logger(
         'info',
         `Track ${TrackType[trackType]} has ended, notifying the SFU`,
       );
@@ -284,17 +262,23 @@ export class Publisher {
             : undefined;
 
       let preferredCodec = opts.preferredCodec;
-      if (!preferredCodec && trackType === TrackType.VIDEO && isReactNative()) {
-        const osName = getOSInfo()?.name.toLowerCase();
-        if (osName === 'ipados') {
-          // in ipads it was noticed that if vp8 codec is used
-          // then the bytes sent is 0 in the outbound-rtp
-          // so we are forcing h264 codec for ipads
-          preferredCodec = 'H264';
-        } else if (osName === 'android') {
-          preferredCodec = 'VP8';
+      if (!preferredCodec && trackType === TrackType.VIDEO) {
+        if (isReactNative()) {
+          const osName = getOSInfo()?.name.toLowerCase();
+          if (osName === 'ipados') {
+            // in ipads it was noticed that if vp8 codec is used
+            // then the bytes sent is 0 in the outbound-rtp
+            // so we are forcing h264 codec for ipads
+            preferredCodec = 'H264';
+          } else if (osName === 'android') {
+            preferredCodec = 'VP8';
+          }
         }
       }
+      const codecPreferences = this.getCodecPreferences(
+        trackType,
+        preferredCodec,
+      );
 
       // listen for 'ended' event on the track as it might be ended abruptly
       // by an external factor as permission revokes, device disconnected, etc.
@@ -313,17 +297,13 @@ export class Publisher {
         sendEncodings: videoEncodings,
       });
 
-      this.logger('debug', `Added ${TrackType[trackType]} transceiver`);
+      logger('debug', `Added ${TrackType[trackType]} transceiver`);
       this.transceiverInitOrder.push(trackType);
       this.transceiverRegistry[trackType] = transceiver;
       this.publishOptionsPerTrackType.set(trackType, opts);
 
-      const codecPreferences =
-        'setCodecPreferences' in transceiver
-          ? this.getCodecPreferences(trackType, preferredCodec)
-          : undefined;
-      if (codecPreferences) {
-        this.logger(
+      if ('setCodecPreferences' in transceiver && codecPreferences) {
+        logger(
           'info',
           `Setting ${TrackType[trackType]} codec preferences`,
           codecPreferences,
@@ -331,7 +311,7 @@ export class Publisher {
         try {
           transceiver.setCodecPreferences(codecPreferences);
         } catch (err) {
-          this.logger('warn', `Couldn't set codec preferences`, err);
+          logger('warn', `Couldn't set codec preferences`, err);
         }
       }
     } else {
@@ -384,10 +364,30 @@ export class Publisher {
    * @param trackType the track type to check.
    */
   isPublishing = (trackType: TrackType): boolean => {
-    const transceiver = this.transceiverRegistry[trackType];
-    if (!transceiver || !transceiver.sender) return false;
-    const track = transceiver.sender.track;
-    return !!track && track.readyState === 'live' && track.enabled;
+    const transceiverForTrackType = this.transceiverRegistry[trackType];
+    if (transceiverForTrackType && transceiverForTrackType.sender) {
+      const sender = transceiverForTrackType.sender;
+      return (
+        !!sender.track &&
+        sender.track.readyState === 'live' &&
+        sender.track.enabled
+      );
+    }
+    return false;
+  };
+
+  /**
+   * Returns true if the given track type is currently live
+   *
+   * @param trackType the track type to check.
+   */
+  isLive = (trackType: TrackType): boolean => {
+    const transceiverForTrackType = this.transceiverRegistry[trackType];
+    if (transceiverForTrackType && transceiverForTrackType.sender) {
+      const sender = transceiverForTrackType.sender;
+      return !!sender.track && sender.track.readyState === 'live';
+    }
+    return false;
   };
 
   private notifyTrackMuteStateChanged = async (
@@ -399,7 +399,6 @@ export class Publisher {
 
     const audioOrVideoOrScreenShareStream =
       trackTypeToParticipantStreamKey(trackType);
-    if (!audioOrVideoOrScreenShareStream) return;
     if (isMuted) {
       this.state.updateParticipant(this.sfuClient.sessionId, (p) => ({
         publishedTracks: p.publishedTracks.filter((t) => t !== trackType),
@@ -420,8 +419,8 @@ export class Publisher {
   /**
    * Stops publishing all tracks and stop all tracks.
    */
-  private stopPublishing = () => {
-    this.logger('debug', 'Stopping publishing all tracks');
+  stopPublishing = () => {
+    logger('debug', 'Stopping publishing all tracks');
     this.pc.getSenders().forEach((s) => {
       s.track?.stop();
       if (this.pc.signalingState !== 'closed') {
@@ -431,7 +430,7 @@ export class Publisher {
   };
 
   updateVideoPublishQuality = async (enabledLayers: VideoLayerSetting[]) => {
-    this.logger(
+    logger(
       'info',
       'Update publish quality, requested layers by SFU:',
       enabledLayers,
@@ -439,13 +438,13 @@ export class Publisher {
 
     const videoSender = this.transceiverRegistry[TrackType.VIDEO]?.sender;
     if (!videoSender) {
-      this.logger('warn', 'Update publish quality, no video sender found.');
+      logger('warn', 'Update publish quality, no video sender found.');
       return;
     }
 
     const params = videoSender.getParameters();
     if (params.encodings.length === 0) {
-      this.logger(
+      logger(
         'warn',
         'Update publish quality, No suitable video encoding quality found',
       );
@@ -470,7 +469,7 @@ export class Publisher {
             layer.scaleResolutionDownBy >= 1 &&
             layer.scaleResolutionDownBy !== enc.scaleResolutionDownBy
           ) {
-            this.logger(
+            logger(
               'debug',
               '[dynascale]: setting scaleResolutionDownBy from server',
               'layer',
@@ -483,7 +482,7 @@ export class Publisher {
           }
 
           if (layer.maxBitrate > 0 && layer.maxBitrate !== enc.maxBitrate) {
-            this.logger(
+            logger(
               'debug',
               '[dynascale] setting max-bitrate from the server',
               'layer',
@@ -499,7 +498,7 @@ export class Publisher {
             layer.maxFramerate > 0 &&
             layer.maxFramerate !== enc.maxFramerate
           ) {
-            this.logger(
+            logger(
               'debug',
               '[dynascale]: setting maxFramerate from server',
               'layer',
@@ -517,13 +516,9 @@ export class Publisher {
     const activeLayers = params.encodings.filter((e) => e.active);
     if (changed) {
       await videoSender.setParameters(params);
-      this.logger(
-        'info',
-        `Update publish quality, enabled rids: `,
-        activeLayers,
-      );
+      logger('info', `Update publish quality, enabled rids: `, activeLayers);
     } else {
-      this.logger('info', `Update publish quality, no change: `, activeLayers);
+      logger('info', `Update publish quality, no change: `, activeLayers);
     }
   };
 
@@ -557,7 +552,7 @@ export class Publisher {
   private onIceCandidate = (e: RTCPeerConnectionIceEvent) => {
     const { candidate } = e;
     if (!candidate) {
-      this.logger('debug', 'null ice candidate');
+      logger('debug', 'null ice candidate');
       return;
     }
     this.sfuClient
@@ -566,7 +561,7 @@ export class Publisher {
         peerType: PeerType.PUBLISHER_UNSPECIFIED,
       })
       .catch((err) => {
-        this.logger('warn', `ICETrickle failed`, err);
+        logger('warn', `ICETrickle failed`, err);
       });
   };
 
@@ -579,24 +574,44 @@ export class Publisher {
     this.sfuClient = sfuClient;
   };
 
+  /**
+   * Performs a migration of this publisher instance to a new SFU.
+   *
+   * Initiates a new `iceRestart` offer/answer exchange with the new SFU.
+   *
+   * @param sfuClient the new SFU client to migrate to.
+   * @param connectionConfig the new connection configuration to use.
+   */
+  migrateTo = async (
+    sfuClient: StreamSfuClient,
+    connectionConfig?: RTCConfiguration,
+  ) => {
+    this.sfuClient = sfuClient;
+    this.pc.setConfiguration(connectionConfig);
+    this._connectionConfiguration = connectionConfig;
+
+    const shouldRestartIce = this.pc.iceConnectionState === 'connected';
+    if (shouldRestartIce) {
+      // negotiate only if there are tracks to publish
+      await this.negotiate({ iceRestart: true });
+    }
+  };
+
   /**
    * Restarts the ICE connection and renegotiates with the SFU.
    */
   restartIce = async () => {
-    this.logger('debug', 'Restarting ICE connection');
+    logger('debug', 'Restarting ICE connection');
     const signalingState = this.pc.signalingState;
     if (this.isIceRestarting || signalingState === 'have-local-offer') {
-      this.logger('debug', 'ICE restart is already in progress');
+      logger('debug', 'ICE restart is already in progress');
       return;
     }
     await this.negotiate({ iceRestart: true });
   };
 
   private onNegotiationNeeded = () => {
-    this.negotiate().catch((err) => {
-      this.logger('warn', `Negotiation failed.`, err);
-      this.onUnrecoverableError?.();
-    });
+    this.negotiate().catch((err) => logger('warn', `Negotiation failed.`, err));
   };
 
   /**
@@ -610,15 +625,28 @@ export class Publisher {
     const offer = await this.pc.createOffer(options);
     let sdp = this.mungeCodecs(offer.sdp);
     if (sdp && this.isPublishing(TrackType.SCREEN_SHARE_AUDIO)) {
-      sdp = this.enableHighQualityAudio(sdp);
+      const transceiver =
+        this.transceiverRegistry[TrackType.SCREEN_SHARE_AUDIO];
+      if (transceiver && transceiver.sender.track) {
+        const mid =
+          transceiver.mid ??
+          this.extractMid(
+            sdp,
+            transceiver.sender.track,
+            TrackType.SCREEN_SHARE_AUDIO,
+          );
+        sdp = enableHighQualityAudio(sdp, mid);
+      }
     }
 
     // set the munged SDP back to the offer
     offer.sdp = sdp;
 
-    const trackInfos = this.getAnnouncedTracks(offer.sdp);
+    const trackInfos = this.getCurrentTrackInfos(offer.sdp);
     if (trackInfos.length === 0) {
-      throw new Error(`Can't negotiate without announcing any tracks`);
+      throw new Error(
+        `Can't initiate negotiation without announcing any tracks`,
+      );
     }
 
     await this.pc.setLocalDescription(offer);
@@ -628,36 +656,32 @@ export class Publisher {
       tracks: trackInfos,
     });
 
-    const { sdp: remoteSdp, error } = response;
     try {
-      await this.pc.setRemoteDescription({ type: 'answer', sdp: remoteSdp });
+      await this.pc.setRemoteDescription({
+        type: 'answer',
+        sdp: response.sdp,
+      });
     } catch (e) {
-      this.logger('error', `setRemoteDescription error`, remoteSdp, error, e);
-      throw e;
-    } finally {
-      this.isIceRestarting = false;
+      logger('error', `setRemoteDescription error`, {
+        sdp: response.sdp,
+        error: e,
+      });
     }
 
+    this.isIceRestarting = false;
+
     this.sfuClient.iceTrickleBuffer.publisherCandidates.subscribe(
       async (candidate) => {
         try {
           const iceCandidate = JSON.parse(candidate.iceCandidate);
           await this.pc.addIceCandidate(iceCandidate);
         } catch (e) {
-          this.logger('warn', `ICE candidate error`, e, candidate);
+          logger('warn', `ICE candidate error`, [e, candidate]);
         }
       },
     );
   };
 
-  private enableHighQualityAudio = (sdp: string) => {
-    const transceiver = this.transceiverRegistry[TrackType.SCREEN_SHARE_AUDIO];
-    if (!transceiver) return sdp;
-
-    const mid = this.extractMid(transceiver, sdp, TrackType.SCREEN_SHARE_AUDIO);
-    return enableHighQualityAudio(sdp, mid);
-  };
-
   private mungeCodecs = (sdp?: string) => {
     if (sdp) {
       sdp = toggleDtx(sdp, this.isDtxEnabled);
@@ -666,23 +690,20 @@ export class Publisher {
   };
 
   private extractMid = (
-    transceiver: RTCRtpTransceiver,
     sdp: string | undefined,
+    track: MediaStreamTrack,
     trackType: TrackType,
   ): string => {
-    if (transceiver.mid) return transceiver.mid;
-
     if (!sdp) {
-      this.logger('warn', 'No SDP found. Returning empty mid');
+      logger('warn', 'No SDP found. Returning empty mid');
       return '';
     }
 
-    this.logger(
+    logger(
       'debug',
       `No 'mid' found for track. Trying to find it from the Offer SDP`,
     );
 
-    const track = transceiver.sender.track!;
     const parsedSdp = SDP.parse(sdp);
     const media = parsedSdp.media.find((m) => {
       return (
@@ -692,9 +713,9 @@ export class Publisher {
       );
     });
     if (typeof media?.mid === 'undefined') {
-      this.logger(
+      logger(
         'debug',
-        `No mid found in SDP for track type ${track.kind} and id ${track.id}. Attempting to find it heuristically`,
+        `No mid found in SDP for track type ${track.kind} and id ${track.id}. Attempting to find a heuristic mid`,
       );
 
       const heuristicMid = this.transceiverInitOrder.indexOf(trackType);
@@ -702,19 +723,13 @@ export class Publisher {
         return String(heuristicMid);
       }
 
-      this.logger('debug', 'No heuristic mid found. Returning empty mid');
+      logger('debug', 'No heuristic mid found. Returning empty mid');
       return '';
     }
     return String(media.mid);
   };
 
-  /**
-   * Returns a list of tracks that are currently being published.
-   *
-   * @internal
-   * @param sdp an optional SDP to extract the `mid` from.
-   */
-  getAnnouncedTracks = (sdp?: string): TrackInfo[] => {
+  getCurrentTrackInfos = (sdp?: string) => {
     sdp = sdp || this.pc.localDescription?.sdp;
 
     const { settings } = this.state;
@@ -732,8 +747,7 @@ export class Publisher {
         );
         const track = transceiver.sender.track!;
         let optimalLayers: OptimalVideoLayer[];
-        const isTrackLive = track.readyState === 'live';
-        if (isTrackLive) {
+        if (track.readyState === 'live') {
           const publishOpts = this.publishOptionsPerTrackType.get(trackType);
           optimalLayers =
             trackType === TrackType.VIDEO
@@ -748,7 +762,7 @@ export class Publisher {
         } else {
           // we report the last known optimal layers for ended tracks
           optimalLayers = this.trackLayersCache[trackType] || [];
-          this.logger(
+          logger(
             'debug',
             `Track ${TrackType[trackType]} is ended. Announcing last known optimal layers`,
             optimalLayers,
@@ -778,12 +792,11 @@ export class Publisher {
           trackId: track.id,
           layers: layers,
           trackType,
-          mid: this.extractMid(transceiver, sdp, trackType),
+          mid: transceiver.mid ?? this.extractMid(sdp, track, trackType),
 
           stereo: isStereo,
           dtx: isAudioTrack && this.isDtxEnabled,
           red: isAudioTrack && this.isRedEnabled,
-          muted: !isTrackLive,
         };
       });
   };
@@ -795,30 +808,53 @@ export class Publisher {
     const iceState = this.pc.iceConnectionState;
     const logLevel =
       iceState === 'connected' || iceState === 'checking' ? 'debug' : 'warn';
-    this.logger(logLevel, `ICE Candidate error`, errorMessage);
+    logger(logLevel, `ICE Candidate error`, errorMessage);
   };
 
   private onIceConnectionStateChange = () => {
     const state = this.pc.iceConnectionState;
-    this.logger('debug', `ICE Connection state changed to`, state);
+    logger('debug', `ICE Connection state changed to`, state);
 
-    if (this.state.callingState === CallingState.RECONNECTING) return;
+    const hasNetworkConnection =
+      this.state.callingState !== CallingState.OFFLINE;
 
-    if (state === 'failed' || state === 'disconnected') {
-      this.logger('debug', `Attempting to restart ICE`);
+    if (state === 'failed') {
+      logger('debug', `Attempting to restart ICE`);
       this.restartIce().catch((e) => {
-        this.logger('error', `ICE restart error`, e);
+        logger('error', `ICE restart error`, e);
         this.onUnrecoverableError?.();
       });
+    } else if (state === 'disconnected' && hasNetworkConnection) {
+      // when in `disconnected` state, the browser may recover automatically,
+      // hence, we delay the ICE restart
+      logger('debug', `Scheduling ICE restart in ${this.iceRestartDelay} ms.`);
+      this.iceRestartTimeout = setTimeout(() => {
+        // check if the state is still `disconnected` or `failed`
+        // as the connection may have recovered (or failed) in the meantime
+        if (
+          this.pc.iceConnectionState === 'disconnected' ||
+          this.pc.iceConnectionState === 'failed'
+        ) {
+          this.restartIce().catch((e) => {
+            logger('error', `ICE restart error`, e);
+            this.onUnrecoverableError?.();
+          });
+        } else {
+          logger(
+            'debug',
+            `Scheduled ICE restart: connection recovered, canceled.`,
+          );
+        }
+      }, this.iceRestartDelay);
     }
   };
 
   private onIceGatheringStateChange = () => {
-    this.logger('debug', `ICE Gathering State`, this.pc.iceGatheringState);
+    logger('debug', `ICE Gathering State`, this.pc.iceGatheringState);
   };
 
   private onSignalingStateChange = () => {
-    this.logger('debug', `Signaling state changed`, this.pc.signalingState);
+    logger('debug', `Signaling state changed`, this.pc.signalingState);
   };
 
   private ridToVideoQuality = (rid: string): VideoQuality => {