@stream-io/video-client 0.1.3 → 0.1.4

package/src/rtc/Publisher.ts

@@ -28,14 +28,19 @@ import {
 } from '../helpers/sdp-munging';
 import { Logger } from '../coordinator/connection/types';
 import { getLogger } from '../logger';
+import { Dispatcher } from './Dispatcher';
+
+const logger: Logger = getLogger(['Publisher']);
 
 export type PublisherOpts = {
   sfuClient: StreamSfuClient;
   state: CallState;
+  dispatcher: Dispatcher;
   connectionConfig?: RTCConfiguration;
   isDtxEnabled: boolean;
   isRedEnabled: boolean;
   preferredVideoCodec?: string;
+  iceRestartDelay?: number;
 };
 
 /**
@@ -45,6 +50,7 @@ export type PublisherOpts = {
 export class Publisher {
   private pc: RTCPeerConnection;
   private readonly state: CallState;
+  private readonly dispatcher: Dispatcher;
 
   private readonly transceiverRegistry: {
     [key in TrackType]: RTCRtpTransceiver | undefined;
@@ -87,7 +93,11 @@ export class Publisher {
   private readonly isDtxEnabled: boolean;
   private readonly isRedEnabled: boolean;
   private readonly preferredVideoCodec?: string;
-  private logger: Logger = getLogger(['Publisher']);
+
+  private readonly unsubscribeOnIceRestart: () => void;
+
+  private readonly iceRestartDelay: number;
+  private isIceRestarting = false;
 
   /**
    * The SFU client instance to use for publishing and signaling.
@@ -100,24 +110,40 @@ export class Publisher {
    * @param connectionConfig the connection configuration to use.
    * @param sfuClient the SFU client to use.
    * @param state the call state to use.
+   * @param dispatcher the dispatcher to use.
    * @param isDtxEnabled whether DTX is enabled.
    * @param isRedEnabled whether RED is enabled.
    * @param preferredVideoCodec the preferred video codec.
+   * @param iceRestartDelay the delay in milliseconds to wait before restarting ICE once connection goes to `disconnected` state.
    */
   constructor({
     connectionConfig,
     sfuClient,
+    dispatcher,
     state,
     isDtxEnabled,
     isRedEnabled,
     preferredVideoCodec,
+    iceRestartDelay = 2500,
   }: PublisherOpts) {
     this.pc = this.createPeerConnection(connectionConfig);
     this.sfuClient = sfuClient;
     this.state = state;
+    this.dispatcher = dispatcher;
     this.isDtxEnabled = isDtxEnabled;
     this.isRedEnabled = isRedEnabled;
     this.preferredVideoCodec = preferredVideoCodec;
+    this.iceRestartDelay = iceRestartDelay;
+
+    this.unsubscribeOnIceRestart = dispatcher.on(
+      'iceRestart',
+      async (message) => {
+        if (message.eventPayload.oneofKind !== 'iceRestart') return;
+        const { iceRestart } = message.eventPayload;
+        if (iceRestart.peerType !== PeerType.PUBLISHER_UNSPECIFIED) return;
+        await this.restartIce();
+      },
+    );
   }
 
   private createPeerConnection = (connectionConfig?: RTCConfiguration) => {
@@ -154,6 +180,7 @@ export class Publisher {
       });
     }
 
+    this.unsubscribeOnIceRestart();
     this.pc.removeEventListener('negotiationneeded', this.onNegotiationNeeded);
     this.pc.close();
   };
@@ -192,7 +219,7 @@ export class Publisher {
      * Once the track has ended, it will notify the SFU and update the state.
      */
     const handleTrackEnded = async () => {
-      this.logger(
+      logger(
         'info',
         `Track ${TrackType[trackType]} has ended, notifying the SFU`,
       );
@@ -233,12 +260,12 @@ export class Publisher {
         sendEncodings: videoEncodings,
       });
 
-      this.logger('debug', `Added ${TrackType[trackType]} transceiver`);
+      logger('debug', `Added ${TrackType[trackType]} transceiver`);
       this.transceiverInitOrder.push(trackType);
       this.transceiverRegistry[trackType] = transceiver;
 
       if ('setCodecPreferences' in transceiver && codecPreferences) {
-        this.logger(
+        logger(
           'info',
           `Setting ${TrackType[trackType]} codec preferences`,
           codecPreferences,
@@ -336,7 +363,7 @@ export class Publisher {
    * Stops publishing all tracks and stop all tracks.
    */
   stopPublishing = () => {
-    this.logger('debug', 'Stopping publishing all tracks');
+    logger('debug', 'Stopping publishing all tracks');
     this.pc.getSenders().forEach((s) => {
       s.track?.stop();
       if (this.pc.signalingState !== 'closed') {
@@ -346,7 +373,7 @@ export class Publisher {
   };
 
   updateVideoPublishQuality = async (enabledRids: string[]) => {
-    this.logger(
+    logger(
       'info',
       'Update publish quality, requested rids by SFU:',
       enabledRids,
@@ -354,13 +381,13 @@ export class Publisher {
 
     const videoSender = this.transceiverRegistry[TrackType.VIDEO]?.sender;
     if (!videoSender) {
-      this.logger('warn', 'Update publish quality, no video sender found.');
+      logger('warn', 'Update publish quality, no video sender found.');
       return;
     }
 
     const params = videoSender.getParameters();
     if (params.encodings.length === 0) {
-      this.logger(
+      logger(
         'warn',
         'Update publish quality, No suitable video encoding quality found',
       );
@@ -383,12 +410,9 @@ export class Publisher {
       .join(', ');
     if (changed) {
       await videoSender.setParameters(params);
-      this.logger(
-        'info',
-        `Update publish quality, enabled rids: ${activeRids}`,
-      );
+      logger('info', `Update publish quality, enabled rids: ${activeRids}`);
     } else {
-      this.logger('info', `Update publish quality, no change: ${activeRids}`);
+      logger('info', `Update publish quality, no change: ${activeRids}`);
     }
   };
 
@@ -422,7 +446,7 @@ export class Publisher {
   private onIceCandidate = async (e: RTCPeerConnectionIceEvent) => {
     const { candidate } = e;
     if (!candidate) {
-      this.logger('debug', 'null ice candidate');
+      logger('debug', 'null ice candidate');
       return;
     }
     await this.sfuClient.iceTrickle({
@@ -457,7 +481,12 @@ export class Publisher {
    * Restarts the ICE connection and renegotiates with the SFU.
    */
   restartIce = async () => {
-    this.logger('debug', 'Restarting ICE connection');
+    logger('debug', 'Restarting ICE connection');
+    const signalingState = this.pc.signalingState;
+    if (this.isIceRestarting || signalingState === 'have-local-offer') {
+      logger('debug', 'ICE restart is already in progress');
+      return;
+    }
     await this.negotiate({ iceRestart: true });
   };
 
@@ -471,6 +500,8 @@ export class Publisher {
    * @param options the optional offer options to use.
    */
   private negotiate = async (options?: RTCOfferOptions) => {
+    this.isIceRestarting = options?.iceRestart ?? false;
+
     const offer = await this.pc.createOffer(options);
     offer.sdp = this.mungeCodecs(offer.sdp);
 
@@ -494,22 +525,21 @@ export class Publisher {
         sdp: response.sdp,
       });
     } catch (e) {
-      this.logger('error', `setRemoteDescription error`, {
+      logger('error', `setRemoteDescription error`, {
         sdp: response.sdp,
         error: e,
       });
     }
 
+    this.isIceRestarting = false;
+
     this.sfuClient.iceTrickleBuffer.publisherCandidates.subscribe(
       async (candidate) => {
         try {
           const iceCandidate = JSON.parse(candidate.iceCandidate);
           await this.pc.addIceCandidate(iceCandidate);
         } catch (e) {
-          this.logger('error', `ICE candidate error`, {
-            error: e,
-            candidate,
-          });
+          logger('warn', `ICE candidate error`, [e, candidate]);
         }
       },
     );
@@ -544,11 +574,11 @@ export class Publisher {
     ): string => {
       if (defaultMid) return defaultMid;
       if (!sdp) {
-        this.logger('warn', 'No SDP found. Returning empty mid');
+        logger('warn', 'No SDP found. Returning empty mid');
         return '';
       }
 
-      this.logger(
+      logger(
         'debug',
         `No 'mid' found for track. Trying to find it from the Offer SDP`,
       );
@@ -562,7 +592,7 @@ export class Publisher {
         );
       });
       if (typeof media?.mid === 'undefined') {
-        this.logger(
+        logger(
           'debug',
           `No mid found in SDP for track type ${track.kind} and id ${track.id}. Attempting to find a heuristic mid`,
         );
@@ -572,7 +602,7 @@ export class Publisher {
           return String(heuristicMid);
         }
 
-        this.logger('debug', 'No heuristic mid found. Returning empty mid');
+        logger('debug', 'No heuristic mid found. Returning empty mid');
         return '';
       }
       return String(media.mid);
@@ -603,7 +633,7 @@ export class Publisher {
         } else {
           // we report the last known optimal layers for ended tracks
           optimalLayers = this.trackLayersCache[trackType] || [];
-          this.logger(
+          logger(
             'debug',
             `Track ${TrackType[trackType]} is ended. Announcing last known optimal layers`,
             optimalLayers,
@@ -639,22 +669,22 @@ export class Publisher {
     const errorMessage =
       e instanceof RTCPeerConnectionIceErrorEvent &&
       `${e.errorCode}: ${e.errorText}`;
-    this.logger('error', `ICE Candidate error`, errorMessage);
+    logger('error', `ICE Candidate error`, errorMessage);
   };
 
   private onIceConnectionStateChange = () => {
     const state = this.pc.iceConnectionState;
-    this.logger('debug', `ICE Connection state changed to`, state);
+    logger('debug', `ICE Connection state changed to`, state);
 
     if (state === 'failed') {
-      this.logger('warn', `Attempting to restart ICE`);
+      logger('warn', `Attempting to restart ICE`);
       this.restartIce().catch((e) => {
-        this.logger('error', `ICE restart error`, e);
+        logger('error', `ICE restart error`, e);
       });
     } else if (state === 'disconnected') {
       // when in `disconnected` state, the browser may recover automatically,
       // hence, we delay the ICE restart
-      this.logger('warn', `Scheduling ICE restart in 5 seconds`);
+      logger('warn', `Scheduling ICE restart in ${this.iceRestartDelay} ms.`);
       setTimeout(() => {
         // check if the state is still `disconnected` or `failed`
         // as the connection may have recovered (or failed) in the meantime
@@ -663,19 +693,24 @@ export class Publisher {
           this.pc.iceConnectionState === 'failed'
         ) {
           this.restartIce().catch((e) => {
-            this.logger('error', `ICE restart error`, e);
+            logger('error', `ICE restart error`, e);
           });
+        } else {
+          logger(
+            'debug',
+            `Scheduled ICE restart: connection recovered, canceled.`,
+          );
         }
-      }, 5000);
+      }, this.iceRestartDelay);
     }
   };
 
   private onIceGatheringStateChange = () => {
-    this.logger('debug', `ICE Gathering State`, this.pc.iceGatheringState);
+    logger('debug', `ICE Gathering State`, this.pc.iceGatheringState);
   };
 
   private onSignalingStateChange = () => {
-    this.logger('debug', `Signaling state changed`, this.pc.signalingState);
+    logger('debug', `Signaling state changed`, this.pc.signalingState);
   };
 
   private ridToVideoQuality = (rid: string): VideoQuality => {

package/src/rtc/Subscriber.ts

@@ -11,6 +11,7 @@ export type SubscriberOpts = {
   dispatcher: Dispatcher;
   state: CallState;
   connectionConfig?: RTCConfiguration;
+  iceRestartDelay?: number;
 };
 
 const logger = getLogger(['Subscriber']);
@@ -21,11 +22,16 @@ const logger = getLogger(['Subscriber']);
  */
 export class Subscriber {
   private pc: RTCPeerConnection;
-  private readonly unregisterOnSubscriberOffer: () => void;
   private sfuClient: StreamSfuClient;
   private dispatcher: Dispatcher;
   private state: CallState;
 
+  private readonly unregisterOnSubscriberOffer: () => void;
+  private readonly unregisterOnIceRestart: () => void;
+
+  private readonly iceRestartDelay: number;
+  private isIceRestarting = false;
+
   /**
    * Constructs a new `Subscriber` instance.
    *
@@ -33,16 +39,19 @@ export class Subscriber {
    * @param dispatcher the dispatcher to use.
    * @param state the state of the call.
    * @param connectionConfig the connection configuration to use.
+   * @param iceRestartDelay the delay in milliseconds to wait before restarting ICE when connection goes to `disconnected` state.
    */
   constructor({
     sfuClient,
     dispatcher,
     state,
     connectionConfig,
+    iceRestartDelay = 2500,
   }: SubscriberOpts) {
     this.sfuClient = sfuClient;
     this.dispatcher = dispatcher;
     this.state = state;
+    this.iceRestartDelay = iceRestartDelay;
 
     this.pc = this.createPeerConnection(connectionConfig);
 
@@ -54,6 +63,16 @@ export class Subscriber {
         await this.negotiate(subscriberOffer);
       },
     );
+
+    this.unregisterOnIceRestart = dispatcher.on(
+      'iceRestart',
+      async (message) => {
+        if (message.eventPayload.oneofKind !== 'iceRestart') return;
+        const { iceRestart } = message.eventPayload;
+        if (iceRestart.peerType !== PeerType.SUBSCRIBER) return;
+        await this.restartIce();
+      },
+    );
   }
 
   /**
@@ -84,6 +103,7 @@ export class Subscriber {
    */
   close = () => {
     this.unregisterOnSubscriberOffer();
+    this.unregisterOnIceRestart();
     this.pc.close();
   };
 
@@ -177,9 +197,23 @@ export class Subscriber {
   /**
    * Restarts the ICE connection and renegotiates with the SFU.
    */
-  restartIce = () => {
+  restartIce = async () => {
     logger('debug', 'Restarting ICE connection');
-    this.pc.restartIce();
+    if (this.pc.signalingState === 'have-remote-offer') {
+      logger('debug', 'ICE restart is already in progress');
+      return;
+    }
+    const previousIsIceRestarting = this.isIceRestarting;
+    try {
+      this.isIceRestarting = true;
+      await this.sfuClient.iceRestart({
+        peerType: PeerType.SUBSCRIBER,
+      });
+    } catch (e) {
+      // restore the previous state, as our intent for restarting ICE failed
+      this.isIceRestarting = previousIsIceRestarting;
+      throw e;
+    }
   };
 
   private handleOnTrack = (e: RTCTrackEvent) => {
@@ -280,12 +314,11 @@ export class Subscriber {
           const iceCandidate = JSON.parse(candidate.iceCandidate);
           await this.pc.addIceCandidate(iceCandidate);
         } catch (e) {
-          logger('error', `ICE candidate error`, [e, candidate]);
+          logger('warn', `ICE candidate error`, [e, candidate]);
         }
       },
     );
 
-    // apply ice candidates
     const answer = await this.pc.createAnswer();
     await this.pc.setLocalDescription(answer);
 
@@ -293,14 +326,48 @@ export class Subscriber {
       peerType: PeerType.SUBSCRIBER,
       sdp: answer.sdp || '',
     });
+
+    this.isIceRestarting = false;
   };
 
   private onIceConnectionStateChange = () => {
-    logger('info', `ICE connection state changed`, this.pc.iceConnectionState);
+    const state = this.pc.iceConnectionState;
+    logger('debug', `ICE connection state changed`, state);
+
+    // do nothing when ICE is restarting
+    if (this.isIceRestarting) return;
+
+    if (state === 'failed') {
+      logger('warn', `Attempting to restart ICE`);
+      this.restartIce().catch((e) => {
+        logger('error', `ICE restart failed`, e);
+      });
+    } else if (state === 'disconnected') {
+      // when in `disconnected` state, the browser may recover automatically,
+      // hence, we delay the ICE restart
+      logger('warn', `Scheduling ICE restart in ${this.iceRestartDelay} ms.`);
+      setTimeout(() => {
+        // check if the state is still `disconnected` or `failed`
+        // as the connection may have recovered (or failed) in the meantime
+        if (
+          this.pc.iceConnectionState === 'disconnected' ||
+          this.pc.iceConnectionState === 'failed'
+        ) {
+          this.restartIce().catch((e) => {
+            logger('error', `ICE restart failed`, e);
+          });
+        } else {
+          logger(
+            'debug',
+            `Scheduled ICE restart: connection recovered, canceled.`,
+          );
+        }
+      }, 5000);
+    }
   };
 
   private onIceGatheringStateChange = () => {
-    logger('info', `ICE gathering state changed`, this.pc.iceGatheringState);
+    logger('debug', `ICE gathering state changed`, this.pc.iceGatheringState);
   };
 
   private onIceCandidateError = (e: Event) => {

package/src/rtc/__tests__/Publisher.test.ts

@@ -5,8 +5,9 @@ import { Publisher } from '../Publisher';
 import { CallState } from '../../store';
 import { StreamSfuClient } from '../../StreamSfuClient';
 import { Dispatcher } from '../Dispatcher';
-import { TrackType } from '../../gen/video/sfu/models/models';
+import { PeerType, TrackType } from '../../gen/video/sfu/models/models';
 import { IceTrickleBuffer } from '../IceTrickleBuffer';
+import { SfuEvent } from '../../gen/video/sfu/event/events';
 
 vi.mock('../../StreamSfuClient', () => {
   console.log('MOCKING StreamSfuClient');
@@ -33,9 +34,10 @@ describe('Publisher', () => {
   let publisher: Publisher;
   let sfuClient: StreamSfuClient;
   let state: CallState;
+  let dispatcher: Dispatcher;
 
   beforeEach(() => {
-    const dispatcher = new Dispatcher();
+    dispatcher = new Dispatcher();
     sfuClient = new StreamSfuClient({
       dispatcher,
       sfuServer: {
@@ -52,15 +54,18 @@ describe('Publisher', () => {
     state = new CallState();
     publisher = new Publisher({
       sfuClient,
+      dispatcher,
       state,
       isDtxEnabled: true,
       isRedEnabled: true,
+      iceRestartDelay: 100,
     });
   });
 
   afterEach(() => {
     vi.clearAllMocks();
     vi.resetModules();
+    dispatcher.offAll();
   });
 
   it('can publish, re-publish and un-publish a stream', async () => {
@@ -209,4 +214,79 @@ describe('Publisher', () => {
       expect(sfuClient.setPublisher).toHaveBeenCalled();
     });
   });
+
+  describe('Publisher ICE Restart', () => {
+    it('should perform ICE restart when iceRestart event is received', () => {
+      vi.spyOn(publisher, 'restartIce').mockResolvedValue();
+      dispatcher.dispatch(
+        SfuEvent.create({
+          eventPayload: {
+            oneofKind: 'iceRestart',
+            iceRestart: {
+              peerType: PeerType.PUBLISHER_UNSPECIFIED,
+            },
+          },
+        }),
+      );
+      expect(publisher.restartIce).toHaveBeenCalled();
+    });
+
+    it('should not perform ICE restart when iceRestart event is received for a different peer type', () => {
+      vi.spyOn(publisher, 'restartIce').mockResolvedValue();
+      dispatcher.dispatch(
+        SfuEvent.create({
+          eventPayload: {
+            oneofKind: 'iceRestart',
+            iceRestart: {
+              peerType: PeerType.SUBSCRIBER,
+            },
+          },
+        }),
+      );
+      expect(publisher.restartIce).not.toHaveBeenCalled();
+    });
+
+    it(`should drop consequent ICE restart requests`, async () => {
+      // @ts-ignore
+      publisher['pc'].signalingState = 'have-local-offer';
+      // @ts-ignore
+      vi.spyOn(publisher, 'negotiate').mockResolvedValue();
+
+      await publisher.restartIce();
+      expect(publisher['negotiate']).not.toHaveBeenCalled();
+    });
+
+    it(`should perform ICE restart when connection state changes to 'failed'`, () => {
+      vi.spyOn(publisher, 'restartIce').mockResolvedValue();
+      // @ts-ignore
+      publisher['pc'].iceConnectionState = 'failed';
+      publisher['onIceConnectionStateChange']();
+      expect(publisher.restartIce).toHaveBeenCalled();
+    });
+
+    it(`should perform ICE restart when connection state changes to 'disconnected'`, () => {
+      vi.spyOn(publisher, 'restartIce').mockResolvedValue();
+      vi.useFakeTimers();
+
+      // @ts-ignore
+      publisher['pc'].iceConnectionState = 'disconnected';
+      publisher['onIceConnectionStateChange']();
+      vi.runAllTimers();
+      expect(publisher.restartIce).toHaveBeenCalled();
+    });
+
+    it(`should bail-out from ICE restart once connection recovers before timeout`, () => {
+      vi.spyOn(publisher, 'restartIce').mockResolvedValue();
+      vi.useFakeTimers();
+
+      // @ts-ignore
+      publisher['pc'].iceConnectionState = 'disconnected';
+      publisher['onIceConnectionStateChange']();
+      // @ts-ignore
+      publisher['pc'].iceConnectionState = 'connected';
+
+      vi.runAllTimers();
+      expect(publisher.restartIce).not.toHaveBeenCalled();
+    });
+  });
 });