@stream-io/video-client 1.48.0 → 1.50.0

package/src/rtc/BasePeerConnection.ts

@@ -13,7 +13,12 @@ import { StreamSfuClient } from '../StreamSfuClient';
 import { AllSfuEvents, Dispatcher } from './Dispatcher';
 import { withoutConcurrency } from '../helpers/concurrency';
 import { StatsTracer, Tracer, traceRTCPeerConnection } from '../stats';
-import type { BasePeerConnectionOpts, OnReconnectionNeeded } from './types';
+import {
+  BasePeerConnectionOpts,
+  OnIceConnected,
+  OnReconnectionNeeded,
+  ReconnectReason,
+} from './types';
 import type { ClientPublishOptions } from '../types';
 
 /**
@@ -31,8 +36,11 @@ export abstract class BasePeerConnection {
   protected sfuClient: StreamSfuClient;
 
   private onReconnectionNeeded?: OnReconnectionNeeded;
+  private onIceConnected?: OnIceConnected;
   private readonly iceRestartDelay: number;
+  private iceHasEverConnected = false;
   private iceRestartTimeout?: NodeJS.Timeout;
+  private preConnectStuckTimeout?: NodeJS.Timeout;
   protected isIceRestarting = false;
   private isDisposed = false;
 
@@ -56,6 +64,7 @@ export abstract class BasePeerConnection {
       state,
       dispatcher,
       onReconnectionNeeded,
+      onIceConnected,
       tag,
       enableTracing,
       clientPublishOptions,
@@ -70,6 +79,7 @@ export abstract class BasePeerConnection {
     this.clientPublishOptions = clientPublishOptions;
     this.tag = tag;
     this.onReconnectionNeeded = onReconnectionNeeded;
+    this.onIceConnected = onIceConnected;
     this.logger = videoLoggerSystem.getLogger(
       peerType === PeerType.SUBSCRIBER ? 'Subscriber' : 'Publisher',
       { tags: [tag] },
@@ -108,7 +118,10 @@ export abstract class BasePeerConnection {
   dispose() {
     clearTimeout(this.iceRestartTimeout);
     this.iceRestartTimeout = undefined;
+    clearTimeout(this.preConnectStuckTimeout);
+    this.preConnectStuckTimeout = undefined;
     this.onReconnectionNeeded = undefined;
+    this.onIceConnected = undefined;
     this.isDisposed = true;
     this.detachEventHandlers();
     this.pc.close();
@@ -145,14 +158,17 @@ export abstract class BasePeerConnection {
    */
   protected tryRestartIce = () => {
     this.restartIce().catch((e) => {
-      const reason = 'restartICE() failed, initiating reconnect';
-      this.logger.error(reason, e);
+      this.logger.error('restartICE() failed, initiating reconnect', e);
       const strategy =
         e instanceof NegotiationError &&
         e.error.code === ErrorCode.PARTICIPANT_SIGNAL_LOST
           ? WebsocketReconnectStrategy.FAST
           : WebsocketReconnectStrategy.REJOIN;
-      this.onReconnectionNeeded?.(strategy, reason, this.peerType);
+      this.onReconnectionNeeded?.(
+        strategy,
+        ReconnectReason.RESTART_ICE_FAILED,
+        this.peerType,
+      );
     });
   };
 
@@ -239,6 +255,20 @@ export abstract class BasePeerConnection {
     return !failedStates.has(iceState) && !failedStates.has(connectionState);
   };
 
+  /**
+   * Returns true only when the peer connection is currently fully established
+   * (ICE `connected`/`completed` AND connection state `connected`).
+   * Transient states like `disconnected`, `checking`, or `new` return false.
+   */
+  isStable = () => {
+    const iceState = this.pc.iceConnectionState;
+    const connectionState = this.pc.connectionState;
+    return (
+      (iceState === 'connected' || iceState === 'completed') &&
+      connectionState === 'connected'
+    );
+  };
+
   /**
    * Handles the ICECandidate event and
    * Initiates an ICE Trickle process with the SFU.
@@ -292,7 +322,7 @@ export abstract class BasePeerConnection {
     if (state === 'failed') {
       this.onReconnectionNeeded?.(
         WebsocketReconnectStrategy.REJOIN,
-        'Connection failed',
+        ReconnectReason.CONNECTION_FAILED,
         this.peerType,
       );
       return;
@@ -320,6 +350,54 @@ export abstract class BasePeerConnection {
     // do nothing when ICE is restarting
     if (this.isIceRestarting) return;
 
+    // Pre-connect handling: ICE has never reached `connected`/`completed`.
+    // Restart is futile here (the data plane was never established), but
+    // these two terminal-ish states need different treatment:
+    // - `failed` is terminal, escalate to REJOIN so a new SFU/credentials
+    //   /PC configuration gets a chance, and let `Call.reconnect` count
+    //   this toward the unsupported-network budget.
+    // - `disconnected` is transient, the browser may yet move back to
+    //   `checking`/`connected`. Don't restart, don't escalate; wait it
+    //   out. If it ultimately fails, ICE will transition to `failed` and
+    //   the branch above will take over.
+    if (!this.iceHasEverConnected) {
+      if (state === 'failed') {
+        this.logger.info('ICE failed before connected, escalating to REJOIN');
+        clearTimeout(this.preConnectStuckTimeout);
+        this.preConnectStuckTimeout = undefined;
+        this.onReconnectionNeeded?.(
+          WebsocketReconnectStrategy.REJOIN,
+          ReconnectReason.ICE_NEVER_CONNECTED,
+          this.peerType,
+        );
+        return;
+      }
+      if (state === 'disconnected') {
+        this.logger.info('ICE disconnected before connected, wait to recover');
+        // Watchdog: if the browser stays in `disconnected` without ever
+        // reaching `connected` or transitioning to `failed`, escalate to
+        // REJOIN ourselves so we don't wait silently forever. Rare but
+        // observed on flaky mobile networks.
+        clearTimeout(this.preConnectStuckTimeout);
+        this.preConnectStuckTimeout = setTimeout(() => {
+          if (
+            !this.iceHasEverConnected &&
+            this.pc.iceConnectionState === 'disconnected'
+          ) {
+            this.logger.info(
+              'ICE stuck in pre-connect disconnected, escalating to REJOIN',
+            );
+            this.onReconnectionNeeded?.(
+              WebsocketReconnectStrategy.REJOIN,
+              ReconnectReason.ICE_NEVER_CONNECTED,
+              this.peerType,
+            );
+          }
+        }, this.iceRestartDelay * 2);
+        return;
+      }
+    }
+
     switch (state) {
       case 'failed':
         // in the `failed` state, we try to restart ICE immediately
@@ -341,12 +419,24 @@ export abstract class BasePeerConnection {
         break;
 
       case 'connected':
-        // in the `connected` state, we clear the ice restart timeout if it exists
+      case 'completed':
+        // Fire `onIceConnected` exactly once per peer-connection lifetime —
+        // the first time ICE reaches `connected`/`completed` end-to-end.
+        // Used by `Call` to reset the unsupported-network failure counter
+        // only after WebRTC has actually recovered, not merely on SFU join.
+        if (!this.iceHasEverConnected) {
+          this.iceHasEverConnected = true;
+          this.onIceConnected?.(this.peerType);
+        }
+        // clear any scheduled restartICE since the connection is healthy
         if (this.iceRestartTimeout) {
           this.logger.info('connected connection, canceling restartICE');
           clearTimeout(this.iceRestartTimeout);
           this.iceRestartTimeout = undefined;
         }
+        // clear the pre-connect watchdog if it was armed
+        clearTimeout(this.preConnectStuckTimeout);
+        this.preConnectStuckTimeout = undefined;
         break;
     }
   };

package/src/rtc/Publisher.ts

@@ -20,6 +20,7 @@ import {
   toVideoLayers,
 } from './layers';
 import { isSvcCodec } from './codecs';
+import { toRTCDegradationPreference } from './helpers/degradationPreference';
 import { isAudioTrackType } from './helpers/tracks';
 import { extractMid, removeCodecsExcept, setStartBitrate } from './helpers/sdp';
 import { withoutConcurrency } from '../helpers/concurrency';
@@ -135,7 +136,9 @@ export class Publisher extends BasePeerConnection {
     });
 
     const params = transceiver.sender.getParameters();
-    params.degradationPreference = 'maintain-framerate';
+    params.degradationPreference =
+      toRTCDegradationPreference(publishOption.degradationPreference) ??
+      'maintain-framerate';
     await transceiver.sender.setParameters(params);
 
     const trackType = publishOption.trackType;
@@ -248,6 +251,38 @@ export class Publisher extends BasePeerConnection {
     return false;
   };
 
+  /**
+   * Re-arms the encoder for the given track type by detaching and
+   * reattaching the currently published track on each matching sender.
+   *
+   * Workaround for a WebKit / iOS Safari quirk: after a system audio
+   * session interruption (Siri, PSTN call), the `RTCRtpSender` encoder
+   * can stop producing RTP packets even though the underlying
+   * `MediaStreamTrack` is `live` and `track.muted === false`.
+   * `replaceTrack(null)` followed by `replaceTrack(track)` resets the
+   * sender's encoder pipeline without renegotiation, restoring packet
+   * flow with the same SSRC.
+   *
+   * No-op when nothing is published for the given track type.
+   *
+   * @param trackType the track type to refresh.
+   */
+  refreshTrack = async (trackType: TrackType) => {
+    for (const item of this.transceiverCache.items()) {
+      if (item.publishOption.trackType !== trackType) continue;
+      const { sender } = item.transceiver;
+      const track = sender.track;
+      if (!track || track.readyState !== 'live') continue;
+      try {
+        await sender.replaceTrack(null);
+        await sender.replaceTrack(track);
+        this.logger.debug(`Refreshed ${TrackType[trackType]} sender`);
+      } catch (err) {
+        this.logger.warn(`Failed to refresh ${TrackType[trackType]}`, err);
+      }
+    }
+  };
+
   /**
    * Stops the cloned track that is being published to the SFU.
    */
@@ -344,6 +379,17 @@ export class Publisher extends BasePeerConnection {
       }
     }
 
+    const degradationPreference = toRTCDegradationPreference(
+      videoSender.degradationPreference,
+    );
+    if (
+      degradationPreference &&
+      params.degradationPreference !== degradationPreference
+    ) {
+      params.degradationPreference = degradationPreference;
+      changed = true;
+    }
+
     const activeEncoders = params.encodings.filter((e) => e.active);
     if (!changed) {
       return this.logger.info(`${tag} no change:`, activeEncoders);
@@ -460,7 +506,8 @@ export class Publisher extends BasePeerConnection {
     const trackInfos: TrackInfo[] = [];
     for (const publishOption of this.publishOptions) {
       const bundle = this.transceiverCache.get(publishOption);
-      if (!bundle || !bundle.transceiver.sender.track) continue;
+      const track = bundle?.transceiver.sender.track;
+      if (!bundle || !track || track.readyState !== 'live') continue;
       trackInfos.push(this.toTrackInfo(bundle, sdp));
     }
     return trackInfos;

package/src/rtc/Subscriber.ts

@@ -1,9 +1,10 @@
 import { BasePeerConnection } from './BasePeerConnection';
 import { BasePeerConnectionOpts } from './types';
 import { NegotiationError } from './NegotiationError';
-import { PeerType } from '../gen/video/sfu/models/models';
+import { PeerType, TrackType } from '../gen/video/sfu/models/models';
 import { SubscriberOffer } from '../gen/video/sfu/event/events';
 import { toTrackType, trackTypeToParticipantStreamKey } from './helpers/tracks';
+import { pushToIfMissing, removeFromIfPresent } from '../helpers/array';
 import { enableStereo, removeCodecsExcept } from './helpers/sdp';
 
 /**
@@ -67,7 +68,8 @@ export class Subscriber extends BasePeerConnection {
   };
 
   private handleOnTrack = (e: RTCTrackEvent) => {
-    const [primaryStream] = e.streams;
+    const { streams, track } = e;
+    const [primaryStream] = streams;
     // example: `e3f6aaf8-b03d-4911-be36-83f47d37a76a:TRACK_TYPE_VIDEO`
     const [trackId, rawTrackType] = primaryStream.id.split(':');
     const participantToUpdate = this.state.participants.find(
@@ -75,30 +77,35 @@ export class Subscriber extends BasePeerConnection {
     );
     this.logger.debug(
       `[onTrack]: Got remote ${rawTrackType} track for userId: ${participantToUpdate?.userId}`,
-      e.track.id,
-      e.track,
+      track.id,
+      track,
     );
 
+    const trackType = toTrackType(rawTrackType);
+    if (!trackType) {
+      return this.logger.error(`Unknown track type: ${rawTrackType}`);
+    }
+
     const trackDebugInfo = `${participantToUpdate?.userId} ${rawTrackType}:${trackId}`;
-    e.track.addEventListener('mute', () => {
+    track.addEventListener('mute', () => {
       this.logger.info(`[onTrack]: Track muted: ${trackDebugInfo}`);
+      this.setRemoteTrackInterrupted(trackId, trackType, true);
     });
-
-    e.track.addEventListener('unmute', () => {
+    track.addEventListener('unmute', () => {
       this.logger.info(`[onTrack]: Track unmuted: ${trackDebugInfo}`);
+      this.setRemoteTrackInterrupted(trackId, trackType, false);
     });
-
-    e.track.addEventListener('ended', () => {
+    track.addEventListener('ended', () => {
       this.logger.info(`[onTrack]: Track ended: ${trackDebugInfo}`);
+      this.setRemoteTrackInterrupted(trackId, trackType, false);
       this.state.removeOrphanedTrack(primaryStream.id);
     });
 
-    const trackType = toTrackType(rawTrackType);
-    if (!trackType) {
-      return this.logger.error(`Unknown track type: ${rawTrackType}`);
+    if (track.muted) {
+      this.setRemoteTrackInterrupted(trackId, trackType, true);
     }
 
-    this.trackIdToTrackType.set(e.track.id, trackType);
+    this.trackIdToTrackType.set(track.id, trackType);
 
     if (!participantToUpdate) {
       this.logger.warn(
@@ -133,7 +140,7 @@ export class Subscriber extends BasePeerConnection {
     // now, dispose the previous stream if it exists
     if (previousStream) {
       this.logger.info(
-        `[onTrack]: Cleaning up previous remote ${e.track.kind} tracks for userId: ${participantToUpdate.userId}`,
+        `[onTrack]: Cleaning up previous remote ${track.kind} tracks for userId: ${participantToUpdate.userId}`,
       );
       previousStream.getTracks().forEach((t) => {
         t.stop();
@@ -142,6 +149,27 @@ export class Subscriber extends BasePeerConnection {
     }
   };
 
+  private setRemoteTrackInterrupted = (
+    trackId: string,
+    trackType: TrackType,
+    interrupted: boolean,
+  ) => {
+    if (trackType !== TrackType.AUDIO) return;
+    const target = this.state.participants.find(
+      (p) => p.trackLookupPrefix === trackId,
+    );
+    if (!target) return;
+    this.state.updateParticipant(target.sessionId, (p) => {
+      const current = p.interruptedTracks ?? [];
+      const has = current.includes(trackType);
+      if (interrupted === has) return {};
+      const next = interrupted
+        ? pushToIfMissing([...current], trackType)
+        : removeFromIfPresent([...current], trackType);
+      return { interruptedTracks: next };
+    });
+  };
+
   private negotiate = async (subscriberOffer: SubscriberOffer) => {
     await this.pc.setRemoteDescription({
       type: 'offer',