@stream-io/video-client 1.55.1 → 1.56.0

package/src/reporting/__tests__/ClientEventReporter.test.ts

@@ -7,6 +7,7 @@ import { ErrorFromResponse } from '../../coordinator/connection/types';
 import { SfuTimeoutError } from '../../errors';
 import type { AxiosResponse } from 'axios';
 import { PeerType, TrackType } from '../../gen/video/sfu/models/models';
+import type { PeerConnectionStateChangeEvent } from '../../rtc';
 
 vi.mock('../../devices', () => ({
   getAudioBrowserPermission: () => ({ asStateObservable: () => of('granted') }),
@@ -15,6 +16,27 @@ vi.mock('../../devices', () => ({
 
 const flush = () => new Promise((resolve) => setTimeout(resolve, 0));
 
+const peerConnectionEvent = (
+  peerType: PeerType,
+  state: RTCPeerConnectionState,
+  iceConnectionState: RTCIceConnectionState,
+): PeerConnectionStateChangeEvent => ({
+  peerType,
+  stateType: 'peerConnection',
+  state,
+  iceConnectionState,
+});
+
+const iceEvent = (
+  peerType: PeerType,
+  state: RTCIceConnectionState,
+): PeerConnectionStateChangeEvent => ({
+  peerType,
+  stateType: 'ice',
+  state,
+  iceConnectionState: state,
+});
+
 describe('ClientEventReporter', () => {
   const cid = 'default:call-1';
   let doAxiosRequest: ReturnType<typeof vi.fn>;
@@ -24,6 +46,21 @@ describe('ClientEventReporter', () => {
   const postedEvents = (): Array<Record<string, any>> =>
     doAxiosRequest.mock.calls.map((call) => call[2].events[0]);
 
+  const reportPeerConnectionState = (
+    peerType: PeerType,
+    state: RTCPeerConnectionState,
+    iceConnectionState: RTCIceConnectionState,
+  ) => {
+    reporter.onPeerConnectionStateChange(
+      cid,
+      peerConnectionEvent(peerType, state, iceConnectionState),
+    );
+  };
+
+  const reportIceState = (peerType: PeerType, state: RTCIceConnectionState) => {
+    reporter.onPeerConnectionStateChange(cid, iceEvent(peerType, state));
+  };
+
   beforeEach(() => {
     doAxiosRequest = vi.fn().mockResolvedValue({});
     const streamClient = fromPartial<StreamClient>({
@@ -465,16 +502,16 @@ describe('ClientEventReporter', () => {
 
   it('tracks a publisher peer connection connect', async () => {
     reporter.startCorrelation(cid, 'first-attempt');
-    reporter.onPeerConnectionStateChange(cid, {
-      peerType: PeerType.PUBLISHER_UNSPECIFIED,
-      stateType: 'peerConnection',
-      state: 'connecting',
-    });
-    reporter.onPeerConnectionStateChange(cid, {
-      peerType: PeerType.PUBLISHER_UNSPECIFIED,
-      stateType: 'peerConnection',
-      state: 'connected',
-    });
+    reportPeerConnectionState(
+      PeerType.PUBLISHER_UNSPECIFIED,
+      'connecting',
+      'checking',
+    );
+    reportPeerConnectionState(
+      PeerType.PUBLISHER_UNSPECIFIED,
+      'connected',
+      'connected',
+    );
     await flush();
 
     const events = postedEvents().filter(
@@ -497,16 +534,8 @@ describe('ClientEventReporter', () => {
 
   it('reports an ICE failure on the peer connection', async () => {
     reporter.startCorrelation(cid, 'first-attempt');
-    reporter.onPeerConnectionStateChange(cid, {
-      peerType: PeerType.SUBSCRIBER,
-      stateType: 'peerConnection',
-      state: 'connecting',
-    });
-    reporter.onPeerConnectionStateChange(cid, {
-      peerType: PeerType.SUBSCRIBER,
-      stateType: 'ice',
-      state: 'failed',
-    });
+    reportPeerConnectionState(PeerType.SUBSCRIBER, 'connecting', 'checking');
+    reportIceState(PeerType.SUBSCRIBER, 'failed');
     await flush();
 
     const completed = postedEvents().filter(
@@ -522,25 +551,111 @@ describe('ClientEventReporter', () => {
     });
   });
 
-  it('opens a fresh peer connection pair after a new correlation', async () => {
+  it('reports a DTLS failure with the real ICE state (connected)', async () => {
     reporter.startCorrelation(cid, 'first-attempt');
-    reporter.onPeerConnectionStateChange(cid, {
-      peerType: PeerType.PUBLISHER_UNSPECIFIED,
-      stateType: 'peerConnection',
-      state: 'connecting',
+    reportPeerConnectionState(
+      PeerType.PUBLISHER_UNSPECIFIED,
+      'connecting',
+      'checking',
+    );
+    reportIceState(PeerType.PUBLISHER_UNSPECIFIED, 'connected');
+    // Aggregate connection fails while ICE stays connected -> genuine DTLS.
+    reportPeerConnectionState(
+      PeerType.PUBLISHER_UNSPECIFIED,
+      'failed',
+      'connected',
+    );
+    await flush();
+
+    const completed = postedEvents().filter(
+      (e) =>
+        e.stage === 'PeerConnectionConnect' && e.event_type === 'completed',
+    );
+    expect(completed).toHaveLength(1);
+    expect(completed[0]).toMatchObject({
+      outcome: 'failure',
+      peer_connection: 'publish',
+      retry_failure_code: 'DTLS_CONNECTIVITY_FAILED',
+      ice_state: 'CONNECTED',
     });
+  });
 
-    reporter.startCorrelation(cid, 'full-rejoin');
-    reporter.onPeerConnectionStateChange(cid, {
-      peerType: PeerType.PUBLISHER_UNSPECIFIED,
-      stateType: 'peerConnection',
-      state: 'connecting',
+  it('reports a DTLS failure with the real ICE state (never connected)', async () => {
+    reporter.startCorrelation(cid, 'first-attempt');
+    reportPeerConnectionState(
+      PeerType.PUBLISHER_UNSPECIFIED,
+      'connecting',
+      'checking',
+    );
+    // ICE is still `checking` when the aggregate peer-connection state fails:
+    // the reported ice_state must be NOT_CONNECTED, not an assumed CONNECTED.
+    reportPeerConnectionState(
+      PeerType.PUBLISHER_UNSPECIFIED,
+      'failed',
+      'checking',
+    );
+    await flush();
+
+    const completed = postedEvents().filter(
+      (e) =>
+        e.stage === 'PeerConnectionConnect' && e.event_type === 'completed',
+    );
+    expect(completed).toHaveLength(1);
+    expect(completed[0]).toMatchObject({
+      outcome: 'failure',
+      peer_connection: 'publish',
+      retry_failure_code: 'DTLS_CONNECTIVITY_FAILED',
+      ice_state: 'NOT_CONNECTED',
     });
-    reporter.onPeerConnectionStateChange(cid, {
-      peerType: PeerType.PUBLISHER_UNSPECIFIED,
-      stateType: 'peerConnection',
-      state: 'connected',
+  });
+
+  it('reports the ICE state captured when the pair opened, with no ICE event', async () => {
+    reporter.startCorrelation(cid, 'first-attempt');
+    // The pair opens with ICE already connected (carried on the connecting
+    // event); no separate ICE event is ever delivered.
+    reportPeerConnectionState(
+      PeerType.PUBLISHER_UNSPECIFIED,
+      'connecting',
+      'connected',
+    );
+    reportPeerConnectionState(
+      PeerType.PUBLISHER_UNSPECIFIED,
+      'failed',
+      'connected',
+    );
+    await flush();
+
+    const completed = postedEvents().filter(
+      (e) =>
+        e.stage === 'PeerConnectionConnect' && e.event_type === 'completed',
+    );
+    expect(completed).toHaveLength(1);
+    expect(completed[0]).toMatchObject({
+      outcome: 'failure',
+      retry_failure_code: 'DTLS_CONNECTIVITY_FAILED',
+      ice_state: 'CONNECTED',
     });
+  });
+
+  it('opens a fresh peer connection pair after a new correlation', async () => {
+    reporter.startCorrelation(cid, 'first-attempt');
+    reportPeerConnectionState(
+      PeerType.PUBLISHER_UNSPECIFIED,
+      'connecting',
+      'checking',
+    );
+
+    reporter.startCorrelation(cid, 'full-rejoin');
+    reportPeerConnectionState(
+      PeerType.PUBLISHER_UNSPECIFIED,
+      'connecting',
+      'checking',
+    );
+    reportPeerConnectionState(
+      PeerType.PUBLISHER_UNSPECIFIED,
+      'connected',
+      'connected',
+    );
     await flush();
 
     const pc = postedEvents().filter(
@@ -566,16 +681,16 @@ describe('ClientEventReporter', () => {
   it('marks a peer connection as previously connected on reconnect', async () => {
     reporter.startCorrelation(cid, 'first-attempt');
     const connect = () => {
-      reporter.onPeerConnectionStateChange(cid, {
-        peerType: PeerType.PUBLISHER_UNSPECIFIED,
-        stateType: 'peerConnection',
-        state: 'connecting',
-      });
-      reporter.onPeerConnectionStateChange(cid, {
-        peerType: PeerType.PUBLISHER_UNSPECIFIED,
-        stateType: 'peerConnection',
-        state: 'connected',
-      });
+      reportPeerConnectionState(
+        PeerType.PUBLISHER_UNSPECIFIED,
+        'connecting',
+        'checking',
+      );
+      reportPeerConnectionState(
+        PeerType.PUBLISHER_UNSPECIFIED,
+        'connected',
+        'connected',
+      );
     };
     connect();
     connect();

package/src/rtc/BasePeerConnection.ts

@@ -362,6 +362,7 @@ export abstract class BasePeerConnection {
     try {
       this.onPeerConnectionStateChange?.({
         peerType: this.peerType,
+        iceConnectionState: this.pc.iceConnectionState,
         ...event,
       });
     } catch (err) {

package/src/rtc/Publisher.ts

@@ -40,6 +40,7 @@ export class Publisher extends BasePeerConnection {
   private readonly clonedTracks = new Set<MediaStreamTrack>();
   private publishOptions: PublishOption[];
   private readonly selfSubEnabled: boolean;
+  private readonly transceiverLockKey = `pub.tx.${this.lock}`;
 
   /**
    * Constructs a new `Publisher` instance.
@@ -107,28 +108,31 @@ export class Publisher extends BasePeerConnection {
     trackType: TrackType,
     options: TrackPublishOptions = {},
   ) => {
-    if (!this.publishOptions.some((o) => o.trackType === trackType)) {
-      throw new Error(`No publish options found for ${TrackType[trackType]}`);
-    }
-
-    for (const publishOption of this.publishOptions) {
-      if (publishOption.trackType !== trackType) continue;
-
-      // create a clone of the track as otherwise the same trackId will
-      // appear in the SDP in multiple transceivers
-      const trackToPublish = this.cloneTrack(track);
+    return withoutConcurrency(this.transceiverLockKey, async () => {
+      const publishOptions = this.publishOptions.filter(
+        (o) => o.trackType === trackType,
+      );
+      if (!publishOptions.length) {
+        throw new Error(`No publish options found for ${TrackType[trackType]}`);
+      }
 
-      const bundle = this.transceiverCache.get(publishOption);
-      if (!bundle) {
-        await this.addTransceiver(trackToPublish, publishOption, options);
-      } else {
-        const previousTrack = bundle.transceiver.sender.track;
-        await this.updateTransceiver(bundle, trackToPublish, options);
-        if (!isReactNative()) {
-          this.stopTrack(previousTrack);
+      for (const publishOption of publishOptions) {
+        // create a clone of the track as otherwise the same trackId will
+        // appear in the SDP in multiple transceivers
+        const trackToPublish = this.cloneTrack(track);
+
+        const bundle = this.transceiverCache.get(publishOption);
+        if (!bundle) {
+          await this.addTransceiver(trackToPublish, publishOption, options);
+        } else {
+          const previousTrack = bundle.transceiver.sender.track;
+          await this.updateTransceiver(bundle, trackToPublish, options);
+          if (!isReactNative()) {
+            this.stopTrack(previousTrack);
+          }
         }
       }
-    }
+    });
   };
 
   /**
@@ -139,6 +143,7 @@ export class Publisher extends BasePeerConnection {
     publishOption: PublishOption,
     options: TrackPublishOptions,
   ) => {
+    const trackType = publishOption.trackType;
     const encodings = isAudioTrackType(publishOption.trackType)
       ? computeAudioLayers(publishOption, options)
       : computeVideoLayers(track, publishOption);
@@ -149,6 +154,12 @@ export class Publisher extends BasePeerConnection {
       direction: 'sendonly',
       sendEncodings,
     });
+    // claim the cache entry in the same task that created the transceiver:
+    // an await in between would let a concurrent caller add a second one
+    // for the same publish option
+    this.transceiverCache.add({ publishOption, transceiver, options });
+    this.trackIdToTrackType.set(track.id, trackType);
+    this.logger.debug(`Added ${TrackType[trackType]} transceiver`);
 
     const params = transceiver.sender.getParameters();
     params.degradationPreference =
@@ -156,11 +167,6 @@ export class Publisher extends BasePeerConnection {
       'maintain-framerate';
     await transceiver.sender.setParameters(params);
 
-    const trackType = publishOption.trackType;
-    this.logger.debug(`Added ${TrackType[trackType]} transceiver`);
-    this.transceiverCache.add({ publishOption, transceiver, options });
-    this.trackIdToTrackType.set(track.id, trackType);
-
     await this.negotiate();
   };
 
@@ -225,38 +231,40 @@ export class Publisher extends BasePeerConnection {
    * Synchronizes the current Publisher state with the provided publish options.
    */
   private syncPublishOptions = async () => {
-    // enable publishing with new options -> [av1, vp9]
-    for (const publishOption of this.publishOptions) {
-      const { trackType } = publishOption;
-      if (!this.isPublishing(trackType)) continue;
-      if (this.transceiverCache.has(publishOption)) continue;
-
-      const item = this.transceiverCache.find(
-        (i) =>
-          !!i.transceiver.sender.track &&
-          i.publishOption.trackType === trackType,
-      );
-      if (!item) continue;
-
-      // take the track from the existing transceiver for the same track type,
-      // clone it and publish it with the new publish options
-      const track = this.cloneTrack(item.transceiver.sender.track!);
-      await this.addTransceiver(track, publishOption, item.options);
-    }
+    return withoutConcurrency(this.transceiverLockKey, async () => {
+      // enable publishing with new options -> [av1, vp9]
+      for (const publishOption of this.publishOptions) {
+        const { trackType } = publishOption;
+        if (!this.isPublishing(trackType)) continue;
+        if (this.transceiverCache.has(publishOption)) continue;
+
+        const item = this.transceiverCache.find(
+          (i) =>
+            !!i.transceiver.sender.track &&
+            i.publishOption.trackType === trackType,
+        );
+        if (!item) continue;
+
+        // take the track from the existing transceiver for the same track type,
+        // clone it and publish it with the new publish options
+        const track = this.cloneTrack(item.transceiver.sender.track!);
+        await this.addTransceiver(track, publishOption, item.options);
+      }
 
-    // stop publishing with options not required anymore -> [vp9]
-    for (const item of this.transceiverCache.items()) {
-      const { publishOption, transceiver } = item;
-      const hasPublishOption = this.publishOptions.some(
-        (option) =>
-          option.id === publishOption.id &&
-          option.trackType === publishOption.trackType,
-      );
-      if (hasPublishOption) continue;
-      // it is safe to stop the track here, it is a clone
-      this.stopTrack(transceiver.sender.track);
-      await this.updateTransceiver(item, null);
-    }
+      // stop publishing with options not required anymore -> [vp9]
+      for (const item of this.transceiverCache.items()) {
+        const { publishOption, transceiver } = item;
+        const hasPublishOption = this.publishOptions.some(
+          (option) =>
+            option.id === publishOption.id &&
+            option.trackType === publishOption.trackType,
+        );
+        if (hasPublishOption) continue;
+        // it is safe to stop the track here, it is a clone
+        this.stopTrack(transceiver.sender.track);
+        await this.updateTransceiver(item, null);
+      }
+    });
   };
 
   /**
@@ -456,13 +464,25 @@ export class Publisher extends BasePeerConnection {
   private negotiate = async (options?: RTCOfferOptions): Promise<void> => {
     return withoutConcurrency(`publisher.negotiate.${this.lock}`, async () => {
       const offer = await this.pc.createOffer(options);
-      const tracks = this.getAnnouncedTracks(offer.sdp);
-      if (!tracks.length) throw new Error(`Can't negotiate without any tracks`);
-
       try {
         this.isIceRestarting = options?.iceRestart ?? false;
         await this.pc.setLocalDescription(offer);
 
+        // the announced tracks have to be read after `setLocalDescription`.
+        // `transceiver.mid` is null until the offer is applied, and reading the
+        // tracks earlier makes `extractMid` guess the mid from the offer SDP.
+        // That guess goes stale after a rolled-back negotiation, and a mid that
+        // doesn't match the m-section carrying the track stops the SFU from
+        // correlating the two: it then answers with every offered codec instead
+        // of the announced one, and we publish the first codec of that list.
+        // Per https://www.w3.org/TR/webrtc/#set-description, applying a local
+        // offer associates the transceiver with its m-section and sets
+        // `[[Mid]]` to `[[JsepMid]]`; a rollback disassociates a transceiver
+        // that wasn't associated before and resets both slots to null.
+        const tracks = this.getAnnouncedTracks(offer.sdp);
+        if (!tracks.length)
+          throw new Error(`Can't negotiate without any tracks`);
+
         const { sdp: baseSdp = '' } = offer;
         const {
           dangerouslyForceCodec,