@stream-io/video-client 1.24.0 → 1.25.0

package/src/rtc/__tests__/Publisher.test.ts

@@ -1,21 +1,26 @@
 import './mocks/webrtc.mocks';
 
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { anyString } from 'vitest-mock-extended';
+import { NegotiationError } from '../NegotiationError';
 import { Publisher } from '../Publisher';
 import { CallState } from '../../store';
 import { StreamSfuClient } from '../../StreamSfuClient';
 import { DispatchableMessage, Dispatcher } from '../Dispatcher';
 import {
+  ErrorCode,
   PeerType,
   PublishOption,
-  SdkType,
   TrackInfo,
   TrackType,
+  WebsocketReconnectStrategy,
 } from '../../gen/video/sfu/models/models';
+import { SetPublisherResponse } from '../../gen/video/sfu/signal_rpc/signal';
 import { SfuEvent } from '../../gen/video/sfu/event/events';
 import { IceTrickleBuffer } from '../IceTrickleBuffer';
 import { StreamClient } from '../../coordinator/connection/client';
 import { TransceiverCache } from '../TransceiverCache';
+import { promiseWithResolvers } from '../../helpers/promise';
 
 vi.mock('../../StreamSfuClient', () => {
   console.log('MOCKING StreamSfuClient');
@@ -47,7 +52,7 @@ describe('Publisher', () => {
         ice_servers: [],
       },
       logTag: 'test',
-      enableTracing: false,
+      enableTracing: true,
     });
 
     // @ts-expect-error readonly field
@@ -63,18 +68,6 @@ describe('Publisher', () => {
       state,
       logTag: 'test',
       enableTracing: false,
-      clientDetails: {
-        sdk: {
-          type: SdkType.PLAIN_JAVASCRIPT,
-          major: '1',
-          minor: '0',
-          patch: '0',
-        },
-        device: {
-          name: 'test-device',
-          version: '1.0.0',
-        },
-      },
       publishOptions: [
         {
           id: 1,
@@ -242,20 +235,169 @@ describe('Publisher', () => {
       expect(publisher['negotiate']).not.toHaveBeenCalled();
     });
 
+    it('should initiate new negotiation when ICE restart is requested', async () => {
+      // @ts-expect-error private method
+      vi.spyOn(publisher, 'negotiate').mockResolvedValue();
+
+      await publisher.restartIce();
+      expect(publisher['negotiate']).toHaveBeenCalled();
+    });
+
     it(`should perform ICE restart when connection state changes to 'failed'`, () => {
-      publisher['onUnrecoverableError'] = vi.fn();
+      vi.spyOn(publisher, 'restartIce').mockResolvedValue();
       // @ts-expect-error private api
       publisher['pc'].iceConnectionState = 'failed';
       publisher['onIceConnectionStateChange']();
-      expect(publisher['onUnrecoverableError']).toHaveBeenCalled();
+      expect(publisher.restartIce).toHaveBeenCalled();
     });
 
-    it(`should perform ICE restart when connection state changes to 'disconnected'`, () => {
+    it(`should perform rejoin when ICE restart fails after connection state changes to 'failed'`, async () => {
+      const { promise: lock, resolve: unlock } = promiseWithResolvers<void>();
+      publisher['onReconnectionNeeded'] = vi
+        .fn()
+        .mockImplementation(() => unlock());
+      vi.spyOn(publisher, 'restartIce').mockRejectedValue('ICE restart failed');
+      // @ts-expect-error private api
+      publisher['pc'].iceConnectionState = 'failed';
+      publisher['onIceConnectionStateChange']();
+
+      await lock;
+      expect(publisher.restartIce).toHaveBeenCalled();
+      expect(publisher['onReconnectionNeeded']).toHaveBeenCalled();
+    });
+
+    it(`should perform fast reconnect when ICE restart fails with SIGNAL_LOST error`, async () => {
+      const { promise: lock, resolve: unlock } = promiseWithResolvers<void>();
+      publisher['onReconnectionNeeded'] = vi
+        .fn()
+        .mockImplementation(() => unlock());
+      publisher.getAnnouncedTracks = vi
+        .fn()
+        .mockReturnValue([
+          { trackId: '123', trackType: TrackType.VIDEO, mid: '0' },
+        ]);
+
+      // @ts-expect-error private api
+      vi.spyOn(publisher, 'negotiate');
+      vi.spyOn(publisher, 'restartIce');
+
+      const pc = publisher['pc'];
+
+      sfuClient.setPublisher = vi.fn().mockImplementation(() => {
+        // @ts-expect-error private api
+        pc.signalingState = 'have-local-offer';
+        return {
+          response: {
+            error: {
+              code: ErrorCode.PARTICIPANT_SIGNAL_LOST,
+              message: 'Signal lost',
+              shouldRetry: true,
+            },
+          } as SetPublisherResponse,
+        };
+      });
+
+      // @ts-expect-error private api
+      pc.iceConnectionState = 'failed';
+      publisher['onIceConnectionStateChange']();
+
+      await lock;
+      expect(publisher.restartIce).toHaveBeenCalled();
+      expect(publisher['negotiate']).toHaveBeenCalled();
+      expect(publisher['onReconnectionNeeded']).toHaveBeenCalledWith(
+        WebsocketReconnectStrategy.FAST,
+        anyString(),
+      );
+
+      expect(pc.setLocalDescription).toHaveBeenCalledTimes(2);
+      expect(pc.setLocalDescription).toHaveBeenLastCalledWith({
+        type: 'rollback',
+      });
+      expect(pc.setRemoteDescription).not.toHaveBeenCalled();
+    });
+
+    it(`should perform REJOIN reconnect when ICE restart fails with any other error code`, async () => {
+      const { promise: lock, resolve: unlock } = promiseWithResolvers<void>();
+      publisher['onReconnectionNeeded'] = vi
+        .fn()
+        .mockImplementation(() => unlock());
+      publisher.getAnnouncedTracks = vi
+        .fn()
+        .mockReturnValue([
+          { trackId: '123', trackType: TrackType.VIDEO, mid: '0' },
+        ]);
+
+      // @ts-expect-error private api
+      vi.spyOn(publisher, 'negotiate');
+      vi.spyOn(publisher, 'restartIce');
+
+      sfuClient.setPublisher = vi.fn().mockResolvedValue({
+        response: {
+          error: {
+            code: ErrorCode.PARTICIPANT_NOT_FOUND,
+            message: 'participant not found',
+            shouldRetry: true,
+          },
+        } as SetPublisherResponse,
+      });
+
+      // @ts-expect-error private api
+      publisher['pc'].iceConnectionState = 'failed';
+      publisher['onIceConnectionStateChange']();
+
+      await lock;
+      expect(publisher.restartIce).toHaveBeenCalled();
+      expect(publisher['negotiate']).toHaveBeenCalled();
+      await expect(publisher.restartIce).rejects.toThrowError(NegotiationError);
+      expect(publisher['onReconnectionNeeded']).toHaveBeenCalledWith(
+        WebsocketReconnectStrategy.REJOIN,
+        anyString(),
+      );
+    });
+
+    it(`should schedule ICE restart when connection state changes to 'disconnected'`, () => {
       vi.spyOn(publisher, 'restartIce').mockResolvedValue();
+      vi.useFakeTimers();
+      // @ts-expect-error private api
+      publisher['pc'].iceConnectionState = 'disconnected';
+      publisher['onIceConnectionStateChange']();
+
+      vi.runOnlyPendingTimers();
+      expect(publisher.restartIce).toHaveBeenCalled();
+    });
+
+    it(`should perform rejoin when scheduled ICE restart fails`, async () => {
+      vi.spyOn(publisher, 'restartIce').mockRejectedValue('ICE restart failed');
+      const { promise: lock, resolve } = promiseWithResolvers<void>();
+      publisher['onReconnectionNeeded'] = vi
+        .fn()
+        .mockImplementation(() => resolve());
+      vi.useFakeTimers();
       // @ts-expect-error private api
       publisher['pc'].iceConnectionState = 'disconnected';
       publisher['onIceConnectionStateChange']();
+
+      vi.runOnlyPendingTimers();
+
+      await lock;
       expect(publisher.restartIce).toHaveBeenCalled();
+      expect(publisher['onReconnectionNeeded']).toHaveBeenCalled();
+    });
+
+    it(`should schedule ICE restart but cancel it if connection recovers in the meantime`, () => {
+      vi.spyOn(publisher, 'restartIce').mockResolvedValue();
+      vi.useFakeTimers();
+      // @ts-expect-error private api
+      publisher['pc'].iceConnectionState = 'disconnected';
+      publisher['onIceConnectionStateChange']();
+
+      // @ts-expect-error private api
+      publisher['pc'].iceConnectionState = 'connected';
+      publisher['onIceConnectionStateChange']();
+
+      vi.runOnlyPendingTimers();
+      expect(publisher.restartIce).not.toHaveBeenCalled();
+      expect(publisher['iceRestartTimeout']).toBeUndefined();
     });
   });
 
@@ -721,6 +863,7 @@ describe('Publisher', () => {
     it('isPublishing should return true if there are active tracks', () => {
       expect(publisher.isPublishing(TrackType.VIDEO)).toBe(true);
       expect(publisher.isPublishing(TrackType.SCREEN_SHARE_AUDIO)).toBe(false);
+      expect(publisher.isPublishing()).toBe(true);
     });
 
     it('getTrackType should return the track type', () => {

package/src/rtc/__tests__/Subscriber.test.ts

@@ -6,11 +6,13 @@ import { StreamSfuClient } from '../../StreamSfuClient';
 import { Subscriber } from '../Subscriber';
 import { CallState } from '../../store';
 import { SfuEvent, SubscriberOffer } from '../../gen/video/sfu/event/events';
+import { ICERestartResponse } from '../../gen/video/sfu/signal_rpc/signal';
 import {
+  ErrorCode,
   PeerType,
-  SdkType,
   TrackType,
 } from '../../gen/video/sfu/models/models';
+import { NegotiationError } from '../NegotiationError';
 import { IceTrickleBuffer } from '../IceTrickleBuffer';
 import { StreamClient } from '../../coordinator/connection/client';
 
@@ -43,7 +45,7 @@ describe('Subscriber', () => {
         token: 'token',
         ice_servers: [],
       },
-      enableTracing: false,
+      enableTracing: true,
     });
     // @ts-expect-error readonly field
     sfuClient.iceTrickleBuffer = new IceTrickleBuffer();
@@ -54,15 +56,7 @@ describe('Subscriber', () => {
       state,
       connectionConfig: { iceServers: [] },
       logTag: 'test',
-      enableTracing: false,
-      clientDetails: {
-        sdk: {
-          type: SdkType.PLAIN_JAVASCRIPT,
-          major: '1',
-          minor: '0',
-          patch: '1',
-        },
-      },
+      enableTracing: true,
     });
   });
 
@@ -92,7 +86,7 @@ describe('Subscriber', () => {
     });
 
     it('should ask the SFU for ICE restart', async () => {
-      sfuClient.iceRestart = vi.fn();
+      sfuClient.iceRestart = vi.fn().mockResolvedValue({ response: {} });
       // @ts-expect-error - private field
       subscriber['pc'].connectionState = 'connected';
 
@@ -103,20 +97,43 @@ describe('Subscriber', () => {
     });
 
     it(`should perform ICE restart when connection state changes to 'failed'`, () => {
-      subscriber['onUnrecoverableError'] = vi.fn();
+      vi.spyOn(subscriber, 'restartIce').mockResolvedValue();
       // @ts-expect-error - private field
       subscriber['pc'].iceConnectionState = 'failed';
       subscriber['onIceConnectionStateChange']();
-      expect(subscriber['onUnrecoverableError']).toHaveBeenCalled();
+      expect(subscriber['restartIce']).toHaveBeenCalled();
     });
 
     it(`should perform ICE restart when connection state changes to 'disconnected'`, () => {
       vi.spyOn(subscriber, 'restartIce').mockResolvedValue();
+      vi.useFakeTimers();
       // @ts-expect-error - private field
       subscriber['pc'].iceConnectionState = 'disconnected';
       subscriber['onIceConnectionStateChange']();
+      vi.runOnlyPendingTimers();
       expect(subscriber.restartIce).toHaveBeenCalled();
     });
+
+    it(`should throw NegotiationError when SFU returns an error`, async () => {
+      sfuClient.iceRestart = vi.fn().mockResolvedValue({
+        response: {
+          error: {
+            code: ErrorCode.PARTICIPANT_SIGNAL_LOST,
+            message: 'Signal lost',
+          },
+        } as ICERestartResponse,
+      });
+
+      // @ts-expect-error - private field
+      subscriber['pc'].connectionState = 'connected';
+
+      await expect(subscriber.restartIce()).rejects.toThrowError(
+        NegotiationError,
+      );
+      expect(sfuClient.iceRestart).toHaveBeenCalledWith({
+        peerType: PeerType.SUBSCRIBER,
+      });
+    });
   });
 
   describe('OnTrack', () => {

package/src/rtc/helpers/__tests__/sdp.stereo.test.ts

@@ -0,0 +1,120 @@
+import { describe, expect, it } from 'vitest';
+import { enableStereo } from '../sdp';
+
+const offerSdp = `o=- 5087842825318906515 1750254551 IN IP4 127.0.0.1
+s=-
+t=0 0
+a=group:BUNDLE 0 1
+a=extmap-allow-mixed
+a=msid-semantic: WMS a1e5f21f716affb7:TRACK_TYPE_AUDIO:eO a1e5f21f716affb7:TRACK_TYPE_VIDEO:Nk
+a=ice-lite
+m=audio 51808 UDP/TLS/RTP/SAVPF 111 63 (19 more lines) direction=sendrecv mid=0
+c=IN IP4 18.119.157.125
+a=rtcp:51808 IN IP4 18.119.157.125
+a=candidate:965407073 1 udp 2130706431 18.119.157.125 51808 typ host generation 0
+a=candidate:965407073 2 udp 2130706431 18.119.157.125 51808 typ host generation 0
+a=ice-ufrag:AFJnYPvMfEaZeHdt
+a=ice-pwd:mbwUwrcoSApXwOyGrQOAsipWsfFGHcww
+a=fingerprint:sha-256 13:79:52:41:12:BB:A7:5D:39:F0:9B:1A:95:58:94:D6:F9:D3:1E:00:A4:9D:CA:12:26:AE:7C:2A:E1:FC:42:F4
+a=setup:actpass
+a=mid:0
+a=sendrecv
+a=msid:a1e5f21f716affb7:TRACK_TYPE_AUDIO:eO 5040549b-8458-4646-892d-ad08f4475568
+a=rtcp-mux
+a=rtcp-rsize
+a=rtpmap:111 opus/48000/2
+a=fmtp:111 maxaveragebitrate=510000;minptime=10;sprop-stereo=1;stereo=1;useinbandfec=1
+a=rtpmap:63 red/48000/2
+a=fmtp:63 111/111
+a=ssrc:1826085709 cname:a1e5f21f716affb7:TRACK_TYPE_AUDIO:eO
+a=ssrc:1826085709 msid:a1e5f21f716affb7:TRACK_TYPE_AUDIO:eO 5040549b-8458-4646-892d-ad08f4475568
+m=video 9 UDP/TLS/RTP/SAVPF 96 97 direction=sendrecv mid=1
+c=IN IP4 0.0.0.0
+a=rtcp:9 IN IP4 0.0.0.0
+a=ice-ufrag:AFJnYPvMfEaZeHdt
+a=ice-pwd:mbwUwrcoSApXwOyGrQOAsipWsfFGHcww
+a=fingerprint:sha-256 13:79:52:41:12:BB:A7:5D:39:F0:9B:1A:95:58:94:D6:F9:D3:1E:00:A4:9D:CA:12:26:AE:7C:2A:E1:FC:42:F4
+a=setup:actpass
+a=mid:1
+a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01
+a=extmap:1 https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension
+a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time
+a=sendrecv
+a=msid:a1e5f21f716affb7:TRACK_TYPE_VIDEO:Nk 40c4a11d-fa67-49b6-a4fa-1625b4bde4a5
+a=rtcp-mux
+a=rtcp-rsize
+a=rtpmap:96 VP8/90000
+a=rtcp-fb:96 ccm fir
+a=rtcp-fb:96 nack
+a=rtcp-fb:96 nack pli
+a=rtcp-fb:96 goog-remb
+a=rtcp-fb:96 transport-cc
+a=rtpmap:97 rtx/90000
+a=fmtp:97 apt=96
+a=ssrc-group:FID 3718708632 2750164767
+a=ssrc:3718708632 cname:a1e5f21f716affb7:TRACK_TYPE_VIDEO:Nk
+a=ssrc:3718708632 msid:a1e5f21f716affb7:TRACK_TYPE_VIDEO:Nk 40c4a11d-fa67-49b6-a4fa-1625b4bde4a5
+a=ssrc:2750164767 cname:a1e5f21f716affb7:TRACK_TYPE_VIDEO:Nk
+a=ssrc:2750164767 msid:a1e5f21f716affb7:TRACK_TYPE_VIDEO:Nk 40c4a11d-fa67-49b6-a4fa-1625b4bde4a5
+`;
+
+const answerSdp = `v=0
+o=- 6793916097087762106 2 IN IP4 127.0.0.1
+s=-
+t=0 0
+a=group:BUNDLE 0 1
+a=extmap-allow-mixed
+a=msid-semantic: WMS
+m=audio 9 UDP/TLS/RTP/SAVPF 111 63
+c=IN IP4 0.0.0.0
+a=rtcp:9 IN IP4 0.0.0.0
+a=ice-ufrag:zcgT
+a=ice-pwd:v559SXDwx4y9yAv7oeCvjsDR
+a=ice-options:trickle
+a=fingerprint:sha-256 F7:C8:B3:87:4A:AD:5A:86:48:1B:51:04:BE:CE:3B:D6:D3:7C:25:63:3E:9C:2B:F6:5B:8C:65:1F:72:8A:11:61
+a=setup:active
+a=mid:0
+a=recvonly
+a=rtcp-mux
+a=rtcp-rsize
+a=rtpmap:111 opus/48000/2
+a=fmtp:111 minptime=10;useinbandfec=1
+a=rtpmap:63 red/48000/2
+a=fmtp:63 111/111
+m=video 9 UDP/TLS/RTP/SAVPF 96 97
+c=IN IP4 0.0.0.0
+a=rtcp:9 IN IP4 0.0.0.0
+a=ice-ufrag:zcgT
+a=ice-pwd:v559SXDwx4y9yAv7oeCvjsDR
+a=ice-options:trickle
+a=fingerprint:sha-256 F7:C8:B3:87:4A:AD:5A:86:48:1B:51:04:BE:CE:3B:D6:D3:7C:25:63:3E:9C:2B:F6:5B:8C:65:1F:72:8A:11:61
+a=setup:active
+a=mid:1
+a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time
+a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01
+a=extmap:1 https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension
+a=recvonly
+a=rtcp-mux
+a=rtcp-rsize
+a=rtpmap:96 VP8/90000
+a=rtcp-fb:96 goog-remb
+a=rtcp-fb:96 transport-cc
+a=rtcp-fb:96 ccm fir
+a=rtcp-fb:96 nack
+a=rtcp-fb:96 nack pli
+a=rtpmap:97 rtx/90000
+a=fmtp:97 apt=96
+`;
+
+describe('sdp - enableStereo', () => {
+  it('should enable stereo in the answer SDP based on the offered stereo in the offer SDP', () => {
+    const result = enableStereo(offerSdp, answerSdp);
+    expect(result).toContain('a=fmtp:111 minptime=10;useinbandfec=1;stereo=1');
+  });
+
+  it('should not modify the answer SDP if no stereo is offered', () => {
+    const modifiedOfferSdp = offerSdp.replace(/stereo=1/g, 'stereo=0');
+    const result = enableStereo(modifiedOfferSdp, answerSdp);
+    expect(result).toBe(answerSdp);
+  });
+});

package/src/rtc/helpers/sdp.ts

@@ -1,4 +1,4 @@
-import { parse } from 'sdp-transform';
+import { parse, write } from 'sdp-transform';
 
 /**
  * Extracts the mid from the transceiver or the SDP.
@@ -28,3 +28,45 @@ export const extractMid = (
   if (transceiverInitIndex < 0) return '';
   return String(transceiverInitIndex);
 };
+
+/**
+ * Enables stereo in the answer SDP based on the offered stereo in the offer SDP.
+ *
+ * @param offerSdp the offer SDP containing the stereo configuration.
+ * @param answerSdp the answer SDP to be modified.
+ */
+export const enableStereo = (offerSdp: string, answerSdp: string): string => {
+  const offeredStereoMids = new Set<string>();
+  const parsedOfferSdp = parse(offerSdp);
+  for (const media of parsedOfferSdp.media) {
+    if (media.type !== 'audio') continue;
+
+    const opus = media.rtp.find((r) => r.codec === 'opus');
+    if (!opus) continue;
+
+    for (const fmtp of media.fmtp) {
+      if (fmtp.payload === opus.payload && fmtp.config.includes('stereo=1')) {
+        offeredStereoMids.add(media.mid!);
+      }
+    }
+  }
+
+  // No stereo offered, return the original answerSdp
+  if (offeredStereoMids.size === 0) return answerSdp;
+
+  const parsedAnswerSdp = parse(answerSdp);
+  for (const media of parsedAnswerSdp.media) {
+    if (media.type !== 'audio' || !offeredStereoMids.has(media.mid!)) continue;
+
+    const opus = media.rtp.find((r) => r.codec === 'opus');
+    if (!opus) continue;
+
+    for (const fmtp of media.fmtp) {
+      if (fmtp.payload === opus.payload && !fmtp.config.includes('stereo=1')) {
+        fmtp.config += ';stereo=1';
+      }
+    }
+  }
+
+  return write(parsedAnswerSdp);
+};

package/src/types.ts

@@ -215,6 +215,18 @@ export type ScreenShareSettings = {
    * Defaults to 3000000 (3Mbps).
    */
   maxBitrate?: number;
+
+  /**
+   * The content hint to use when publishing the screen share.
+   * This can be used to optimize the video quality for different types of content.
+   *
+   * Defaults to '' (no hint, browser's default behavior).
+   * Use 'motion' for video content, 'detail' for presentations or documents, and 'text' for text-heavy content.
+   *
+   * Please read the documentation for more information on content hints:
+   * - https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack/contentHint
+   */
+  contentHint?: '' | 'motion' | 'detail' | 'text';
 };
 
 export type CallLeaveOptions = {