@stream-io/video-client 1.11.7 → 1.11.9

package/dist/src/rtc/codecs.d.ts

@@ -7,7 +7,7 @@ import type { PreferredCodec } from '../types';
  * @param codecToRemove the codec to exclude from the list.
  * @param codecPreferencesSource the source of the codec preferences.
  */
-export declare const getPreferredCodecs: (kind: "audio" | "video", preferredCodec: string, codecToRemove?: string, codecPreferencesSource?: "sender" | "receiver") => RTCRtpCodec[] | undefined;
+export declare const getPreferredCodecs: (kind: "audio" | "video", preferredCodec: string, codecToRemove: string | undefined, codecPreferencesSource: "sender" | "receiver") => RTCRtpCodec[] | undefined;
 /**
  * Returns a generic SDP for the given direction.
  * We use this SDP to send it as part of our JoinRequest so that the SFU

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stream-io/video-client",
-  "version": "1.11.7",
+  "version": "1.11.9",
   "packageManager": "yarn@3.2.4",
   "main": "dist/index.cjs.js",
   "module": "dist/index.es.js",

package/src/devices/InputMediaDeviceManager.ts

@@ -215,6 +215,7 @@ export abstract class InputMediaDeviceManager<
       await this.applySettingsToStream();
     } catch (error) {
       this.state.setDevice(prevDeviceId);
+      await this.applySettingsToStream();
       throw error;
     }
   }

package/src/devices/devices.ts

@@ -184,7 +184,7 @@ export const getAudioStream = async (
   const constraints: MediaStreamConstraints = {
     audio: {
       ...audioDeviceConstraints.audio,
-      ...trackConstraints,
+      ...normalizeContraints(trackConstraints),
     },
   };
 
@@ -195,16 +195,6 @@ export const getAudioStream = async (
     });
     return await getStream(constraints);
   } catch (error) {
-    if (error instanceof OverconstrainedError && trackConstraints?.deviceId) {
-      const { deviceId, ...relaxedContraints } = trackConstraints;
-      getLogger(['devices'])(
-        'warn',
-        'Failed to get audio stream, will try again with relaxed contraints',
-        { error, constraints, relaxedContraints },
-      );
-      return getAudioStream(relaxedContraints);
-    }
-
     getLogger(['devices'])('error', 'Failed to get audio stream', {
       error,
       constraints,
@@ -227,7 +217,7 @@ export const getVideoStream = async (
   const constraints: MediaStreamConstraints = {
     video: {
       ...videoDeviceConstraints.video,
-      ...trackConstraints,
+      ...normalizeContraints(trackConstraints),
     },
   };
   try {
@@ -237,16 +227,6 @@ export const getVideoStream = async (
     });
     return await getStream(constraints);
   } catch (error) {
-    if (error instanceof OverconstrainedError && trackConstraints?.deviceId) {
-      const { deviceId, ...relaxedContraints } = trackConstraints;
-      getLogger(['devices'])(
-        'warn',
-        'Failed to get video stream, will try again with relaxed contraints',
-        { error, constraints, relaxedContraints },
-      );
-      return getVideoStream(relaxedContraints);
-    }
-
     getLogger(['devices'])('error', 'Failed to get video stream', {
       error,
       constraints,
@@ -255,6 +235,20 @@ export const getVideoStream = async (
   }
 };
 
+function normalizeContraints(constraints: MediaTrackConstraints | undefined) {
+  if (
+    constraints?.deviceId === 'default' ||
+    (typeof constraints?.deviceId === 'object' &&
+      'exact' in constraints.deviceId &&
+      constraints.deviceId.exact === 'default')
+  ) {
+    const { deviceId, ...contraintsWithoutDeviceId } = constraints;
+    return contraintsWithoutDeviceId;
+  }
+
+  return constraints;
+}
+
 /**
  * Prompts the user for a permission to share a screen.
  * If the user grants the permission, a screen sharing stream is returned. Throws otherwise.

package/src/helpers/__tests__/sdp-munging.test.ts

@@ -188,4 +188,96 @@ a=simulcast:send q;h;f`;
     expect(target).not.toContain('VP9');
     expect(target).not.toContain('AV1');
   });
+
+  it('works with iOS RN vp8', () => {
+    const sdp = `v=0
+o=- 2055959380019004946 2 IN IP4 127.0.0.1
+s=-
+t=0 0
+a=group:BUNDLE 0
+a=extmap-allow-mixed
+a=msid-semantic: WMS FE2B3B06-61D7-4ACC-A4EF-76441C116E47
+m=video 9 UDP/TLS/RTP/SAVPF 96 97 98 99 100 101 127 103 35 36 104 105 106
+c=IN IP4 0.0.0.0
+a=rtcp:9 IN IP4 0.0.0.0
+a=ice-ufrag:gCgh
+a=ice-pwd:bz18EOLBL9+kSJfLiVOyU4RP
+a=ice-options:trickle renomination
+a=fingerprint:sha-256 6B:04:36:6D:E6:92:B5:68:DA:30:CF:53:46:14:49:5B:48:3E:B9:F7:06:B4:E8:85:B1:8C:B3:1C:EB:E8:F8:16
+a=setup:actpass
+a=mid:0
+a=extmap:1 urn:ietf:params:rtp-hdrext:toffset
+a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time
+a=extmap:3 urn:3gpp:video-orientation
+a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01
+a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay
+a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type
+a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing
+a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space
+a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid
+a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id
+a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id
+a=extmap:12 https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension
+a=extmap:14 http://www.webrtc.org/experiments/rtp-hdrext/video-layers-allocation00
+a=sendonly
+a=msid:FE2B3B06-61D7-4ACC-A4EF-76441C116E47 93FCE555-1DA2-4721-901C-5D263E11DF23
+a=rtcp-mux
+a=rtcp-rsize
+a=rtpmap:96 H264/90000
+a=rtcp-fb:96 goog-remb
+a=rtcp-fb:96 transport-cc
+a=rtcp-fb:96 ccm fir
+a=rtcp-fb:96 nack
+a=rtcp-fb:96 nack pli
+a=fmtp:96 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640c29
+a=rtpmap:97 rtx/90000
+a=fmtp:97 apt=96
+a=rtpmap:98 H264/90000
+a=rtcp-fb:98 goog-remb
+a=rtcp-fb:98 transport-cc
+a=rtcp-fb:98 ccm fir
+a=rtcp-fb:98 nack
+a=rtcp-fb:98 nack pli
+a=fmtp:98 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e029
+a=rtpmap:99 rtx/90000
+a=fmtp:99 apt=98
+a=rtpmap:100 VP8/90000
+a=rtcp-fb:100 goog-remb
+a=rtcp-fb:100 transport-cc
+a=rtcp-fb:100 ccm fir
+a=rtcp-fb:100 nack
+a=rtcp-fb:100 nack pli
+a=rtpmap:101 rtx/90000
+a=fmtp:101 apt=100
+a=rtpmap:127 VP9/90000
+a=rtcp-fb:127 goog-remb
+a=rtcp-fb:127 transport-cc
+a=rtcp-fb:127 ccm fir
+a=rtcp-fb:127 nack
+a=rtcp-fb:127 nack pli
+a=rtpmap:103 rtx/90000
+a=fmtp:103 apt=127
+a=rtpmap:35 AV1/90000
+a=rtcp-fb:35 goog-remb
+a=rtcp-fb:35 transport-cc
+a=rtcp-fb:35 ccm fir
+a=rtcp-fb:35 nack
+a=rtcp-fb:35 nack pli
+a=rtpmap:36 rtx/90000
+a=fmtp:36 apt=35
+a=rtpmap:104 red/90000
+a=rtpmap:105 rtx/90000
+a=fmtp:105 apt=104
+a=rtpmap:106 ulpfec/90000
+a=rid:q send
+a=rid:h send
+a=rid:f send
+a=simulcast:send q;h;f`;
+    const target = preserveCodec(sdp, '0', {
+      clockRate: 90000,
+      mimeType: 'video/VP8',
+    });
+    expect(target).toContain('VP8');
+    expect(target).not.toContain('VP9');
+  });
 });

package/src/helpers/sdp-munging.ts

@@ -156,12 +156,16 @@ export const preserveCodec = (
     // find the payload id of the desired codec
     const payloads = new Set<number>();
     for (const rtp of media.rtp) {
-      if (
-        rtp.codec.toLowerCase() === codecName &&
-        media.fmtp.some(
-          (f) => f.payload === rtp.payload && equal(toSet(f.config), codecFmtp),
-        )
-      ) {
+      if (rtp.codec.toLowerCase() !== codecName) continue;
+      const match =
+        // vp8 doesn't have any fmtp, we preserve it without any additional checks
+        codecName === 'vp8'
+          ? true
+          : media.fmtp.some(
+              (f) =>
+                f.payload === rtp.payload && equal(toSet(f.config), codecFmtp),
+            );
+      if (match) {
         payloads.add(rtp.payload);
       }
     }

package/src/rtc/Publisher.ts

@@ -29,6 +29,8 @@ import { Dispatcher } from './Dispatcher';
 import { VideoLayerSetting } from '../gen/video/sfu/event/events';
 import { TargetResolutionResponse } from '../gen/shims';
 import { withoutConcurrency } from '../helpers/concurrency';
+import { isReactNative } from '../helpers/platforms';
+import { isFirefox } from '../helpers/browsers';
 
 export type PublisherConstructorOpts = {
   sfuClient: StreamSfuClient;
@@ -256,6 +258,7 @@ export class Publisher {
     const codecPreferences = this.getCodecPreferences(
       trackType,
       trackType === TrackType.VIDEO ? codecInUse : undefined,
+      'receiver',
     );
     if (!codecPreferences) return;
 
@@ -458,13 +461,14 @@ export class Publisher {
 
   private getCodecPreferences = (
     trackType: TrackType,
-    preferredCodec?: string,
-    codecPreferencesSource?: 'sender' | 'receiver',
+    preferredCodec: string | undefined,
+    codecPreferencesSource: 'sender' | 'receiver',
   ) => {
     if (trackType === TrackType.VIDEO) {
       return getPreferredCodecs(
         'video',
         preferredCodec || 'vp8',
+        undefined,
         codecPreferencesSource,
       );
     }
@@ -475,6 +479,7 @@ export class Publisher {
         'audio',
         preferredCodec ?? defaultAudioCodec,
         codecToRemove,
+        codecPreferencesSource,
       );
     }
   };
@@ -578,7 +583,9 @@ export class Publisher {
 
   private removeUnpreferredCodecs(sdp: string, trackType: TrackType): string {
     const opts = this.publishOptsForTrack.get(trackType);
-    if (!opts || !opts.forceSingleCodec) return sdp;
+    const forceSingleCodec =
+      !!opts?.forceSingleCodec || isReactNative() || isFirefox();
+    if (!opts || !forceSingleCodec) return sdp;
 
     const codec = opts.forceCodec || getOptimalVideoCodec(opts.preferredCodec);
     const orderedCodecs = this.getCodecPreferences(trackType, codec, 'sender');

package/src/rtc/__tests__/codecs.test.ts

@@ -5,7 +5,7 @@ import './mocks/webrtc.mocks';
 describe('codecs', () => {
   it('should return preferred audio codec', () => {
     RTCRtpReceiver.getCapabilities = vi.fn().mockReturnValue(audioCodecs);
-    const codecs = getPreferredCodecs('audio', 'red');
+    const codecs = getPreferredCodecs('audio', 'red', undefined, 'receiver');
     expect(codecs).toBeDefined();
     expect(codecs?.map((c) => c.mimeType)).toEqual([
       'audio/red',
@@ -20,7 +20,7 @@ describe('codecs', () => {
 
   it('should return preferred video codec', () => {
     RTCRtpReceiver.getCapabilities = vi.fn().mockReturnValue(videoCodecs);
-    const codecs = getPreferredCodecs('video', 'vp8');
+    const codecs = getPreferredCodecs('video', 'vp8', undefined, 'receiver');
     expect(codecs).toBeDefined();
     // prettier-ignore
     expect(codecs?.map((c) => [c.mimeType, c.sdpFmtpLine])).toEqual([
@@ -40,12 +40,12 @@ describe('codecs', () => {
 
   it('should pick the baseline H264 codec', () => {
     RTCRtpReceiver.getCapabilities = vi.fn().mockReturnValue(videoCodecs);
-    const codecs = getPreferredCodecs('video', 'h264');
+    const codecs = getPreferredCodecs('video', 'h264', undefined, 'receiver');
     expect(codecs).toBeDefined();
     // prettier-ignore
     expect(codecs?.map((c) => [c.mimeType, c.sdpFmtpLine])).toEqual([
-      ['video/H264', 'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f'],
       ['video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f'],
+      ['video/H264', 'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f'],
       ['video/H264', 'level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640c1f'],
       ['video/H264', 'level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=640c1f'],
       ['video/rtx', undefined],
@@ -62,12 +62,12 @@ describe('codecs', () => {
     RTCRtpReceiver.getCapabilities = vi
       .fn()
       .mockReturnValue(videoCodecsFirefox);
-    const codecs = getPreferredCodecs('video', 'h264');
+    const codecs = getPreferredCodecs('video', 'h264', undefined, 'receiver');
     expect(codecs).toBeDefined();
     // prettier-ignore
     expect(codecs?.map((c) => [c.mimeType, c.sdpFmtpLine])).toEqual([
-      ['video/H264', 'profile-level-id=42e01f;level-asymmetry-allowed=1'],
       ['video/H264', 'profile-level-id=42e01f;level-asymmetry-allowed=1;packetization-mode=1'],
+      ['video/H264', 'profile-level-id=42e01f;level-asymmetry-allowed=1'],
       ['video/VP8', 'max-fs=12288;max-fr=60'],
       ['video/rtx', undefined],
       ['video/VP9', 'max-fs=12288;max-fr=60'],

package/src/rtc/codecs.ts

@@ -14,8 +14,8 @@ import type { PreferredCodec } from '../types';
 export const getPreferredCodecs = (
   kind: 'audio' | 'video',
   preferredCodec: string,
-  codecToRemove?: string,
-  codecPreferencesSource: 'sender' | 'receiver' = 'receiver',
+  codecToRemove: string | undefined,
+  codecPreferencesSource: 'sender' | 'receiver',
 ): RTCRtpCodec[] | undefined => {
   const source =
     codecPreferencesSource === 'receiver' ? RTCRtpReceiver : RTCRtpSender;
@@ -60,12 +60,7 @@ export const getPreferredCodecs = (
       continue;
     }
 
-    // packetization-mode mode is optional; when not present it defaults to 0:
-    // https://datatracker.ietf.org/doc/html/rfc6184#section-6.2
-    if (
-      sdpFmtpLine.includes('packetization-mode=0') ||
-      !sdpFmtpLine.includes('packetization-mode')
-    ) {
+    if (sdpFmtpLine.includes('packetization-mode=1')) {
       preferred.unshift(codec);
     } else {
       preferred.push(codec);
@@ -107,7 +102,9 @@ export const getOptimalVideoCodec = (
   if (isReactNative()) {
     const os = getOSInfo()?.name.toLowerCase();
     if (os === 'android') return preferredOr(preferredCodec, 'vp8');
-    if (os === 'ios' || os === 'ipados') return 'h264';
+    if (os === 'ios' || os === 'ipados') {
+      return supportsH264Baseline() ? 'h264' : 'vp8';
+    }
     return preferredOr(preferredCodec, 'h264');
   }
   if (isSafari()) return 'h264';
@@ -139,6 +136,20 @@ const preferredOr = (
     : fallback;
 };
 
+/**
+ * Returns whether the platform supports the H264 baseline codec.
+ */
+const supportsH264Baseline = (): boolean => {
+  if (!('getCapabilities' in RTCRtpSender)) return false;
+  const capabilities = RTCRtpSender.getCapabilities('video');
+  if (!capabilities) return false;
+  return capabilities.codecs.some(
+    (c) =>
+      c.mimeType.toLowerCase() === 'video/h264' &&
+      c.sdpFmtpLine?.includes('profile-level-id=42e01f'),
+  );
+};
+
 /**
  * Returns whether the codec is an SVC codec.
  *