@stream-io/video-client 1.11.3 → 1.11.4

package/dist/src/helpers/sdp-munging.d.ts

@@ -2,6 +2,10 @@
  * Returns an SDP with DTX enabled or disabled.
  */
 export declare const toggleDtx: (sdp: string, enable: boolean) => string;
+/**
+ * Returns and SDP with all the codecs except the given codec removed.
+ */
+export declare const preserveCodec: (sdp: string, mid: string, codec: RTCRtpCodec) => string;
 /**
  * Enables high-quality audio through SDP munging for the given trackMid.
  *

package/dist/src/rtc/Publisher.d.ts

@@ -124,6 +124,7 @@ export declare class Publisher {
      * @param options the optional offer options to use.
      */
     private negotiate;
+    private removeUnpreferredCodecs;
     private enableHighQualityAudio;
     /**
      * Returns a list of tracks that are currently being published.

package/dist/src/types.d.ts

@@ -129,6 +129,12 @@ export type PublishOptions = {
      * Use with caution.
      */
     forceCodec?: PreferredCodec;
+    /**
+     * When using a preferred codec, force the use of a single codec.
+     * Enabling this, it will remove all other supported codecs from the SDP.
+     * Defaults to false.
+     */
+    forceSingleCodec?: boolean;
     /**
      * The preferred scalability to use when publishing the video stream.
      * Applicable only for SVC codecs.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stream-io/video-client",
-  "version": "1.11.3",
+  "version": "1.11.4",
   "packageManager": "yarn@3.2.4",
   "main": "dist/index.cjs.js",
   "module": "dist/index.es.js",

package/src/helpers/__tests__/sdp-munging.test.ts

@@ -1,5 +1,9 @@
 import { describe, expect, it } from 'vitest';
-import { enableHighQualityAudio, toggleDtx } from '../sdp-munging';
+import {
+  enableHighQualityAudio,
+  preserveCodec,
+  toggleDtx,
+} from '../sdp-munging';
 import { initialSdp as HQAudioSDP } from './hq-audio-sdp';
 
 describe('sdp-munging', () => {
@@ -21,4 +25,167 @@ a=maxptime:40`;
     expect(sdpWithHighQualityAudio).toContain('maxaveragebitrate=510000');
     expect(sdpWithHighQualityAudio).toContain('stereo=1');
   });
+
+  it('preserves the preferred codec', () => {
+    const sdp = `v=0
+o=- 8608371809202407637 2 IN IP4 127.0.0.1
+s=-
+t=0 0
+a=extmap-allow-mixed
+a=msid-semantic: WMS 52fafc21-b8bb-4f4f-8072-86a29cb6590e
+a=group:BUNDLE 0
+m=video 9 UDP/TLS/RTP/SAVPF 98 100 99 101
+c=IN IP4 0.0.0.0
+a=rtpmap:98 VP9/90000
+a=rtpmap:99 rtx/90000
+a=rtpmap:100 VP9/90000
+a=rtpmap:101 rtx/90000
+a=fmtp:98 profile-id=0
+a=fmtp:99 apt=98
+a=fmtp:100 profile-id=2
+a=fmtp:101 apt=100
+a=rtcp:9 IN IP4 0.0.0.0
+a=rtcp-fb:98 goog-remb
+a=rtcp-fb:98 transport-cc
+a=rtcp-fb:98 ccm fir
+a=rtcp-fb:98 nack
+a=rtcp-fb:98 nack pli
+a=rtcp-fb:100 goog-remb
+a=rtcp-fb:100 transport-cc
+a=rtcp-fb:100 ccm fir
+a=rtcp-fb:100 nack
+a=rtcp-fb:100 nack pli
+a=extmap:1 urn:ietf:params:rtp-hdrext:toffset
+a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time
+a=extmap:3 urn:3gpp:video-orientation
+a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01
+a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay
+a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type
+a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing
+a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space
+a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid
+a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id
+a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id
+a=extmap:12 https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension
+a=extmap:14 http://www.webrtc.org/experiments/rtp-hdrext/video-layers-allocation00
+a=setup:actpass
+a=mid:0
+a=msid:52fafc21-b8bb-4f4f-8072-86a29cb6590e 1bd1c5c2-d3cc-4490-ac0c-70b187242232
+a=sendonly
+a=ice-ufrag:LvRk
+a=ice-pwd:IpBRr2Rrg9TkOgayjYqALhPY
+a=fingerprint:sha-256 18:DE:8F:ED:E6:A2:0C:99:A8:25:AB:C9:F8:3D:91:4C:3E:9F:B4:1F:22:87:A7:3C:85:8F:F3:51:09:A7:E3:FA
+a=ice-options:trickle
+a=ssrc:3192778601 cname:yYSN5R+RG2j3luO7
+a=ssrc:3192778601 msid:52fafc21-b8bb-4f4f-8072-86a29cb6590e 1bd1c5c2-d3cc-4490-ac0c-70b187242232
+a=ssrc:283365205 cname:yYSN5R+RG2j3luO7
+a=ssrc:283365205 msid:52fafc21-b8bb-4f4f-8072-86a29cb6590e 1bd1c5c2-d3cc-4490-ac0c-70b187242232
+a=ssrc-group:FID 3192778601 283365205
+a=rtcp-mux
+a=rtcp-rsize`;
+    const target = preserveCodec(sdp, '0', {
+      mimeType: 'video/VP9',
+      clockRate: 90000,
+      sdpFmtpLine: 'profile-id=0',
+    });
+    expect(target).toContain('VP9');
+    expect(target).not.toContain('profile-id=2');
+  });
+
+  it('handles ios munging', () => {
+    const sdp = `v=0
+o=- 525780719364332676 2 IN IP4 127.0.0.1
+s=-
+t=0 0
+a=group:BUNDLE 0
+a=extmap-allow-mixed
+a=msid-semantic: WMS BF3AFE62-88F8-4189-99D7-7CAE159205E3
+m=video 9 UDP/TLS/RTP/SAVPF 96 97 98 99 100 101 127 103 35 36 104 105 106
+c=IN IP4 0.0.0.0
+a=rtcp:9 IN IP4 0.0.0.0
+a=ice-ufrag:SAkq
+a=ice-pwd:FYHHro0VWRO8CjI/M1VG5vRw
+a=ice-options:trickle renomination
+a=fingerprint:sha-256 03:5B:16:0E:E1:7B:FE:4F:9A:5C:AC:CF:08:21:4B:49:CE:53:79:E6:97:AE:4E:73:F8:43:34:C3:11:F7:6D:E7
+a=setup:actpass
+a=mid:0
+a=extmap:1 urn:ietf:params:rtp-hdrext:toffset
+a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time
+a=extmap:3 urn:3gpp:video-orientation
+a=extmap:4 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01
+a=extmap:5 http://www.webrtc.org/experiments/rtp-hdrext/playout-delay
+a=extmap:6 http://www.webrtc.org/experiments/rtp-hdrext/video-content-type
+a=extmap:7 http://www.webrtc.org/experiments/rtp-hdrext/video-timing
+a=extmap:8 http://www.webrtc.org/experiments/rtp-hdrext/color-space
+a=extmap:9 urn:ietf:params:rtp-hdrext:sdes:mid
+a=extmap:10 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id
+a=extmap:11 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id
+a=extmap:12 https://aomediacodec.github.io/av1-rtp-spec/#dependency-descriptor-rtp-header-extension
+a=extmap:14 http://www.webrtc.org/experiments/rtp-hdrext/video-layers-allocation00
+a=sendonly
+a=msid:BF3AFE62-88F8-4189-99D7-7CAE159205E3 6013DC02-A0A5-43A9-9D41-9D4A89648A42
+a=rtcp-mux
+a=rtcp-rsize
+a=rtpmap:96 H264/90000
+a=rtcp-fb:96 goog-remb
+a=rtcp-fb:96 transport-cc
+a=rtcp-fb:96 ccm fir
+a=rtcp-fb:96 nack
+a=rtcp-fb:96 nack pli
+a=fmtp:96 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640c29
+a=rtpmap:97 rtx/90000
+a=fmtp:97 apt=96
+a=rtpmap:98 H264/90000
+a=rtcp-fb:98 goog-remb
+a=rtcp-fb:98 transport-cc
+a=rtcp-fb:98 ccm fir
+a=rtcp-fb:98 nack
+a=rtcp-fb:98 nack pli
+a=fmtp:98 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e029
+a=rtpmap:99 rtx/90000
+a=fmtp:99 apt=98
+a=rtpmap:100 VP8/90000
+a=rtcp-fb:100 goog-remb
+a=rtcp-fb:100 transport-cc
+a=rtcp-fb:100 ccm fir
+a=rtcp-fb:100 nack
+a=rtcp-fb:100 nack pli
+a=rtpmap:101 rtx/90000
+a=fmtp:101 apt=100
+a=rtpmap:127 VP9/90000
+a=rtcp-fb:127 goog-remb
+a=rtcp-fb:127 transport-cc
+a=rtcp-fb:127 ccm fir
+a=rtcp-fb:127 nack
+a=rtcp-fb:127 nack pli
+a=rtpmap:103 rtx/90000
+a=fmtp:103 apt=127
+a=rtpmap:35 AV1/90000
+a=rtcp-fb:35 goog-remb
+a=rtcp-fb:35 transport-cc
+a=rtcp-fb:35 ccm fir
+a=rtcp-fb:35 nack
+a=rtcp-fb:35 nack pli
+a=rtpmap:36 rtx/90000
+a=fmtp:36 apt=35
+a=rtpmap:104 red/90000
+a=rtpmap:105 rtx/90000
+a=fmtp:105 apt=104
+a=rtpmap:106 ulpfec/90000
+a=rid:q send
+a=rid:h send
+a=rid:f send
+a=simulcast:send q;h;f`;
+    const target = preserveCodec(sdp, '0', {
+      mimeType: 'video/H264',
+      clockRate: 90000,
+      sdpFmtpLine:
+        'profile-level-id=42e029;packetization-mode=1;level-asymmetry-allowed=1',
+    });
+    expect(target).toContain('H264');
+    expect(target).toContain('profile-level-id=42e029');
+    expect(target).not.toContain('profile-level-id=640c29');
+    expect(target).not.toContain('VP9');
+    expect(target).not.toContain('AV1');
+  });
 });

package/src/helpers/sdp-munging.ts

@@ -129,6 +129,61 @@ export const toggleDtx = (sdp: string, enable: boolean): string => {
   return sdp.replace(opusFmtp.original, newFmtp);
 };
 
+/**
+ * Returns and SDP with all the codecs except the given codec removed.
+ */
+export const preserveCodec = (
+  sdp: string,
+  mid: string,
+  codec: RTCRtpCodec,
+): string => {
+  const [kind, codecName] = codec.mimeType.toLowerCase().split('/');
+
+  const toSet = (fmtpLine: string) =>
+    new Set(fmtpLine.split(';').map((f) => f.trim().toLowerCase()));
+
+  const equal = (a: Set<string>, b: Set<string>) => {
+    if (a.size !== b.size) return false;
+    for (const item of a) if (!b.has(item)) return false;
+    return true;
+  };
+
+  const codecFmtp = toSet(codec.sdpFmtpLine || '');
+  const parsedSdp = SDP.parse(sdp);
+  for (const media of parsedSdp.media) {
+    if (media.type !== kind || String(media.mid) !== mid) continue;
+
+    // find the payload id of the desired codec
+    const payloads = new Set<number>();
+    for (const rtp of media.rtp) {
+      if (
+        rtp.codec.toLowerCase() === codecName &&
+        media.fmtp.some(
+          (f) => f.payload === rtp.payload && equal(toSet(f.config), codecFmtp),
+        )
+      ) {
+        payloads.add(rtp.payload);
+      }
+    }
+
+    // find the corresponding rtx codec by matching apt=<preserved-codec-payload>
+    for (const fmtp of media.fmtp) {
+      const match = fmtp.config.match(/(apt)=(\d+)/);
+      if (!match) continue;
+      const [, , preservedCodecPayload] = match;
+      if (payloads.has(Number(preservedCodecPayload))) {
+        payloads.add(fmtp.payload);
+      }
+    }
+
+    media.rtp = media.rtp.filter((r) => payloads.has(r.payload));
+    media.fmtp = media.fmtp.filter((f) => payloads.has(f.payload));
+    media.rtcpFb = media.rtcpFb?.filter((f) => payloads.has(f.payload));
+    media.payloads = Array.from(payloads).join(' ');
+  }
+  return SDP.write(parsedSdp);
+};
+
 /**
  * Enables high-quality audio through SDP munging for the given trackMid.
  *

package/src/rtc/Publisher.ts

@@ -20,6 +20,7 @@ import { PublishOptions } from '../types';
 import {
   enableHighQualityAudio,
   extractMid,
+  preserveCodec,
   toggleDtx,
 } from '../helpers/sdp-munging';
 import { Logger } from '../coordinator/connection/types';
@@ -530,6 +531,12 @@ export class Publisher {
       if (this.isPublishing(TrackType.SCREEN_SHARE_AUDIO)) {
         offer.sdp = this.enableHighQualityAudio(offer.sdp);
       }
+      if (this.isPublishing(TrackType.VIDEO)) {
+        // Hotfix for platforms that don't respect the ordered codec list
+        // (Firefox, Android, Linux, etc...).
+        // We remove all the codecs from the SDP except the one we want to use.
+        offer.sdp = this.removeUnpreferredCodecs(offer.sdp, TrackType.VIDEO);
+      }
     }
 
     const trackInfos = this.getAnnouncedTracks(offer.sdp);
@@ -564,6 +571,23 @@ export class Publisher {
     );
   };
 
+  private removeUnpreferredCodecs(sdp: string, trackType: TrackType): string {
+    const opts = this.publishOptsForTrack.get(trackType);
+    if (!opts || !opts.forceSingleCodec) return sdp;
+
+    const codec = opts.forceCodec || opts.preferredCodec;
+    const orderedCodecs = this.getCodecPreferences(trackType, codec);
+    if (!orderedCodecs || orderedCodecs.length === 0) return sdp;
+
+    const transceiver = this.transceiverCache.get(trackType);
+    if (!transceiver) return sdp;
+
+    const index = this.transceiverInitOrder.indexOf(trackType);
+    const mid = extractMid(transceiver, index, sdp);
+    const [codecToPreserve] = orderedCodecs;
+    return preserveCodec(sdp, mid, codecToPreserve);
+  }
+
   private enableHighQualityAudio = (sdp: string) => {
     const transceiver = this.transceiverCache.get(TrackType.SCREEN_SHARE_AUDIO);
     if (!transceiver) return sdp;

package/src/rtc/codecs.ts

@@ -50,7 +50,7 @@ export const getPreferredCodecs = (
     }
 
     const sdpFmtpLine = codec.sdpFmtpLine;
-    if (!sdpFmtpLine || !sdpFmtpLine.includes('profile-level-id=42e01f')) {
+    if (!sdpFmtpLine || !sdpFmtpLine.includes('profile-level-id=42')) {
       // this is not the baseline h264 codec, prioritize it lower
       partiallyPreferred.push(codec);
       continue;

package/src/types.ts

@@ -167,6 +167,12 @@ export type PublishOptions = {
    * Use with caution.
    */
   forceCodec?: PreferredCodec;
+  /**
+   * When using a preferred codec, force the use of a single codec.
+   * Enabling this, it will remove all other supported codecs from the SDP.
+   * Defaults to false.
+   */
+  forceSingleCodec?: boolean;
   /**
    * The preferred scalability to use when publishing the video stream.
    * Applicable only for SVC codecs.