livekit-client 2.20.0 → 2.20.1

package/src/room/PCTransport.test.ts

@@ -0,0 +1,150 @@
+import { type MediaDescription, parse } from 'sdp-transform';
+import { describe, expect, it } from 'vitest';
+import { conformBundledCodecFmtp, placeholderMidsFromTransceivers } from './PCTransport';
+
+/** Parse the `key[=value]` pairs of an fmtp config into a comparable set. */
+const paramSet = (config: string) => new Set(config.split(';').filter(Boolean));
+
+const fmtpOf = (media: MediaDescription[], mid: string, payload: number) => {
+  const m = media.find((section) => `${section.mid}` === mid)!;
+  return m.fmtp.find((f) => f.payload === payload)?.config;
+};
+
+/** Predicate for conformBundledCodecFmtp based on an explicit set of mids. */
+const placeholders = (mids: string[]) => {
+  const set = new Set(mids);
+  return (m: MediaDescription) => set.has(`${m.mid}`);
+};
+
+// One publisher bundle: a published mic + camera (real sends, with msid/ssrc)
+// and their pre-populated recvonly placeholders.
+const PUBLISHER_SDP = `v=0
+o=- 0 0 IN IP4 127.0.0.1
+s=-
+t=0 0
+a=group:BUNDLE 0 1 2 3
+m=audio 9 UDP/TLS/RTP/SAVPF 111
+c=IN IP4 0.0.0.0
+a=mid:0
+a=sendonly
+a=msid:s mic
+a=ssrc:1111 cname:a
+a=rtpmap:111 opus/48000/2
+a=fmtp:111 minptime=10;useinbandfec=1;usedtx=1
+m=audio 9 UDP/TLS/RTP/SAVPF 111
+c=IN IP4 0.0.0.0
+a=mid:1
+a=recvonly
+a=rtpmap:111 opus/48000/2
+a=fmtp:111 minptime=10;useinbandfec=1
+m=video 9 UDP/TLS/RTP/SAVPF 49
+c=IN IP4 0.0.0.0
+a=mid:2
+a=sendonly
+a=msid:s cam
+a=ssrc:2222 cname:b
+a=rtpmap:49 H265/90000
+a=fmtp:49 level-id=186;profile-id=1;tier-flag=0;tx-mode=SRST
+m=video 9 UDP/TLS/RTP/SAVPF 49
+c=IN IP4 0.0.0.0
+a=mid:3
+a=recvonly
+a=rtpmap:49 H265/90000
+a=fmtp:49 level-id=180;profile-id=1;tier-flag=0;tx-mode=SRST`;
+
+describe('placeholderMidsFromTransceivers', () => {
+  const tr = (mid: string | null, track: unknown) =>
+    ({ mid, sender: { track } }) as unknown as RTCRtpTransceiver;
+
+  it('includes transceivers with a mid and no outgoing track', () => {
+    const mids = placeholderMidsFromTransceivers([
+      tr('0', {}), // real send
+      tr('1', null), // pre-populated placeholder
+      tr('2', null), // reverted after unpublish (msid may linger in SDP, but no track)
+      tr(null, null), // not yet negotiated — no mid, excluded
+    ]);
+    expect(mids).toEqual(new Set(['1', '2']));
+  });
+});
+
+describe('conformBundledCodecFmtp', () => {
+  it('copies opus and H.265 fmtp from real sections onto placeholders', () => {
+    const { media } = parse(PUBLISHER_SDP);
+
+    conformBundledCodecFmtp(media, placeholders(['1', '3']));
+
+    expect(fmtpOf(media, '1', 111)).toBe(fmtpOf(media, '0', 111));
+    expect(paramSet(fmtpOf(media, '1', 111)!)).toContain('usedtx=1');
+    expect(fmtpOf(media, '3', 49)).toBe(fmtpOf(media, '2', 49));
+    expect(paramSet(fmtpOf(media, '3', 49)!)).toContain('level-id=186');
+  });
+
+  it('never rewrites the real (non-placeholder) sections', () => {
+    const { media } = parse(PUBLISHER_SDP);
+
+    conformBundledCodecFmtp(media, placeholders(['1', '3']));
+
+    expect(fmtpOf(media, '0', 111)).toBe('minptime=10;useinbandfec=1;usedtx=1');
+    expect(paramSet(fmtpOf(media, '2', 49)!)).toContain('level-id=186');
+  });
+
+  it('converges divergent placeholders even when no real section declares the payload', () => {
+    // Two placeholders that disagree (a reverted one kept level 186, a fresh one
+    // has 180) with no active send present.
+    const { media } = parse(`v=0
+o=- 0 0 IN IP4 127.0.0.1
+s=-
+t=0 0
+a=group:BUNDLE 10 11
+m=video 9 UDP/TLS/RTP/SAVPF 49
+a=mid:10
+a=rtpmap:49 H265/90000
+a=fmtp:49 level-id=186;profile-id=1;tier-flag=0;tx-mode=SRST
+m=video 9 UDP/TLS/RTP/SAVPF 49
+a=mid:11
+a=rtpmap:49 H265/90000
+a=fmtp:49 level-id=180;profile-id=1;tier-flag=0;tx-mode=SRST`);
+
+    conformBundledCodecFmtp(media, placeholders(['10', '11']));
+
+    expect(fmtpOf(media, '10', 49)).toBe(fmtpOf(media, '11', 49));
+  });
+
+  it('conforms a reverted section that kept its msid (second-publish case)', () => {
+    // The real second publish: a live camera send (level 186), plus the section
+    // left over from the first (unpublished) track. Chrome keeps that section's
+    // `a=msid`/`a=ssrc` even though it no longer sends, so it must be identified
+    // as a placeholder by its transceiver (no track), not by SDP.
+    const { media } = parse(`v=0
+o=- 0 0 IN IP4 127.0.0.1
+s=-
+t=0 0
+a=group:BUNDLE 3 4 5
+m=video 9 UDP/TLS/RTP/SAVPF 49
+a=mid:3
+a=sendonly
+a=msid:s cam
+a=ssrc:9 cname:c
+a=rtpmap:49 H265/90000
+a=fmtp:49 level-id=186;profile-id=1;tier-flag=0;tx-mode=SRST
+m=video 9 UDP/TLS/RTP/SAVPF 49
+a=mid:4
+a=sendonly
+a=msid:s stale
+a=ssrc:8 cname:d
+a=rtpmap:49 H265/90000
+a=fmtp:49 level-id=180;profile-id=1;tier-flag=0;tx-mode=SRST
+m=video 9 UDP/TLS/RTP/SAVPF 49
+a=mid:5
+a=recvonly
+a=rtpmap:49 H265/90000
+a=fmtp:49 level-id=180;profile-id=1;tier-flag=0;tx-mode=SRST`);
+
+    // mid 4 still has msid/ssrc in the SDP, but its transceiver has no track.
+    conformBundledCodecFmtp(media, placeholders(['4', '5']));
+
+    expect(paramSet(fmtpOf(media, '3', 49)!)).toContain('level-id=186'); // real send untouched
+    expect(fmtpOf(media, '4', 49)).toBe(fmtpOf(media, '3', 49)); // stale conformed
+    expect(fmtpOf(media, '5', 49)).toBe(fmtpOf(media, '3', 49)); // placeholder conformed
+  });
+});

package/src/room/PCTransport.ts

@@ -234,6 +234,17 @@ export default class PCTransport extends (EventEmitter as new () => TypedEmitter
           });
         }
       });
+      // The server's answer sets per-track codec params (e.g. opus `usedtx`) on
+      // the sections mapped to a published track, but leaves the pre-populated
+      // recvonly placeholder sections with defaults. Conform the placeholders so
+      // each shared payload type is consistent across the bundle, otherwise
+      // libwebrtc flags a "bundled payload type collision".
+      const placeholderMids = this.getPlaceholderMids();
+      if (placeholderMids.size > 0) {
+        conformBundledCodecFmtp(sdpParsed.media, (media) =>
+          placeholderMids.has(getMidString(media.mid!)),
+        );
+      }
       mungedSDP = write(sdpParsed);
     }
     await this.setMungedSDP(sd, mungedSDP, true);
@@ -400,6 +411,17 @@ export default class PCTransport extends (EventEmitter as new () => TypedEmitter
           });
         }
       });
+      // Conform the placeholder sections (pre-populated, or reverted after an
+      // unpublish) so every shared payload type carries identical fmtp across the
+      // bundle, otherwise libwebrtc flags a "bundled payload type collision".
+      // Detection is by transceiver (mids are stable across renegotiations) since
+      // an unpublished section keeps its `a=msid`.
+      const placeholderMids = this.getPlaceholderMids();
+      if (placeholderMids.size > 0) {
+        conformBundledCodecFmtp(sdpParsed.media, (media) =>
+          placeholderMids.has(getMidString(media.mid!)),
+        );
+      }
       if (this.latestOfferId > offerId) {
         this.log.warn('latestOfferId mismatch', {
           ...this.logContext,
@@ -428,6 +450,17 @@ export default class PCTransport extends (EventEmitter as new () => TypedEmitter
     return answer;
   }
 
+  /**
+   * Returns the mids of transceivers that carry no outgoing track on this
+   * (publisher) connection: the pre-populated placeholders added by
+   * `RTCEngine.applyInitialPublisherLayout`, plus any transceiver that was used
+   * for a track and reverted on unpublish. Their codec fmtp is conformed to the
+   * published tracks so a shared payload type stays consistent across the bundle.
+   */
+  private getPlaceholderMids(): Set<string> {
+    return placeholderMidsFromTransceivers(this._pc?.getTransceivers() ?? []);
+  }
+
   createDataChannel(label: string, dataChannelDict: RTCDataChannelInit) {
     return this.pc.createDataChannel(label, dataChannelDict);
   }
@@ -703,6 +736,86 @@ function ensureAudioNackAndStereo(
   }
 }
 
+/**
+ * Returns the mids of transceivers that carry no outgoing track: the
+ * pre-populated placeholders added by `RTCEngine.applyInitialPublisherLayout`,
+ * plus any transceiver that was used for a track and reverted on unpublish. The
+ * `sender.track` check is the reliable signal — an unpublished section keeps its
+ * `a=msid` (and its stale send-derived fmtp), so it can't be told apart from a
+ * real send by SDP alone. Transceiver mids are stable across renegotiations, so
+ * this works for every offer/answer after the first.
+ * @internal
+ */
+export function placeholderMidsFromTransceivers(
+  transceivers: readonly RTCRtpTransceiver[],
+): Set<string> {
+  const mids = new Set<string>();
+  for (const transceiver of transceivers) {
+    if (transceiver.mid && !transceiver.sender.track) {
+      mids.add(transceiver.mid);
+    }
+  }
+  return mids;
+}
+
+/**
+ * Within a BUNDLE group a payload type must map to identical codec parameters
+ * across every m-line. When the same payload type carries different fmtp between
+ * sections — e.g. opus `usedtx=1` on the published microphone but not on the
+ * pre-populated recvonly placeholders, or H.265 with different `level-id` between
+ * a published video track and a placeholder — libwebrtc flags a "bundled payload
+ * type collision".
+ *
+ * Rewrite the placeholder sections so every shared payload type carries the
+ * same fmtp. Real (non-placeholder) sections always win the canonical value, so
+ * a published track's encoder parameters are never altered. When no real
+ * section declares a payload type — e.g. a placeholder that was reused for a
+ * track and then reverted to recvonly keeps its send-derived `level-id` while
+ * fresh placeholders use the default — the placeholders still converge on the
+ * first value seen, so two placeholders can't disagree either. Only placeholder
+ * sections are ever rewritten, and it is codec-agnostic (opus, H.265, ...).
+ * `isPlaceholder` identifies the sections to conform.
+ * @internal
+ */
+export function conformBundledCodecFmtp(
+  media: MediaDescription[],
+  isPlaceholder: (media: MediaDescription) => boolean,
+) {
+  // Canonical fmtp per payload type. Payload types are unique within a BUNDLE
+  // group, so keying by payload alone (across audio and video) is safe. A real
+  // section's value always takes precedence; otherwise the first placeholder
+  // value seen is used so divergent placeholders still converge.
+  const canonicalByPayload = new Map<number, string>();
+  const fromRealSection = new Set<number>();
+  for (const m of media) {
+    const placeholder = isPlaceholder(m);
+    for (const fmtp of m.fmtp ?? []) {
+      if (!placeholder) {
+        canonicalByPayload.set(fmtp.payload, fmtp.config);
+        fromRealSection.add(fmtp.payload);
+      } else if (!canonicalByPayload.has(fmtp.payload)) {
+        canonicalByPayload.set(fmtp.payload, fmtp.config);
+      }
+    }
+  }
+  if (canonicalByPayload.size === 0) {
+    return;
+  }
+
+  // Conform placeholder sections to the canonical fmtp for each shared payload.
+  for (const m of media) {
+    if (!isPlaceholder(m)) {
+      continue;
+    }
+    for (const fmtp of m.fmtp ?? []) {
+      const config = canonicalByPayload.get(fmtp.payload);
+      if (config !== undefined && fmtp.config !== config) {
+        fmtp.config = config;
+      }
+    }
+  }
+}
+
 function extractStereoAndNackAudioFromOffer(offer: RTCSessionDescriptionInit): {
   stereoMids: string[];
   nackMids: string[];

package/src/room/RTCEngine.ts

@@ -61,6 +61,7 @@ import {
 } from '../frameMetadata/utils';
 import log, { LoggerNames, getLogger } from '../logger';
 import type { InternalRoomOptions } from '../options';
+import type { NonSharedUint8Array } from '../type-polyfills/non-shared-typed-arrays';
 import TypedPromise from '../utils/TypedPromise';
 import { DataPacketBuffer } from '../utils/dataPacketBuffer';
 import { TTLMap } from '../utils/ttlmap';

package/src/room/Room.ts

@@ -46,6 +46,7 @@ import type {
   RoomConnectOptions,
   RoomOptions,
 } from '../options';
+import type { NonSharedUint8Array } from '../type-polyfills/non-shared-typed-arrays';
 import TypedPromise from '../utils/TypedPromise';
 import { getBrowser } from '../utils/browserParser';
 import { CLIENT_PROTOCOL_DEFAULT } from '../version';

package/src/room/data-track/depacketizer.ts

@@ -1,5 +1,6 @@
 import { type Throws } from '@livekit/throws-transformer/throws';
 import { LoggerNames, getLogger } from '../../logger';
+import type { NonSharedUint8Array } from '../../type-polyfills/non-shared-typed-arrays';
 import { LivekitReasonedError } from '../errors';
 import { type DataTrackFrameInternal } from './frame';
 import { DataTrackPacket, FrameMarker } from './packet';

package/src/room/data-track/frame.ts

@@ -1,3 +1,4 @@
+import type { NonSharedUint8Array } from '../../type-polyfills/non-shared-typed-arrays';
 import { DataTrackExtensions, DataTrackUserTimestampExtension } from './packet/extensions';
 
 /** A pair of payload bytes and packet extensions which can be fed into a {@link DataTrackPacketizer}. */

package/src/room/data-track/outgoing/types.ts

@@ -1,3 +1,4 @@
+import type { NonSharedUint8Array } from '../../../type-polyfills/non-shared-typed-arrays';
 import type LocalDataTrack from '../LocalDataTrack';
 import { type DataTrackHandle } from '../handle';
 import { type DataTrackInfo, type DataTrackSid } from '../types';

package/src/room/data-track/packet/extensions.ts

@@ -1,4 +1,5 @@
 import { type Throws } from '@livekit/throws-transformer/throws';
+import type { NonSharedUint8Array } from '../../../type-polyfills/non-shared-typed-arrays';
 import { coerceToDataView } from '../utils';
 import { EXT_TAG_PADDING, U8_LENGTH_BYTES, U64_LENGTH_BYTES } from './constants';
 import { DataTrackDeserializeError, DataTrackDeserializeErrorReason } from './errors';

package/src/room/data-track/packet/index.ts

@@ -1,4 +1,5 @@
 import { type Throws } from '@livekit/throws-transformer/throws';
+import type { NonSharedUint8Array } from '../../../type-polyfills/non-shared-typed-arrays';
 import { DataTrackHandle, DataTrackHandleError, DataTrackHandleErrorReason } from '../handle';
 import {
   DataTrackTimestamp,

package/src/room/data-track/packet/serializable.ts

@@ -1,4 +1,5 @@
 import { type Throws } from '@livekit/throws-transformer/throws';
+import type { NonSharedUint8Array } from '../../../type-polyfills/non-shared-typed-arrays';
 import { DataTrackSerializeError } from './errors';
 
 /** An abstract class implementing common behavior related to data track binary serialization. */

package/src/room/participant/LocalParticipant.ts

@@ -29,6 +29,7 @@ import {
   isFrameMetadataSupported,
 } from '../../frameMetadata/utils';
 import type { InternalRoomOptions } from '../../options';
+import type { NonSharedUint8Array } from '../../type-polyfills/non-shared-typed-arrays';
 import TypedPromise from '../../utils/TypedPromise';
 import { PCTransportState } from '../PCTransportManager';
 import type RTCEngine from '../RTCEngine';

package/src/room/participant/Participant.ts

@@ -12,6 +12,7 @@ import {
 import { EventEmitter } from 'events';
 import type TypedEmitter from 'typed-emitter';
 import log, { LoggerNames, type StructuredLogger, getLogger } from '../../logger';
+import type { NonSharedUint8Array } from '../../type-polyfills/non-shared-typed-arrays';
 import { ParticipantEvent, TrackEvent } from '../events';
 import type LocalTrackPublication from '../track/LocalTrackPublication';
 import type LocalVideoTrack from '../track/LocalVideoTrack';

package/src/room/track/Track.ts

@@ -9,6 +9,7 @@ import { EventEmitter } from 'events';
 import type TypedEventEmitter from 'typed-emitter';
 import type { SignalClient } from '../../api/SignalClient';
 import log, { LoggerNames, type StructuredLogger, getLogger } from '../../logger';
+import type { NonSharedUint8Array } from '../../type-polyfills/non-shared-typed-arrays';
 import { TrackEvent } from '../events';
 import type { LoggerOptions } from '../types';
 import { isFireFox, isSafari, isWeb } from '../utils';

package/src/room/track/record.ts

@@ -1,3 +1,4 @@
+import type { NonSharedUint8Array } from '../../type-polyfills/non-shared-typed-arrays';
 import type LocalTrack from './LocalTrack';
 
 // Check if MediaRecorder is available

package/src/room/utils.ts

@@ -7,6 +7,7 @@ import {
   Transcription as TranscriptionModel,
 } from '@livekit/protocol';
 import { type Throws } from '@livekit/throws-transformer/throws';
+import type { NonSharedUint8Array } from '../type-polyfills/non-shared-typed-arrays';
 import TypedPromise from '../utils/TypedPromise';
 import { getBrowser } from '../utils/browserParser';
 import type { BrowserDetails } from '../utils/browserParser';

package/src/type-polyfills/{non-shared-typed-arrays.d.ts → non-shared-typed-arrays.ts}

@@ -3,4 +3,9 @@
 // only accept the non-shared variant `Uint8Array<ArrayBuffer>`. Using `ReturnType<typeof Uint8Array.from>`
 // resolves to that non-shared variant on TS versions that support the generic, while remaining
 // equivalent to plain `Uint8Array` on older versions — so this alias works across the range we support.
-type NonSharedUint8Array = ReturnType<typeof Uint8Array.from>;
+//
+// Exported as a normal type (rather than an ambient global) so it is emitted into the published `dist`
+// types and imported explicitly wherever it is used. An earlier ambient `.d.ts` declaration was resolved
+// during our own build but never shipped, so consumers' type-checkers (and tools like API Extractor)
+// failed with "Cannot find name 'NonSharedUint8Array'" on our emitted `.d.ts` files.
+export type NonSharedUint8Array = ReturnType<typeof Uint8Array.from>;

package/src/utils/dataPacketBuffer.ts

@@ -1,3 +1,5 @@
+import type { NonSharedUint8Array } from '../type-polyfills/non-shared-typed-arrays';
+
 export interface DataPacketItem {
   data: NonSharedUint8Array;
   sequence: number;