livekit-client 2.20.0 → 2.20.2

package/src/e2ee/E2eeManager.ts

@@ -1,6 +1,7 @@
 import { Encryption_Type, TrackInfo } from '@livekit/protocol';
 import { EventEmitter } from 'events';
 import type TypedEventEmitter from 'typed-emitter';
+import type { FrameMetadata } from '../frameMetadata/types';
 import { hasFrameMetadataPublishOptions } from '../frameMetadata/utils';
 import log, { LogLevel, workerLogger } from '../logger';
 import type RTCEngine from '../room/RTCEngine';
@@ -20,6 +21,7 @@ import {
   isScriptTransformSupportedForWorker,
   isVideoTrack,
 } from '../room/utils';
+import type { NonSharedUint8Array } from '../type-polyfills/non-shared-typed-arrays';
 import type { BaseKeyProvider } from './KeyProvider';
 import { E2EE_FLAG } from './constants';
 import { type E2EEManagerCallbacks, EncryptionEvent, KeyProviderEvent } from './events';
@@ -246,7 +248,7 @@ export class E2EEManager
     trackId: string,
     rtpTimestamp: number,
     ssrc: number,
-    metadata: { userTimestamp: bigint; frameId: number },
+    metadata: FrameMetadata,
   ) {
     if (!this.room) {
       return;

package/src/e2ee/types.ts

@@ -2,6 +2,7 @@ import type { FrameMetadataPayload } from '../frameMetadata/frameMetadata';
 import type { FrameMetadataPublishOptions } from '../frameMetadata/types';
 import type { LogLevel } from '../logger';
 import type { VideoCodec } from '../room/track/options';
+import type { NonSharedUint8Array } from '../type-polyfills/non-shared-typed-arrays';
 import type { BaseE2EEManager } from './E2eeManager';
 import type { BaseKeyProvider } from './KeyProvider';
 

package/src/e2ee/utils.ts

@@ -1,4 +1,5 @@
 import { type DataPacket, EncryptedPacketPayload } from '@livekit/protocol';
+import type { NonSharedUint8Array } from '../type-polyfills/non-shared-typed-arrays';
 import { ENCRYPTION_ALGORITHM } from './constants';
 import type { KeyProviderOptions } from './types';
 

package/src/e2ee/worker/DataCryptor.ts

@@ -1,4 +1,5 @@
 import { workerLogger } from '../../logger';
+import type { NonSharedUint8Array } from '../../type-polyfills/non-shared-typed-arrays';
 import { ENCRYPTION_ALGORITHM } from '../constants';
 import { CryptorError, CryptorErrorReason } from '../errors';
 import type { DecodeRatchetOptions, KeySet, RatchetResult } from '../types';

package/src/e2ee/worker/FrameCryptor.test.ts

@@ -1,6 +1,7 @@
 import { afterEach, describe, expect, it, vitest } from 'vitest';
 import { appendPacketTrailer, extractPacketTrailer } from '../../frameMetadata/frameMetadata';
 import type { FrameMetadataPublishOptions } from '../../frameMetadata/types';
+import type { NonSharedUint8Array } from '../../type-polyfills/non-shared-typed-arrays';
 import { IV_LENGTH, KEY_PROVIDER_DEFAULTS } from '../constants';
 import { CryptorEvent } from '../events';
 import type { KeyProviderOptions } from '../types';

package/src/e2ee/worker/FrameCryptor.ts

@@ -10,6 +10,7 @@ import { hasFrameMetadataPublishOptions } from '../../frameMetadata/utils';
 import { workerLogger } from '../../logger';
 import { type VideoCodec, videoCodecs } from '../../room/track/options';
 import { mimeTypeToVideoCodecString } from '../../room/track/utils';
+import type { NonSharedUint8Array } from '../../type-polyfills/non-shared-typed-arrays';
 import { ENCRYPTION_ALGORITHM, IV_LENGTH, UNENCRYPTED_BYTES } from '../constants';
 import { CryptorError, CryptorErrorReason } from '../errors';
 import { type CryptorCallbacks, CryptorEvent } from '../events';

package/src/e2ee/worker/ParticipantKeyHandler.test.ts

@@ -1,4 +1,5 @@
 import { describe, expect, it, test, vitest } from 'vitest';
+import type { NonSharedUint8Array } from '../../type-polyfills/non-shared-typed-arrays';
 import { ENCRYPTION_ALGORITHM, KEY_PROVIDER_DEFAULTS } from '../constants';
 import { KeyHandlerEvent } from '../events';
 import { createKeyMaterialFromString, importKey } from '../utils';

package/src/e2ee/worker/e2ee.worker.ts

@@ -1,5 +1,6 @@
 import { workerLogger } from '../../logger';
 import type { VideoCodec } from '../../room/track/options';
+import type { NonSharedUint8Array } from '../../type-polyfills/non-shared-typed-arrays';
 import { AsyncQueue } from '../../utils/AsyncQueue';
 import { KEY_PROVIDER_DEFAULTS } from '../constants';
 import { CryptorErrorReason } from '../errors';

package/src/e2ee/worker/naluUtils.ts

@@ -2,6 +2,7 @@
  * NALU (Network Abstraction Layer Unit) utilities for H.264 and H.265 video processing
  * Contains functions for parsing and working with NALUs in video frames
  */
+import type { NonSharedUint8Array } from '../../type-polyfills/non-shared-typed-arrays';
 
 /**
  * Mask for extracting NALU type from H.264 header byte

package/src/e2ee/worker/sifPayload.ts

@@ -1,4 +1,5 @@
 import type { VideoCodec } from '../..';
+import type { NonSharedUint8Array } from '../../type-polyfills/non-shared-typed-arrays';
 
 //  Payload definitions taken from https://github.com/livekit/livekit/blob/master/pkg/sfu/downtrack.go#L104
 

package/src/frameMetadata/frameMetadata.test.ts

@@ -1,11 +1,57 @@
 import { afterEach, describe, expect, it, vi } from 'vitest';
 import {
+  PACKET_TRAILER_ENVELOPE_SIZE,
+  PACKET_TRAILER_FRAME_ID_TAG,
+  PACKET_TRAILER_MAGIC,
+  PACKET_TRAILER_TIMESTAMP_TAG,
+  PACKET_TRAILER_USER_DATA_TAG,
   appendPacketTrailer,
   appendPacketTrailerToEncodedFrame,
   extractPacketTrailer,
   processPacketTrailer,
 } from './frameMetadata';
 
+/**
+ * Builds a packet trailer (XORed TLVs + envelope) for the given fields. The
+ * SDK only writes timestamp/frameId, so this mirrors the native sender's TLV
+ * encoding to exercise user_data extraction.
+ */
+function buildTrailer(
+  payload: Uint8Array,
+  fields: { userTimestamp?: bigint; frameId?: number; userData?: Uint8Array },
+): Uint8Array {
+  const tlvs: number[] = [];
+
+  if (fields.userTimestamp !== undefined) {
+    tlvs.push(PACKET_TRAILER_TIMESTAMP_TAG ^ 0xff, 8 ^ 0xff);
+    for (let i = 7; i >= 0; i -= 1) {
+      tlvs.push(Number((fields.userTimestamp >> BigInt(i * 8)) & BigInt(0xff)) ^ 0xff);
+    }
+  }
+
+  if (fields.frameId !== undefined) {
+    tlvs.push(PACKET_TRAILER_FRAME_ID_TAG ^ 0xff, 4 ^ 0xff);
+    for (let i = 3; i >= 0; i -= 1) {
+      tlvs.push(((fields.frameId >> (i * 8)) & 0xff) ^ 0xff);
+    }
+  }
+
+  if (fields.userData !== undefined) {
+    tlvs.push(PACKET_TRAILER_USER_DATA_TAG ^ 0xff, fields.userData.length ^ 0xff);
+    for (const byte of fields.userData) {
+      tlvs.push(byte ^ 0xff);
+    }
+  }
+
+  const trailerLength = tlvs.length + PACKET_TRAILER_ENVELOPE_SIZE;
+  const result = new Uint8Array(payload.length + trailerLength);
+  result.set(payload, 0);
+  result.set(tlvs, payload.length);
+  result[payload.length + tlvs.length] = trailerLength ^ 0xff;
+  result.set(PACKET_TRAILER_MAGIC, payload.length + tlvs.length + 1);
+  return result;
+}
+
 describe('packetTrailer', () => {
   afterEach(() => {
     vi.useRealTimers();
@@ -47,6 +93,43 @@ describe('packetTrailer', () => {
     });
   });
 
+  it('extracts user_data from a trailer carrying only user_data', () => {
+    const payload = Uint8Array.from([1, 2, 3, 4]);
+    const userData = Uint8Array.from([0x00, 0x01, 0xfe, 0xff, 0x42]);
+    const trailer = buildTrailer(payload, { userData });
+    const extracted = extractPacketTrailer(trailer);
+
+    expect(Array.from(extracted.data)).toEqual(Array.from(payload));
+    expect(extracted.metadata?.userTimestamp).toBe(0n);
+    expect(extracted.metadata?.frameId).toBe(0);
+    expect(extracted.metadata?.userData).toBeInstanceOf(Uint8Array);
+    expect(Array.from(extracted.metadata!.userData!)).toEqual(Array.from(userData));
+  });
+
+  it('extracts user_data alongside timestamp and frameId', () => {
+    const payload = Uint8Array.from([9, 8, 7, 6, 5]);
+    const userData = Uint8Array.from([10, 20, 30, 40]);
+    const trailer = buildTrailer(payload, {
+      userTimestamp: 1_744_249_600_123_456n,
+      frameId: 42,
+      userData,
+    });
+    const extracted = extractPacketTrailer(trailer);
+
+    expect(Array.from(extracted.data)).toEqual(Array.from(payload));
+    expect(extracted.metadata?.userTimestamp).toBe(1_744_249_600_123_456n);
+    expect(extracted.metadata?.frameId).toBe(42);
+    expect(Array.from(extracted.metadata!.userData!)).toEqual(Array.from(userData));
+  });
+
+  it('leaves userData undefined when no user_data tag is present', () => {
+    const payload = Uint8Array.from([1, 2, 3, 4]);
+    const trailer = appendPacketTrailer(payload, 1_744_249_600_123_456n, 42);
+    const extracted = extractPacketTrailer(trailer);
+
+    expect(extracted.metadata?.userData).toBeUndefined();
+  });
+
   it('returns data unchanged when both timestamp and frameId are 0', () => {
     const payload = Uint8Array.from([1, 2, 3, 4]);
     const result = appendPacketTrailer(payload, 0n, 0);

package/src/frameMetadata/frameMetadata.ts

@@ -10,6 +10,7 @@ export const PACKET_TRAILER_MAGIC = Uint8Array.from([
 
 export const PACKET_TRAILER_TIMESTAMP_TAG = 0x01;
 export const PACKET_TRAILER_FRAME_ID_TAG = 0x02;
+export const PACKET_TRAILER_USER_DATA_TAG = 0x03;
 export const PACKET_TRAILER_ENVELOPE_SIZE = 5;
 
 const TIMESTAMP_TLV_SIZE = 10;
@@ -127,6 +128,13 @@ export function extractPacketTrailer(data: ArrayBuffer | Uint8Array): ExtractPac
     } else if (tag === PACKET_TRAILER_FRAME_ID_TAG && length === 4) {
       metadata.frameId = readUint32Xor(bytes, offset, length);
       foundAny = true;
+    } else if (tag === PACKET_TRAILER_USER_DATA_TAG) {
+      const userData = new Uint8Array(length);
+      for (let index = 0; index < length; index += 1) {
+        userData[index] = bytes[offset + index] ^ 0xff;
+      }
+      metadata.userData = userData;
+      foundAny = true;
     }
 
     offset += length;

package/src/frameMetadata/types.ts

@@ -3,6 +3,7 @@ import type { FrameMetadataPayload } from './frameMetadata';
 export interface FrameMetadata {
   userTimestamp: bigint;
   frameId: number;
+  userData?: Uint8Array;
 }
 
 /** @deprecated Use {@link FrameMetadata} instead. */

package/src/logger.ts

@@ -22,6 +22,8 @@ export enum LoggerNames {
   PCTransport = 'livekit-pc-transport',
   E2EE = 'lk-e2ee',
   DataTracks = 'livekit-data-tracks',
+  Region = 'livekit-region',
+  ICE = 'livekit-ice',
 }
 
 type LogLevelString = keyof typeof LogLevel;

package/src/room/PCTransport.test.ts

@@ -0,0 +1,150 @@
+import { type MediaDescription, parse } from 'sdp-transform';
+import { describe, expect, it } from 'vitest';
+import { conformBundledCodecFmtp, placeholderMidsFromTransceivers } from './PCTransport';
+
+/** Parse the `key[=value]` pairs of an fmtp config into a comparable set. */
+const paramSet = (config: string) => new Set(config.split(';').filter(Boolean));
+
+const fmtpOf = (media: MediaDescription[], mid: string, payload: number) => {
+  const m = media.find((section) => `${section.mid}` === mid)!;
+  return m.fmtp.find((f) => f.payload === payload)?.config;
+};
+
+/** Predicate for conformBundledCodecFmtp based on an explicit set of mids. */
+const placeholders = (mids: string[]) => {
+  const set = new Set(mids);
+  return (m: MediaDescription) => set.has(`${m.mid}`);
+};
+
+// One publisher bundle: a published mic + camera (real sends, with msid/ssrc)
+// and their pre-populated recvonly placeholders.
+const PUBLISHER_SDP = `v=0
+o=- 0 0 IN IP4 127.0.0.1
+s=-
+t=0 0
+a=group:BUNDLE 0 1 2 3
+m=audio 9 UDP/TLS/RTP/SAVPF 111
+c=IN IP4 0.0.0.0
+a=mid:0
+a=sendonly
+a=msid:s mic
+a=ssrc:1111 cname:a
+a=rtpmap:111 opus/48000/2
+a=fmtp:111 minptime=10;useinbandfec=1;usedtx=1
+m=audio 9 UDP/TLS/RTP/SAVPF 111
+c=IN IP4 0.0.0.0
+a=mid:1
+a=recvonly
+a=rtpmap:111 opus/48000/2
+a=fmtp:111 minptime=10;useinbandfec=1
+m=video 9 UDP/TLS/RTP/SAVPF 49
+c=IN IP4 0.0.0.0
+a=mid:2
+a=sendonly
+a=msid:s cam
+a=ssrc:2222 cname:b
+a=rtpmap:49 H265/90000
+a=fmtp:49 level-id=186;profile-id=1;tier-flag=0;tx-mode=SRST
+m=video 9 UDP/TLS/RTP/SAVPF 49
+c=IN IP4 0.0.0.0
+a=mid:3
+a=recvonly
+a=rtpmap:49 H265/90000
+a=fmtp:49 level-id=180;profile-id=1;tier-flag=0;tx-mode=SRST`;
+
+describe('placeholderMidsFromTransceivers', () => {
+  const tr = (mid: string | null, track: unknown) =>
+    ({ mid, sender: { track } }) as unknown as RTCRtpTransceiver;
+
+  it('includes transceivers with a mid and no outgoing track', () => {
+    const mids = placeholderMidsFromTransceivers([
+      tr('0', {}), // real send
+      tr('1', null), // pre-populated placeholder
+      tr('2', null), // reverted after unpublish (msid may linger in SDP, but no track)
+      tr(null, null), // not yet negotiated — no mid, excluded
+    ]);
+    expect(mids).toEqual(new Set(['1', '2']));
+  });
+});
+
+describe('conformBundledCodecFmtp', () => {
+  it('copies opus and H.265 fmtp from real sections onto placeholders', () => {
+    const { media } = parse(PUBLISHER_SDP);
+
+    conformBundledCodecFmtp(media, placeholders(['1', '3']));
+
+    expect(fmtpOf(media, '1', 111)).toBe(fmtpOf(media, '0', 111));
+    expect(paramSet(fmtpOf(media, '1', 111)!)).toContain('usedtx=1');
+    expect(fmtpOf(media, '3', 49)).toBe(fmtpOf(media, '2', 49));
+    expect(paramSet(fmtpOf(media, '3', 49)!)).toContain('level-id=186');
+  });
+
+  it('never rewrites the real (non-placeholder) sections', () => {
+    const { media } = parse(PUBLISHER_SDP);
+
+    conformBundledCodecFmtp(media, placeholders(['1', '3']));
+
+    expect(fmtpOf(media, '0', 111)).toBe('minptime=10;useinbandfec=1;usedtx=1');
+    expect(paramSet(fmtpOf(media, '2', 49)!)).toContain('level-id=186');
+  });
+
+  it('converges divergent placeholders even when no real section declares the payload', () => {
+    // Two placeholders that disagree (a reverted one kept level 186, a fresh one
+    // has 180) with no active send present.
+    const { media } = parse(`v=0
+o=- 0 0 IN IP4 127.0.0.1
+s=-
+t=0 0
+a=group:BUNDLE 10 11
+m=video 9 UDP/TLS/RTP/SAVPF 49
+a=mid:10
+a=rtpmap:49 H265/90000
+a=fmtp:49 level-id=186;profile-id=1;tier-flag=0;tx-mode=SRST
+m=video 9 UDP/TLS/RTP/SAVPF 49
+a=mid:11
+a=rtpmap:49 H265/90000
+a=fmtp:49 level-id=180;profile-id=1;tier-flag=0;tx-mode=SRST`);
+
+    conformBundledCodecFmtp(media, placeholders(['10', '11']));
+
+    expect(fmtpOf(media, '10', 49)).toBe(fmtpOf(media, '11', 49));
+  });
+
+  it('conforms a reverted section that kept its msid (second-publish case)', () => {
+    // The real second publish: a live camera send (level 186), plus the section
+    // left over from the first (unpublished) track. Chrome keeps that section's
+    // `a=msid`/`a=ssrc` even though it no longer sends, so it must be identified
+    // as a placeholder by its transceiver (no track), not by SDP.
+    const { media } = parse(`v=0
+o=- 0 0 IN IP4 127.0.0.1
+s=-
+t=0 0
+a=group:BUNDLE 3 4 5
+m=video 9 UDP/TLS/RTP/SAVPF 49
+a=mid:3
+a=sendonly
+a=msid:s cam
+a=ssrc:9 cname:c
+a=rtpmap:49 H265/90000
+a=fmtp:49 level-id=186;profile-id=1;tier-flag=0;tx-mode=SRST
+m=video 9 UDP/TLS/RTP/SAVPF 49
+a=mid:4
+a=sendonly
+a=msid:s stale
+a=ssrc:8 cname:d
+a=rtpmap:49 H265/90000
+a=fmtp:49 level-id=180;profile-id=1;tier-flag=0;tx-mode=SRST
+m=video 9 UDP/TLS/RTP/SAVPF 49
+a=mid:5
+a=recvonly
+a=rtpmap:49 H265/90000
+a=fmtp:49 level-id=180;profile-id=1;tier-flag=0;tx-mode=SRST`);
+
+    // mid 4 still has msid/ssrc in the SDP, but its transceiver has no track.
+    conformBundledCodecFmtp(media, placeholders(['4', '5']));
+
+    expect(paramSet(fmtpOf(media, '3', 49)!)).toContain('level-id=186'); // real send untouched
+    expect(fmtpOf(media, '4', 49)).toBe(fmtpOf(media, '3', 49)); // stale conformed
+    expect(fmtpOf(media, '5', 49)).toBe(fmtpOf(media, '3', 49)); // placeholder conformed
+  });
+});