livekit-client 2.22.3 → 2.23.0

package/src/e2ee/worker/FrameCryptor.test.ts

@@ -1,6 +1,8 @@
 import { afterEach, describe, expect, it, vitest } from 'vitest';
 import { appendPacketTrailer, extractPacketTrailer } from '../../frameMetadata/frameMetadata';
 import type { FrameMetadataPublishOptions } from '../../frameMetadata/types';
+import { workerLogger } from '../../logger';
+import type { VideoCodec } from '../../room/track/options';
 import type { NonSharedUint8Array } from '../../type-polyfills/non-shared-typed-arrays';
 import { IV_LENGTH, KEY_PROVIDER_DEFAULTS } from '../constants';
 import { CryptorEvent } from '../events';
@@ -71,12 +73,14 @@ function prepareParticipantTestEncoder(
   participantIdentity: string,
   partialKeyProviderOptions: Partial<KeyProviderOptions>,
   packetTrailer?: FrameMetadataPublishOptions,
+  codec?: VideoCodec,
 ) {
   return prepareParticipantTest(
     'encode',
     participantIdentity,
     partialKeyProviderOptions,
     packetTrailer,
+    codec,
   );
 }
 
@@ -85,6 +89,7 @@ function prepareParticipantTest(
   participantIdentity: string,
   partialKeyProviderOptions: Partial<KeyProviderOptions>,
   packetTrailer?: FrameMetadataPublishOptions,
+  codec?: VideoCodec,
 ): {
   keys: ParticipantKeyHandler;
   cryptor: FrameCryptor;
@@ -110,7 +115,7 @@ function prepareParticipantTest(
     new ReadableStream(input),
     new WritableStream(output),
     'testTrack',
-    undefined,
+    codec,
     packetTrailer,
   );
 
@@ -233,81 +238,224 @@ describe('FrameCryptor', () => {
       }
     });
 
-    it('appends packet trailer after encryption', async () => {
-      vitest.useFakeTimers();
-      const now = new Date('2025-04-10T12:00:00.123Z');
-      vitest.setSystemTime(now);
-      try {
-        const { keys, input, output } = prepareParticipantTestEncoder(
-          participantIdentity,
-          {},
-          { timestamp: true, frameId: true },
-        );
-
-        await keys.setKey(await createKeyMaterialFromString('key1'), 1);
+    describe('H.264 start codes (#2140)', () => {
+      const sps = [0x67, 0x64, 0x00, 0x1f, 0xac, 0xd9];
+      const pps = [0x68, 0xee, 0x3c, 0xb0];
+      const idr = [0x65, 0x88, 0x84, 0x21, 0x43, 0x55, 0x66, 0x77];
+
+      async function encryptAndDecrypt(plainTextData: NonSharedUint8Array) {
+        const encoder = prepareParticipantTestEncoder(participantIdentity, {});
+        await encoder.keys.setKey(await createKeyMaterialFromString('key1'), 1);
+        const encrypted = mockRTCEncodedVideoFrame(plainTextData);
+        encoder.input.write(encrypted);
+        await vitest.waitFor(() => expect(encoder.output.chunks).toHaveLength(1));
+        const encryptedData = new Uint8Array(encrypted.data.slice(0));
+
+        const decoder = prepareParticipantTestDecoder(participantIdentity, {});
+        await decoder.keys.setKey(await createKeyMaterialFromString('key1'), 1);
+        const decrypted = mockRTCEncodedVideoFrame(new Uint8Array(encryptedData));
+        decoder.input.write(decrypted);
+        await vitest.waitFor(() => expect(decoder.output.chunks).toHaveLength(1));
+
+        return { encryptedData, decryptedData: new Uint8Array(decrypted.data) };
+      }
 
-        const frame = mockRTCEncodedVideoFrame(
-          new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]),
+      it('rewrites 3-byte start codes in the unencrypted header to 4-byte start codes', async () => {
+        const { encryptedData, decryptedData } = await encryptAndDecrypt(
+          new Uint8Array([0, 0, 1, ...sps, 0, 0, 1, ...pps, 0, 0, 1, ...idr]),
         );
 
-        input.write(frame);
-        await vitest.waitFor(() => expect(output.chunks).toHaveLength(1));
-
-        const extracted = extractPacketTrailer(frame.data);
-        expect(extracted.metadata?.userTimestamp).toBeGreaterThanOrEqual(
-          BigInt(now.getTime()) * BigInt(1000),
+        // The unencrypted header runs through the first 2 bytes of the slice NALU.
+        const expectedHeader = [0, 0, 0, 1, ...sps, 0, 0, 0, 1, ...pps, 0, 0, 0, 1, 0x65, 0x88];
+        expect(Array.from(encryptedData.subarray(0, expectedHeader.length))).toEqual(
+          expectedHeader,
         );
-        expect(extracted.metadata?.frameId).toBe(1);
+        expect(Array.from(decryptedData)).toEqual([
+          0,
+          0,
+          0,
+          1,
+          ...sps,
+          0,
+          0,
+          0,
+          1,
+          ...pps,
+          0,
+          0,
+          0,
+          1,
+          ...idr,
+        ]);
+      });
 
-        const frameTrailer = new Uint8Array(
-          extracted.data.buffer.slice(extracted.data.byteLength - 2),
+      it('rewrites a 3-byte start code on a delta frame', async () => {
+        const pSlice = [0x41, 0x9a, 0x21, 0x43, 0x55, 0x66];
+        const { encryptedData, decryptedData } = await encryptAndDecrypt(
+          new Uint8Array([0, 0, 1, ...pSlice]),
         );
-        expect(frameTrailer[0]).toEqual(IV_LENGTH);
-        expect(frameTrailer[1]).toEqual(1);
-      } finally {
-        vitest.useRealTimers();
-      }
-    });
 
-    it('does not append packet trailer after encryption when no trailer features are enabled', async () => {
-      vitest.useFakeTimers();
-      try {
-        const { keys, input, output } = prepareParticipantTestEncoder(
-          participantIdentity,
-          {},
-          { timestamp: false, frameId: false },
-        );
+        expect(Array.from(encryptedData.subarray(0, 6))).toEqual([0, 0, 0, 1, 0x41, 0x9a]);
+        expect(Array.from(decryptedData)).toEqual([0, 0, 0, 1, ...pSlice]);
+      });
 
-        await keys.setKey(await createKeyMaterialFromString('key1'), 1);
+      it('leaves 4-byte start codes in the unencrypted header unchanged', async () => {
+        const plainTextData = [0, 0, 0, 1, ...sps, 0, 0, 0, 1, ...pps, 0, 0, 0, 1, ...idr];
+        const { encryptedData, decryptedData } = await encryptAndDecrypt(
+          new Uint8Array(plainTextData),
+        );
 
-        const frame = mockRTCEncodedVideoFrame(
-          new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]),
+        const expectedHeader = [0, 0, 0, 1, ...sps, 0, 0, 0, 1, ...pps, 0, 0, 0, 1, 0x65, 0x88];
+        expect(Array.from(encryptedData.subarray(0, expectedHeader.length))).toEqual(
+          expectedHeader,
         );
+        expect(Array.from(decryptedData)).toEqual(plainTextData);
+      });
+      it('logs the frame bytes once when NALU processing fails', async () => {
+        vitest.useFakeTimers();
+        const warn = vitest.spyOn(workerLogger, 'warn');
+        try {
+          const { keys, input, output } = prepareParticipantTestEncoder(
+            participantIdentity,
+            {},
+            undefined,
+            'h264',
+          );
+
+          await keys.setKey(await createKeyMaterialFromString('key1'), 1);
+
+          // No start code at offset 0, so findNALUIndices throws 'byte stream contains
+          // leading data' and the cryptor falls back to VP8 handling.
+          input.write(
+            mockRTCEncodedVideoFrame(new Uint8Array([9, 9, 9, 9, 1, 2, 3, 4, 5, 6, 7, 8])),
+          );
+          await vitest.waitFor(() => expect(output.chunks).toHaveLength(1));
+
+          expect(warn).toHaveBeenCalledWith(
+            'NALU processing failed, falling back to VP8 handling',
+            expect.objectContaining({
+              detectedCodec: 'h264',
+              frameType: 'key',
+              byteLength: 12,
+              firstBytes: expect.stringMatching(/^09 09 09 09 /),
+            }),
+          );
+
+          // same payload type, so the once-per-tuple guard must suppress the second log
+          input.write(
+            mockRTCEncodedVideoFrame(new Uint8Array([8, 8, 8, 8, 1, 2, 3, 4, 5, 6, 7, 8])),
+          );
+          await vitest.waitFor(() => expect(output.chunks).toHaveLength(2));
+
+          expect(warn).toHaveBeenCalledTimes(1);
+        } finally {
+          warn.mockRestore();
+          vitest.useRealTimers();
+        }
+      });
 
-        input.write(frame);
-        await vitest.waitFor(() => expect(output.chunks).toHaveLength(1));
+      it('appends packet trailer after encryption', async () => {
+        vitest.useFakeTimers();
+        const now = new Date('2025-04-10T12:00:00.123Z');
+        vitest.setSystemTime(now);
+        try {
+          const { keys, input, output } = prepareParticipantTestEncoder(
+            participantIdentity,
+            {},
+            { timestamp: true, frameId: true },
+          );
+
+          await keys.setKey(await createKeyMaterialFromString('key1'), 1);
+
+          const frame = mockRTCEncodedVideoFrame(
+            new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]),
+          );
+
+          input.write(frame);
+          await vitest.waitFor(() => expect(output.chunks).toHaveLength(1));
+
+          const extracted = extractPacketTrailer(frame.data);
+          expect(extracted.metadata?.userTimestamp).toBeGreaterThanOrEqual(
+            BigInt(now.getTime()) * BigInt(1000),
+          );
+          expect(extracted.metadata?.frameId).toBe(1);
+
+          const frameTrailer = new Uint8Array(
+            extracted.data.buffer.slice(extracted.data.byteLength - 2),
+          );
+          expect(frameTrailer[0]).toEqual(IV_LENGTH);
+          expect(frameTrailer[1]).toEqual(1);
+        } finally {
+          vitest.useRealTimers();
+        }
+      });
 
-        const extracted = extractPacketTrailer(frame.data);
-        expect(extracted.metadata).toBeUndefined();
-        expect(extracted.data.byteLength).toBe(frame.data.byteLength);
+      it('does not append packet trailer after encryption when no trailer features are enabled', async () => {
+        vitest.useFakeTimers();
+        try {
+          const { keys, input, output } = prepareParticipantTestEncoder(
+            participantIdentity,
+            {},
+            { timestamp: false, frameId: false },
+          );
+
+          await keys.setKey(await createKeyMaterialFromString('key1'), 1);
+
+          const frame = mockRTCEncodedVideoFrame(
+            new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]),
+          );
+
+          input.write(frame);
+          await vitest.waitFor(() => expect(output.chunks).toHaveLength(1));
+
+          const extracted = extractPacketTrailer(frame.data);
+          expect(extracted.metadata).toBeUndefined();
+          expect(extracted.data.byteLength).toBe(frame.data.byteLength);
+
+          const frameTrailer = new Uint8Array(frame.data.slice(frame.data.byteLength - 2));
+          expect(frameTrailer[0]).toEqual(IV_LENGTH);
+          expect(frameTrailer[1]).toEqual(1);
+        } finally {
+          vitest.useRealTimers();
+        }
+      });
 
-        const frameTrailer = new Uint8Array(frame.data.slice(frame.data.byteLength - 2));
-        expect(frameTrailer[0]).toEqual(IV_LENGTH);
-        expect(frameTrailer[1]).toEqual(1);
-      } finally {
-        vitest.useRealTimers();
-      }
-    });
+      it('appends only timestamp packet trailer metadata after encryption', async () => {
+        vitest.useFakeTimers();
+        const now = new Date('2025-04-10T12:00:00.123Z');
+        vitest.setSystemTime(now);
+        try {
+          const { keys, input, output } = prepareParticipantTestEncoder(
+            participantIdentity,
+            {},
+            { timestamp: true },
+          );
+
+          await keys.setKey(await createKeyMaterialFromString('key1'), 1);
+
+          const frame = mockRTCEncodedVideoFrame(
+            new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]),
+          );
+
+          input.write(frame);
+          await vitest.waitFor(() => expect(output.chunks).toHaveLength(1));
+
+          const extracted = extractPacketTrailer(frame.data);
+          expect(frame.data.byteLength - extracted.data.byteLength).toBe(15);
+          expect(extracted.metadata?.userTimestamp).toBeGreaterThanOrEqual(
+            BigInt(now.getTime()) * BigInt(1000),
+          );
+          expect(extracted.metadata?.frameId).toBe(0);
+        } finally {
+          vitest.useRealTimers();
+        }
+      });
 
-    it('appends only timestamp packet trailer metadata after encryption', async () => {
-      vitest.useFakeTimers();
-      const now = new Date('2025-04-10T12:00:00.123Z');
-      vitest.setSystemTime(now);
-      try {
+      it('appends only frame id packet trailer metadata after encryption', async () => {
         const { keys, input, output } = prepareParticipantTestEncoder(
           participantIdentity,
           {},
-          { timestamp: true },
+          { frameId: true },
         );
 
         await keys.setKey(await createKeyMaterialFromString('key1'), 1);
@@ -320,37 +468,11 @@ describe('FrameCryptor', () => {
         await vitest.waitFor(() => expect(output.chunks).toHaveLength(1));
 
         const extracted = extractPacketTrailer(frame.data);
-        expect(frame.data.byteLength - extracted.data.byteLength).toBe(15);
-        expect(extracted.metadata?.userTimestamp).toBeGreaterThanOrEqual(
-          BigInt(now.getTime()) * BigInt(1000),
-        );
-        expect(extracted.metadata?.frameId).toBe(0);
-      } finally {
-        vitest.useRealTimers();
-      }
-    });
-
-    it('appends only frame id packet trailer metadata after encryption', async () => {
-      const { keys, input, output } = prepareParticipantTestEncoder(
-        participantIdentity,
-        {},
-        { frameId: true },
-      );
-
-      await keys.setKey(await createKeyMaterialFromString('key1'), 1);
-
-      const frame = mockRTCEncodedVideoFrame(
-        new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]),
-      );
-
-      input.write(frame);
-      await vitest.waitFor(() => expect(output.chunks).toHaveLength(1));
-
-      const extracted = extractPacketTrailer(frame.data);
-      expect(frame.data.byteLength - extracted.data.byteLength).toBe(11);
-      expect(extracted.metadata).toEqual({
-        userTimestamp: BigInt(0),
-        frameId: 1,
+        expect(frame.data.byteLength - extracted.data.byteLength).toBe(11);
+        expect(extracted.metadata).toEqual({
+          userTimestamp: BigInt(0),
+          frameId: 1,
+        });
       });
     });
   });

package/src/e2ee/worker/FrameCryptor.ts

@@ -25,7 +25,7 @@ import type {
 import { deriveKeys, isVideoFrame, needsRbspUnescaping, parseRbsp, writeRbsp } from '../utils';
 import { ErrorRateLimiter } from './ErrorRateLimiter';
 import type { ParticipantKeyHandler } from './ParticipantKeyHandler';
-import { processNALUsForEncryption } from './naluUtils';
+import { normalizeAnnexBStartCodes, processNALUsForEncryption } from './naluUtils';
 import { identifySifPayload } from './sifPayload';
 
 export const encryptionEnabledMap: Map<string, boolean> = new Map();
@@ -490,7 +490,16 @@ export class FrameCryptor extends BaseFrameCryptor {
       let frameInfo = this.getUnencryptedBytes(encodedFrame);
 
       // Thіs is not encrypted and contains the VP8 payload descriptor or the Opus TOC byte.
-      const frameHeader = new Uint8Array(encodedFrame.data, 0, frameInfo.unencryptedBytes);
+      let frameHeader: NonSharedUint8Array = new Uint8Array(
+        encodedFrame.data,
+        0,
+        frameInfo.unencryptedBytes,
+      );
+      if (frameInfo.requiresNALUProcessing) {
+        // The receiver's depacketizer rebuilds NALUs with 4-byte start codes, so authenticate (and
+        // send) the header in that form or decryption fails for encoders emitting 3-byte ones.
+        frameHeader = normalizeAnnexBStartCodes(frameHeader);
+      }
 
       // Frame trailer contains the R|IV_LENGTH and key index
       const frameTrailer = new Uint8Array(2);
@@ -510,7 +519,7 @@ export class FrameCryptor extends BaseFrameCryptor {
           {
             name: ENCRYPTION_ALGORITHM,
             iv,
-            additionalData: new Uint8Array(encodedFrame.data, 0, frameHeader.byteLength),
+            additionalData: frameHeader,
           },
           encryptionKey,
           new Uint8Array(encodedFrame.data, frameInfo.unencryptedBytes),
@@ -906,7 +915,7 @@ export class FrameCryptor extends BaseFrameCryptor {
         };
       }
     } catch (e) {
-      this.logNALUFallbackOnce(fallbackKey, payloadType, e);
+      this.logNALUFallbackOnce(fallbackKey, frame, payloadType, detectedCodec, e);
     }
 
     // Fallback to VP8 handling
@@ -919,7 +928,9 @@ export class FrameCryptor extends BaseFrameCryptor {
    */
   private logNALUFallbackOnce(
     fallbackKey: string,
+    frame: RTCEncodedVideoFrame,
     payloadType: number | undefined,
+    detectedCodec: VideoCodec | undefined,
     error: unknown,
   ) {
     if (this.loggedNALUFallbacks.has(fallbackKey)) {
@@ -929,6 +940,12 @@ export class FrameCryptor extends BaseFrameCryptor {
     workerLogger.warn('NALU processing failed, falling back to VP8 handling', {
       error,
       payloadType,
+      detectedCodec,
+      frameType: frame.type,
+      byteLength: frame.data.byteLength,
+      firstBytes: Array.from(new Uint8Array(frame.data, 0, Math.min(16, frame.data.byteLength)))
+        .map((b) => b.toString(16).padStart(2, '0'))
+        .join(' '),
       ...this.logContext,
     });
   }

package/src/e2ee/worker/naluUtils.test.ts

@@ -0,0 +1,123 @@
+import { describe, expect, it } from 'vitest';
+import { normalizeAnnexBStartCodes, processNALUsForEncryption } from './naluUtils';
+
+/** Annex-B frame: a 4-byte start code before every NALU. */
+function annexB(...nalus: number[][]) {
+  return Uint8Array.from(nalus.flatMap((nalu) => [0x00, 0x00, 0x00, 0x01, ...nalu]));
+}
+
+// h264: 0x06 SEI, 0x65 IDR slice, 0x67 SPS. h265: 0x26 IDR_W_RADL slice.
+// An h264 SEI byte (0x06) reads as h265 TSA_R, which is why the SEI leads here.
+// findNALUIndices stops at stream.length - 3. A trailing NALU with fewer than 4 payload
+// bytes is never indexed, so its fixture tests nothing. Interior NALUs are fine at any
+// length, because the next start code ends them.
+const h264KeyFrame = annexB([0x06, 0x11, 0x22], [0x65, 0x33, 0x44, 0x55, 0x66]);
+const h265KeyFrame = annexB([0x26, 0x11, 0x22], [0x02, 0x33, 0x44, 0x55, 0x66]);
+const noSliceFrame = annexB([0x67, 0x11, 0x22, 0x33, 0x44, 0x55]);
+// 0x25 is a slice under both: h264 &0x1f = 5 IDR, h265 (b >> 1) & 0x3f = 18 BLA_W_RADL.
+const ambiguousFrame = annexB([0x25, 0x11, 0x22, 0x33, 0x44, 0x55]);
+
+describe('normalizeAnnexBStartCodes', () => {
+  it('rewrites 3-byte start codes to 4-byte start codes', () => {
+    // prettier-ignore
+    const input = new Uint8Array([
+      0x00, 0x00, 0x01, 0x67, 0x64, // SPS
+      0x00, 0x00, 0x01, 0x68, 0xee, // PPS
+      0x00, 0x00, 0x01, 0x65, 0x88, // IDR slice
+    ]);
+    // prettier-ignore
+    const expected = new Uint8Array([
+      0x00, 0x00, 0x00, 0x01, 0x67, 0x64,
+      0x00, 0x00, 0x00, 0x01, 0x68, 0xee,
+      0x00, 0x00, 0x00, 0x01, 0x65, 0x88,
+    ]);
+
+    expect(normalizeAnnexBStartCodes(input)).toEqual(expected);
+  });
+
+  it('rewrites only the 3-byte start codes when mixed with 4-byte ones', () => {
+    // prettier-ignore
+    const input = new Uint8Array([
+      0x00, 0x00, 0x00, 0x01, 0x67, 0x64,
+      0x00, 0x00, 0x01, 0x68, 0xee,
+      0x00, 0x00, 0x00, 0x01, 0x06, 0x05,
+      0x00, 0x00, 0x01, 0x65, 0x88,
+    ]);
+    // prettier-ignore
+    const expected = new Uint8Array([
+      0x00, 0x00, 0x00, 0x01, 0x67, 0x64,
+      0x00, 0x00, 0x00, 0x01, 0x68, 0xee,
+      0x00, 0x00, 0x00, 0x01, 0x06, 0x05,
+      0x00, 0x00, 0x00, 0x01, 0x65, 0x88,
+    ]);
+
+    expect(normalizeAnnexBStartCodes(input)).toEqual(expected);
+  });
+
+  it('returns the same instance when all start codes are already 4-byte', () => {
+    // prettier-ignore
+    const input = new Uint8Array([
+      0x00, 0x00, 0x00, 0x01, 0x67, 0x64,
+      0x00, 0x00, 0x00, 0x01, 0x65, 0x88,
+    ]);
+
+    expect(normalizeAnnexBStartCodes(input)).toBe(input);
+  });
+
+  it('does not add a zero when a start code already has extra leading zeros', () => {
+    const input = new Uint8Array([0x00, 0x00, 0x00, 0x00, 0x01, 0x65, 0x88]);
+
+    expect(normalizeAnnexBStartCodes(input)).toBe(input);
+  });
+});
+
+describe('processNALUsForEncryption', () => {
+  it('slices at the h264 slice NALU even when seeded with h265', () => {
+    // The whole point: the h265 seed must not win, or the offset lands at 6 (inside the
+    // SEI) and the slice header gets encrypted with no error raised.
+    expect(processNALUsForEncryption(h264KeyFrame, 'h265')).toEqual({
+      unencryptedBytes: 13,
+      detectedCodec: 'h264',
+      requiresNALUProcessing: true,
+    });
+  });
+
+  it('reads an h264 frame whose SEI precedes the slice as h264', () => {
+    expect(processNALUsForEncryption(h264KeyFrame)).toEqual({
+      unencryptedBytes: 13,
+      detectedCodec: 'h264',
+      requiresNALUProcessing: true,
+    });
+  });
+
+  it('still reads an h265 frame as h265', () => {
+    expect(processNALUsForEncryption(h265KeyFrame)).toEqual({
+      unencryptedBytes: 6,
+      detectedCodec: 'h265',
+      requiresNALUProcessing: true,
+    });
+    expect(processNALUsForEncryption(h265KeyFrame, 'h264')).toEqual({
+      unencryptedBytes: 6,
+      detectedCodec: 'h265',
+      requiresNALUProcessing: true,
+    });
+  });
+
+  it('resolves a byte that is a slice under both codecs as h264', () => {
+    // Only the h264-first ordering decides this one. Swap the two loops back and it reads h265.
+    expect(processNALUsForEncryption(ambiguousFrame)).toEqual({
+      unencryptedBytes: 6,
+      detectedCodec: 'h264',
+      requiresNALUProcessing: true,
+    });
+  });
+
+  it('throws with a seed, and returns quietly without one, when no slice NALU is present', () => {
+    expect(() => processNALUsForEncryption(noSliceFrame, 'h264')).toThrow('Could not find NALU');
+    expect(processNALUsForEncryption(noSliceFrame)).toEqual({
+      unencryptedBytes: 0,
+      detectedCodec: 'unknown',
+      requiresNALUProcessing: false,
+    });
+  });
+});

package/src/e2ee/worker/naluUtils.ts

@@ -202,8 +202,14 @@ export interface NALUProcessingResult {
  * @returns Detected codec type
  */
 function detectCodecFromNALUs(data: NonSharedUint8Array, naluIndices: number[]): DetectedCodec {
+  // h264 runs to completion first. Several non-slice h264 NALUs read as h265 slice types
+  // (AUD 0x09 -> STSA_N, SEI 0x06 -> TSA_R), so interleaving the two checks lets one of
+  // those outrank the real h264 slice that follows it. The reverse cannot happen: h265
+  // header bytes are even at nuh_layer_id 0, so they never mask to an h264 slice type.
   for (const naluIndex of naluIndices) {
     if (isH264SliceNALU(parseH264NALUType(data[naluIndex]))) return 'h264';
+  }
+  for (const naluIndex of naluIndices) {
     if (isH265SliceNALU(parseH265NALUType(data[naluIndex]))) return 'h265';
   }
   return 'unknown';
@@ -306,7 +312,7 @@ function findNALUIndices(stream: NonSharedUint8Array): number[] {
 /**
  * Process NALU data for frame encryption, detecting codec and finding unencrypted bytes
  * @param data Frame data
- * @param knownCodec Known codec from other sources (optional)
+ * @param knownCodec Codec from other sources, used only when the bytes are inconclusive (optional)
  * @returns NALU processing result
  */
 export function processNALUsForEncryption(
@@ -314,7 +320,16 @@ export function processNALUsForEncryption(
   knownCodec?: 'h264' | 'h265',
 ): NALUProcessingResult {
   const naluIndices = findNALUIndices(data);
-  const detectedCodec = knownCodec ?? detectCodecFromNALUs(data, naluIndices);
+  // The bytes outrank knownCodec, which can be the codec the client asked for rather than the
+  // one that was negotiated. Parsing the wrong one mis-slices without raising anything: the
+  // offset lands before the real slice header, so the header itself gets encrypted.
+  // That holds in one direction only. An h264 access unit carrying no slice NALU but carrying
+  // an AUD or SEI that reads as an h265 slice now slices there instead of throwing. No real
+  // WebRTC access unit can reach it, because every one of them carries a type 1 or 5 slice.
+  let detectedCodec = detectCodecFromNALUs(data, naluIndices);
+  if (detectedCodec === 'unknown') {
+    detectedCodec = knownCodec ?? 'unknown';
+  }
 
   if (detectedCodec === 'unknown') {
     return { unencryptedBytes: 0, detectedCodec, requiresNALUProcessing: false };
@@ -327,3 +342,43 @@ export function processNALUsForEncryption(
 
   return { unencryptedBytes, detectedCodec, requiresNALUProcessing: true };
 }
+
+/**
+ * Rewrite every 3-byte Annex B start code (00 00 01) in `data` to the 4-byte form (00 00 00 01).
+ *
+ * WebRTC's H.264/H.265 depacketizer always reconstructs NALUs with 4-byte start codes, regardless
+ * of what the encoder emitted. Some encoders (e.g. Intel Quick Sync) emit 3-byte start codes, so
+ * the unencrypted header a receiver sees would differ from what the sender authenticated as
+ * AES-GCM additionalData. Normalizing on the sender makes both sides agree.
+ *
+ * Returns `data` unchanged (same instance) when it contains no 3-byte start codes.
+ */
+export function normalizeAnnexBStartCodes(data: NonSharedUint8Array): NonSharedUint8Array {
+  const shortStartCodes: number[] = [];
+  for (let i = 0; i + 2 < data.length;) {
+    if (data[i + 2] === 0) {
+      i += 1;
+      continue;
+    }
+    if (data[i + 2] === 1 && data[i] === 0 && data[i + 1] === 0 && (i === 0 || data[i - 1] !== 0)) {
+      shortStartCodes.push(i);
+    }
+    // data[i + 2] is non-zero, so no start code can begin at i + 1 or i + 2 either.
+    i += 3;
+  }
+  if (shortStartCodes.length === 0) {
+    return data;
+  }
+
+  // The new array is zero-filled, so the one-byte gap left before each short start code becomes
+  // the inserted leading 00.
+  const result = new Uint8Array(data.length + shortStartCodes.length);
+  let from = 0;
+  for (let inserted = 0; inserted < shortStartCodes.length; inserted++) {
+    const to = shortStartCodes[inserted];
+    result.set(data.subarray(from, to), from + inserted);
+    from = to;
+  }
+  result.set(data.subarray(from), from + shortStartCodes.length);
+  return result;
+}

package/src/index.ts

@@ -56,6 +56,7 @@ import {
   isSVCCodec,
   isVideoCodec,
   isVideoTrack,
+  releaseEmptyAudioStreamTrack,
   supportsAV1,
   supportsAdaptiveStream,
   supportsAudioOutputSelection,
@@ -145,6 +146,7 @@ export {
   getEmptyVideoStreamTrack,
   getLogger,
   isBrowserSupported,
+  releaseEmptyAudioStreamTrack,
   setLogExtension,
   setLogLevel,
   supportsAV1,

package/src/options.ts

@@ -166,6 +166,22 @@ export interface InternalRoomConnectOptions {
    */
   rtcConfig?: RTCConfiguration;
 
+  /**
+   * when the server runs ICE lite, asks it to use a full ICE agent for this participant.
+   * this is a server-side setting that the server may ignore, defaults to false
+   */
+  disableIceLite: boolean;
+
+  /**
+   * whether the SFU may pause subscribed video tracks when this participant's downstream
+   * connection is congested, so that other tracks can continue to stream smoothly.
+   * paused tracks resume automatically when conditions allow, and `RoomEvent.TrackStreamStateChanged`
+   * fires on each transition.
+   * when left undefined, the server's default applies (the project setting on LiveKit Cloud,
+   * or `rtc.congestion_control.allow_pause` when self-hosting)
+   */
+  subscriberAllowPause?: boolean;
+
   /** specifies how often an initial join connection is allowed to retry (only applicable if server is not reachable) */
   maxRetries: number;