livekit-client 2.18.8 → 2.18.10

package/src/room/RTCEngine.ts

@@ -54,9 +54,14 @@ import {
   toProtoSessionDescription,
 } from '../api/SignalClient';
 import type { BaseE2EEManager } from '../e2ee/E2eeManager';
-import { asEncryptablePacket } from '../e2ee/utils';
+import { asEncryptablePacket, isInsertableStreamSupported } from '../e2ee/utils';
 import log, { LoggerNames, getLogger } from '../logger';
 import type { InternalRoomOptions } from '../options';
+import {
+  hasPacketTrailerPublishOptions,
+  isPacketTrailerSupported,
+  shouldUsePacketTrailerScriptTransform,
+} from '../packetTrailer/utils';
 import TypedPromise from '../utils/TypedPromise';
 import { DataPacketBuffer } from '../utils/dataPacketBuffer';
 import { TTLMap } from '../utils/ttlmap';
@@ -762,7 +767,14 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
   ): RTCConfiguration {
     const rtcConfig = { ...this.rtcConfig };
 
-    if (this.signalOpts?.e2eeEnabled) {
+    // E2EE and packet trailer extraction both rely on encoded frame transforms.
+    // Only opt into the createEncodedStreams flavor when that path will be
+    // used; RTCRtpScriptTransform does not need the PeerConnection flag.
+    const needsInsertableStreams =
+      this.signalOpts?.e2eeEnabled ||
+      (this.options.packetTrailer?.worker && !shouldUsePacketTrailerScriptTransform());
+
+    if (needsInsertableStreams && isInsertableStreamSupported()) {
       this.log.debug('E2EE - setting up transports with insertable streams');
       //  this makes sure that no data is sent before the transforms are ready
       // @ts-ignore
@@ -940,8 +952,8 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
           return;
         }
         const decryptedData = await this.e2eeManager?.handleEncryptedData(
-          dp.value.value.encryptedValue,
-          dp.value.value.iv,
+          dp.value.value.encryptedValue as NonSharedUint8Array,
+          dp.value.value.iv as NonSharedUint8Array,
           dp.participantIdentity,
           dp.value.value.keyIndex,
         );
@@ -1004,16 +1016,17 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
     opts: TrackPublishOptions,
     encodings?: RTCRtpEncodingParameters[],
   ) {
+    let sender: RTCRtpSender;
     if (supportsTransceiver()) {
-      const sender = await this.createTransceiverRTCRtpSender(track, opts, encodings);
-      return sender;
-    }
-    if (supportsAddTrack()) {
+      sender = await this.createTransceiverRTCRtpSender(track, opts, encodings);
+    } else if (supportsAddTrack()) {
       this.log.warn('using add-track fallback');
-      const sender = await this.createRTCRtpSender(track.mediaStreamTrack);
-      return sender;
+      sender = await this.createRTCRtpSender(track.mediaStreamTrack);
+    } else {
+      throw new UnexpectedConnectionState('Required webRTC APIs not supported on this device');
     }
-    throw new UnexpectedConnectionState('Required webRTC APIs not supported on this device');
+    this.setupPacketTrailerSender(sender, opts);
+    return sender;
   }
 
   async createSimulcastSender(
@@ -1022,16 +1035,67 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
     opts: TrackPublishOptions,
     encodings?: RTCRtpEncodingParameters[],
   ) {
-    // store RTCRtpSender
+    let sender: RTCRtpSender | undefined;
     if (supportsTransceiver()) {
-      return this.createSimulcastTransceiverSender(track, simulcastTrack, opts, encodings);
-    }
-    if (supportsAddTrack()) {
+      sender = await this.createSimulcastTransceiverSender(track, simulcastTrack, opts, encodings);
+    } else if (supportsAddTrack()) {
       this.log.debug('using add-track fallback');
-      return this.createRTCRtpSender(track.mediaStreamTrack);
+      sender = await this.createRTCRtpSender(track.mediaStreamTrack);
+    } else {
+      throw new UnexpectedConnectionState('Cannot stream on this device');
     }
+    if (sender) {
+      this.setupPacketTrailerSender(sender, opts);
+    }
+    return sender;
+  }
 
-    throw new UnexpectedConnectionState('Cannot stream on this device');
+  private setupPacketTrailerSender(sender: RTCRtpSender, opts: TrackPublishOptions = {}) {
+    if (!this.options.packetTrailer?.worker || this.signalOpts?.e2eeEnabled) {
+      return;
+    }
+
+    const packetTrailer = opts.packetTrailer;
+    const hasPacketTrailer = hasPacketTrailerPublishOptions(packetTrailer);
+
+    if (shouldUsePacketTrailerScriptTransform()) {
+      if (hasPacketTrailer) {
+        // @ts-ignore
+        sender.transform = new RTCRtpScriptTransform(this.options.packetTrailer.worker, {
+          kind: 'encode',
+          packetTrailer,
+        });
+      }
+      return;
+    }
+
+    if (
+      !isPacketTrailerSupported(this.options.packetTrailer) ||
+      !('createEncodedStreams' in sender)
+    ) {
+      if (hasPacketTrailer) {
+        this.log.warn('packet trailer transform not supported; skipping write', this.logContext);
+      }
+      return;
+    }
+
+    // @ts-ignore
+    const { readable, writable } = sender.createEncodedStreams();
+    if (hasPacketTrailer) {
+      this.options.packetTrailer.worker.postMessage(
+        {
+          kind: 'encode',
+          data: {
+            readableStream: readable,
+            writableStream: writable,
+            packetTrailer,
+          },
+        },
+        [readable, writable],
+      );
+    } else {
+      readable.pipeTo(writable);
+    }
   }
 
   private async createTransceiverRTCRtpSender(
@@ -1447,7 +1511,9 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
     if (this.e2eeManager && this.e2eeManager.isDataChannelEncryptionEnabled) {
       const encryptablePacket = asEncryptablePacket(packet);
       if (encryptablePacket) {
-        const encryptedData = await this.e2eeManager.encryptData(encryptablePacket.toBinary());
+        const encryptedData = await this.e2eeManager.encryptData(
+          encryptablePacket.toBinary() as NonSharedUint8Array,
+        );
         packet.value = {
           case: 'encryptedPacket',
           value: new EncryptedPacket({
@@ -1464,7 +1530,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
       this.reliableDataSequence += 1;
     }
 
-    const msg = packet.toBinary();
+    const msg = packet.toBinary() as Uint8Array<ArrayBuffer>;
 
     switch (kind) {
       case DataChannelKind.LOSSY:
@@ -1491,7 +1557,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
 
   /* @internal */
   async sendLossyBytes(
-    bytes: Uint8Array,
+    bytes: NonSharedUint8Array,
     kind: Exclude<DataChannelKind, DataChannelKind.RELIABLE>,
     bufferStatusLowBehavior: 'drop' | 'wait' = 'drop',
   ) {

package/src/room/Room.test.ts

@@ -1,5 +1,7 @@
-import { describe, expect, it } from 'vitest';
+import { ClientInfo_Capability, JoinResponse } from '@livekit/protocol';
+import { describe, expect, it, vi } from 'vitest';
 import Room from './Room';
+import { roomConnectOptionDefaults, roomOptionDefaults } from './defaults';
 import { RoomEvent } from './events';
 
 describe('Active device switch', () => {
@@ -28,3 +30,62 @@ describe('Active device switch', () => {
     expect(kind).toBe('audioinput');
   });
 });
+
+describe('Room signaling options', () => {
+  it('advertises packet trailer capability when E2EE can handle trailers', async () => {
+    const room = new Room();
+    const join = vi.fn().mockResolvedValue({
+      joinResponse: new JoinResponse({
+        room: { name: 'test-room', sid: 'room-sid' },
+        participant: { sid: 'participant-sid', identity: 'test-user' },
+      }),
+      serverInfo: { version: '1.0.0' },
+    });
+    const engine = { join };
+
+    (
+      room as unknown as {
+        e2eeManager: unknown;
+        connectSignal: (
+          url: string,
+          token: string,
+          engine: unknown,
+          connectOptions: typeof roomConnectOptionDefaults,
+          roomOptions: typeof roomOptionDefaults,
+          abortController: AbortController,
+        ) => Promise<JoinResponse>;
+      }
+    ).e2eeManager = {};
+
+    await (
+      room as unknown as {
+        connectSignal: (
+          url: string,
+          token: string,
+          engine: unknown,
+          connectOptions: typeof roomConnectOptionDefaults,
+          roomOptions: typeof roomOptionDefaults,
+          abortController: AbortController,
+        ) => Promise<JoinResponse>;
+      }
+    ).connectSignal(
+      'wss://test.livekit.io',
+      'test-token',
+      engine,
+      roomConnectOptionDefaults,
+      roomOptionDefaults,
+      new AbortController(),
+    );
+
+    expect(join).toHaveBeenCalledWith(
+      'wss://test.livekit.io',
+      'test-token',
+      expect.objectContaining({
+        clientInfoCapabilities: [ClientInfo_Capability.CAP_PACKET_TRAILER],
+        e2eeEnabled: true,
+      }),
+      expect.any(AbortSignal),
+      false,
+    );
+  });
+});

package/src/room/Room.ts

@@ -1,6 +1,7 @@
 import { Mutex } from '@livekit/mutex';
 import {
   ChatMessage as ChatMessageModel,
+  ClientInfo_Capability,
   ConnectionQualityUpdate,
   type DataPacket,
   DataPacket_Kind,
@@ -43,6 +44,8 @@ import type {
   RoomConnectOptions,
   RoomOptions,
 } from '../options';
+import { PacketTrailerManager } from '../packetTrailer/PacketTrailerManager';
+import { isPacketTrailerSupported } from '../packetTrailer/utils';
 import TypedPromise from '../utils/TypedPromise';
 import { getBrowser } from '../utils/browserParser';
 import { BackOffStrategy } from './BackOffStrategy';
@@ -187,6 +190,8 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
 
   private e2eeManager: BaseE2EEManager | undefined;
 
+  private packetTrailerManager: PacketTrailerManager | undefined;
+
   private e2eeStateMutex: Mutex = new Mutex();
 
   private connectionReconcileInterval?: ReturnType<typeof setInterval>;
@@ -307,6 +312,8 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       this.outgoingDataTrackManager,
     );
 
+    this.setupPacketTrailer();
+
     if (this.options.e2ee || this.options.encryption) {
       this.setupE2EE();
     }
@@ -465,6 +472,13 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     }
   }
 
+  private setupPacketTrailer() {
+    // The manager is always created so tracks that advertise packet trailer
+    // features can be wired up when the app passes a packet trailer worker.
+    this.packetTrailerManager = new PacketTrailerManager(this.options.packetTrailer);
+    this.packetTrailerManager.setup(this);
+  }
+
   private get logContext() {
     return {
       room: this.name,
@@ -914,6 +928,10 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
         autoSubscribe: connectOptions.autoSubscribe,
         adaptiveStream:
           typeof roomOptions.adaptiveStream === 'object' ? true : roomOptions.adaptiveStream,
+        clientInfoCapabilities:
+          isPacketTrailerSupported(roomOptions.packetTrailer) || !!this.e2eeManager
+            ? [ClientInfo_Capability.CAP_PACKET_TRAILER]
+            : undefined,
         maxRetries: connectOptions.maxRetries,
         e2eeEnabled: !!this.e2eeManager,
         websocketTimeout: connectOptions.websocketTimeout,
@@ -946,7 +964,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
 
     if (this.e2eeManager) {
       try {
-        this.e2eeManager.setSifTrailer(joinResponse.sifTrailer);
+        this.e2eeManager.setSifTrailer(joinResponse.sifTrailer as NonSharedUint8Array);
       } catch (e: any) {
         this.log.error(e instanceof Error ? e.message : 'Could not set SifTrailer', {
           error: e,
@@ -1825,7 +1843,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
         this.handleParticipantDisconnected(info.identity, remoteParticipant);
       } else {
         // create participant if doesn't exist
-        remoteParticipant = this.getOrCreateParticipant(info.identity, info);
+        this.getOrCreateParticipant(info.identity, info);
       }
     }
 
@@ -2018,7 +2036,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
   ) => {
     this.emit(
       RoomEvent.DataReceived,
-      userPacket.payload,
+      userPacket.payload as NonSharedUint8Array,
       participant,
       kind,
       userPacket.topic,
@@ -2026,7 +2044,12 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     );
 
     // also emit on the participant
-    participant?.emit(ParticipantEvent.DataReceived, userPacket.payload, kind, encryptionType);
+    participant?.emit(
+      ParticipantEvent.DataReceived,
+      userPacket.payload as NonSharedUint8Array,
+      kind,
+      encryptionType,
+    );
   };
 
   private handleSipDtmf = (participant: RemoteParticipant | undefined, dtmf: SipDTMF) => {
@@ -2874,7 +2897,7 @@ export type RoomEventCallbacks = {
   activeSpeakersChanged: (speakers: Array<Participant>) => void;
   roomMetadataChanged: (metadata: string) => void;
   dataReceived: (
-    payload: Uint8Array,
+    payload: NonSharedUint8Array,
     participant?: RemoteParticipant,
     kind?: DataPacket_Kind,
     topic?: string,

package/src/room/data-track/LocalDataTrack.ts

@@ -4,7 +4,7 @@ import { type DataTrackFrame, DataTrackFrameInternal } from './frame';
 import type { DataTrackHandle } from './handle';
 import type OutgoingDataTrackManager from './outgoing/OutgoingDataTrackManager';
 import { DataTrackPushFrameError } from './outgoing/errors';
-import type { DataTrackOptions } from './outgoing/types';
+import type { DataTrackOptions, EventPacketsFlushedChange } from './outgoing/types';
 import {
   DataTrackSymbol,
   type IDataTrack,
@@ -32,6 +32,8 @@ export default class LocalDataTrack implements ILocalTrack, IDataTrack {
   /** Resolves once the data track has sent all pending packets the rtc data channel buffer. */
   protected flushedFuture = new Future<void, never>();
 
+  protected isFlushed = true;
+
   /** @internal */
   constructor(options: DataTrackOptions, manager: OutgoingDataTrackManager) {
     this.options = options;
@@ -39,7 +41,7 @@ export default class LocalDataTrack implements ILocalTrack, IDataTrack {
 
     this.log = getLogger(LoggerNames.DataTracks);
 
-    this.manager.on('packetsFlushed', this.handleManagerPacketsFlushed);
+    this.manager.on('packetsFlushedChange', this.handleManagerPacketsFlushedChange);
     this.manager.on('reset', this.handleManagerReset);
   }
 
@@ -47,15 +49,18 @@ export default class LocalDataTrack implements ILocalTrack, IDataTrack {
     // When the associated manager resets, mark any in flight flushes as complete
     // There's nothing actionable a user can do to get these to complete so no
     // error is being thrown.
-    this.handleManagerPacketsFlushed();
+    this.flushedFuture.resolve?.();
 
-    this.manager.off('packetsFlushed', this.handleManagerReset);
+    this.manager.off('packetsFlushedChange', this.handleManagerPacketsFlushedChange);
     this.manager.off('reset', this.handleManagerReset);
   };
 
-  private handleManagerPacketsFlushed = () => {
-    this.flushedFuture.resolve?.();
-    this.flushedFuture = new Future();
+  private handleManagerPacketsFlushedChange = (event: EventPacketsFlushedChange) => {
+    this.isFlushed = event.isFlushed;
+    if (event.isFlushed) {
+      this.flushedFuture.resolve?.();
+      this.flushedFuture = new Future();
+    }
   };
 
   /** @internal */
@@ -152,6 +157,9 @@ export default class LocalDataTrack implements ILocalTrack, IDataTrack {
    * ```
    **/
   async flush(): Promise<void> {
+    if (this.isFlushed) {
+      return;
+    }
     return this.flushedFuture.promise;
   }
 

package/src/room/data-track/RemoteDataTrack.ts

@@ -7,7 +7,7 @@ import {
   type IRemoteTrack,
   TrackSymbol,
 } from './track-interfaces';
-import { type DataTrackInfo } from './types';
+import { type DataTrackInfo, type RemoteDataTrackPipelineOptions } from './types';
 
 type RemoteDataTrackOptions = {
   publisherIdentity: Participant['identity'];
@@ -80,4 +80,11 @@ export default class RemoteDataTrack implements IRemoteTrack, IDataTrack {
       throw err;
     }
   }
+
+  /** Configure how incoming frames for this track are processed before they are handed out to
+   * subscribers (the "pipeline"). These options apply to all current and future subscriptions
+   * of this track, and may be set at any time. */
+  setPipelineOptions(options: RemoteDataTrackPipelineOptions): void {
+    this.manager.setPipelineOptions(this.info.sid, options);
+  }
 }