livekit-client 2.18.9 → 2.19.0

package/src/packetTrailer/worker/packetTrailer.worker.ts

@@ -0,0 +1,155 @@
+import { appendPacketTrailerToEncodedFrame, processPacketTrailer } from '../packetTrailer';
+import type {
+  PTMetadataMessage,
+  PTScriptTransformOptions,
+  PTWorkerMessage,
+  PacketTrailerPublishOptions,
+} from '../types';
+import { hasPacketTrailerPublishOptions } from '../utils';
+
+/**
+ * Holds the trackId currently associated with a pipeline. A mutable
+ * wrapper is used so the transform closure always reads the latest
+ * trackId after a receiver gets re-bound to a new track.
+ */
+interface PipelineState {
+  trackId: string;
+  hasPacketTrailer: boolean;
+}
+
+const pipelines = new Map<string, PipelineState>();
+
+onmessage = (ev: MessageEvent<PTWorkerMessage>) => {
+  const msg = ev.data;
+
+  switch (msg.kind) {
+    case 'init':
+      postMessage({ kind: 'initAck' });
+      break;
+
+    case 'decode':
+      setupDecodeTransform(
+        msg.data.readableStream,
+        msg.data.writableStream,
+        msg.data.trackId,
+        msg.data.hasPacketTrailer,
+      );
+      break;
+
+    case 'encode':
+      setupEncodeTransform(
+        msg.data.readableStream,
+        msg.data.writableStream,
+        msg.data.packetTrailer,
+      );
+      break;
+
+    case 'updateTrackId':
+      updateTrackId(msg.data.oldTrackId, msg.data.newTrackId, msg.data.hasPacketTrailer);
+      break;
+
+    default:
+      break;
+  }
+};
+
+function setupDecodeTransform(
+  readable: ReadableStream,
+  writable: WritableStream,
+  trackId: string,
+  hasPacketTrailer: boolean,
+) {
+  const state: PipelineState = { trackId, hasPacketTrailer };
+  pipelines.set(trackId, state);
+
+  const transform = new TransformStream({
+    transform(
+      frame: RTCEncodedVideoFrame,
+      controller: TransformStreamDefaultController<RTCEncodedVideoFrame>,
+    ) {
+      try {
+        if (state.hasPacketTrailer) {
+          const result = processPacketTrailer(frame, state.trackId);
+          if (result.data) {
+            frame.data = result.data;
+          }
+          if (result.payload) {
+            const msg: PTMetadataMessage = { kind: 'metadata', data: result.payload };
+            postMessage(msg);
+          }
+        }
+      } catch {
+        // Never drop frames on trailer-extraction failure — pass through so
+        // video keeps decoding even if metadata is lost for this frame.
+      }
+      controller.enqueue(frame);
+    },
+  });
+
+  readable
+    .pipeThrough(transform)
+    .pipeTo(writable)
+    .catch(() => {
+      pipelines.delete(state.trackId);
+    });
+}
+
+function setupEncodeTransform(
+  readable: ReadableStream,
+  writable: WritableStream,
+  packetTrailer?: PacketTrailerPublishOptions,
+) {
+  if (!hasPacketTrailerPublishOptions(packetTrailer)) {
+    readable.pipeTo(writable).catch(() => {});
+    return;
+  }
+
+  let frameId = 0;
+  const transform = new TransformStream({
+    transform(
+      frame: RTCEncodedVideoFrame,
+      controller: TransformStreamDefaultController<RTCEncodedVideoFrame>,
+    ) {
+      try {
+        if (packetTrailer?.frameId) {
+          frameId = frameId === 0xffffffff ? 1 : frameId + 1;
+        }
+        appendPacketTrailerToEncodedFrame(frame, packetTrailer, frameId);
+      } catch {
+        // Never drop frames on trailer-write failure.
+      }
+      controller.enqueue(frame);
+    },
+  });
+
+  readable
+    .pipeThrough(transform)
+    .pipeTo(writable)
+    .catch(() => {});
+}
+
+function updateTrackId(oldTrackId: string, newTrackId: string, hasPacketTrailer: boolean) {
+  const state = pipelines.get(oldTrackId);
+  if (state) {
+    state.trackId = newTrackId;
+    state.hasPacketTrailer = hasPacketTrailer;
+    pipelines.delete(oldTrackId);
+    pipelines.set(newTrackId, state);
+  }
+}
+
+// Operations using RTCRtpScriptTransform.
+// @ts-ignore
+if (self.RTCTransformEvent) {
+  // @ts-ignore
+  self.onrtctransform = (event: RTCTransformEvent) => {
+    // @ts-ignore
+    const transformer = event.transformer;
+    const options = transformer.options as PTScriptTransformOptions;
+    if (options.kind === 'encode') {
+      setupEncodeTransform(transformer.readable, transformer.writable, options.packetTrailer);
+    } else {
+      setupDecodeTransform(transformer.readable, transformer.writable, options.trackId, true);
+    }
+  };
+}

package/src/packetTrailer/worker/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "extends": "../../../tsconfig.json",
+  "compilerOptions": {
+    "lib": [
+      "DOM",
+      "DOM.Iterable",
+      "ES2017",
+      "ES2018.Promise",
+      "WebWorker",
+      "ES2021.WeakRef",
+      "DOM.AsyncIterable"
+    ]
+  }
+}

package/src/room/BackOffStrategy.test.ts

@@ -6,7 +6,7 @@ vi.mock('./utils', async () => {
   const actual = await vi.importActual('./utils');
   return {
     ...actual,
-    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+
     sleep: vi.fn((ms: number) => Promise.resolve()),
     extractProjectFromUrl: vi.fn((url: URL) => {
       // @ts-ignore

package/src/room/RTCEngine.test.ts

@@ -0,0 +1,219 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import RTCEngine from './RTCEngine';
+import { roomOptionDefaults } from './defaults';
+
+describe('RTCEngine', () => {
+  const originalRTCRtpSender = window.RTCRtpSender;
+  const originalRTCRtpScriptTransform = (window as unknown as { RTCRtpScriptTransform?: unknown })
+    .RTCRtpScriptTransform;
+  const originalUserAgent = navigator.userAgent;
+
+  afterEach(() => {
+    Object.defineProperty(window, 'RTCRtpSender', {
+      configurable: true,
+      value: originalRTCRtpSender,
+      writable: true,
+    });
+    Object.defineProperty(window, 'RTCRtpScriptTransform', {
+      configurable: true,
+      value: originalRTCRtpScriptTransform,
+      writable: true,
+    });
+    Object.defineProperty(window.navigator, 'userAgent', {
+      configurable: true,
+      value: originalUserAgent,
+    });
+  });
+
+  function stubInsertableStreamsSupport() {
+    class MockRTCRtpSender {
+      createEncodedStreams() {}
+    }
+
+    Object.defineProperty(window, 'RTCRtpSender', {
+      configurable: true,
+      value: MockRTCRtpSender,
+      writable: true,
+    });
+  }
+
+  function stubScriptTransformSupport() {
+    Object.defineProperty(window, 'RTCRtpScriptTransform', {
+      configurable: true,
+      value: class MockRTCRtpScriptTransform {},
+      writable: true,
+    });
+    Object.defineProperty(window.navigator, 'userAgent', {
+      configurable: true,
+      value:
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15',
+    });
+  }
+
+  function makeRTCConfiguration(engine: RTCEngine) {
+    return (
+      engine as unknown as { makeRTCConfiguration: () => RTCConfiguration }
+    ).makeRTCConfiguration();
+  }
+
+  function setupPacketTrailerSender(engine: RTCEngine, sender: RTCRtpSender, opts = {}) {
+    (
+      engine as unknown as {
+        setupPacketTrailerSender: (sender: RTCRtpSender, opts?: unknown) => void;
+      }
+    ).setupPacketTrailerSender(sender, opts);
+  }
+
+  it('does not enable encoded insertable streams without E2EE or a packet trailer worker', () => {
+    stubInsertableStreamsSupport();
+
+    const engine = new RTCEngine(roomOptionDefaults);
+
+    expect(makeRTCConfiguration(engine).encodedInsertableStreams).toBeUndefined();
+  });
+
+  it('enables encoded insertable streams when a packet trailer worker is configured', () => {
+    stubInsertableStreamsSupport();
+
+    const engine = new RTCEngine({
+      ...roomOptionDefaults,
+      packetTrailer: { worker: {} as Worker },
+    });
+
+    expect(makeRTCConfiguration(engine).encodedInsertableStreams).toBe(true);
+  });
+
+  it('does not enable encoded insertable streams for packet trailers when script transforms are supported', () => {
+    stubInsertableStreamsSupport();
+    stubScriptTransformSupport();
+
+    const engine = new RTCEngine({
+      ...roomOptionDefaults,
+      packetTrailer: { worker: {} as Worker },
+    });
+
+    expect(makeRTCConfiguration(engine).encodedInsertableStreams).toBeUndefined();
+  });
+
+  it('enables encoded insertable streams for E2EE', () => {
+    stubInsertableStreamsSupport();
+
+    const engine = new RTCEngine(roomOptionDefaults);
+    (
+      engine as unknown as {
+        signalOpts: {
+          autoSubscribe: boolean;
+          maxRetries: number;
+          e2eeEnabled: boolean;
+          websocketTimeout: number;
+        };
+      }
+    ).signalOpts = {
+      autoSubscribe: true,
+      maxRetries: 1,
+      e2eeEnabled: true,
+      websocketTimeout: 15_000,
+    };
+
+    expect(makeRTCConfiguration(engine).encodedInsertableStreams).toBe(true);
+  });
+
+  it('does not create sender encoded streams when packetTrailer has no worker', () => {
+    const engine = new RTCEngine({
+      ...roomOptionDefaults,
+      packetTrailer: {} as never,
+    });
+    const createEncodedStreams = vi.fn();
+    const sender = {
+      createEncodedStreams,
+    } as unknown as RTCRtpSender;
+
+    setupPacketTrailerSender(engine, sender);
+
+    expect(createEncodedStreams).not.toHaveBeenCalled();
+  });
+
+  it('does not create sender passthrough streams for packet trailers when script transforms are supported', () => {
+    stubScriptTransformSupport();
+
+    const engine = new RTCEngine({
+      ...roomOptionDefaults,
+      packetTrailer: { worker: {} as Worker },
+    });
+    const createEncodedStreams = vi.fn();
+    const sender = {
+      createEncodedStreams,
+    } as unknown as RTCRtpSender;
+
+    setupPacketTrailerSender(engine, sender);
+
+    expect(createEncodedStreams).not.toHaveBeenCalled();
+  });
+
+  it('posts sender encode streams to the packet trailer worker when write features are enabled', () => {
+    stubInsertableStreamsSupport();
+
+    const worker = { postMessage: vi.fn() } as unknown as Worker;
+    const engine = new RTCEngine({
+      ...roomOptionDefaults,
+      packetTrailer: { worker },
+    });
+    const readable = {} as ReadableStream;
+    const writable = {} as WritableStream;
+    const createEncodedStreams = vi.fn(() => ({ readable, writable }));
+    const sender = {
+      createEncodedStreams,
+    } as unknown as RTCRtpSender;
+
+    setupPacketTrailerSender(engine, sender, { packetTrailer: { timestamp: true, frameId: true } });
+
+    expect(createEncodedStreams).toHaveBeenCalledTimes(1);
+    expect(worker.postMessage).toHaveBeenCalledWith(
+      {
+        kind: 'encode',
+        data: {
+          readableStream: readable,
+          writableStream: writable,
+          packetTrailer: { timestamp: true, frameId: true },
+        },
+      },
+      [readable, writable],
+    );
+  });
+
+  it('uses RTCRtpScriptTransform for sender packet trailer writes when supported', () => {
+    stubScriptTransformSupport();
+
+    const transform = {};
+    const RTCRtpScriptTransform = vi.fn(() => transform);
+    Object.defineProperty(window, 'RTCRtpScriptTransform', {
+      configurable: true,
+      value: RTCRtpScriptTransform,
+      writable: true,
+    });
+    Object.defineProperty(globalThis, 'RTCRtpScriptTransform', {
+      configurable: true,
+      value: RTCRtpScriptTransform,
+      writable: true,
+    });
+
+    const worker = {} as Worker;
+    const engine = new RTCEngine({
+      ...roomOptionDefaults,
+      packetTrailer: { worker },
+    });
+    const createEncodedStreams = vi.fn();
+    const sender = {
+      createEncodedStreams,
+    } as unknown as RTCRtpSender;
+
+    setupPacketTrailerSender(engine, sender, { packetTrailer: { timestamp: true } });
+
+    expect(RTCRtpScriptTransform).toHaveBeenCalledWith(worker, {
+      kind: 'encode',
+      packetTrailer: { timestamp: true },
+    });
+    expect((sender as unknown as { transform: unknown }).transform).toBe(transform);
+    expect(createEncodedStreams).not.toHaveBeenCalled();
+  });
+});

package/src/room/RTCEngine.ts

@@ -26,7 +26,6 @@ import {
   Room as RoomModel,
   RoomMovedResponse,
   RpcAck,
-  RpcResponse,
   ServerInfo,
   SessionDescription,
   SignalTarget,
@@ -54,9 +53,14 @@ import {
   toProtoSessionDescription,
 } from '../api/SignalClient';
 import type { BaseE2EEManager } from '../e2ee/E2eeManager';
-import { asEncryptablePacket } from '../e2ee/utils';
+import { asEncryptablePacket, isInsertableStreamSupported } from '../e2ee/utils';
 import log, { LoggerNames, getLogger } from '../logger';
 import type { InternalRoomOptions } from '../options';
+import {
+  hasPacketTrailerPublishOptions,
+  isPacketTrailerSupported,
+  shouldUsePacketTrailerScriptTransform,
+} from '../packetTrailer/utils';
 import TypedPromise from '../utils/TypedPromise';
 import { DataPacketBuffer } from '../utils/dataPacketBuffer';
 import { TTLMap } from '../utils/ttlmap';
@@ -74,7 +78,6 @@ import {
   UnexpectedConnectionState,
 } from './errors';
 import { EngineEvent } from './events';
-import { RpcError } from './rpc';
 import CriticalTimers from './timers';
 import type LocalTrack from './track/LocalTrack';
 import type LocalTrackPublication from './track/LocalTrackPublication';
@@ -762,7 +765,14 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
   ): RTCConfiguration {
     const rtcConfig = { ...this.rtcConfig };
 
-    if (this.signalOpts?.e2eeEnabled) {
+    // E2EE and packet trailer extraction both rely on encoded frame transforms.
+    // Only opt into the createEncodedStreams flavor when that path will be
+    // used; RTCRtpScriptTransform does not need the PeerConnection flag.
+    const needsInsertableStreams =
+      this.signalOpts?.e2eeEnabled ||
+      (this.options.packetTrailer?.worker && !shouldUsePacketTrailerScriptTransform());
+
+    if (needsInsertableStreams && isInsertableStreamSupported()) {
       this.log.debug('E2EE - setting up transports with insertable streams');
       //  this makes sure that no data is sent before the transforms are ready
       // @ts-ignore
@@ -940,8 +950,8 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
           return;
         }
         const decryptedData = await this.e2eeManager?.handleEncryptedData(
-          dp.value.value.encryptedValue,
-          dp.value.value.iv,
+          dp.value.value.encryptedValue as NonSharedUint8Array,
+          dp.value.value.iv as NonSharedUint8Array,
           dp.participantIdentity,
           dp.value.value.keyIndex,
         );
@@ -1004,16 +1014,17 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
     opts: TrackPublishOptions,
     encodings?: RTCRtpEncodingParameters[],
   ) {
+    let sender: RTCRtpSender;
     if (supportsTransceiver()) {
-      const sender = await this.createTransceiverRTCRtpSender(track, opts, encodings);
-      return sender;
-    }
-    if (supportsAddTrack()) {
+      sender = await this.createTransceiverRTCRtpSender(track, opts, encodings);
+    } else if (supportsAddTrack()) {
       this.log.warn('using add-track fallback');
-      const sender = await this.createRTCRtpSender(track.mediaStreamTrack);
-      return sender;
+      sender = await this.createRTCRtpSender(track.mediaStreamTrack);
+    } else {
+      throw new UnexpectedConnectionState('Required webRTC APIs not supported on this device');
     }
-    throw new UnexpectedConnectionState('Required webRTC APIs not supported on this device');
+    this.setupPacketTrailerSender(sender, opts);
+    return sender;
   }
 
   async createSimulcastSender(
@@ -1022,16 +1033,67 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
     opts: TrackPublishOptions,
     encodings?: RTCRtpEncodingParameters[],
   ) {
-    // store RTCRtpSender
+    let sender: RTCRtpSender | undefined;
     if (supportsTransceiver()) {
-      return this.createSimulcastTransceiverSender(track, simulcastTrack, opts, encodings);
-    }
-    if (supportsAddTrack()) {
+      sender = await this.createSimulcastTransceiverSender(track, simulcastTrack, opts, encodings);
+    } else if (supportsAddTrack()) {
       this.log.debug('using add-track fallback');
-      return this.createRTCRtpSender(track.mediaStreamTrack);
+      sender = await this.createRTCRtpSender(track.mediaStreamTrack);
+    } else {
+      throw new UnexpectedConnectionState('Cannot stream on this device');
+    }
+    if (sender) {
+      this.setupPacketTrailerSender(sender, opts);
+    }
+    return sender;
+  }
+
+  private setupPacketTrailerSender(sender: RTCRtpSender, opts: TrackPublishOptions = {}) {
+    if (!this.options.packetTrailer?.worker || this.signalOpts?.e2eeEnabled) {
+      return;
     }
 
-    throw new UnexpectedConnectionState('Cannot stream on this device');
+    const packetTrailer = opts.packetTrailer;
+    const hasPacketTrailer = hasPacketTrailerPublishOptions(packetTrailer);
+
+    if (shouldUsePacketTrailerScriptTransform()) {
+      if (hasPacketTrailer) {
+        // @ts-ignore
+        sender.transform = new RTCRtpScriptTransform(this.options.packetTrailer.worker, {
+          kind: 'encode',
+          packetTrailer,
+        });
+      }
+      return;
+    }
+
+    if (
+      !isPacketTrailerSupported(this.options.packetTrailer) ||
+      !('createEncodedStreams' in sender)
+    ) {
+      if (hasPacketTrailer) {
+        this.log.warn('packet trailer transform not supported; skipping write', this.logContext);
+      }
+      return;
+    }
+
+    // @ts-ignore
+    const { readable, writable } = sender.createEncodedStreams();
+    if (hasPacketTrailer) {
+      this.options.packetTrailer.worker.postMessage(
+        {
+          kind: 'encode',
+          data: {
+            readableStream: readable,
+            writableStream: writable,
+            packetTrailer,
+          },
+        },
+        [readable, writable],
+      );
+    } else {
+      readable.pipeTo(writable);
+    }
   }
 
   private async createTransceiverRTCRtpSender(
@@ -1400,30 +1462,6 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
     });
   };
 
-  /** @internal */
-  async publishRpcResponse(
-    destinationIdentity: string,
-    requestId: string,
-    payload: string | null,
-    error: RpcError | null,
-  ) {
-    const packet = new DataPacket({
-      destinationIdentities: [destinationIdentity],
-      kind: DataPacket_Kind.RELIABLE,
-      value: {
-        case: 'rpcResponse',
-        value: new RpcResponse({
-          requestId,
-          value: error
-            ? { case: 'error', value: error.toProto() }
-            : { case: 'payload', value: payload ?? '' },
-        }),
-      },
-    });
-
-    await this.sendDataPacket(packet, DataChannelKind.RELIABLE);
-  }
-
   /** @internal */
   async publishRpcAck(destinationIdentity: string, requestId: string) {
     const packet = new DataPacket({
@@ -1447,7 +1485,9 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
     if (this.e2eeManager && this.e2eeManager.isDataChannelEncryptionEnabled) {
       const encryptablePacket = asEncryptablePacket(packet);
       if (encryptablePacket) {
-        const encryptedData = await this.e2eeManager.encryptData(encryptablePacket.toBinary());
+        const encryptedData = await this.e2eeManager.encryptData(
+          encryptablePacket.toBinary() as NonSharedUint8Array,
+        );
         packet.value = {
           case: 'encryptedPacket',
           value: new EncryptedPacket({
@@ -1464,7 +1504,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
       this.reliableDataSequence += 1;
     }
 
-    const msg = packet.toBinary();
+    const msg = packet.toBinary() as Uint8Array<ArrayBuffer>;
 
     switch (kind) {
       case DataChannelKind.LOSSY:
@@ -1491,7 +1531,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit
 
   /* @internal */
   async sendLossyBytes(
-    bytes: Uint8Array,
+    bytes: NonSharedUint8Array,
     kind: Exclude<DataChannelKind, DataChannelKind.RELIABLE>,
     bufferStatusLowBehavior: 'drop' | 'wait' = 'drop',
   ) {

package/src/room/Room.test.ts

@@ -1,5 +1,7 @@
-import { describe, expect, it } from 'vitest';
+import { ClientInfo_Capability, JoinResponse } from '@livekit/protocol';
+import { describe, expect, it, vi } from 'vitest';
 import Room from './Room';
+import { roomConnectOptionDefaults, roomOptionDefaults } from './defaults';
 import { RoomEvent } from './events';
 
 describe('Active device switch', () => {
@@ -28,3 +30,62 @@ describe('Active device switch', () => {
     expect(kind).toBe('audioinput');
   });
 });
+
+describe('Room signaling options', () => {
+  it('advertises packet trailer capability when E2EE can handle trailers', async () => {
+    const room = new Room();
+    const join = vi.fn().mockResolvedValue({
+      joinResponse: new JoinResponse({
+        room: { name: 'test-room', sid: 'room-sid' },
+        participant: { sid: 'participant-sid', identity: 'test-user' },
+      }),
+      serverInfo: { version: '1.0.0' },
+    });
+    const engine = { join };
+
+    (
+      room as unknown as {
+        e2eeManager: unknown;
+        connectSignal: (
+          url: string,
+          token: string,
+          engine: unknown,
+          connectOptions: typeof roomConnectOptionDefaults,
+          roomOptions: typeof roomOptionDefaults,
+          abortController: AbortController,
+        ) => Promise<JoinResponse>;
+      }
+    ).e2eeManager = {};
+
+    await (
+      room as unknown as {
+        connectSignal: (
+          url: string,
+          token: string,
+          engine: unknown,
+          connectOptions: typeof roomConnectOptionDefaults,
+          roomOptions: typeof roomOptionDefaults,
+          abortController: AbortController,
+        ) => Promise<JoinResponse>;
+      }
+    ).connectSignal(
+      'wss://test.livekit.io',
+      'test-token',
+      engine,
+      roomConnectOptionDefaults,
+      roomOptionDefaults,
+      new AbortController(),
+    );
+
+    expect(join).toHaveBeenCalledWith(
+      'wss://test.livekit.io',
+      'test-token',
+      expect.objectContaining({
+        clientInfoCapabilities: [ClientInfo_Capability.CAP_PACKET_TRAILER],
+        e2eeEnabled: true,
+      }),
+      expect.any(AbortSignal),
+      false,
+    );
+  });
+});