livekit-client 2.18.8 → 2.18.10

package/src/packetTrailer/types.ts

@@ -0,0 +1,75 @@
+import type { PacketTrailerFramePayload } from './packetTrailer';
+
+export interface PacketTrailerMetadata {
+  userTimestamp: bigint;
+  frameId: number;
+}
+
+export interface PacketTrailerPublishOptions {
+  timestamp?: boolean;
+  frameId?: boolean;
+}
+
+export interface PTBaseMessage {
+  kind: string;
+  data?: unknown;
+}
+
+export interface PTInitMessage extends PTBaseMessage {
+  kind: 'init';
+}
+
+export interface PTInitAck extends PTBaseMessage {
+  kind: 'initAck';
+}
+
+export interface PTDecodeMessage extends PTBaseMessage {
+  kind: 'decode';
+  data: {
+    readableStream: ReadableStream;
+    writableStream: WritableStream;
+    trackId: string;
+    hasPacketTrailer: boolean;
+  };
+}
+
+export interface PTEncodeMessage extends PTBaseMessage {
+  kind: 'encode';
+  data: {
+    readableStream: ReadableStream;
+    writableStream: WritableStream;
+    packetTrailer?: PacketTrailerPublishOptions;
+  };
+}
+
+export type PTScriptTransformOptions =
+  | {
+      kind: 'decode';
+      trackId: string;
+    }
+  | {
+      kind: 'encode';
+      packetTrailer?: PacketTrailerPublishOptions;
+    };
+
+export interface PTMetadataMessage extends PTBaseMessage {
+  kind: 'metadata';
+  data: PacketTrailerFramePayload;
+}
+
+export interface PTUpdateTrackIdMessage extends PTBaseMessage {
+  kind: 'updateTrackId';
+  data: {
+    oldTrackId: string;
+    newTrackId: string;
+    hasPacketTrailer: boolean;
+  };
+}
+
+export type PTWorkerMessage =
+  | PTInitMessage
+  | PTInitAck
+  | PTDecodeMessage
+  | PTEncodeMessage
+  | PTUpdateTrackIdMessage
+  | PTMetadataMessage;

package/src/packetTrailer/utils.test.ts

@@ -0,0 +1,105 @@
+import { PacketTrailerFeature } from '@livekit/protocol';
+import { afterEach, describe, expect, it } from 'vitest';
+import {
+  getPacketTrailerFeatures,
+  getPacketTrailerPublishOptions,
+  isPacketTrailerSupported,
+} from './utils';
+
+describe('packet trailer support', () => {
+  const originalRTCRtpSender = window.RTCRtpSender;
+  const originalRTCRtpScriptTransform = (window as unknown as { RTCRtpScriptTransform?: unknown })
+    .RTCRtpScriptTransform;
+  const originalUserAgent = navigator.userAgent;
+
+  afterEach(() => {
+    Object.defineProperty(window, 'RTCRtpSender', {
+      configurable: true,
+      value: originalRTCRtpSender,
+      writable: true,
+    });
+    Object.defineProperty(window, 'RTCRtpScriptTransform', {
+      configurable: true,
+      value: originalRTCRtpScriptTransform,
+      writable: true,
+    });
+    Object.defineProperty(window.navigator, 'userAgent', {
+      configurable: true,
+      value: originalUserAgent,
+    });
+  });
+
+  function stubScriptTransformSupport(userAgent: string) {
+    Object.defineProperty(window, 'RTCRtpSender', {
+      configurable: true,
+      value: undefined,
+      writable: true,
+    });
+    Object.defineProperty(window, 'RTCRtpScriptTransform', {
+      configurable: true,
+      value: class MockRTCRtpScriptTransform {},
+      writable: true,
+    });
+    Object.defineProperty(window.navigator, 'userAgent', {
+      configurable: true,
+      value: userAgent,
+    });
+  }
+
+  it('supports packet trailers with RTCRtpScriptTransform on Safari', () => {
+    stubScriptTransformSupport(
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15',
+    );
+
+    expect(isPacketTrailerSupported({ worker: {} as Worker })).toBe(true);
+  });
+
+  it('supports packet trailers with RTCRtpScriptTransform on Firefox', () => {
+    stubScriptTransformSupport(
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 14.0; rv:144.0) Gecko/20100101 Firefox/144.0',
+    );
+
+    expect(isPacketTrailerSupported({ worker: {} as Worker })).toBe(true);
+  });
+
+  it('does not use RTCRtpScriptTransform support on Chromium-based browsers', () => {
+    stubScriptTransformSupport(
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36',
+    );
+
+    expect(isPacketTrailerSupported({ worker: {} as Worker })).toBe(false);
+  });
+});
+
+describe('packet trailer publish features', () => {
+  it('maps publish options to protocol features', () => {
+    expect(getPacketTrailerFeatures({ timestamp: true, frameId: true })).toEqual([
+      PacketTrailerFeature.PTF_USER_TIMESTAMP,
+      PacketTrailerFeature.PTF_FRAME_ID,
+    ]);
+    expect(getPacketTrailerFeatures({ timestamp: true })).toEqual([
+      PacketTrailerFeature.PTF_USER_TIMESTAMP,
+    ]);
+    expect(getPacketTrailerFeatures({ frameId: true })).toEqual([
+      PacketTrailerFeature.PTF_FRAME_ID,
+    ]);
+    expect(getPacketTrailerFeatures()).toEqual([]);
+  });
+
+  it('maps protocol features to publish options', () => {
+    expect(
+      getPacketTrailerPublishOptions([
+        PacketTrailerFeature.PTF_USER_TIMESTAMP,
+        PacketTrailerFeature.PTF_FRAME_ID,
+      ]),
+    ).toEqual({ timestamp: true, frameId: true });
+    expect(getPacketTrailerPublishOptions([PacketTrailerFeature.PTF_USER_TIMESTAMP])).toEqual({
+      timestamp: true,
+    });
+    expect(getPacketTrailerPublishOptions([PacketTrailerFeature.PTF_FRAME_ID])).toEqual({
+      frameId: true,
+    });
+    expect(getPacketTrailerPublishOptions()).toBeUndefined();
+    expect(getPacketTrailerPublishOptions([])).toBeUndefined();
+  });
+});

package/src/packetTrailer/utils.ts

@@ -0,0 +1,50 @@
+import { PacketTrailerFeature } from '@livekit/protocol';
+import { isInsertableStreamSupported } from '../e2ee/utils';
+import { isScriptTransformSupportedForWorker } from '../room/utils';
+import type { PacketTrailerOptions } from './PacketTrailerManager';
+import type { PacketTrailerPublishOptions } from './types';
+
+export function shouldUsePacketTrailerScriptTransform() {
+  return isScriptTransformSupportedForWorker();
+}
+
+export function isPacketTrailerSupported(options?: PacketTrailerOptions) {
+  return (
+    !!options?.worker && (isInsertableStreamSupported() || shouldUsePacketTrailerScriptTransform())
+  );
+}
+
+export function hasPacketTrailerPublishOptions(options?: PacketTrailerPublishOptions): boolean {
+  return !!(options?.timestamp || options?.frameId);
+}
+
+export function getPacketTrailerFeatures(
+  options?: PacketTrailerPublishOptions,
+): PacketTrailerFeature[] {
+  const features: PacketTrailerFeature[] = [];
+  if (options?.timestamp) {
+    features.push(PacketTrailerFeature.PTF_USER_TIMESTAMP);
+  }
+  if (options?.frameId) {
+    features.push(PacketTrailerFeature.PTF_FRAME_ID);
+  }
+  return features;
+}
+
+export function getPacketTrailerPublishOptions(
+  features?: PacketTrailerFeature[],
+): PacketTrailerPublishOptions | undefined {
+  if (!features || features.length === 0) {
+    return undefined;
+  }
+
+  const options: PacketTrailerPublishOptions = {};
+  if (features.includes(PacketTrailerFeature.PTF_USER_TIMESTAMP)) {
+    options.timestamp = true;
+  }
+  if (features.includes(PacketTrailerFeature.PTF_FRAME_ID)) {
+    options.frameId = true;
+  }
+
+  return options.timestamp || options.frameId ? options : undefined;
+}

package/src/packetTrailer/worker/packetTrailer.worker.ts

@@ -0,0 +1,155 @@
+import { appendPacketTrailerToEncodedFrame, processPacketTrailer } from '../packetTrailer';
+import type {
+  PTMetadataMessage,
+  PTScriptTransformOptions,
+  PTWorkerMessage,
+  PacketTrailerPublishOptions,
+} from '../types';
+import { hasPacketTrailerPublishOptions } from '../utils';
+
+/**
+ * Holds the trackId currently associated with a pipeline. A mutable
+ * wrapper is used so the transform closure always reads the latest
+ * trackId after a receiver gets re-bound to a new track.
+ */
+interface PipelineState {
+  trackId: string;
+  hasPacketTrailer: boolean;
+}
+
+const pipelines = new Map<string, PipelineState>();
+
+onmessage = (ev: MessageEvent<PTWorkerMessage>) => {
+  const msg = ev.data;
+
+  switch (msg.kind) {
+    case 'init':
+      postMessage({ kind: 'initAck' });
+      break;
+
+    case 'decode':
+      setupDecodeTransform(
+        msg.data.readableStream,
+        msg.data.writableStream,
+        msg.data.trackId,
+        msg.data.hasPacketTrailer,
+      );
+      break;
+
+    case 'encode':
+      setupEncodeTransform(
+        msg.data.readableStream,
+        msg.data.writableStream,
+        msg.data.packetTrailer,
+      );
+      break;
+
+    case 'updateTrackId':
+      updateTrackId(msg.data.oldTrackId, msg.data.newTrackId, msg.data.hasPacketTrailer);
+      break;
+
+    default:
+      break;
+  }
+};
+
+function setupDecodeTransform(
+  readable: ReadableStream,
+  writable: WritableStream,
+  trackId: string,
+  hasPacketTrailer: boolean,
+) {
+  const state: PipelineState = { trackId, hasPacketTrailer };
+  pipelines.set(trackId, state);
+
+  const transform = new TransformStream({
+    transform(
+      frame: RTCEncodedVideoFrame,
+      controller: TransformStreamDefaultController<RTCEncodedVideoFrame>,
+    ) {
+      try {
+        if (state.hasPacketTrailer) {
+          const result = processPacketTrailer(frame, state.trackId);
+          if (result.data) {
+            frame.data = result.data;
+          }
+          if (result.payload) {
+            const msg: PTMetadataMessage = { kind: 'metadata', data: result.payload };
+            postMessage(msg);
+          }
+        }
+      } catch {
+        // Never drop frames on trailer-extraction failure — pass through so
+        // video keeps decoding even if metadata is lost for this frame.
+      }
+      controller.enqueue(frame);
+    },
+  });
+
+  readable
+    .pipeThrough(transform)
+    .pipeTo(writable)
+    .catch(() => {
+      pipelines.delete(state.trackId);
+    });
+}
+
+function setupEncodeTransform(
+  readable: ReadableStream,
+  writable: WritableStream,
+  packetTrailer?: PacketTrailerPublishOptions,
+) {
+  if (!hasPacketTrailerPublishOptions(packetTrailer)) {
+    readable.pipeTo(writable).catch(() => {});
+    return;
+  }
+
+  let frameId = 0;
+  const transform = new TransformStream({
+    transform(
+      frame: RTCEncodedVideoFrame,
+      controller: TransformStreamDefaultController<RTCEncodedVideoFrame>,
+    ) {
+      try {
+        if (packetTrailer?.frameId) {
+          frameId = frameId === 0xffffffff ? 1 : frameId + 1;
+        }
+        appendPacketTrailerToEncodedFrame(frame, packetTrailer, frameId);
+      } catch {
+        // Never drop frames on trailer-write failure.
+      }
+      controller.enqueue(frame);
+    },
+  });
+
+  readable
+    .pipeThrough(transform)
+    .pipeTo(writable)
+    .catch(() => {});
+}
+
+function updateTrackId(oldTrackId: string, newTrackId: string, hasPacketTrailer: boolean) {
+  const state = pipelines.get(oldTrackId);
+  if (state) {
+    state.trackId = newTrackId;
+    state.hasPacketTrailer = hasPacketTrailer;
+    pipelines.delete(oldTrackId);
+    pipelines.set(newTrackId, state);
+  }
+}
+
+// Operations using RTCRtpScriptTransform.
+// @ts-ignore
+if (self.RTCTransformEvent) {
+  // @ts-ignore
+  self.onrtctransform = (event: RTCTransformEvent) => {
+    // @ts-ignore
+    const transformer = event.transformer;
+    const options = transformer.options as PTScriptTransformOptions;
+    if (options.kind === 'encode') {
+      setupEncodeTransform(transformer.readable, transformer.writable, options.packetTrailer);
+    } else {
+      setupDecodeTransform(transformer.readable, transformer.writable, options.trackId, true);
+    }
+  };
+}

package/src/packetTrailer/worker/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "extends": "../../../tsconfig.json",
+  "compilerOptions": {
+    "lib": [
+      "DOM",
+      "DOM.Iterable",
+      "ES2017",
+      "ES2018.Promise",
+      "WebWorker",
+      "ES2021.WeakRef",
+      "DOM.AsyncIterable"
+    ]
+  }
+}

package/src/room/BackOffStrategy.test.ts

@@ -6,7 +6,7 @@ vi.mock('./utils', async () => {
   const actual = await vi.importActual('./utils');
   return {
     ...actual,
-    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+
     sleep: vi.fn((ms: number) => Promise.resolve()),
     extractProjectFromUrl: vi.fn((url: URL) => {
       // @ts-ignore

package/src/room/RTCEngine.test.ts

@@ -0,0 +1,219 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import RTCEngine from './RTCEngine';
+import { roomOptionDefaults } from './defaults';
+
+describe('RTCEngine', () => {
+  const originalRTCRtpSender = window.RTCRtpSender;
+  const originalRTCRtpScriptTransform = (window as unknown as { RTCRtpScriptTransform?: unknown })
+    .RTCRtpScriptTransform;
+  const originalUserAgent = navigator.userAgent;
+
+  afterEach(() => {
+    Object.defineProperty(window, 'RTCRtpSender', {
+      configurable: true,
+      value: originalRTCRtpSender,
+      writable: true,
+    });
+    Object.defineProperty(window, 'RTCRtpScriptTransform', {
+      configurable: true,
+      value: originalRTCRtpScriptTransform,
+      writable: true,
+    });
+    Object.defineProperty(window.navigator, 'userAgent', {
+      configurable: true,
+      value: originalUserAgent,
+    });
+  });
+
+  function stubInsertableStreamsSupport() {
+    class MockRTCRtpSender {
+      createEncodedStreams() {}
+    }
+
+    Object.defineProperty(window, 'RTCRtpSender', {
+      configurable: true,
+      value: MockRTCRtpSender,
+      writable: true,
+    });
+  }
+
+  function stubScriptTransformSupport() {
+    Object.defineProperty(window, 'RTCRtpScriptTransform', {
+      configurable: true,
+      value: class MockRTCRtpScriptTransform {},
+      writable: true,
+    });
+    Object.defineProperty(window.navigator, 'userAgent', {
+      configurable: true,
+      value:
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15',
+    });
+  }
+
+  function makeRTCConfiguration(engine: RTCEngine) {
+    return (
+      engine as unknown as { makeRTCConfiguration: () => RTCConfiguration }
+    ).makeRTCConfiguration();
+  }
+
+  function setupPacketTrailerSender(engine: RTCEngine, sender: RTCRtpSender, opts = {}) {
+    (
+      engine as unknown as {
+        setupPacketTrailerSender: (sender: RTCRtpSender, opts?: unknown) => void;
+      }
+    ).setupPacketTrailerSender(sender, opts);
+  }
+
+  it('does not enable encoded insertable streams without E2EE or a packet trailer worker', () => {
+    stubInsertableStreamsSupport();
+
+    const engine = new RTCEngine(roomOptionDefaults);
+
+    expect(makeRTCConfiguration(engine).encodedInsertableStreams).toBeUndefined();
+  });
+
+  it('enables encoded insertable streams when a packet trailer worker is configured', () => {
+    stubInsertableStreamsSupport();
+
+    const engine = new RTCEngine({
+      ...roomOptionDefaults,
+      packetTrailer: { worker: {} as Worker },
+    });
+
+    expect(makeRTCConfiguration(engine).encodedInsertableStreams).toBe(true);
+  });
+
+  it('does not enable encoded insertable streams for packet trailers when script transforms are supported', () => {
+    stubInsertableStreamsSupport();
+    stubScriptTransformSupport();
+
+    const engine = new RTCEngine({
+      ...roomOptionDefaults,
+      packetTrailer: { worker: {} as Worker },
+    });
+
+    expect(makeRTCConfiguration(engine).encodedInsertableStreams).toBeUndefined();
+  });
+
+  it('enables encoded insertable streams for E2EE', () => {
+    stubInsertableStreamsSupport();
+
+    const engine = new RTCEngine(roomOptionDefaults);
+    (
+      engine as unknown as {
+        signalOpts: {
+          autoSubscribe: boolean;
+          maxRetries: number;
+          e2eeEnabled: boolean;
+          websocketTimeout: number;
+        };
+      }
+    ).signalOpts = {
+      autoSubscribe: true,
+      maxRetries: 1,
+      e2eeEnabled: true,
+      websocketTimeout: 15_000,
+    };
+
+    expect(makeRTCConfiguration(engine).encodedInsertableStreams).toBe(true);
+  });
+
+  it('does not create sender encoded streams when packetTrailer has no worker', () => {
+    const engine = new RTCEngine({
+      ...roomOptionDefaults,
+      packetTrailer: {} as never,
+    });
+    const createEncodedStreams = vi.fn();
+    const sender = {
+      createEncodedStreams,
+    } as unknown as RTCRtpSender;
+
+    setupPacketTrailerSender(engine, sender);
+
+    expect(createEncodedStreams).not.toHaveBeenCalled();
+  });
+
+  it('does not create sender passthrough streams for packet trailers when script transforms are supported', () => {
+    stubScriptTransformSupport();
+
+    const engine = new RTCEngine({
+      ...roomOptionDefaults,
+      packetTrailer: { worker: {} as Worker },
+    });
+    const createEncodedStreams = vi.fn();
+    const sender = {
+      createEncodedStreams,
+    } as unknown as RTCRtpSender;
+
+    setupPacketTrailerSender(engine, sender);
+
+    expect(createEncodedStreams).not.toHaveBeenCalled();
+  });
+
+  it('posts sender encode streams to the packet trailer worker when write features are enabled', () => {
+    stubInsertableStreamsSupport();
+
+    const worker = { postMessage: vi.fn() } as unknown as Worker;
+    const engine = new RTCEngine({
+      ...roomOptionDefaults,
+      packetTrailer: { worker },
+    });
+    const readable = {} as ReadableStream;
+    const writable = {} as WritableStream;
+    const createEncodedStreams = vi.fn(() => ({ readable, writable }));
+    const sender = {
+      createEncodedStreams,
+    } as unknown as RTCRtpSender;
+
+    setupPacketTrailerSender(engine, sender, { packetTrailer: { timestamp: true, frameId: true } });
+
+    expect(createEncodedStreams).toHaveBeenCalledTimes(1);
+    expect(worker.postMessage).toHaveBeenCalledWith(
+      {
+        kind: 'encode',
+        data: {
+          readableStream: readable,
+          writableStream: writable,
+          packetTrailer: { timestamp: true, frameId: true },
+        },
+      },
+      [readable, writable],
+    );
+  });
+
+  it('uses RTCRtpScriptTransform for sender packet trailer writes when supported', () => {
+    stubScriptTransformSupport();
+
+    const transform = {};
+    const RTCRtpScriptTransform = vi.fn(() => transform);
+    Object.defineProperty(window, 'RTCRtpScriptTransform', {
+      configurable: true,
+      value: RTCRtpScriptTransform,
+      writable: true,
+    });
+    Object.defineProperty(globalThis, 'RTCRtpScriptTransform', {
+      configurable: true,
+      value: RTCRtpScriptTransform,
+      writable: true,
+    });
+
+    const worker = {} as Worker;
+    const engine = new RTCEngine({
+      ...roomOptionDefaults,
+      packetTrailer: { worker },
+    });
+    const createEncodedStreams = vi.fn();
+    const sender = {
+      createEncodedStreams,
+    } as unknown as RTCRtpSender;
+
+    setupPacketTrailerSender(engine, sender, { packetTrailer: { timestamp: true } });
+
+    expect(RTCRtpScriptTransform).toHaveBeenCalledWith(worker, {
+      kind: 'encode',
+      packetTrailer: { timestamp: true },
+    });
+    expect((sender as unknown as { transform: unknown }).transform).toBe(transform);
+    expect(createEncodedStreams).not.toHaveBeenCalled();
+  });
+});