livekit-client 2.18.9 → 2.18.10

package/src/room/participant/LocalParticipant.test.ts

@@ -0,0 +1,81 @@
+import { PacketTrailerFeature } from '@livekit/protocol';
+import { describe, expect, it, vi } from 'vitest';
+import type LocalTrack from '../track/LocalTrack';
+import { Track } from '../track/Track';
+import type { TrackPublishOptions } from '../track/options';
+import LocalParticipant from './LocalParticipant';
+
+type PacketTrailerTestParticipant = {
+  canPublishPacketTrailer: () => boolean;
+  log: { warn: ReturnType<typeof vi.fn> };
+  normalizeRequestedPacketTrailerOptions: (
+    track: LocalTrack,
+    opts: TrackPublishOptions,
+  ) => PacketTrailerFeature[];
+};
+
+function makeParticipant(canPublishPacketTrailer: boolean) {
+  const participant = Object.create(LocalParticipant.prototype) as PacketTrailerTestParticipant;
+  participant.canPublishPacketTrailer = () => canPublishPacketTrailer;
+  participant.log = { warn: vi.fn() };
+  return participant;
+}
+
+function makeTrack(kind: Track.Kind) {
+  return {
+    kind,
+    sid: 'track-sid',
+    source: kind === Track.Kind.Video ? Track.Source.Camera : Track.Source.Microphone,
+    isMuted: false,
+    mediaStreamID: 'stream-id',
+    mediaStreamTrack: {
+      enabled: true,
+      id: 'media-track-id',
+    },
+  } as unknown as LocalTrack;
+}
+
+describe('LocalParticipant packet trailer publish options', () => {
+  it('normalizes requested video packet trailer options to advertised features', () => {
+    const participant = makeParticipant(true);
+    const opts: TrackPublishOptions = { packetTrailer: { timestamp: true, frameId: true } };
+
+    const features = participant.normalizeRequestedPacketTrailerOptions(
+      makeTrack(Track.Kind.Video),
+      opts,
+    );
+
+    expect(features).toEqual([
+      PacketTrailerFeature.PTF_USER_TIMESTAMP,
+      PacketTrailerFeature.PTF_FRAME_ID,
+    ]);
+    expect(opts.packetTrailer).toEqual({ timestamp: true, frameId: true });
+  });
+
+  it('clears packet trailer options for non-video tracks', () => {
+    const participant = makeParticipant(true);
+    const opts: TrackPublishOptions = { packetTrailer: { timestamp: true } };
+
+    const features = participant.normalizeRequestedPacketTrailerOptions(
+      makeTrack(Track.Kind.Audio),
+      opts,
+    );
+
+    expect(features).toEqual([]);
+    expect(opts.packetTrailer).toBeUndefined();
+  });
+
+  it('clears packet trailer options when publishing packet trailers is unsupported', () => {
+    const participant = makeParticipant(false);
+    const opts: TrackPublishOptions = { packetTrailer: { frameId: true } };
+
+    const features = participant.normalizeRequestedPacketTrailerOptions(
+      makeTrack(Track.Kind.Video),
+      opts,
+    );
+
+    expect(features).toEqual([]);
+    expect(opts.packetTrailer).toBeUndefined();
+    expect(participant.log.warn).toHaveBeenCalledOnce();
+  });
+});

package/src/room/participant/LocalParticipant.ts

@@ -9,6 +9,7 @@ import {
   DataPacket_Kind,
   Encryption_Type,
   JoinResponse,
+  PacketTrailerFeature,
   ParticipantInfo,
   RequestResponse,
   RequestResponse_Reason,
@@ -25,6 +26,12 @@ import {
 } from '@livekit/protocol';
 import { SignalConnectionState } from '../../api/SignalClient';
 import type { InternalRoomOptions } from '../../options';
+import {
+  getPacketTrailerFeatures,
+  getPacketTrailerPublishOptions,
+  hasPacketTrailerPublishOptions,
+  isPacketTrailerSupported,
+} from '../../packetTrailer/utils';
 import TypedPromise from '../../utils/TypedPromise';
 import { PCTransportState } from '../PCTransportManager';
 import type RTCEngine from '../RTCEngine';
@@ -343,7 +350,7 @@ export default class LocalParticipant extends Participant {
             break;
           default:
             error = DataTrackPublishError.unknown(response.reason, response.message);
-            return;
+            break;
         }
 
         this.roomOutgoingDataTrackManager.receivedSfuPublishResponse(
@@ -1109,6 +1116,8 @@ export default class LocalParticipant extends Participant {
     if (isLocalAudioTrack(track) && track.hasPreConnectBuffer) {
       audioFeatures.push(AudioTrackFeature.TF_PRECONNECT_BUFFER);
     }
+    const packetTrailerFeatures: PacketTrailerFeature[] =
+      this.normalizeRequestedPacketTrailerOptions(track, opts);
 
     // create track publication from track
     const req = new AddTrackRequest({
@@ -1125,15 +1134,13 @@ export default class LocalParticipant extends Participant {
       stream: opts?.stream,
       backupCodecPolicy: opts?.backupCodecPolicy as BackupCodecPolicy,
       audioFeatures,
+      packetTrailerFeatures,
     });
 
     // compute encodings and layers for video
     let encodings: RTCRtpEncodingParameters[] | undefined;
     if (track.kind === Track.Kind.Video) {
-      let dims: Track.Dimensions = {
-        width: 0,
-        height: 0,
-      };
+      let dims: Track.Dimensions;
       try {
         dims = await track.waitForDimensions();
       } catch (e) {
@@ -1240,6 +1247,9 @@ export default class LocalParticipant extends Participant {
       }
 
       track.sender = await this.engine.createSender(track, opts, encodings);
+      if (isLocalVideoTrack(track)) {
+        track.publishOptions = opts;
+      }
       this.emit(ParticipantEvent.LocalSenderCreated, track.sender, track);
 
       if (isLocalVideoTrack(track)) {
@@ -1307,7 +1317,7 @@ export default class LocalParticipant extends Participant {
         reject(err);
       }
     });
-    if (this.enabledPublishVideoCodecs.length > 0) {
+    if (this.enabledPublishVideoCodecs.length > 0 && packetTrailerFeatures.length === 0) {
       const rets = await Promise.all([addTrackPromise, negotiate()]);
       ti = rets[0];
     } else {
@@ -1438,6 +1448,34 @@ export default class LocalParticipant extends Participant {
     return publication;
   }
 
+  private canPublishPacketTrailer() {
+    return !!(
+      this.roomOptions.e2ee ||
+      this.roomOptions.encryption ||
+      isPacketTrailerSupported(this.roomOptions.packetTrailer)
+    );
+  }
+
+  private normalizeRequestedPacketTrailerOptions(track: LocalTrack, opts: TrackPublishOptions) {
+    if (track.kind !== Track.Kind.Video || !hasPacketTrailerPublishOptions(opts.packetTrailer)) {
+      opts.packetTrailer = undefined;
+      return [];
+    }
+
+    if (!this.canPublishPacketTrailer()) {
+      this.log.warn('packet trailer transform not supported; not advertising features', {
+        ...this.logContext,
+        ...getLogContextFromTrack(track),
+      });
+      opts.packetTrailer = undefined;
+      return [];
+    }
+
+    const features = getPacketTrailerFeatures(opts.packetTrailer);
+    opts.packetTrailer = getPacketTrailerPublishOptions(features);
+    return features;
+  }
+
   override get isLocal(): boolean {
     return true;
   }
@@ -1490,12 +1528,15 @@ export default class LocalParticipant extends Participant {
     if (!simulcastTrack) {
       return;
     }
+    const packetTrailerFeatures = this.normalizeRequestedPacketTrailerOptions(track, opts);
+
     const req = new AddTrackRequest({
       cid: simulcastTrack.mediaStreamTrack.id,
       type: Track.kindToProto(track.kind),
       muted: track.isMuted,
       source: Track.sourceToProto(track.source),
       sid: track.sid,
+      packetTrailerFeatures,
       simulcastCodecs: [
         {
           codec: opts.videoCodec,
@@ -1703,7 +1744,7 @@ export default class LocalParticipant extends Participant {
    * @param data Uint8Array of the payload. To send string data, use TextEncoder.encode
    * @param options optionally specify a `reliable`, `topic` and `destination`
    */
-  async publishData(data: Uint8Array, options: DataPublishOptions = {}): Promise<void> {
+  async publishData(data: NonSharedUint8Array, options: DataPublishOptions = {}): Promise<void> {
     const kind = options.reliable ? DataChannelKind.RELIABLE : DataChannelKind.LOSSY;
     const dataPacketKind = options.reliable ? DataPacket_Kind.RELIABLE : DataPacket_Kind.LOSSY;
     const destinationIdentities = options.destinationIdentities;

package/src/room/participant/Participant.ts

@@ -413,7 +413,7 @@ export type ParticipantEventCallbacks = {
   participantMetadataChanged: (prevMetadata: string | undefined, participant?: any) => void;
   participantNameChanged: (name: string) => void;
   dataReceived: (
-    payload: Uint8Array,
+    payload: NonSharedUint8Array,
     kind: DataPacket_Kind,
     encryptionType?: Encryption_Type,
   ) => void;

package/src/room/participant/publishUtils.ts

@@ -194,7 +194,7 @@ export function computeVideoEncodings(
     return [videoEncoding];
   }
 
-  let presets: Array<VideoPreset> = [];
+  let presets: Array<VideoPreset>;
   if (isScreenShare) {
     presets =
       sortPresets(options?.screenShareSimulcastLayers) ??

package/src/room/track/PacketTrailerExtractor.ts

@@ -0,0 +1,43 @@
+import type { PacketTrailerMetadata } from '../../packetTrailer/types';
+
+const MAX_ENTRIES = 300;
+
+/**
+ * Caches packet trailer metadata extracted from received video frames,
+ * keyed by RTP timestamp so it can be looked up when the frame is displayed.
+ *
+ * Metadata is populated either by the packet trailer worker managed by
+ * `PacketTrailerManager` (non-E2EE) or by the E2EE FrameCryptor worker
+ * after decryption (E2EE).
+ *
+ * @experimental
+ */
+export class PacketTrailerExtractor {
+  private metadataMap = new Map<number, PacketTrailerMetadata>();
+
+  private activeSsrc: number = 0;
+
+  storeMetadata(rtpTimestamp: number, ssrc: number, metadata: PacketTrailerMetadata) {
+    // Simulcast layer switch: SSRC changed, flush stale entries from old layer.
+    if (this.activeSsrc !== 0 && this.activeSsrc !== ssrc) {
+      this.metadataMap.clear();
+    }
+    this.activeSsrc = ssrc;
+
+    while (this.metadataMap.size >= MAX_ENTRIES) {
+      const evicted = this.metadataMap.keys().next().value!;
+      this.metadataMap.delete(evicted);
+    }
+
+    this.metadataMap.set(rtpTimestamp, metadata);
+  }
+
+  lookupMetadata(rtpTimestamp: number): PacketTrailerMetadata | undefined {
+    return this.metadataMap.get(rtpTimestamp);
+  }
+
+  dispose() {
+    this.metadataMap.clear();
+    this.activeSsrc = 0;
+  }
+}

package/src/room/track/RemoteVideoTrack.ts

@@ -1,3 +1,4 @@
+import type { PacketTrailerMetadata } from '../../packetTrailer/types';
 import { debounce } from '../debounce';
 import { TrackEvent } from '../events';
 import type { VideoReceiverStats } from '../stats';
@@ -6,6 +7,7 @@ import CriticalTimers from '../timers';
 import type { LoggerOptions } from '../types';
 import type { ObservableMediaElement } from '../utils';
 import { getDevicePixelRatio, getIntersectionObserver, getResizeObserver, isWeb } from '../utils';
+import type { PacketTrailerExtractor } from './PacketTrailerExtractor';
 import RemoteTrack from './RemoteTrack';
 import { Track, attachToElement, detachTrack } from './Track';
 import type { AdaptiveStreamSettings } from './types';
@@ -23,6 +25,9 @@ export default class RemoteVideoTrack extends RemoteTrack<Track.Kind.Video> {
 
   private lastDimensions?: Track.Dimensions;
 
+  /** @internal */
+  packetTrailerExtractor?: PacketTrailerExtractor;
+
   constructor(
     mediaTrack: MediaStreamTrack,
     sid: string,
@@ -38,6 +43,23 @@ export default class RemoteVideoTrack extends RemoteTrack<Track.Kind.Video> {
     return this.adaptiveStreamSettings !== undefined;
   }
 
+  /**
+   * Look up frame-level metadata for a given RTP timestamp.
+   * Use with the `TrackEvent.TimeSyncUpdate` event to correlate displayed frames
+   * with their capture-time metadata.
+   *
+   * Requires the room to be configured with the `packetTrailer` worker option
+   * and the publishing track to have packet trailer features enabled.
+   *
+   */
+  lookupFrameMetadata({
+    rtpTimestamp,
+  }: {
+    rtpTimestamp: number;
+  }): PacketTrailerMetadata | undefined {
+    return this.packetTrailerExtractor?.lookupMetadata(rtpTimestamp);
+  }
+
   override setStreamState(value: Track.StreamState) {
     super.setStreamState(value);
     this.log.debug('setStreamState', value);
@@ -140,12 +162,11 @@ export default class RemoteVideoTrack extends RemoteTrack<Track.Kind.Video> {
   detach(): HTMLMediaElement[];
   detach(element: HTMLMediaElement): HTMLMediaElement;
   detach(element?: HTMLMediaElement): HTMLMediaElement | HTMLMediaElement[] {
-    let detachedElements: HTMLMediaElement[] = [];
     if (element) {
       this.stopObservingElement(element);
       return super.detach(element);
     }
-    detachedElements = super.detach();
+    const detachedElements = super.detach();
 
     for (const e of detachedElements) {
       this.stopObservingElement(e);

package/src/room/track/Track.ts

@@ -537,6 +537,6 @@ export type TrackEventCallbacks = {
   trackProcessorUpdate: (processor?: TrackProcessor<Track.Kind, any>) => void;
   audioTrackFeatureUpdate: (track: any, feature: AudioTrackFeature, enabled: boolean) => void;
   timeSyncUpdate: (update: { timestamp: number; rtpTimestamp: number }) => void;
-  preConnectBufferFlushed: (buffer: Uint8Array[]) => void;
+  preConnectBufferFlushed: (buffer: NonSharedUint8Array[]) => void;
   cpuConstrained: () => void;
 };

package/src/room/track/create.ts

@@ -111,10 +111,6 @@ export async function createLocalTracks(
     return await Promise.all(
       stream.getTracks().map(async (mediaStreamTrack) => {
         const isAudio = mediaStreamTrack.kind === 'audio';
-        let trackOptions = isAudio ? opts!.audio : opts!.video;
-        if (typeof trackOptions === 'boolean' || !trackOptions) {
-          trackOptions = {};
-        }
         let trackConstraints: MediaTrackConstraints | undefined;
         const conOrBool = isAudio ? constraints.audio : constraints.video;
         if (typeof conOrBool !== 'boolean') {

package/src/room/track/options.ts

@@ -1,3 +1,4 @@
+import type { PacketTrailerPublishOptions } from '../../packetTrailer/types';
 import type { Track } from './Track';
 import type {
   AudioProcessorOptions,
@@ -128,6 +129,16 @@ export interface TrackPublishDefaults {
    * Defaults to false.
    */
   preConnectBuffer?: boolean;
+
+  /**
+   * Packet trailer metadata to append to published video frames.
+   *
+   * Requires either room-level packet trailer worker configuration or E2EE,
+   * because encoded frame transforms are used to write the trailer.
+   *
+   * @experimental
+   */
+  packetTrailer?: PacketTrailerPublishOptions;
 }
 
 /**

package/src/room/track/record.ts

@@ -51,7 +51,7 @@ export class LocalTrackRecorder<T extends LocalTrack> extends RecorderBase {
       start: (controller) => {
         streamController = controller;
         dataListener = async (event: BlobEvent) => {
-          let data: Uint8Array;
+          let data: NonSharedUint8Array;
 
           if (event.data.arrayBuffer) {
             const arrayBuffer = await event.data.arrayBuffer();

package/src/room/track/utils.ts

@@ -280,7 +280,10 @@ export function getLogContextFromTrack(track: Track | TrackPublication): Record<
 }
 
 export function supportsSynchronizationSources(): boolean {
-  return typeof RTCRtpReceiver !== 'undefined' && 'getSynchronizationSources' in RTCRtpReceiver;
+  return (
+    typeof RTCRtpReceiver !== 'undefined' &&
+    typeof RTCRtpReceiver.prototype.getSynchronizationSources === 'function'
+  );
 }
 
 export function diffAttributes(

package/src/room/utils.test.ts

@@ -1,5 +1,6 @@
+import { ClientInfo_Capability } from '@livekit/protocol';
 import { describe, expect, it } from 'vitest';
-import { extractMaxAgeFromRequestHeaders, splitUtf8, toWebsocketUrl } from './utils';
+import { extractMaxAgeFromRequestHeaders, getClientInfo, splitUtf8, toWebsocketUrl } from './utils';
 
 describe('toWebsocketUrl', () => {
   it('leaves wss urls alone', () => {
@@ -15,6 +16,18 @@ describe('toWebsocketUrl', () => {
   });
 });
 
+describe('getClientInfo', () => {
+  it('does not advertise packet trailer capability by default', () => {
+    expect(getClientInfo().capabilities).toEqual([]);
+  });
+
+  it('advertises packet trailer capability when provided', () => {
+    expect(getClientInfo([ClientInfo_Capability.CAP_PACKET_TRAILER]).capabilities).toEqual([
+      ClientInfo_Capability.CAP_PACKET_TRAILER,
+    ]);
+  });
+});
+
 describe('splitUtf8', () => {
   it('splits a string into chunks of the given size', () => {
     expect(splitUtf8('hello world', 5)).toEqual([

package/src/room/utils.ts

@@ -1,6 +1,7 @@
 import {
   ChatMessage as ChatMessageModel,
   ClientInfo,
+  ClientInfo_Capability,
   ClientInfo_SDK,
   DisconnectReason,
   Transcription as TranscriptionModel,
@@ -179,6 +180,18 @@ export function isChromiumBased(): boolean {
   return !!browser && browser.name === 'Chrome' && browser.os !== 'iOS';
 }
 
+export function isScriptTransformSupportedForWorker(): boolean {
+  // Chrome occasionally throws an `InvalidState` error when using script transforms directly after introducing this API in 141.
+  // Disabling it for Chrome based browsers until the API has stabilized.
+  // @ts-ignore
+  return (
+    typeof window !== 'undefined' &&
+    // @ts-ignore
+    typeof window.RTCRtpScriptTransform !== 'undefined' &&
+    !isChromiumBased()
+  );
+}
+
 export function isSafari(): boolean {
   return getBrowser()?.name === 'Safari';
 }
@@ -364,8 +377,9 @@ export interface ObservableMediaElement extends HTMLMediaElement {
   handleVisibilityChanged: (entry: IntersectionObserverEntry) => void;
 }
 
-export function getClientInfo(): ClientInfo {
+export function getClientInfo(capabilities?: ClientInfo_Capability[]): ClientInfo {
   const info = new ClientInfo({
+    capabilities,
     sdk: ClientInfo_SDK.JS,
     protocol: protocolVersion,
     version,
@@ -740,12 +754,12 @@ export function isRemoteParticipant(p: Participant): p is RemoteParticipant {
   return !p.isLocal;
 }
 
-export function splitUtf8(s: string, n: number): Uint8Array[] {
+export function splitUtf8(s: string, n: number): NonSharedUint8Array[] {
   if (n < 4) {
     throw new Error('n must be at least 4 due to utf8 encoding rules');
   }
   // adapted from https://stackoverflow.com/a/6043797
-  const result: Uint8Array[] = [];
+  const result: NonSharedUint8Array[] = [];
   let encoded = new TextEncoder().encode(s);
   while (encoded.length > n) {
     let k = n;

package/src/test/MockMediaStreamTrack.ts

@@ -1,4 +1,3 @@
-/* eslint-disable @typescript-eslint/no-unused-vars */
 // @ts-ignore
 export default class MockMediaStreamTrack implements MediaStreamTrack {
   contentHint: string = '';

package/src/type-polyfills/non-shared-typed-arrays.d.ts

@@ -0,0 +1,6 @@
+// As of TS 5.7, `Uint8Array` is generic over its backing buffer (`Uint8Array<ArrayBufferLike>`),
+// which includes `SharedArrayBuffer`. Many Web APIs (WebCrypto, structured clone, RTCDataChannel)
+// only accept the non-shared variant `Uint8Array<ArrayBuffer>`. Using `ReturnType<typeof Uint8Array.from>`
+// resolves to that non-shared variant on TS versions that support the generic, while remaining
+// equivalent to plain `Uint8Array` on older versions — so this alias works across the range we support.
+type NonSharedUint8Array = ReturnType<typeof Uint8Array.from>;

package/src/utils/dataPacketBuffer.ts

@@ -1,5 +1,5 @@
 export interface DataPacketItem {
-  data: Uint8Array;
+  data: NonSharedUint8Array;
   sequence: number;
 }
 

package/src/version.ts

@@ -1,4 +1,4 @@
 import { version as v } from '../package.json';
 
 export const version = v;
-export const protocolVersion = 16;
+export const protocolVersion = 17;