livekit-client 2.18.9 → 2.19.0

package/src/room/data-track/frame.ts

@@ -2,13 +2,13 @@ import { DataTrackExtensions, DataTrackUserTimestampExtension } from './packet/e
 
 /** A pair of payload bytes and packet extensions which can be fed into a {@link DataTrackPacketizer}. */
 export type DataTrackFrame = {
-  payload: Uint8Array;
+  payload: NonSharedUint8Array;
   userTimestamp?: bigint;
 };
 
 /** An internal representation o data track frame which contains all SFU metadata. */
 export type DataTrackFrameInternal = {
-  payload: Uint8Array;
+  payload: NonSharedUint8Array;
   extensions: DataTrackExtensions;
 };
 

package/src/room/data-track/incoming/IncomingDataTrackManager.test.ts

@@ -936,5 +936,199 @@ describe('DataTrackIncomingManager', () => {
         process.off('unhandledRejection', onUnhandled);
       }
     });
+
+    it('should depacketize multiple interleaved partial frames when setMaxPartialFrames is called before subscribe', async () => {
+      const manager = new IncomingDataTrackManager();
+      const managerEvents = subscribeToEvents<DataTrackIncomingManagerCallbacks>(manager, [
+        'sfuUpdateSubscription',
+        'trackPublished',
+      ]);
+
+      const senderIdentity = 'identity';
+      const sid = 'data track sid';
+      const handle = DataTrackHandle.fromNumber(5);
+
+      await manager.receiveSfuPublicationUpdates(
+        new Map([[senderIdentity, [{ sid, pubHandle: handle, name: 'test', usesE2ee: false }]]]),
+      );
+      const trackPublishedEvent = await managerEvents.waitFor('trackPublished');
+
+      // Configure the track BEFORE any subscribe.
+      trackPublishedEvent.track.setPipelineOptions({ maxPartialFrames: 3 });
+
+      const [stream, sfuSubscriptionComplete] = manager.openSubscriptionStream(sid);
+      const reader = stream.getReader();
+
+      await managerEvents.waitFor('sfuUpdateSubscription');
+      manager.receivedSfuSubscriberHandles(new Map([[handle, sid]]));
+      await sfuSubscriptionComplete;
+
+      // Two interleaved partial frames: Start(1), Start(2), Final(1), Final(2). With the default
+      // maxPartialFrames=1 frame 1 would be evicted by frame 2; with maxPartialFrames=3 both
+      // frames coexist and emerge.
+      pushInterleavedTwoFramePair(manager, handle, {
+        frameOneNumber: 1,
+        frameOneStartSequence: 0,
+        frameOnePayloads: [new Uint8Array([0xa1]), new Uint8Array([0xa2])],
+        frameTwoNumber: 2,
+        frameTwoStartSequence: 100,
+        frameTwoPayloads: [new Uint8Array([0xb1]), new Uint8Array([0xb2])],
+      });
+
+      const first = await reader.read();
+      expect(first.done).toStrictEqual(false);
+      expect(first.value?.payload).toStrictEqual(new Uint8Array([0xa1, 0xa2]));
+
+      const second = await reader.read();
+      expect(second.done).toStrictEqual(false);
+      expect(second.value?.payload).toStrictEqual(new Uint8Array([0xb1, 0xb2]));
+    });
+
+    it('should pick up setMaxPartialFrames live on an already-active subscription', async () => {
+      const manager = new IncomingDataTrackManager();
+      const managerEvents = subscribeToEvents<DataTrackIncomingManagerCallbacks>(manager, [
+        'sfuUpdateSubscription',
+        'trackPublished',
+      ]);
+
+      const senderIdentity = 'identity';
+      const sid = 'data track sid';
+      const handle = DataTrackHandle.fromNumber(5);
+
+      await manager.receiveSfuPublicationUpdates(
+        new Map([[senderIdentity, [{ sid, pubHandle: handle, name: 'test', usesE2ee: false }]]]),
+      );
+      const trackPublishedEvent = await managerEvents.waitFor('trackPublished');
+
+      const [stream, sfuSubscriptionComplete] = manager.openSubscriptionStream(sid);
+      const reader = stream.getReader();
+
+      await managerEvents.waitFor('sfuUpdateSubscription');
+      manager.receivedSfuSubscriberHandles(new Map([[handle, sid]]));
+      await sfuSubscriptionComplete;
+
+      // Subscription is now active; flip the cap on the live pipeline.
+      trackPublishedEvent.track.setPipelineOptions({ maxPartialFrames: 3 });
+
+      pushInterleavedTwoFramePair(manager, handle, {
+        frameOneNumber: 1,
+        frameOneStartSequence: 0,
+        frameOnePayloads: [new Uint8Array([0xa1]), new Uint8Array([0xa2])],
+        frameTwoNumber: 2,
+        frameTwoStartSequence: 100,
+        frameTwoPayloads: [new Uint8Array([0xb1]), new Uint8Array([0xb2])],
+      });
+
+      const first = await reader.read();
+      expect(first.value?.payload).toStrictEqual(new Uint8Array([0xa1, 0xa2]));
+
+      const second = await reader.read();
+      expect(second.value?.payload).toStrictEqual(new Uint8Array([0xb1, 0xb2]));
+    });
+
+    it('should drop the older partial frame by default (no setMaxPartialFrames call)', async () => {
+      const manager = new IncomingDataTrackManager();
+      const managerEvents = subscribeToEvents<DataTrackIncomingManagerCallbacks>(manager, [
+        'sfuUpdateSubscription',
+        'trackPublished',
+      ]);
+
+      const senderIdentity = 'identity';
+      const sid = 'data track sid';
+      const handle = DataTrackHandle.fromNumber(5);
+
+      await manager.receiveSfuPublicationUpdates(
+        new Map([[senderIdentity, [{ sid, pubHandle: handle, name: 'test', usesE2ee: false }]]]),
+      );
+      await managerEvents.waitFor('trackPublished');
+
+      const [stream, sfuSubscriptionComplete] = manager.openSubscriptionStream(sid);
+      const reader = stream.getReader();
+
+      await managerEvents.waitFor('sfuUpdateSubscription');
+      manager.receivedSfuSubscriberHandles(new Map([[handle, sid]]));
+      await sfuSubscriptionComplete;
+
+      // Default cap of 1: Start(2) evicts Start(1), so Final(1) is unknown and only frame 2
+      // makes it through.
+      pushInterleavedTwoFramePair(manager, handle, {
+        frameOneNumber: 1,
+        frameOneStartSequence: 0,
+        frameOnePayloads: [new Uint8Array([0xa1]), new Uint8Array([0xa2])],
+        frameTwoNumber: 2,
+        frameTwoStartSequence: 100,
+        frameTwoPayloads: [new Uint8Array([0xb1]), new Uint8Array([0xb2])],
+      });
+
+      const onlyFrame = await reader.read();
+      expect(onlyFrame.done).toStrictEqual(false);
+      expect(onlyFrame.value?.payload).toStrictEqual(new Uint8Array([0xb1, 0xb2]));
+    });
   });
 });
+
+/** Pushes Start(frame1), Start(frame2), Final(frame1), Final(frame2) packets through the manager
+ * to exercise the depacketizer's concurrent-partial-frame handling. */
+function pushInterleavedTwoFramePair(
+  manager: IncomingDataTrackManager,
+  trackHandle: DataTrackHandle,
+  args: {
+    frameOneNumber: number;
+    frameOneStartSequence: number;
+    frameOnePayloads: [Uint8Array, Uint8Array];
+    frameTwoNumber: number;
+    frameTwoStartSequence: number;
+    frameTwoPayloads: [Uint8Array, Uint8Array];
+  },
+) {
+  const buildPacket = (
+    frameNumber: number,
+    sequence: number,
+    marker: FrameMarker,
+    payload: Uint8Array,
+  ) =>
+    new DataTrackPacket(
+      new DataTrackPacketHeader({
+        extensions: new DataTrackExtensions(),
+        frameNumber: WrapAroundUnsignedInt.u16(frameNumber),
+        marker,
+        sequence: WrapAroundUnsignedInt.u16(sequence),
+        timestamp: DataTrackTimestamp.fromRtpTicks(0),
+        trackHandle,
+      }),
+      payload,
+    ).toBinary();
+
+  manager.packetReceived(
+    buildPacket(
+      args.frameOneNumber,
+      args.frameOneStartSequence,
+      FrameMarker.Start,
+      args.frameOnePayloads[0],
+    ),
+  );
+  manager.packetReceived(
+    buildPacket(
+      args.frameTwoNumber,
+      args.frameTwoStartSequence,
+      FrameMarker.Start,
+      args.frameTwoPayloads[0],
+    ),
+  );
+  manager.packetReceived(
+    buildPacket(
+      args.frameOneNumber,
+      args.frameOneStartSequence + 1,
+      FrameMarker.Final,
+      args.frameOnePayloads[1],
+    ),
+  );
+  manager.packetReceived(
+    buildPacket(
+      args.frameTwoNumber,
+      args.frameTwoStartSequence + 1,
+      FrameMarker.Final,
+      args.frameTwoPayloads[1],
+    ),
+  );
+}

package/src/room/data-track/incoming/IncomingDataTrackManager.ts

@@ -13,7 +13,11 @@ import { DataTrackDepacketizerDropError } from '../depacketizer';
 import { type DataTrackFrame, DataTrackFrameInternal } from '../frame';
 import { DataTrackHandle } from '../handle';
 import { DataTrackPacket } from '../packet';
-import { type DataTrackInfo, type DataTrackSid } from '../types';
+import {
+  type DataTrackInfo,
+  type DataTrackSid,
+  type RemoteDataTrackPipelineOptions,
+} from '../types';
 import { DataTrackSubscribeError } from './errors';
 import IncomingDataTrackPipeline from './pipeline';
 import {
@@ -65,6 +69,7 @@ type Descriptor<S extends SubscriptionState> = {
   info: DataTrackInfo;
   publisherIdentity: Participant['identity'];
   subscription: S;
+  pipelineOptions: RemoteDataTrackPipelineOptions;
 };
 
 type IncomingDataTrackManagerOptions = {
@@ -113,6 +118,19 @@ export default class IncomingDataTrackManager extends (EventEmitter as new () =>
     }
   }
 
+  /** @internal */
+  setPipelineOptions(sid: DataTrackSid, options: RemoteDataTrackPipelineOptions): void {
+    const descriptor = this.descriptors.get(sid);
+    if (!descriptor) {
+      log.warn(`Unknown track ${sid}, cannot set pipeline options.`);
+      return;
+    }
+    descriptor.pipelineOptions = options;
+    if (descriptor.subscription.type === 'active') {
+      descriptor.subscription.pipeline.setOptions(options);
+    }
+  }
+
   /** Allocates a ReadableStream which emits when a new {@link DataTrackFrame} is received from the
    * SFU. The SFU subscription is initiated lazily when the stream is created.
    *
@@ -473,6 +491,7 @@ export default class IncomingDataTrackManager extends (EventEmitter as new () =>
       info,
       publisherIdentity,
       subscription: { type: 'none' },
+      pipelineOptions: {},
     };
     this.descriptors.set(descriptor.info.sid, descriptor);
 
@@ -530,6 +549,7 @@ export default class IncomingDataTrackManager extends (EventEmitter as new () =>
           info: descriptor.info,
           publisherIdentity: descriptor.publisherIdentity,
           e2eeManager: this.e2eeManager,
+          pipelineOptions: descriptor.pipelineOptions,
         });
 
         const previousDescriptorSubscription = descriptor.subscription;

package/src/room/data-track/incoming/pipeline.ts

@@ -4,7 +4,7 @@ import { LoggerNames, getLogger } from '../../../logger';
 import DataTrackDepacketizer, { DataTrackDepacketizerDropError } from '../depacketizer';
 import type { DataTrackFrameInternal } from '../frame';
 import { DataTrackPacket } from '../packet';
-import { type DataTrackInfo } from '../types';
+import { type DataTrackInfo, type RemoteDataTrackPipelineOptions } from '../types';
 
 const log = getLogger(LoggerNames.DataTracks);
 
@@ -15,6 +15,7 @@ type Options = {
   info: DataTrackInfo;
   publisherIdentity: string;
   e2eeManager: BaseE2EEManager | null;
+  pipelineOptions?: RemoteDataTrackPipelineOptions;
 };
 
 /**
@@ -27,6 +28,8 @@ export default class IncomingDataTrackPipeline {
 
   private depacketizer: DataTrackDepacketizer;
 
+  private options: RemoteDataTrackPipelineOptions;
+
   /**
    * Creates a new pipeline with the given options.
    */
@@ -44,12 +47,17 @@ export default class IncomingDataTrackPipeline {
     this.publisherIdentity = options.publisherIdentity;
     this.e2eeManager = options.e2eeManager ?? null;
     this.depacketizer = depacketizer;
+    this.options = options.pipelineOptions ?? {};
   }
 
   updateE2eeManager(e2eeManager: BaseE2EEManager | null) {
     this.e2eeManager = e2eeManager;
   }
 
+  setOptions(options: RemoteDataTrackPipelineOptions): void {
+    this.options = options;
+  }
+
   async processPacket(
     packet: DataTrackPacket,
   ): Promise<Throws<DataTrackFrameInternal | null, DataTrackDepacketizerDropError>> {
@@ -74,7 +82,10 @@ export default class IncomingDataTrackPipeline {
   ): Throws<DataTrackFrameInternal | null, DataTrackDepacketizerDropError> {
     let frame: DataTrackFrameInternal | null;
     try {
-      frame = this.depacketizer.push(packet);
+      frame = this.depacketizer.push(packet, {
+        throwOnInterruption: false,
+        maxPartialFrames: this.options.maxPartialFrames,
+      });
     } catch (err) {
       // In a future version, use this to maintain drop statistics.
       // FIXME: is this a good idea?

package/src/room/data-track/outgoing/types.ts

@@ -17,7 +17,8 @@ export type SfuPublishResponseResult =
         | DataTrackPublishError<DataTrackPublishErrorReason.NotAllowed>
         | DataTrackPublishError<DataTrackPublishErrorReason.DuplicateName>
         | DataTrackPublishError<DataTrackPublishErrorReason.InvalidName>
-        | DataTrackPublishError<DataTrackPublishErrorReason.LimitReached>;
+        | DataTrackPublishError<DataTrackPublishErrorReason.LimitReached>
+        | DataTrackPublishError<DataTrackPublishErrorReason.Unknown>;
     };
 
 /** Request sent to the SFU to publish a track. */
@@ -36,7 +37,7 @@ export type EventSfuUnpublishRequest = {
 export type EventPacketAvailable = {
   /** The handle associated with the data track which this packet bytes belong to. */
   handle: DataTrackHandle;
-  bytes: Uint8Array;
+  bytes: NonSharedUint8Array;
 };
 
 /** A track has been created by a local participant and is available to be

package/src/room/data-track/packet/extensions.ts

@@ -75,9 +75,9 @@ export class DataTrackE2eeExtension extends DataTrackExtension {
 
   keyIndex: number;
 
-  iv: Uint8Array; /* NOTE: According to the rust implementation, this should be 12 bytes long. */
+  iv: NonSharedUint8Array; /* NOTE: According to the rust implementation, this should be 12 bytes long. */
 
-  constructor(keyIndex: number, iv: Uint8Array) {
+  constructor(keyIndex: number, iv: NonSharedUint8Array) {
     super();
     this.keyIndex = keyIndex;
     this.iv = iv;

package/src/room/data-track/packet/index.ts

@@ -300,9 +300,9 @@ export enum FrameMarker {
 export class DataTrackPacket extends Serializable {
   header: DataTrackPacketHeader;
 
-  payload: Uint8Array;
+  payload: NonSharedUint8Array;
 
-  constructor(header: DataTrackPacketHeader, payload: Uint8Array) {
+  constructor(header: DataTrackPacketHeader, payload: NonSharedUint8Array) {
     super();
     this.header = header;
     this.payload = payload;
@@ -349,10 +349,10 @@ export class DataTrackPacket extends Serializable {
       dataView.byteOffset + dataView.byteLength,
     );
 
-    return [new DataTrackPacket(header, new Uint8Array(payload)), dataView.byteLength] as [
-      DataTrackPacket,
-      number,
-    ];
+    return [
+      new DataTrackPacket(header, new Uint8Array(payload) as NonSharedUint8Array),
+      dataView.byteLength,
+    ] as [DataTrackPacket, number];
   }
 
   toJSON() {

package/src/room/data-track/packet/serializable.ts

@@ -10,7 +10,7 @@ export default abstract class Serializable {
   abstract toBinaryInto(dataView: DataView): Throws<number, DataTrackSerializeError>;
 
   /** Encodes the instance as binary and returns the data as a Uint8Array. */
-  toBinary(): Throws<Uint8Array, DataTrackSerializeError> {
+  toBinary(): Throws<NonSharedUint8Array, DataTrackSerializeError> {
     const lengthBytes = this.toBinaryLengthBytes();
     const output = new ArrayBuffer(lengthBytes);
     const view = new DataView(output);

package/src/room/data-track/types.ts

@@ -11,6 +11,14 @@ export type DataTrackInfo = {
   usesE2ee: boolean;
 };
 
+export type RemoteDataTrackPipelineOptions = {
+  /** Set the maximum number of in-flight partial frames the depacketizer will track
+   * concurrently for this track. Higher values give more out-of-order tolerance for
+   * high-frequency senders. Defaults to 1.
+   */
+  maxPartialFrames?: number;
+};
+
 export const DataTrackInfo = {
   from(protocolInfo: ProtocolDataTrackInfo): DataTrackInfo {
     return {

package/src/room/events.ts

@@ -225,7 +225,7 @@ export enum RoomEvent {
    * Data packets provides the ability to use LiveKit to send/receive arbitrary payloads.
    * All participants in the room will receive the messages sent to the room.
    *
-   * args: (payload: Uint8Array, participant: [[Participant]], kind: [[DataPacket_Kind]], topic?: string)
+   * args: (payload: NonSharedUint8Array, participant: [[Participant]], kind: [[DataPacket_Kind]], topic?: string)
    */
   DataReceived = 'dataReceived',
 
@@ -489,7 +489,7 @@ export enum ParticipantEvent {
    * Data packets provides the ability to use LiveKit to send/receive arbitrary payloads.
    * All participants in the room will receive the messages sent to the room.
    *
-   * args: (payload: Uint8Array, kind: [[DataPacket_Kind]])
+   * args: (payload: NonSharedUint8Array, kind: [[DataPacket_Kind]])
    */
   DataReceived = 'dataReceived',
 

package/src/room/participant/LocalParticipant.test.ts

@@ -0,0 +1,81 @@
+import { PacketTrailerFeature } from '@livekit/protocol';
+import { describe, expect, it, vi } from 'vitest';
+import type LocalTrack from '../track/LocalTrack';
+import { Track } from '../track/Track';
+import type { TrackPublishOptions } from '../track/options';
+import LocalParticipant from './LocalParticipant';
+
+type PacketTrailerTestParticipant = {
+  canPublishPacketTrailer: () => boolean;
+  log: { warn: ReturnType<typeof vi.fn> };
+  normalizeRequestedPacketTrailerOptions: (
+    track: LocalTrack,
+    opts: TrackPublishOptions,
+  ) => PacketTrailerFeature[];
+};
+
+function makeParticipant(canPublishPacketTrailer: boolean) {
+  const participant = Object.create(LocalParticipant.prototype) as PacketTrailerTestParticipant;
+  participant.canPublishPacketTrailer = () => canPublishPacketTrailer;
+  participant.log = { warn: vi.fn() };
+  return participant;
+}
+
+function makeTrack(kind: Track.Kind) {
+  return {
+    kind,
+    sid: 'track-sid',
+    source: kind === Track.Kind.Video ? Track.Source.Camera : Track.Source.Microphone,
+    isMuted: false,
+    mediaStreamID: 'stream-id',
+    mediaStreamTrack: {
+      enabled: true,
+      id: 'media-track-id',
+    },
+  } as unknown as LocalTrack;
+}
+
+describe('LocalParticipant packet trailer publish options', () => {
+  it('normalizes requested video packet trailer options to advertised features', () => {
+    const participant = makeParticipant(true);
+    const opts: TrackPublishOptions = { packetTrailer: { timestamp: true, frameId: true } };
+
+    const features = participant.normalizeRequestedPacketTrailerOptions(
+      makeTrack(Track.Kind.Video),
+      opts,
+    );
+
+    expect(features).toEqual([
+      PacketTrailerFeature.PTF_USER_TIMESTAMP,
+      PacketTrailerFeature.PTF_FRAME_ID,
+    ]);
+    expect(opts.packetTrailer).toEqual({ timestamp: true, frameId: true });
+  });
+
+  it('clears packet trailer options for non-video tracks', () => {
+    const participant = makeParticipant(true);
+    const opts: TrackPublishOptions = { packetTrailer: { timestamp: true } };
+
+    const features = participant.normalizeRequestedPacketTrailerOptions(
+      makeTrack(Track.Kind.Audio),
+      opts,
+    );
+
+    expect(features).toEqual([]);
+    expect(opts.packetTrailer).toBeUndefined();
+  });
+
+  it('clears packet trailer options when publishing packet trailers is unsupported', () => {
+    const participant = makeParticipant(false);
+    const opts: TrackPublishOptions = { packetTrailer: { frameId: true } };
+
+    const features = participant.normalizeRequestedPacketTrailerOptions(
+      makeTrack(Track.Kind.Video),
+      opts,
+    );
+
+    expect(features).toEqual([]);
+    expect(opts.packetTrailer).toBeUndefined();
+    expect(participant.log.warn).toHaveBeenCalledOnce();
+  });
+});