livekit-client 2.18.0 → 2.18.1

package/src/room/Room.ts

@@ -47,7 +47,7 @@ import TypedPromise from '../utils/TypedPromise';
 import { getBrowser } from '../utils/browserParser';
 import { BackOffStrategy } from './BackOffStrategy';
 import DeviceManager from './DeviceManager';
-import RTCEngine from './RTCEngine';
+import RTCEngine, { DataChannelKind } from './RTCEngine';
 import { RegionUrlProvider } from './RegionUrlProvider';
 import IncomingDataStreamManager from './data-stream/incoming/IncomingDataStreamManager';
 import {
@@ -55,6 +55,11 @@ import {
   type TextStreamHandler,
 } from './data-stream/incoming/StreamReader';
 import OutgoingDataStreamManager from './data-stream/outgoing/OutgoingDataStreamManager';
+import type LocalDataTrack from './data-track/LocalDataTrack';
+import type RemoteDataTrack from './data-track/RemoteDataTrack';
+import IncomingDataTrackManager from './data-track/incoming/IncomingDataTrackManager';
+import OutgoingDataTrackManager from './data-track/outgoing/OutgoingDataTrackManager';
+import { DataTrackInfo, type DataTrackSid } from './data-track/types';
 import {
   audioDefaults,
   publishDefaults,
@@ -70,7 +75,7 @@ import {
 } from './errors';
 import { EngineEvent, ParticipantEvent, RoomEvent, TrackEvent } from './events';
 import LocalParticipant from './participant/LocalParticipant';
-import type Participant from './participant/Participant';
+import Participant from './participant/Participant';
 import { type ConnectionQuality, ParticipantKind } from './participant/Participant';
 import RemoteParticipant from './participant/RemoteParticipant';
 import { MAX_PAYLOAD_BYTES, RpcError, type RpcInvocationData, byteLength } from './rpc';
@@ -205,6 +210,10 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
 
   private outgoingDataStreamManager: OutgoingDataStreamManager;
 
+  private incomingDataTrackManager: IncomingDataTrackManager;
+
+  private outgoingDataTrackManager: OutgoingDataTrackManager;
+
   private rpcHandlers: Map<string, (data: RpcInvocationData) => Promise<string>> = new Map();
 
   get hasE2EESetup(): boolean {
@@ -243,6 +252,46 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     this.incomingDataStreamManager = new IncomingDataStreamManager();
     this.outgoingDataStreamManager = new OutgoingDataStreamManager(this.engine, this.log);
 
+    this.incomingDataTrackManager = new IncomingDataTrackManager({ e2eeManager: this.e2eeManager });
+    this.incomingDataTrackManager
+      .on('sfuUpdateSubscription', (event) => {
+        this.engine.client.sendUpdateDataSubscription(event.sid, event.subscribe);
+      })
+      .on('trackPublished', (event) => {
+        if (event.track.publisherIdentity === this.localParticipant.identity) {
+          // Only advertize tracks from other participants
+          return;
+        }
+        this.emit(RoomEvent.DataTrackPublished, event.track);
+        this.remoteParticipants.get(event.track.publisherIdentity)?.addRemoteDataTrack(event.track);
+      })
+      .on('trackUnpublished', (event) => {
+        if (event.publisherIdentity === this.localParticipant.identity) {
+          // Only advertize tracks from other participants
+          return;
+        }
+        this.emit(RoomEvent.DataTrackUnpublished, event.sid);
+        this.remoteParticipants.get(event.publisherIdentity)?.removeRemoteDataTrack(event.sid);
+      });
+
+    this.outgoingDataTrackManager = new OutgoingDataTrackManager({ e2eeManager: this.e2eeManager });
+    this.outgoingDataTrackManager
+      .on('sfuPublishRequest', (event) => {
+        this.engine.client.sendPublishDataTrackRequest(event.handle, event.name, event.usesE2ee);
+      })
+      .on('sfuUnpublishRequest', (event) => {
+        this.engine.client.sendUnPublishDataTrackRequest(event.handle);
+      })
+      .on('trackPublished', (event) => {
+        this.emit(RoomEvent.LocalDataTrackPublished, event.track);
+      })
+      .on('trackUnpublished', (event) => {
+        this.emit(RoomEvent.LocalDataTrackUnpublished, event.sid);
+      })
+      .on('packetAvailable', ({ bytes }) => {
+        this.engine.sendLossyBytes(bytes, DataChannelKind.DATA_TRACK_LOSSY, 'wait');
+      });
+
     this.disconnectLock = new Mutex();
 
     this.localParticipant = new LocalParticipant(
@@ -252,6 +301,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       this.options,
       this.rpcHandlers,
       this.outgoingDataStreamManager,
+      this.outgoingDataTrackManager,
     );
 
     if (this.options.e2ee || this.options.encryption) {
@@ -259,6 +309,8 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     }
 
     this.engine.e2eeManager = this.e2eeManager;
+    this.incomingDataTrackManager.updateE2eeManager(this.e2eeManager ?? null);
+    this.outgoingDataTrackManager.updateE2eeManager(this.e2eeManager ?? null);
 
     if (this.options.videoCaptureDefaults.deviceId) {
       this.localParticipant.activeDeviceMap.set(
@@ -515,6 +567,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
         }
       })
       .on(EngineEvent.Restarting, this.handleRestarting)
+      .on(EngineEvent.Restarted, this.handleRestarted)
       .on(EngineEvent.SignalRestarted, this.handleSignalRestarted)
       .on(EngineEvent.Offline, () => {
         if (this.setAndEmitConnectionState(ConnectionState.Reconnecting)) {
@@ -560,6 +613,66 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
         } else {
           this.handleParticipantUpdates(roomMoved.otherParticipants);
         }
+      })
+      .on(EngineEvent.PublishDataTrackResponse, (event) => {
+        if (!event.info) {
+          this.log.warn(
+            `received PublishDataTrackResponse, but event.info was ${event.info}, so skipping.`,
+            this.logContext,
+          );
+          return;
+        }
+
+        this.outgoingDataTrackManager.receivedSfuPublishResponse(event.info.pubHandle, {
+          type: 'ok',
+          data: {
+            sid: event.info.sid,
+            pubHandle: event.info.pubHandle,
+            name: event.info.name,
+            usesE2ee: event.info.encryption !== Encryption_Type.NONE,
+          },
+        });
+      })
+      .on(EngineEvent.UnPublishDataTrackResponse, (event) => {
+        if (!event.info) {
+          this.log.warn(
+            `received UnPublishDataTrackResponse, but event.info was ${event.info}, so skipping.`,
+            this.logContext,
+          );
+          return;
+        }
+
+        this.outgoingDataTrackManager.receivedSfuUnpublishResponse(event.info.pubHandle);
+      })
+      .on(EngineEvent.DataTrackSubscriberHandles, (event) => {
+        const handleToSidMapping = new Map(
+          Object.entries(event.subHandles).map(([key, value]) => {
+            return [parseInt(key, 10), value.trackSid];
+          }),
+        );
+
+        this.incomingDataTrackManager.receivedSfuSubscriberHandles(handleToSidMapping);
+      })
+      .on(EngineEvent.DataTrackPacketReceived, (packetBytes) => {
+        try {
+          this.incomingDataTrackManager.packetReceived(packetBytes);
+        } catch (err) {
+          // NOTE: wrapping in the bare try/catch like this means that the Throws<...> type doesn't
+          // propagate upwards into the public interface.
+          throw err;
+        }
+      })
+      .on(EngineEvent.Joined, (joinResponse) => {
+        // Ingest data track publication updates into data tracks infrastructure
+        const mapped = new Map(
+          joinResponse.otherParticipants.map((participant) => {
+            return [
+              participant.identity,
+              participant.dataTracks.map((info) => DataTrackInfo.from(info)),
+            ];
+          }),
+        );
+        this.incomingDataTrackManager.receiveSfuPublicationUpdates(mapped);
       });
 
     if (this.localParticipant) {
@@ -1456,10 +1569,13 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     ) as RemoteParticipant | undefined;
 
     if (!participant) {
-      this.log.error(
-        `Tried to add a track for a participant, that's not present. Sid: ${participantSid}`,
-        this.logContext,
-      );
+      // server could require extra media sections to accelerate subscription.
+      if (participantSid.startsWith('PA')) {
+        this.log.error(
+          `Tried to add a track for a participant, that's not present. Sid: ${participantSid}`,
+          this.logContext,
+        );
+      }
       return;
     }
 
@@ -1527,6 +1643,11 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     }
   };
 
+  private handleRestarted = () => {
+    this.outgoingDataTrackManager.sfuWillRepublishTracks();
+    this.incomingDataTrackManager.resendSubscriptionUpdates();
+  };
+
   private handleSignalRestarted = async (joinResponse: JoinResponse) => {
     this.log.debug(`signal reconnected to server, region ${joinResponse.serverRegion}`, {
       ...this.logContext,
@@ -1640,10 +1761,10 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
 
   private handleParticipantUpdates = (participantInfos: ParticipantInfo[]) => {
     // handle changes to participant state, and send events
-    participantInfos.forEach((info) => {
+    for (const info of participantInfos) {
       if (info.identity === this.localParticipant.identity) {
         this.localParticipant.updateInfo(info);
-        return;
+        continue;
       }
 
       // LiveKit server doesn't send identity info prior to version 1.5.2 in disconnect updates
@@ -1661,7 +1782,17 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
         // create participant if doesn't exist
         remoteParticipant = this.getOrCreateParticipant(info.identity, info);
       }
-    });
+    }
+
+    // Ingest data track publication updates into data tracks infrastructure
+    const mapped = new Map(
+      participantInfos
+        .filter((p) => p.identity !== this.localParticipant.identity)
+        .map((info) => {
+          return [info.identity, info.dataTracks.map((dataTrack) => DataTrackInfo.from(dataTrack))];
+        }),
+    );
+    this.incomingDataTrackManager.receiveSfuPublicationUpdates(mapped);
   };
 
   private handleParticipantDisconnected(identity: string, participant?: RemoteParticipant) {
@@ -1672,6 +1803,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
     }
 
     this.incomingDataStreamManager.validateParticipantHasNoActiveDataStreams(identity);
+    this.incomingDataTrackManager.handleRemoteParticipantDisconnected(identity);
 
     participant.trackPublications.forEach((publication) => {
       participant.unpublishTrack(publication.trackSid, true);
@@ -2192,7 +2324,7 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
             track.on(TrackEvent.VideoPlaybackFailed, this.handleVideoPlaybackFailed);
             track.on(TrackEvent.VideoPlaybackStarted, this.handleVideoPlaybackStarted);
           }
-          this.emit(RoomEvent.TrackSubscribed, track, publication, participant);
+          this.emitWhenConnected(RoomEvent.TrackSubscribed, track, publication, participant);
         },
       )
       .on(ParticipantEvent.TrackUnpublished, (publication: RemoteTrackPublication) => {
@@ -2270,7 +2402,8 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
       return acc;
     }, [] as RemoteTrackPublication[]);
     const localTracks = this.localParticipant.getTrackPublications() as LocalTrackPublication[]; // FIXME would be nice to have this return LocalTrackPublications directly instead of the type cast
-    this.engine.sendSyncState(remoteTracks, localTracks);
+    const localDataTrackInfos = this.outgoingDataTrackManager.queryPublished();
+    this.engine.sendSyncState(remoteTracks, localTracks, localDataTrackInfos);
   }
 
   /**
@@ -2732,10 +2865,14 @@ export type RoomEventCallbacks = {
   recordingStatusChanged: (recording: boolean) => void;
   participantEncryptionStatusChanged: (encrypted: boolean, participant?: Participant) => void;
   encryptionError: (error: Error, participant?: Participant) => void;
-  dcBufferStatusChanged: (isLow: boolean, kind: DataPacket_Kind) => void;
+  dcBufferStatusChanged: (isLow: boolean, kind: DataChannelKind) => void;
   activeDeviceChanged: (kind: MediaDeviceKind, deviceId: string) => void;
   chatMessage: (message: ChatMessage, participant?: RemoteParticipant | LocalParticipant) => void;
   localTrackSubscribed: (publication: LocalTrackPublication, participant: LocalParticipant) => void;
   metricsReceived: (metrics: MetricsBatch, participant?: Participant) => void;
   participantActive: (participant: Participant) => void;
+  dataTrackPublished: (track: RemoteDataTrack) => void;
+  dataTrackUnpublished: (sid: DataTrackSid) => void;
+  localDataTrackPublished: (track: LocalDataTrack) => void;
+  localDataTrackUnpublished: (sid: DataTrackSid) => void;
 };

package/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts

@@ -1,7 +1,6 @@
 import { Mutex } from '@livekit/mutex';
 import {
   DataPacket,
-  DataPacket_Kind,
   DataStream_ByteHeader,
   DataStream_Chunk,
   DataStream_Header,
@@ -12,6 +11,7 @@ import {
 } from '@livekit/protocol';
 import { type StructuredLogger } from '../../../logger';
 import type RTCEngine from '../../RTCEngine';
+import { DataChannelKind } from '../../RTCEngine';
 import { EngineEvent } from '../../events';
 import type {
   ByteStreamInfo,
@@ -137,7 +137,7 @@ export default class OutgoingDataStreamManager {
         value: header,
       },
     });
-    await this.engine.sendDataPacket(packet, DataPacket_Kind.RELIABLE);
+    await this.engine.sendDataPacket(packet, DataChannelKind.RELIABLE);
 
     let chunkId = 0;
     const engine = this.engine;
@@ -158,7 +158,7 @@ export default class OutgoingDataStreamManager {
               value: chunk,
             },
           });
-          await engine.sendDataPacket(chunkPacket, DataPacket_Kind.RELIABLE);
+          await engine.sendDataPacket(chunkPacket, DataChannelKind.RELIABLE);
 
           chunkId += 1;
         }
@@ -174,7 +174,7 @@ export default class OutgoingDataStreamManager {
             value: trailer,
           },
         });
-        await engine.sendDataPacket(trailerPacket, DataPacket_Kind.RELIABLE);
+        await engine.sendDataPacket(trailerPacket, DataChannelKind.RELIABLE);
       },
       abort(err) {
         console.log('Sink error:', err);
@@ -262,7 +262,7 @@ export default class OutgoingDataStreamManager {
       },
     });
 
-    await this.engine.sendDataPacket(packet, DataPacket_Kind.RELIABLE);
+    await this.engine.sendDataPacket(packet, DataChannelKind.RELIABLE);
 
     let chunkId = 0;
     const writeMutex = new Mutex();
@@ -288,7 +288,7 @@ export default class OutgoingDataStreamManager {
                 }),
               },
             });
-            await engine.sendDataPacket(chunkPacket, DataPacket_Kind.RELIABLE);
+            await engine.sendDataPacket(chunkPacket, DataChannelKind.RELIABLE);
             chunkId += 1;
             byteOffset += subChunk.byteLength;
           }
@@ -307,7 +307,7 @@ export default class OutgoingDataStreamManager {
             value: trailer,
           },
         });
-        await engine.sendDataPacket(trailerPacket, DataPacket_Kind.RELIABLE);
+        await engine.sendDataPacket(trailerPacket, DataChannelKind.RELIABLE);
       },
       abort(err) {
         logLocal.error('Sink error:', err);

package/src/room/data-track/LocalDataTrack.ts

@@ -1,5 +1,9 @@
-import type { DataTrackFrame } from './frame';
+import log, { LoggerNames, type StructuredLogger, getLogger } from '../../logger';
+import { type DataTrackFrame, DataTrackFrameInternal } from './frame';
+import type { DataTrackHandle } from './handle';
 import type OutgoingDataTrackManager from './outgoing/OutgoingDataTrackManager';
+import { DataTrackPushFrameError } from './outgoing/errors';
+import type { DataTrackOptions } from './outgoing/types';
 import {
   DataTrackSymbol,
   type IDataTrack,
@@ -15,23 +19,66 @@ export default class LocalDataTrack implements ILocalTrack, IDataTrack {
 
   readonly typeSymbol = DataTrackSymbol;
 
-  info: DataTrackInfo;
+  protected options: DataTrackOptions;
+
+  /** Represents the currently active {@link DataTrackHandle} for the publication. */
+  protected handle: DataTrackHandle | null = null;
 
   protected manager: OutgoingDataTrackManager;
 
+  protected log: StructuredLogger = log;
+
   /** @internal */
-  constructor(info: DataTrackInfo, manager: OutgoingDataTrackManager) {
-    this.info = info;
+  constructor(options: DataTrackOptions, manager: OutgoingDataTrackManager) {
+    this.options = options;
     this.manager = manager;
+
+    this.log = getLogger(LoggerNames.DataTracks);
+  }
+
+  /** @internal */
+  static withExplicitHandle(
+    options: DataTrackOptions,
+    manager: OutgoingDataTrackManager,
+    handle: DataTrackHandle,
+  ) {
+    const track = new LocalDataTrack(options, manager);
+    track.handle = handle;
+    return track;
+  }
+
+  /** Metrics about the data track publication. */
+  get info() {
+    const descriptor = this.descriptor;
+    if (descriptor?.type === 'active') {
+      return descriptor.info;
+    } else {
+      return undefined;
+    }
   }
 
   /** The raw descriptor from the manager containing the internal state for this local track. */
   protected get descriptor() {
-    return this.manager.getDescriptor(this.info.pubHandle);
+    return this.handle ? this.manager.getDescriptor(this.handle) : null;
+  }
+
+  /**
+   * Publish the track to the SFU. This must be done before calling {@link tryPush} for the first time.
+   * @internal
+   * */
+  async publish(signal?: AbortSignal) {
+    try {
+      this.handle = await this.manager.publishRequest(this.options, signal);
+    } catch (err) {
+      // NOTE: Rethrow errors to break Throws<...> type boundary
+      throw err;
+    }
   }
 
-  isPublished() {
-    return this.descriptor?.type === 'active';
+  isPublished(): this is { info: DataTrackInfo } {
+    // NOTE: a track which is internally in the "resubscribing" state is still considered
+    // published from the public API perspective.
+    return this.descriptor?.type === 'active' && this.descriptor.publishState !== 'unpublished';
   }
 
   /** Try pushing a frame to subscribers of the track.
@@ -41,12 +88,38 @@ export default class LocalDataTrack implements ILocalTrack, IDataTrack {
    * - The track has been unpublished by the local participant or SFU
    * - The room is no longer connected
    */
-  tryPush(payload: DataTrackFrame['payload']) {
+  tryPush(frame: DataTrackFrame) {
+    if (!this.handle) {
+      throw DataTrackPushFrameError.trackUnpublished();
+    }
+
+    const internalFrame = DataTrackFrameInternal.from(frame);
+
     try {
-      return this.manager.tryProcessAndSend(this.info.pubHandle, payload);
+      return this.manager.tryProcessAndSend(this.handle, internalFrame);
     } catch (err) {
       // NOTE: wrapping in the bare try/catch like this means that the Throws<...> type doesn't
-      // propegate upwards into the public interface.
+      // propagate upwards into the public interface.
+      throw err;
+    }
+  }
+
+  /**
+   * Unpublish the track from the SFU. Once this is called, any further calls to {@link tryPush}
+   * will fail.
+   * */
+  async unpublish() {
+    if (!this.handle) {
+      log.warn(
+        `Data track "${this.options.name}" is not published, so unpublishing has no effect.`,
+      );
+      return;
+    }
+
+    try {
+      await this.manager.unpublishRequest(this.handle);
+    } catch (err) {
+      // NOTE: Rethrow errors to break Throws<...> type boundary
       throw err;
     }
   }

package/src/room/data-track/RemoteDataTrack.ts

@@ -1,5 +1,5 @@
 import type Participant from '../participant/Participant';
-import type { DataTrackFrame } from './frame';
+import { type DataTrackFrame } from './frame';
 import type IncomingDataTrackManager from './incoming/IncomingDataTrackManager';
 import {
   DataTrackSymbol,
@@ -13,12 +13,12 @@ type RemoteDataTrackOptions = {
   publisherIdentity: Participant['identity'];
 };
 
-export type RemoteDataTrackSubscribeOptions = {
+export type DataTrackSubscribeOptions = {
   signal?: AbortSignal;
 
   /** The number of {@link DataTrackFrame}s to hold in the ReadableStream before disgarding extra
-   * frames. Defaults to 4, but this may not be good enough for especially high frequency data. */
-  highWaterMark?: number;
+   * frames. Defaults to 16, but this may not be good enough for especially high frequency data. */
+  bufferSize?: number;
 };
 
 export default class RemoteDataTrack implements IRemoteTrack, IDataTrack {
@@ -64,14 +64,12 @@ export default class RemoteDataTrack implements IRemoteTrack, IDataTrack {
    * Note that newly created subscriptions only receive frames published after
    * the initial subscription is established.
    */
-  async subscribe(
-    options?: RemoteDataTrackSubscribeOptions,
-  ): Promise<ReadableStream<DataTrackFrame>> {
+  subscribe(options?: DataTrackSubscribeOptions): ReadableStream<DataTrackFrame> {
     try {
-      const stream = await this.manager.subscribeRequest(
+      const [stream] = this.manager.openSubscriptionStream(
         this.info.sid,
         options?.signal,
-        options?.highWaterMark,
+        options?.bufferSize,
       );
       return stream;
     } catch (err) {

package/src/room/data-track/depacketizer.ts

@@ -1,7 +1,7 @@
 import { type Throws } from '@livekit/throws-transformer/throws';
 import { LoggerNames, getLogger } from '../../logger';
 import { LivekitReasonedError } from '../errors';
-import { type DataTrackFrame } from './frame';
+import { type DataTrackFrameInternal } from './frame';
 import { DataTrackPacket, FrameMarker } from './packet';
 import { DataTrackExtensions } from './packet/extensions';
 import { U16_MAX_SIZE, WrapAroundUnsignedInt } from './utils';
@@ -38,9 +38,9 @@ export class DataTrackDepacketizerDropError<
     this.frameNumber = frameNumber;
   }
 
-  static interrupted(frameNumber: number) {
+  static interrupted(frameNumber: number, newFrameNumber: number) {
     return new DataTrackDepacketizerDropError(
-      'Interrupted by the start of a new frame',
+      `Interrupted by the start of a new frame ${newFrameNumber}`,
       DataTrackDepacketizerDropReason.Interrupted,
       frameNumber,
     );
@@ -94,12 +94,12 @@ export default class DataTrackDepacketizer {
   /** Should be repeatedly called with received {@link DataTrackPacket}s - intermediate calls
    * aggregate the packet's state internally, and return null.
    *
-   * Once this method is called with the final packet to form a frame, a new {@link DataTrackFrame}
+   * Once this method is called with the final packet to form a frame, a new {@link DataTrackFrameInternal}
    * is returned.*/
   push(
     packet: DataTrackPacket,
     options?: PushOptions,
-  ): Throws<DataTrackFrame | null, DataTrackDepacketizerDropError> {
+  ): Throws<DataTrackFrameInternal | null, DataTrackDepacketizerDropError> {
     switch (packet.header.marker) {
       case FrameMarker.Single:
         return this.frameFromSingle(packet, options);
@@ -119,7 +119,7 @@ export default class DataTrackDepacketizer {
     packet: DataTrackPacket,
     options?: PushOptions,
   ): Throws<
-    DataTrackFrame | null,
+    DataTrackFrameInternal | null,
     DataTrackDepacketizerDropError<DataTrackDepacketizerDropReason.Interrupted>
   > {
     if (packet.header.marker !== FrameMarker.Single) {
@@ -133,7 +133,10 @@ export default class DataTrackDepacketizer {
       if (options?.errorOnPartialFrames) {
         const frameNumber = this.partial.frameNumber;
         this.reset();
-        throw DataTrackDepacketizerDropError.interrupted(frameNumber);
+        throw DataTrackDepacketizerDropError.interrupted(
+          frameNumber,
+          packet.header.frameNumber.value,
+        );
       } else {
         log.warn(
           `Data track frame ${this.partial.frameNumber} was interrupted by the start of a new frame, dropping.`,
@@ -161,10 +164,13 @@ export default class DataTrackDepacketizer {
       if (options?.errorOnPartialFrames) {
         const frameNumber = this.partial.frameNumber;
         this.reset();
-        throw DataTrackDepacketizerDropError.interrupted(frameNumber);
+        throw DataTrackDepacketizerDropError.interrupted(
+          frameNumber,
+          packet.header.frameNumber.value,
+        );
       } else {
         log.warn(
-          `Data track frame ${this.partial.frameNumber} was interrupted by the start of a new frame, dropping.`,
+          `Data track frame ${this.partial.frameNumber} was interrupted by the start of a new frame ${packet.header.frameNumber.value}, dropping.`,
         );
       }
     }
@@ -185,7 +191,7 @@ export default class DataTrackDepacketizer {
   /** Push to the existing partial frame. */
   private pushToPartial(
     packet: DataTrackPacket,
-  ): Throws<DataTrackFrame | null, DataTrackDepacketizerDropError> {
+  ): Throws<DataTrackFrameInternal | null, DataTrackDepacketizerDropError> {
     if (packet.header.marker !== FrameMarker.Inter && packet.header.marker !== FrameMarker.Final) {
       // @throws-transformer ignore - this should be treated as a "panic" and not be caught
       throw new Error(
@@ -201,7 +207,10 @@ export default class DataTrackDepacketizer {
     if (packet.header.frameNumber.value !== this.partial.frameNumber) {
       const frameNumber = this.partial.frameNumber;
       this.reset();
-      throw DataTrackDepacketizerDropError.interrupted(frameNumber);
+      throw DataTrackDepacketizerDropError.interrupted(
+        frameNumber,
+        packet.header.frameNumber.value,
+      );
     }
 
     // NOTE: this check will block reprocessing packets with duplicate sequence values if the
@@ -234,7 +243,7 @@ export default class DataTrackDepacketizer {
     partial: PartialFrame,
     endSequence: number,
   ): Throws<
-    DataTrackFrame,
+    DataTrackFrameInternal,
     DataTrackDepacketizerDropError<DataTrackDepacketizerDropReason.Incomplete>
   > {
     const received = partial.payloads.size;

package/src/room/data-track/frame.ts

@@ -1,8 +1,34 @@
-import { DataTrackExtensions } from './packet/extensions';
-import DataTrackPacketizer from './packetizer';
+import { DataTrackExtensions, DataTrackUserTimestampExtension } from './packet/extensions';
 
 /** A pair of payload bytes and packet extensions which can be fed into a {@link DataTrackPacketizer}. */
 export type DataTrackFrame = {
+  payload: Uint8Array;
+  userTimestamp?: bigint;
+};
+
+/** An internal representation o data track frame which contains all SFU metadata. */
+export type DataTrackFrameInternal = {
   payload: Uint8Array;
   extensions: DataTrackExtensions;
 };
+
+export const DataTrackFrameInternal = {
+  from(frame: DataTrackFrame) {
+    return {
+      payload: frame.payload,
+      extensions: new DataTrackExtensions({
+        userTimestamp: frame.userTimestamp
+          ? new DataTrackUserTimestampExtension(frame.userTimestamp)
+          : undefined,
+      }),
+    };
+  },
+  /** Converts from a DataTrackFrameInternal -> DataTrackFrame. Some internal information is
+   * discarded like e2ee encrption extension data. */
+  lossyIntoFrame(frame: DataTrackFrameInternal): DataTrackFrame {
+    return {
+      payload: frame.payload,
+      userTimestamp: frame.extensions.userTimestamp?.timestamp,
+    };
+  },
+};