livekit-client 2.15.4 → 2.15.6

package/src/room/participant/LocalParticipant.ts

@@ -1,4 +1,3 @@
-import { Mutex } from '@livekit/mutex';
 import {
   AddTrackRequest,
   AudioTrackFeature,
@@ -7,12 +6,6 @@ import {
   Codec,
   DataPacket,
   DataPacket_Kind,
-  DataStream_ByteHeader,
-  DataStream_Chunk,
-  DataStream_Header,
-  DataStream_OperationType,
-  DataStream_TextHeader,
-  DataStream_Trailer,
   Encryption_Type,
   JoinResponse,
   ParticipantInfo,
@@ -34,7 +27,8 @@ import { SignalConnectionState } from '../../api/SignalClient';
 import type { InternalRoomOptions } from '../../options';
 import { PCTransportState } from '../PCTransportManager';
 import type RTCEngine from '../RTCEngine';
-import { ByteStreamWriter, TextStreamWriter } from '../StreamWriter';
+import type OutgoingDataStreamManager from '../data-stream/outgoing/OutgoingDataStreamManager';
+import type { TextStreamWriter } from '../data-stream/outgoing/StreamWriter';
 import { defaultVideoCodec } from '../defaults';
 import {
   DeviceUnsupportedError,
@@ -76,10 +70,11 @@ import {
   sourceToKind,
 } from '../track/utils';
 import {
-  type ByteStreamInfo,
   type ChatMessage,
   type DataPublishOptions,
+  type SendFileOptions,
   type SendTextOptions,
+  type StreamBytesOptions,
   type StreamTextOptions,
   type TextStreamInfo,
 } from '../types';
@@ -96,9 +91,7 @@ import {
   isSafari17Based,
   isVideoTrack,
   isWeb,
-  numberToBigInt,
   sleep,
-  splitUtf8,
   supportsAV1,
   supportsVP9,
 } from '../utils';
@@ -112,8 +105,6 @@ import {
   getDefaultDegradationPreference,
 } from './publishUtils';
 
-const STREAM_CHUNK_SIZE = 15_000;
-
 export default class LocalParticipant extends Participant {
   audioTrackPublications: Map<string, LocalTrackPublication>;
 
@@ -157,6 +148,8 @@ export default class LocalParticipant extends Participant {
 
   private rpcHandlers: Map<string, (data: RpcInvocationData) => Promise<string>>;
 
+  private roomOutgoingDataStreamManager: OutgoingDataStreamManager;
+
   private pendingSignalRequests: Map<
     number,
     {
@@ -185,6 +178,7 @@ export default class LocalParticipant extends Participant {
     engine: RTCEngine,
     options: InternalRoomOptions,
     roomRpcHandlers: Map<string, (data: RpcInvocationData) => Promise<string>>,
+    roomOutgoingDataStreamManager: OutgoingDataStreamManager,
   ) {
     super(sid, identity, undefined, undefined, undefined, {
       loggerName: options.loggerName,
@@ -203,6 +197,7 @@ export default class LocalParticipant extends Participant {
     ]);
     this.pendingSignalRequests = new Map();
     this.rpcHandlers = roomRpcHandlers;
+    this.roomOutgoingDataStreamManager = roomOutgoingDataStreamManager;
   }
 
   get lastCameraError(): Error | undefined {
@@ -1690,6 +1685,7 @@ export default class LocalParticipant extends Participant {
     await this.engine.sendDataPacket(packet, DataPacket_Kind.RELIABLE);
   }
 
+  /** @deprecated Consider migrating to {@link sendText} */
   async sendChatMessage(text: string, options?: SendTextOptions): Promise<ChatMessage> {
     const msg = {
       id: crypto.randomUUID(),
@@ -1712,6 +1708,7 @@ export default class LocalParticipant extends Participant {
     return msg;
   }
 
+  /** @deprecated Consider migrating to {@link sendText} */
   async editChatMessage(editText: string, originalMessage: ChatMessage) {
     const msg = {
       ...originalMessage,
@@ -1733,300 +1730,47 @@ export default class LocalParticipant extends Participant {
     return msg;
   }
 
+  /**
+   * Sends the given string to participants in the room via the data channel.
+   * For longer messages, consider using {@link streamText} instead.
+   *
+   * @param text The text payload
+   * @param options.topic Topic identifier used to route the stream to appropriate handlers.
+   */
   async sendText(text: string, options?: SendTextOptions): Promise<TextStreamInfo> {
-    const streamId = crypto.randomUUID();
-    const textInBytes = new TextEncoder().encode(text);
-    const totalTextLength = textInBytes.byteLength;
-
-    const fileIds = options?.attachments?.map(() => crypto.randomUUID());
-
-    const progresses = new Array<number>(fileIds ? fileIds.length + 1 : 1).fill(0);
-
-    const handleProgress = (progress: number, idx: number) => {
-      progresses[idx] = progress;
-      const totalProgress = progresses.reduce((acc, val) => acc + val, 0);
-      options?.onProgress?.(totalProgress);
-    };
-
-    const writer = await this.streamText({
-      streamId,
-      totalSize: totalTextLength,
-      destinationIdentities: options?.destinationIdentities,
-      topic: options?.topic,
-      attachedStreamIds: fileIds,
-      attributes: options?.attributes,
-    });
-
-    await writer.write(text);
-    // set text part of progress to 1
-    handleProgress(1, 0);
-
-    await writer.close();
-
-    if (options?.attachments && fileIds) {
-      await Promise.all(
-        options.attachments.map(async (file, idx) =>
-          this._sendFile(fileIds[idx], file, {
-            topic: options.topic,
-            mimeType: file.type,
-            onProgress: (progress) => {
-              handleProgress(progress, idx + 1);
-            },
-          }),
-        ),
-      );
-    }
-    return writer.info;
+    return this.roomOutgoingDataStreamManager.sendText(text, options);
   }
 
   /**
+   * Creates a new TextStreamWriter which can be used to stream text incrementally
+   * to participants in the room via the data channel.
+   *
+   * @param options.topic Topic identifier used to route the stream to appropriate handlers.
+   *
    * @internal
    * @experimental CAUTION, might get removed in a minor release
    */
   async streamText(options?: StreamTextOptions): Promise<TextStreamWriter> {
-    const streamId = options?.streamId ?? crypto.randomUUID();
-
-    const info: TextStreamInfo = {
-      id: streamId,
-      mimeType: 'text/plain',
-      timestamp: Date.now(),
-      topic: options?.topic ?? '',
-      size: options?.totalSize,
-      attributes: options?.attributes,
-    };
-    const header = new DataStream_Header({
-      streamId,
-      mimeType: info.mimeType,
-      topic: info.topic,
-      timestamp: numberToBigInt(info.timestamp),
-      totalLength: numberToBigInt(options?.totalSize),
-      attributes: info.attributes,
-      contentHeader: {
-        case: 'textHeader',
-        value: new DataStream_TextHeader({
-          version: options?.version,
-          attachedStreamIds: options?.attachedStreamIds,
-          replyToStreamId: options?.replyToStreamId,
-          operationType:
-            options?.type === 'update'
-              ? DataStream_OperationType.UPDATE
-              : DataStream_OperationType.CREATE,
-        }),
-      },
-    });
-    const destinationIdentities = options?.destinationIdentities;
-    const packet = new DataPacket({
-      destinationIdentities,
-      value: {
-        case: 'streamHeader',
-        value: header,
-      },
-    });
-    await this.engine.sendDataPacket(packet, DataPacket_Kind.RELIABLE);
-
-    let chunkId = 0;
-    const localP = this;
-
-    const writableStream = new WritableStream<string>({
-      // Implement the sink
-      async write(text) {
-        for (const textByteChunk of splitUtf8(text, STREAM_CHUNK_SIZE)) {
-          await localP.engine.waitForBufferStatusLow(DataPacket_Kind.RELIABLE);
-          const chunk = new DataStream_Chunk({
-            content: textByteChunk,
-            streamId,
-            chunkIndex: numberToBigInt(chunkId),
-          });
-          const chunkPacket = new DataPacket({
-            destinationIdentities,
-            value: {
-              case: 'streamChunk',
-              value: chunk,
-            },
-          });
-          await localP.engine.sendDataPacket(chunkPacket, DataPacket_Kind.RELIABLE);
-
-          chunkId += 1;
-        }
-      },
-      async close() {
-        const trailer = new DataStream_Trailer({
-          streamId,
-        });
-        const trailerPacket = new DataPacket({
-          destinationIdentities,
-          value: {
-            case: 'streamTrailer',
-            value: trailer,
-          },
-        });
-        await localP.engine.sendDataPacket(trailerPacket, DataPacket_Kind.RELIABLE);
-      },
-      abort(err) {
-        console.log('Sink error:', err);
-        // TODO handle aborts to signal something to receiver side
-      },
-    });
-
-    let onEngineClose = async () => {
-      await writer.close();
-    };
-
-    localP.engine.once(EngineEvent.Closing, onEngineClose);
-
-    const writer = new TextStreamWriter(writableStream, info, () =>
-      this.engine.off(EngineEvent.Closing, onEngineClose),
-    );
-
-    return writer;
+    return this.roomOutgoingDataStreamManager.streamText(options);
   }
 
-  async sendFile(
-    file: File,
-    options?: {
-      mimeType?: string;
-      topic?: string;
-      destinationIdentities?: Array<string>;
-      onProgress?: (progress: number) => void;
-    },
-  ): Promise<{ id: string }> {
-    const streamId = crypto.randomUUID();
-    await this._sendFile(streamId, file, options);
-    return { id: streamId };
-  }
-
-  private async _sendFile(
-    streamId: string,
-    file: File,
-    options?: {
-      mimeType?: string;
-      topic?: string;
-      encryptionType?: Encryption_Type.NONE;
-      destinationIdentities?: Array<string>;
-      onProgress?: (progress: number) => void;
-    },
-  ) {
-    const writer = await this.streamBytes({
-      streamId,
-      totalSize: file.size,
-      name: file.name,
-      mimeType: options?.mimeType ?? file.type,
-      topic: options?.topic,
-      destinationIdentities: options?.destinationIdentities,
-    });
-    const reader = file.stream().getReader();
-    while (true) {
-      const { done, value } = await reader.read();
-      if (done) {
-        break;
-      }
-      await writer.write(value);
-    }
-    await writer.close();
-    return writer.info;
+  /** Send a File to all participants in the room via the data channel.
+   * @param file The File object payload
+   * @param options.topic Topic identifier used to route the stream to appropriate handlers.
+   * @param options.onProgress A callback function used to monitor the upload progress percentage.
+   */
+  async sendFile(file: File, options?: SendFileOptions): Promise<{ id: string }> {
+    return this.roomOutgoingDataStreamManager.sendFile(file, options);
   }
 
-  async streamBytes(options?: {
-    name?: string;
-    topic?: string;
-    attributes?: Record<string, string>;
-    destinationIdentities?: Array<string>;
-    streamId?: string;
-    mimeType?: string;
-    totalSize?: number;
-  }) {
-    const streamId = options?.streamId ?? crypto.randomUUID();
-    const destinationIdentities = options?.destinationIdentities;
-
-    const info: ByteStreamInfo = {
-      id: streamId,
-      mimeType: options?.mimeType ?? 'application/octet-stream',
-      topic: options?.topic ?? '',
-      timestamp: Date.now(),
-      attributes: options?.attributes,
-      size: options?.totalSize,
-      name: options?.name ?? 'unknown',
-    };
-
-    const header = new DataStream_Header({
-      totalLength: numberToBigInt(info.size ?? 0),
-      mimeType: info.mimeType,
-      streamId,
-      topic: info.topic,
-      timestamp: numberToBigInt(Date.now()),
-      attributes: info.attributes,
-      contentHeader: {
-        case: 'byteHeader',
-        value: new DataStream_ByteHeader({
-          name: info.name,
-        }),
-      },
-    });
-
-    const packet = new DataPacket({
-      destinationIdentities,
-      value: {
-        case: 'streamHeader',
-        value: header,
-      },
-    });
-
-    await this.engine.sendDataPacket(packet, DataPacket_Kind.RELIABLE);
-
-    let chunkId = 0;
-    const writeMutex = new Mutex();
-    const engine = this.engine;
-    const log = this.log;
-
-    const writableStream = new WritableStream<Uint8Array>({
-      async write(chunk) {
-        const unlock = await writeMutex.lock();
-
-        let byteOffset = 0;
-        try {
-          while (byteOffset < chunk.byteLength) {
-            const subChunk = chunk.slice(byteOffset, byteOffset + STREAM_CHUNK_SIZE);
-            await engine.waitForBufferStatusLow(DataPacket_Kind.RELIABLE);
-            const chunkPacket = new DataPacket({
-              destinationIdentities,
-              value: {
-                case: 'streamChunk',
-                value: new DataStream_Chunk({
-                  content: subChunk,
-                  streamId,
-                  chunkIndex: numberToBigInt(chunkId),
-                }),
-              },
-            });
-            await engine.sendDataPacket(chunkPacket, DataPacket_Kind.RELIABLE);
-            chunkId += 1;
-            byteOffset += subChunk.byteLength;
-          }
-        } finally {
-          unlock();
-        }
-      },
-      async close() {
-        const trailer = new DataStream_Trailer({
-          streamId,
-        });
-        const trailerPacket = new DataPacket({
-          destinationIdentities,
-          value: {
-            case: 'streamTrailer',
-            value: trailer,
-          },
-        });
-        await engine.sendDataPacket(trailerPacket, DataPacket_Kind.RELIABLE);
-      },
-      abort(err) {
-        log.error('Sink error:', err);
-      },
-    });
-
-    const byteWriter = new ByteStreamWriter(writableStream, info);
-
-    return byteWriter;
+  /**
+   * Stream bytes incrementally to participants in the room via the data channel.
+   * For sending files, consider using {@link sendFile} instead.
+   *
+   * @param options.topic Topic identifier used to route the stream to appropriate handlers.
+   */
+  async streamBytes(options?: StreamBytesOptions) {
+    return this.roomOutgoingDataStreamManager.streamBytes(options);
   }
 
   /**

package/src/room/track/LocalAudioTrack.ts

@@ -169,7 +169,7 @@ export default class LocalAudioTrack extends LocalTrack<Track.Kind.Audio> {
   };
 
   async setProcessor(processor: TrackProcessor<Track.Kind.Audio, AudioProcessorOptions>) {
-    const unlock = await this.processorLock.lock();
+    const unlock = await this.trackChangeLock.lock();
     try {
       if (!isReactNative() && !this.audioContext) {
         throw Error(
@@ -177,7 +177,7 @@ export default class LocalAudioTrack extends LocalTrack<Track.Kind.Audio> {
         );
       }
       if (this.processor) {
-        await this.stopProcessor();
+        await this.internalStopProcessor();
       }
 
       const processorOptions = {

package/src/room/track/LocalTrack.ts

@@ -57,15 +57,13 @@ export default abstract class LocalTrack<
 
   protected processor?: TrackProcessor<TrackKind, any>;
 
-  protected processorLock: Mutex;
-
   protected audioContext?: AudioContext;
 
   protected manuallyStopped: boolean = false;
 
   protected localTrackRecorder: LocalTrackRecorder<typeof this> | undefined;
 
-  private restartLock: Mutex;
+  protected trackChangeLock: Mutex;
 
   /**
    *
@@ -86,9 +84,14 @@ export default abstract class LocalTrack<
     this.providedByUser = userProvidedTrack;
     this.muteLock = new Mutex();
     this.pauseUpstreamLock = new Mutex();
-    this.processorLock = new Mutex();
-    this.restartLock = new Mutex();
-    this.setMediaStreamTrack(mediaTrack, true);
+    this.trackChangeLock = new Mutex();
+    this.trackChangeLock.lock().then(async (unlock) => {
+      try {
+        await this.setMediaStreamTrack(mediaTrack, true);
+      } finally {
+        unlock();
+      }
+    });
 
     // added to satisfy TS compiler, constraints are synced with MediaStreamTrack
     this._constraints = mediaTrack.getConstraints();
@@ -171,27 +174,22 @@ export default abstract class LocalTrack<
     }
     let processedTrack: MediaStreamTrack | undefined;
     if (this.processor && newTrack) {
-      const unlock = await this.processorLock.lock();
-      try {
-        this.log.debug('restarting processor', this.logContext);
-        if (this.kind === 'unknown') {
-          throw TypeError('cannot set processor on track of unknown kind');
-        }
+      this.log.debug('restarting processor', this.logContext);
+      if (this.kind === 'unknown') {
+        throw TypeError('cannot set processor on track of unknown kind');
+      }
 
-        if (this.processorElement) {
-          attachToElement(newTrack, this.processorElement);
-          // ensure the processorElement itself stays muted
-          this.processorElement.muted = true;
-        }
-        await this.processor.restart({
-          track: newTrack,
-          kind: this.kind,
-          element: this.processorElement,
-        });
-        processedTrack = this.processor.processedTrack;
-      } finally {
-        unlock();
+      if (this.processorElement) {
+        attachToElement(newTrack, this.processorElement);
+        // ensure the processorElement itself stays muted
+        this.processorElement.muted = true;
       }
+      await this.processor.restart({
+        track: newTrack,
+        kind: this.kind,
+        element: this.processorElement,
+      });
+      processedTrack = this.processor.processedTrack;
     }
     if (this.sender && this.sender.transport?.state !== 'closed') {
       await this.sender.replaceTrack(processedTrack ?? newTrack);
@@ -290,36 +288,42 @@ export default abstract class LocalTrack<
     track: MediaStreamTrack,
     userProvidedOrOptions: boolean | ReplaceTrackOptions | undefined,
   ) {
-    if (!this.sender) {
-      throw new TrackInvalidError('unable to replace an unpublished track');
-    }
+    const unlock = await this.trackChangeLock.lock();
+    try {
+      if (!this.sender) {
+        throw new TrackInvalidError('unable to replace an unpublished track');
+      }
 
-    let userProvidedTrack: boolean | undefined;
-    let stopProcessor: boolean | undefined;
+      let userProvidedTrack: boolean | undefined;
+      let stopProcessor: boolean | undefined;
 
-    if (typeof userProvidedOrOptions === 'boolean') {
-      userProvidedTrack = userProvidedOrOptions;
-    } else if (userProvidedOrOptions !== undefined) {
-      userProvidedTrack = userProvidedOrOptions.userProvidedTrack;
-      stopProcessor = userProvidedOrOptions.stopProcessor;
-    }
+      if (typeof userProvidedOrOptions === 'boolean') {
+        userProvidedTrack = userProvidedOrOptions;
+      } else if (userProvidedOrOptions !== undefined) {
+        userProvidedTrack = userProvidedOrOptions.userProvidedTrack;
+        stopProcessor = userProvidedOrOptions.stopProcessor;
+      }
 
-    this.providedByUser = userProvidedTrack ?? true;
+      this.providedByUser = userProvidedTrack ?? true;
 
-    this.log.debug('replace MediaStreamTrack', this.logContext);
-    await this.setMediaStreamTrack(track);
-    // this must be synced *after* setting mediaStreamTrack above, since it relies
-    // on the previous state in order to cleanup
+      this.log.debug('replace MediaStreamTrack', this.logContext);
+      await this.setMediaStreamTrack(track);
+      // this must be synced *after* setting mediaStreamTrack above, since it relies
+      // on the previous state in order to cleanup
 
-    if (stopProcessor && this.processor) {
-      await this.stopProcessor();
+      if (stopProcessor && this.processor) {
+        await this.internalStopProcessor();
+      }
+      return this;
+    } finally {
+      unlock();
     }
-    return this;
   }
 
   protected async restart(constraints?: MediaTrackConstraints) {
     this.manuallyStopped = false;
-    const unlock = await this.restartLock.lock();
+    const unlock = await this.trackChangeLock.lock();
+
     try {
       if (!constraints) {
         constraints = this._constraints;
@@ -335,7 +339,7 @@ export default abstract class LocalTrack<
       if (this.kind === Track.Kind.Video) {
         streamConstraints.video = deviceId || facingMode ? { deviceId, facingMode } : true;
       } else {
-        streamConstraints.audio = deviceId ? { deviceId } : true;
+        streamConstraints.audio = deviceId ? { deviceId, ...otherConstraints } : true;
       }
 
       // these steps are duplicated from setMediaStreamTrack because we must stop
@@ -352,7 +356,10 @@ export default abstract class LocalTrack<
       // create new track and attach
       const mediaStream = await navigator.mediaDevices.getUserMedia(streamConstraints);
       const newTrack = mediaStream.getTracks()[0];
-      await newTrack.applyConstraints(otherConstraints);
+      if (this.kind === Track.Kind.Video) {
+        // we already captured the audio track with the constraints, so we only need to apply the video constraints
+        await newTrack.applyConstraints(otherConstraints);
+      }
       newTrack.addEventListener('ended', this.handleEnded);
       this.log.debug('re-acquired MediaStreamTrack', this.logContext);
 
@@ -518,7 +525,7 @@ export default abstract class LocalTrack<
    * @returns
    */
   async setProcessor(processor: TrackProcessor<TrackKind>, showProcessedStreamLocally = true) {
-    const unlock = await this.processorLock.lock();
+    const unlock = await this.trackChangeLock.lock();
     try {
       this.log.debug('setting up processor', this.logContext);
 
@@ -534,7 +541,7 @@ export default abstract class LocalTrack<
       this.log.debug('processor initialized', this.logContext);
 
       if (this.processor) {
-        await this.stopProcessor();
+        await this.internalStopProcessor();
       }
       if (this.kind === 'unknown') {
         throw TypeError('cannot set processor on track of unknown kind');
@@ -589,8 +596,21 @@ export default abstract class LocalTrack<
    * @returns
    */
   async stopProcessor(keepElement = true) {
-    if (!this.processor) return;
+    const unlock = await this.trackChangeLock.lock();
+    try {
+      await this.internalStopProcessor(keepElement);
+    } finally {
+      unlock();
+    }
+  }
 
+  /**
+   * @internal
+   * This method assumes the caller has acquired a trackChangeLock already.
+   * The public facing method for stopping the processor is `stopProcessor` and it wraps this method in the trackChangeLock.
+   */
+  protected async internalStopProcessor(keepElement = true) {
+    if (!this.processor) return;
     this.log.debug('stopping processor', this.logContext);
     this.processor.processedTrack?.stop();
     await this.processor.destroy();

package/src/room/track/RemoteVideoTrack.ts

@@ -38,6 +38,16 @@ export default class RemoteVideoTrack extends RemoteTrack<Track.Kind.Video> {
     return this.adaptiveStreamSettings !== undefined;
   }
 
+  override setStreamState(value: Track.StreamState) {
+    super.setStreamState(value);
+    console.log('setStreamState', value);
+    if (value === Track.StreamState.Active) {
+      // update visibility for adaptive stream tracks when stream state received from server is active
+      // this is needed to ensure the track is stopped when there's no element attached to it at all
+      this.updateVisibility();
+    }
+  }
+
   /**
    * Note: When using adaptiveStream, you need to use remoteVideoTrack.attach() to add the track to a HTMLVideoElement, otherwise your video tracks might never start
    */
@@ -220,7 +230,7 @@ export default class RemoteVideoTrack extends RemoteTrack<Track.Kind.Video> {
     this.updateDimensions();
   }, REACTION_DELAY);
 
-  private updateVisibility() {
+  private updateVisibility(forceEmit?: boolean) {
     const lastVisibilityChange = this.elementInfos.reduce(
       (prev, info) => Math.max(prev, info.visibilityChangedAt || 0),
       0,
@@ -234,7 +244,7 @@ export default class RemoteVideoTrack extends RemoteTrack<Track.Kind.Video> {
     const isVisible =
       (this.elementInfos.some((info) => info.visible) && !backgroundPause) || isPiPMode;
 
-    if (this.lastVisible === isVisible) {
+    if (this.lastVisible === isVisible && !forceEmit) {
       return;
     }