livekit-client 2.15.3 → 2.15.5

package/src/room/participant/LocalParticipant.ts

@@ -1,4 +1,3 @@
-import { Mutex } from '@livekit/mutex';
 import {
   AddTrackRequest,
   AudioTrackFeature,
@@ -7,12 +6,6 @@ import {
   Codec,
   DataPacket,
   DataPacket_Kind,
-  DataStream_ByteHeader,
-  DataStream_Chunk,
-  DataStream_Header,
-  DataStream_OperationType,
-  DataStream_TextHeader,
-  DataStream_Trailer,
   Encryption_Type,
   JoinResponse,
   ParticipantInfo,
@@ -34,7 +27,8 @@ import { SignalConnectionState } from '../../api/SignalClient';
 import type { InternalRoomOptions } from '../../options';
 import { PCTransportState } from '../PCTransportManager';
 import type RTCEngine from '../RTCEngine';
-import { ByteStreamWriter, TextStreamWriter } from '../StreamWriter';
+import type OutgoingDataStreamManager from '../data-stream/outgoing/OutgoingDataStreamManager';
+import type { TextStreamWriter } from '../data-stream/outgoing/StreamWriter';
 import { defaultVideoCodec } from '../defaults';
 import {
   DeviceUnsupportedError,
@@ -76,10 +70,11 @@ import {
   sourceToKind,
 } from '../track/utils';
 import {
-  type ByteStreamInfo,
   type ChatMessage,
   type DataPublishOptions,
+  type SendFileOptions,
   type SendTextOptions,
+  type StreamBytesOptions,
   type StreamTextOptions,
   type TextStreamInfo,
 } from '../types';
@@ -96,9 +91,7 @@ import {
   isSafari17Based,
   isVideoTrack,
   isWeb,
-  numberToBigInt,
   sleep,
-  splitUtf8,
   supportsAV1,
   supportsVP9,
 } from '../utils';
@@ -112,8 +105,6 @@ import {
   getDefaultDegradationPreference,
 } from './publishUtils';
 
-const STREAM_CHUNK_SIZE = 15_000;
-
 export default class LocalParticipant extends Participant {
   audioTrackPublications: Map<string, LocalTrackPublication>;
 
@@ -157,6 +148,8 @@ export default class LocalParticipant extends Participant {
 
   private rpcHandlers: Map<string, (data: RpcInvocationData) => Promise<string>>;
 
+  private roomOutgoingDataStreamManager: OutgoingDataStreamManager;
+
   private pendingSignalRequests: Map<
     number,
     {
@@ -185,6 +178,7 @@ export default class LocalParticipant extends Participant {
     engine: RTCEngine,
     options: InternalRoomOptions,
     roomRpcHandlers: Map<string, (data: RpcInvocationData) => Promise<string>>,
+    roomOutgoingDataStreamManager: OutgoingDataStreamManager,
   ) {
     super(sid, identity, undefined, undefined, undefined, {
       loggerName: options.loggerName,
@@ -203,6 +197,7 @@ export default class LocalParticipant extends Participant {
     ]);
     this.pendingSignalRequests = new Map();
     this.rpcHandlers = roomRpcHandlers;
+    this.roomOutgoingDataStreamManager = roomOutgoingDataStreamManager;
   }
 
   get lastCameraError(): Error | undefined {
@@ -261,7 +256,7 @@ export default class LocalParticipant extends Participant {
       .on(EngineEvent.Resuming, this.handleReconnecting)
       .on(EngineEvent.LocalTrackUnpublished, this.handleLocalTrackUnpublished)
       .on(EngineEvent.SubscribedQualityUpdate, this.handleSubscribedQualityUpdate)
-      .on(EngineEvent.Disconnected, this.handleDisconnected)
+      .on(EngineEvent.Closing, this.handleClosing)
       .on(EngineEvent.SignalRequestResponse, this.handleSignalRequestResponse)
       .on(EngineEvent.DataPacketReceived, this.handleDataPacket);
   }
@@ -278,7 +273,7 @@ export default class LocalParticipant extends Participant {
     this.updateTrackSubscriptionPermissions();
   };
 
-  private handleDisconnected = () => {
+  private handleClosing = () => {
     if (this.reconnectFuture) {
       this.reconnectFuture.promise.catch((e) => this.log.warn(e.message, this.logContext));
       this.reconnectFuture?.reject?.('Got disconnected during reconnection attempt');
@@ -558,10 +553,14 @@ export default class LocalParticipant extends Participant {
         }
 
         for (const localTrack of localTracks) {
+          const opts: TrackPublishOptions = {
+            ...this.roomOptions.publishDefaults,
+            ...options,
+          };
           if (
             source === Track.Source.Microphone &&
             isAudioTrack(localTrack) &&
-            publishOptions?.preConnectBuffer
+            opts.preConnectBuffer
           ) {
             this.log.info('starting preconnect buffer for microphone', {
               ...this.logContext,
@@ -1302,6 +1301,7 @@ export default class LocalParticipant extends Participant {
       ti.audioFeatures.includes(AudioTrackFeature.TF_PRECONNECT_BUFFER)
     ) {
       const stream = track.getPreConnectBuffer();
+      const mimeType = track.getPreConnectBufferMimeType();
       // TODO: we're registering the listener after negotiation, so there might be a race
       this.on(ParticipantEvent.LocalTrackSubscribed, (pub) => {
         if (pub.trackSid === ti.sid) {
@@ -1335,7 +1335,7 @@ export default class LocalParticipant extends Participant {
             });
             const writer = await this.streamBytes({
               name: 'preconnect-buffer',
-              mimeType: 'audio/opus',
+              mimeType,
               topic: 'lk.agent.pre-connect-audio-buffer',
               destinationIdentities: [agent.identity],
               attributes: {
@@ -1685,6 +1685,7 @@ export default class LocalParticipant extends Participant {
     await this.engine.sendDataPacket(packet, DataPacket_Kind.RELIABLE);
   }
 
+  /** @deprecated Consider migrating to {@link sendText} */
   async sendChatMessage(text: string, options?: SendTextOptions): Promise<ChatMessage> {
     const msg = {
       id: crypto.randomUUID(),
@@ -1707,6 +1708,7 @@ export default class LocalParticipant extends Participant {
     return msg;
   }
 
+  /** @deprecated Consider migrating to {@link sendText} */
   async editChatMessage(editText: string, originalMessage: ChatMessage) {
     const msg = {
       ...originalMessage,
@@ -1728,300 +1730,47 @@ export default class LocalParticipant extends Participant {
     return msg;
   }
 
+  /**
+   * Sends the given string to participants in the room via the data channel.
+   * For longer messages, consider using {@link streamText} instead.
+   *
+   * @param text The text payload
+   * @param options.topic Topic identifier used to route the stream to appropriate handlers.
+   */
   async sendText(text: string, options?: SendTextOptions): Promise<TextStreamInfo> {
-    const streamId = crypto.randomUUID();
-    const textInBytes = new TextEncoder().encode(text);
-    const totalTextLength = textInBytes.byteLength;
-
-    const fileIds = options?.attachments?.map(() => crypto.randomUUID());
-
-    const progresses = new Array<number>(fileIds ? fileIds.length + 1 : 1).fill(0);
-
-    const handleProgress = (progress: number, idx: number) => {
-      progresses[idx] = progress;
-      const totalProgress = progresses.reduce((acc, val) => acc + val, 0);
-      options?.onProgress?.(totalProgress);
-    };
-
-    const writer = await this.streamText({
-      streamId,
-      totalSize: totalTextLength,
-      destinationIdentities: options?.destinationIdentities,
-      topic: options?.topic,
-      attachedStreamIds: fileIds,
-      attributes: options?.attributes,
-    });
-
-    await writer.write(text);
-    // set text part of progress to 1
-    handleProgress(1, 0);
-
-    await writer.close();
-
-    if (options?.attachments && fileIds) {
-      await Promise.all(
-        options.attachments.map(async (file, idx) =>
-          this._sendFile(fileIds[idx], file, {
-            topic: options.topic,
-            mimeType: file.type,
-            onProgress: (progress) => {
-              handleProgress(progress, idx + 1);
-            },
-          }),
-        ),
-      );
-    }
-    return writer.info;
+    return this.roomOutgoingDataStreamManager.sendText(text, options);
   }
 
   /**
+   * Creates a new TextStreamWriter which can be used to stream text incrementally
+   * to participants in the room via the data channel.
+   *
+   * @param options.topic Topic identifier used to route the stream to appropriate handlers.
+   *
    * @internal
    * @experimental CAUTION, might get removed in a minor release
    */
   async streamText(options?: StreamTextOptions): Promise<TextStreamWriter> {
-    const streamId = options?.streamId ?? crypto.randomUUID();
-
-    const info: TextStreamInfo = {
-      id: streamId,
-      mimeType: 'text/plain',
-      timestamp: Date.now(),
-      topic: options?.topic ?? '',
-      size: options?.totalSize,
-      attributes: options?.attributes,
-    };
-    const header = new DataStream_Header({
-      streamId,
-      mimeType: info.mimeType,
-      topic: info.topic,
-      timestamp: numberToBigInt(info.timestamp),
-      totalLength: numberToBigInt(options?.totalSize),
-      attributes: info.attributes,
-      contentHeader: {
-        case: 'textHeader',
-        value: new DataStream_TextHeader({
-          version: options?.version,
-          attachedStreamIds: options?.attachedStreamIds,
-          replyToStreamId: options?.replyToStreamId,
-          operationType:
-            options?.type === 'update'
-              ? DataStream_OperationType.UPDATE
-              : DataStream_OperationType.CREATE,
-        }),
-      },
-    });
-    const destinationIdentities = options?.destinationIdentities;
-    const packet = new DataPacket({
-      destinationIdentities,
-      value: {
-        case: 'streamHeader',
-        value: header,
-      },
-    });
-    await this.engine.sendDataPacket(packet, DataPacket_Kind.RELIABLE);
-
-    let chunkId = 0;
-    const localP = this;
-
-    const writableStream = new WritableStream<string>({
-      // Implement the sink
-      async write(text) {
-        for (const textByteChunk of splitUtf8(text, STREAM_CHUNK_SIZE)) {
-          await localP.engine.waitForBufferStatusLow(DataPacket_Kind.RELIABLE);
-          const chunk = new DataStream_Chunk({
-            content: textByteChunk,
-            streamId,
-            chunkIndex: numberToBigInt(chunkId),
-          });
-          const chunkPacket = new DataPacket({
-            destinationIdentities,
-            value: {
-              case: 'streamChunk',
-              value: chunk,
-            },
-          });
-          await localP.engine.sendDataPacket(chunkPacket, DataPacket_Kind.RELIABLE);
-
-          chunkId += 1;
-        }
-      },
-      async close() {
-        const trailer = new DataStream_Trailer({
-          streamId,
-        });
-        const trailerPacket = new DataPacket({
-          destinationIdentities,
-          value: {
-            case: 'streamTrailer',
-            value: trailer,
-          },
-        });
-        await localP.engine.sendDataPacket(trailerPacket, DataPacket_Kind.RELIABLE);
-      },
-      abort(err) {
-        console.log('Sink error:', err);
-        // TODO handle aborts to signal something to receiver side
-      },
-    });
-
-    let onEngineClose = async () => {
-      await writer.close();
-    };
-
-    localP.engine.once(EngineEvent.Closing, onEngineClose);
-
-    const writer = new TextStreamWriter(writableStream, info, () =>
-      this.engine.off(EngineEvent.Closing, onEngineClose),
-    );
-
-    return writer;
+    return this.roomOutgoingDataStreamManager.streamText(options);
   }
 
-  async sendFile(
-    file: File,
-    options?: {
-      mimeType?: string;
-      topic?: string;
-      destinationIdentities?: Array<string>;
-      onProgress?: (progress: number) => void;
-    },
-  ): Promise<{ id: string }> {
-    const streamId = crypto.randomUUID();
-    await this._sendFile(streamId, file, options);
-    return { id: streamId };
-  }
-
-  private async _sendFile(
-    streamId: string,
-    file: File,
-    options?: {
-      mimeType?: string;
-      topic?: string;
-      encryptionType?: Encryption_Type.NONE;
-      destinationIdentities?: Array<string>;
-      onProgress?: (progress: number) => void;
-    },
-  ) {
-    const writer = await this.streamBytes({
-      streamId,
-      totalSize: file.size,
-      name: file.name,
-      mimeType: options?.mimeType ?? file.type,
-      topic: options?.topic,
-      destinationIdentities: options?.destinationIdentities,
-    });
-    const reader = file.stream().getReader();
-    while (true) {
-      const { done, value } = await reader.read();
-      if (done) {
-        break;
-      }
-      await writer.write(value);
-    }
-    await writer.close();
-    return writer.info;
+  /** Send a File to all participants in the room via the data channel.
+   * @param file The File object payload
+   * @param options.topic Topic identifier used to route the stream to appropriate handlers.
+   * @param options.onProgress A callback function used to monitor the upload progress percentage.
+   */
+  async sendFile(file: File, options?: SendFileOptions): Promise<{ id: string }> {
+    return this.roomOutgoingDataStreamManager.sendFile(file, options);
   }
 
-  async streamBytes(options?: {
-    name?: string;
-    topic?: string;
-    attributes?: Record<string, string>;
-    destinationIdentities?: Array<string>;
-    streamId?: string;
-    mimeType?: string;
-    totalSize?: number;
-  }) {
-    const streamId = options?.streamId ?? crypto.randomUUID();
-    const destinationIdentities = options?.destinationIdentities;
-
-    const info: ByteStreamInfo = {
-      id: streamId,
-      mimeType: options?.mimeType ?? 'application/octet-stream',
-      topic: options?.topic ?? '',
-      timestamp: Date.now(),
-      attributes: options?.attributes,
-      size: options?.totalSize,
-      name: options?.name ?? 'unknown',
-    };
-
-    const header = new DataStream_Header({
-      totalLength: numberToBigInt(info.size ?? 0),
-      mimeType: info.mimeType,
-      streamId,
-      topic: info.topic,
-      timestamp: numberToBigInt(Date.now()),
-      attributes: info.attributes,
-      contentHeader: {
-        case: 'byteHeader',
-        value: new DataStream_ByteHeader({
-          name: info.name,
-        }),
-      },
-    });
-
-    const packet = new DataPacket({
-      destinationIdentities,
-      value: {
-        case: 'streamHeader',
-        value: header,
-      },
-    });
-
-    await this.engine.sendDataPacket(packet, DataPacket_Kind.RELIABLE);
-
-    let chunkId = 0;
-    const writeMutex = new Mutex();
-    const engine = this.engine;
-    const log = this.log;
-
-    const writableStream = new WritableStream<Uint8Array>({
-      async write(chunk) {
-        const unlock = await writeMutex.lock();
-
-        let byteOffset = 0;
-        try {
-          while (byteOffset < chunk.byteLength) {
-            const subChunk = chunk.slice(byteOffset, byteOffset + STREAM_CHUNK_SIZE);
-            await engine.waitForBufferStatusLow(DataPacket_Kind.RELIABLE);
-            const chunkPacket = new DataPacket({
-              destinationIdentities,
-              value: {
-                case: 'streamChunk',
-                value: new DataStream_Chunk({
-                  content: subChunk,
-                  streamId,
-                  chunkIndex: numberToBigInt(chunkId),
-                }),
-              },
-            });
-            await engine.sendDataPacket(chunkPacket, DataPacket_Kind.RELIABLE);
-            chunkId += 1;
-            byteOffset += subChunk.byteLength;
-          }
-        } finally {
-          unlock();
-        }
-      },
-      async close() {
-        const trailer = new DataStream_Trailer({
-          streamId,
-        });
-        const trailerPacket = new DataPacket({
-          destinationIdentities,
-          value: {
-            case: 'streamTrailer',
-            value: trailer,
-          },
-        });
-        await engine.sendDataPacket(trailerPacket, DataPacket_Kind.RELIABLE);
-      },
-      abort(err) {
-        log.error('Sink error:', err);
-      },
-    });
-
-    const byteWriter = new ByteStreamWriter(writableStream, info);
-
-    return byteWriter;
+  /**
+   * Stream bytes incrementally to participants in the room via the data channel.
+   * For sending files, consider using {@link sendFile} instead.
+   *
+   * @param options.topic Topic identifier used to route the stream to appropriate handlers.
+   */
+  async streamBytes(options?: StreamBytesOptions) {
+    return this.roomOutgoingDataStreamManager.streamBytes(options);
   }
 
   /**

package/src/room/track/LocalAudioTrack.ts

@@ -169,7 +169,7 @@ export default class LocalAudioTrack extends LocalTrack<Track.Kind.Audio> {
   };
 
   async setProcessor(processor: TrackProcessor<Track.Kind.Audio, AudioProcessorOptions>) {
-    const unlock = await this.processorLock.lock();
+    const unlock = await this.trackChangeLock.lock();
     try {
       if (!isReactNative() && !this.audioContext) {
         throw Error(
@@ -177,7 +177,7 @@ export default class LocalAudioTrack extends LocalTrack<Track.Kind.Audio> {
         );
       }
       if (this.processor) {
-        await this.stopProcessor();
+        await this.internalStopProcessor();
       }
 
       const processorOptions = {

package/src/room/track/LocalTrack.ts

@@ -57,15 +57,13 @@ export default abstract class LocalTrack<
 
   protected processor?: TrackProcessor<TrackKind, any>;
 
-  protected processorLock: Mutex;
-
   protected audioContext?: AudioContext;
 
   protected manuallyStopped: boolean = false;
 
   protected localTrackRecorder: LocalTrackRecorder<typeof this> | undefined;
 
-  private restartLock: Mutex;
+  protected trackChangeLock: Mutex;
 
   /**
    *
@@ -86,9 +84,14 @@ export default abstract class LocalTrack<
     this.providedByUser = userProvidedTrack;
     this.muteLock = new Mutex();
     this.pauseUpstreamLock = new Mutex();
-    this.processorLock = new Mutex();
-    this.restartLock = new Mutex();
-    this.setMediaStreamTrack(mediaTrack, true);
+    this.trackChangeLock = new Mutex();
+    this.trackChangeLock.lock().then(async (unlock) => {
+      try {
+        await this.setMediaStreamTrack(mediaTrack, true);
+      } finally {
+        unlock();
+      }
+    });
 
     // added to satisfy TS compiler, constraints are synced with MediaStreamTrack
     this._constraints = mediaTrack.getConstraints();
@@ -171,27 +174,22 @@ export default abstract class LocalTrack<
     }
     let processedTrack: MediaStreamTrack | undefined;
     if (this.processor && newTrack) {
-      const unlock = await this.processorLock.lock();
-      try {
-        this.log.debug('restarting processor', this.logContext);
-        if (this.kind === 'unknown') {
-          throw TypeError('cannot set processor on track of unknown kind');
-        }
+      this.log.debug('restarting processor', this.logContext);
+      if (this.kind === 'unknown') {
+        throw TypeError('cannot set processor on track of unknown kind');
+      }
 
-        if (this.processorElement) {
-          attachToElement(newTrack, this.processorElement);
-          // ensure the processorElement itself stays muted
-          this.processorElement.muted = true;
-        }
-        await this.processor.restart({
-          track: newTrack,
-          kind: this.kind,
-          element: this.processorElement,
-        });
-        processedTrack = this.processor.processedTrack;
-      } finally {
-        unlock();
+      if (this.processorElement) {
+        attachToElement(newTrack, this.processorElement);
+        // ensure the processorElement itself stays muted
+        this.processorElement.muted = true;
       }
+      await this.processor.restart({
+        track: newTrack,
+        kind: this.kind,
+        element: this.processorElement,
+      });
+      processedTrack = this.processor.processedTrack;
     }
     if (this.sender && this.sender.transport?.state !== 'closed') {
       await this.sender.replaceTrack(processedTrack ?? newTrack);
@@ -290,36 +288,42 @@ export default abstract class LocalTrack<
     track: MediaStreamTrack,
     userProvidedOrOptions: boolean | ReplaceTrackOptions | undefined,
   ) {
-    if (!this.sender) {
-      throw new TrackInvalidError('unable to replace an unpublished track');
-    }
+    const unlock = await this.trackChangeLock.lock();
+    try {
+      if (!this.sender) {
+        throw new TrackInvalidError('unable to replace an unpublished track');
+      }
 
-    let userProvidedTrack: boolean | undefined;
-    let stopProcessor: boolean | undefined;
+      let userProvidedTrack: boolean | undefined;
+      let stopProcessor: boolean | undefined;
 
-    if (typeof userProvidedOrOptions === 'boolean') {
-      userProvidedTrack = userProvidedOrOptions;
-    } else if (userProvidedOrOptions !== undefined) {
-      userProvidedTrack = userProvidedOrOptions.userProvidedTrack;
-      stopProcessor = userProvidedOrOptions.stopProcessor;
-    }
+      if (typeof userProvidedOrOptions === 'boolean') {
+        userProvidedTrack = userProvidedOrOptions;
+      } else if (userProvidedOrOptions !== undefined) {
+        userProvidedTrack = userProvidedOrOptions.userProvidedTrack;
+        stopProcessor = userProvidedOrOptions.stopProcessor;
+      }
 
-    this.providedByUser = userProvidedTrack ?? true;
+      this.providedByUser = userProvidedTrack ?? true;
 
-    this.log.debug('replace MediaStreamTrack', this.logContext);
-    await this.setMediaStreamTrack(track);
-    // this must be synced *after* setting mediaStreamTrack above, since it relies
-    // on the previous state in order to cleanup
+      this.log.debug('replace MediaStreamTrack', this.logContext);
+      await this.setMediaStreamTrack(track);
+      // this must be synced *after* setting mediaStreamTrack above, since it relies
+      // on the previous state in order to cleanup
 
-    if (stopProcessor && this.processor) {
-      await this.stopProcessor();
+      if (stopProcessor && this.processor) {
+        await this.internalStopProcessor();
+      }
+      return this;
+    } finally {
+      unlock();
     }
-    return this;
   }
 
   protected async restart(constraints?: MediaTrackConstraints) {
     this.manuallyStopped = false;
-    const unlock = await this.restartLock.lock();
+    const unlock = await this.trackChangeLock.lock();
+
     try {
       if (!constraints) {
         constraints = this._constraints;
@@ -518,7 +522,7 @@ export default abstract class LocalTrack<
    * @returns
    */
   async setProcessor(processor: TrackProcessor<TrackKind>, showProcessedStreamLocally = true) {
-    const unlock = await this.processorLock.lock();
+    const unlock = await this.trackChangeLock.lock();
     try {
       this.log.debug('setting up processor', this.logContext);
 
@@ -534,7 +538,7 @@ export default abstract class LocalTrack<
       this.log.debug('processor initialized', this.logContext);
 
       if (this.processor) {
-        await this.stopProcessor();
+        await this.internalStopProcessor();
       }
       if (this.kind === 'unknown') {
         throw TypeError('cannot set processor on track of unknown kind');
@@ -589,8 +593,21 @@ export default abstract class LocalTrack<
    * @returns
    */
   async stopProcessor(keepElement = true) {
-    if (!this.processor) return;
+    const unlock = await this.trackChangeLock.lock();
+    try {
+      await this.internalStopProcessor(keepElement);
+    } finally {
+      unlock();
+    }
+  }
 
+  /**
+   * @internal
+   * This method assumes the caller has acquired a trackChangeLock already.
+   * The public facing method for stopping the processor is `stopProcessor` and it wraps this method in the trackChangeLock.
+   */
+  protected async internalStopProcessor(keepElement = true) {
+    if (!this.processor) return;
     this.log.debug('stopping processor', this.logContext);
     this.processor.processedTrack?.stop();
     await this.processor.destroy();
@@ -617,8 +634,13 @@ export default abstract class LocalTrack<
     }
 
     if (!this.localTrackRecorder) {
+      let mimeType = 'audio/webm;codecs=opus';
+      if (!MediaRecorder.isTypeSupported(mimeType)) {
+        // iOS currently only supports video/mp4 as a mime type - even for audio.
+        mimeType = 'video/mp4';
+      }
       this.localTrackRecorder = new LocalTrackRecorder(this, {
-        mimeType: 'audio/webm;codecs=opus',
+        mimeType,
       });
     } else {
       this.log.warn('preconnect buffer already started');
@@ -649,5 +671,9 @@ export default abstract class LocalTrack<
     return this.localTrackRecorder?.byteStream;
   }
 
+  getPreConnectBufferMimeType() {
+    return this.localTrackRecorder?.mimeType;
+  }
+
   protected abstract monitorSender(): void;
 }