livekit-client 2.10.0 → 2.11.1

package/src/room/participant/LocalParticipant.ts

@@ -1,3 +1,4 @@
+import { Mutex } from '@livekit/mutex';
 import {
   AddTrackRequest,
   BackupCodecPolicy,
@@ -27,10 +28,11 @@ import {
   UserPacket,
   protoInt64,
 } from '@livekit/protocol';
+import { SignalConnectionState } from '../../api/SignalClient';
 import type { InternalRoomOptions } from '../../options';
 import { PCTransportState } from '../PCTransportManager';
 import type RTCEngine from '../RTCEngine';
-import { TextStreamWriter } from '../StreamWriter';
+import { ByteStreamWriter, TextStreamWriter } from '../StreamWriter';
 import { defaultVideoCodec } from '../defaults';
 import {
   DeviceUnsupportedError,
@@ -53,6 +55,7 @@ import LocalTrack from '../track/LocalTrack';
 import LocalTrackPublication from '../track/LocalTrackPublication';
 import LocalVideoTrack, { videoLayersFromEncodings } from '../track/LocalVideoTrack';
 import { Track } from '../track/Track';
+import { createLocalTracks } from '../track/create';
 import type {
   AudioCaptureOptions,
   BackupVideoCodec,
@@ -63,8 +66,6 @@ import type {
 } from '../track/options';
 import { ScreenSharePresets, VideoPresets, isBackupCodec } from '../track/options';
 import {
-  constraintsForOptions,
-  extractProcessorsFromOptions,
   getLogContextFromTrack,
   getTrackSourceFromProto,
   mergeDefaultOptions,
@@ -72,6 +73,7 @@ import {
   screenCaptureToDisplayMediaStreamOptions,
 } from '../track/utils';
 import {
+  type ByteStreamInfo,
   type ChatMessage,
   type DataPublishOptions,
   type SendTextOptions,
@@ -104,7 +106,6 @@ import {
   computeTrackBackupEncodings,
   computeVideoEncodings,
   getDefaultDegradationPreference,
-  mediaTrackToLocalTrack,
 } from './publishUtils';
 
 const STREAM_CHUNK_SIZE = 15_000;
@@ -528,9 +529,11 @@ export default class LocalParticipant extends Participant {
               ...this.logContext,
               ...getLogContextFromTrack(localTrack),
             });
+
             publishPromises.push(this.publishTrack(localTrack, publishOptions));
           }
           const publishedTracks = await Promise.all(publishPromises);
+
           // for screen share publications including audio, this will only return the screen share publication, not the screen share audio one
           // revisit if we want to return an array of tracks instead for v2
           [track] = publishedTracks;
@@ -612,61 +615,37 @@ export default class LocalParticipant extends Participant {
       this.roomOptions?.videoCaptureDefaults,
     );
 
-    const { audioProcessor, videoProcessor, optionsWithoutProcessor } =
-      extractProcessorsFromOptions(mergedOptionsWithProcessors);
-
-    const constraints = constraintsForOptions(optionsWithoutProcessor);
-    let stream: MediaStream | undefined;
     try {
-      stream = await navigator.mediaDevices.getUserMedia(constraints);
+      const tracks = await createLocalTracks(mergedOptionsWithProcessors, {
+        loggerName: this.roomOptions.loggerName,
+        loggerContextCb: () => this.logContext,
+      });
+      const localTracks = tracks.map((track) => {
+        if (isAudioTrack(track)) {
+          this.microphoneError = undefined;
+          track.setAudioContext(this.audioContext);
+          track.source = Track.Source.Microphone;
+          this.emit(ParticipantEvent.AudioStreamAcquired);
+        }
+        if (isVideoTrack(track)) {
+          this.cameraError = undefined;
+          track.source = Track.Source.Camera;
+        }
+        return track;
+      });
+      return localTracks;
     } catch (err) {
       if (err instanceof Error) {
-        if (constraints.audio) {
+        if (options.audio) {
           this.microphoneError = err;
         }
-        if (constraints.video) {
+        if (options.video) {
           this.cameraError = err;
         }
       }
 
       throw err;
     }
-
-    if (constraints.audio) {
-      this.microphoneError = undefined;
-      this.emit(ParticipantEvent.AudioStreamAcquired);
-    }
-    if (constraints.video) {
-      this.cameraError = undefined;
-    }
-
-    return Promise.all(
-      stream.getTracks().map(async (mediaStreamTrack) => {
-        const isAudio = mediaStreamTrack.kind === 'audio';
-        let trackConstraints: MediaTrackConstraints | undefined;
-        const conOrBool = isAudio ? constraints.audio : constraints.video;
-        if (typeof conOrBool !== 'boolean') {
-          trackConstraints = conOrBool;
-        }
-        const track = mediaTrackToLocalTrack(mediaStreamTrack, trackConstraints, {
-          loggerName: this.roomOptions.loggerName,
-          loggerContextCb: () => this.logContext,
-        });
-        if (track.kind === Track.Kind.Video) {
-          track.source = Track.Source.Camera;
-        } else if (track.kind === Track.Kind.Audio) {
-          track.source = Track.Source.Microphone;
-          track.setAudioContext(this.audioContext);
-        }
-        track.mediaStream = stream;
-        if (isAudioTrack(track) && audioProcessor) {
-          await track.setProcessor(audioProcessor);
-        } else if (isVideoTrack(track) && videoProcessor) {
-          await track.setProcessor(videoProcessor);
-        }
-        return track;
-      }),
-    );
   }
 
   /**
@@ -862,7 +841,47 @@ export default class LocalParticipant extends Participant {
     if (opts.source) {
       track.source = opts.source;
     }
-    const publishPromise = this.publish(track, opts, isStereo);
+    const publishPromise = new Promise<LocalTrackPublication>(async (resolve, reject) => {
+      try {
+        if (this.engine.client.currentState !== SignalConnectionState.CONNECTED) {
+          this.log.debug('deferring track publication until signal is connected', {
+            ...this.logContext,
+            track: getLogContextFromTrack(track),
+          });
+          const onSignalConnected = async () => {
+            try {
+              const publication = await this.publish(track, opts, isStereo);
+              resolve(publication);
+            } catch (e) {
+              reject(e);
+            }
+          };
+          setTimeout(() => {
+            this.engine.off(EngineEvent.SignalConnected, onSignalConnected);
+            reject(
+              new PublishTrackError(
+                'publishing rejected as engine not connected within timeout',
+                408,
+              ),
+            );
+          }, 15_000);
+          this.engine.once(EngineEvent.SignalConnected, onSignalConnected);
+          this.engine.on(EngineEvent.Closing, () => {
+            this.engine.off(EngineEvent.SignalConnected, onSignalConnected);
+            reject(new PublishTrackError('publishing rejected as engine closed', 499));
+          });
+        } else {
+          try {
+            const publication = await this.publish(track, opts, isStereo);
+            resolve(publication);
+          } catch (e) {
+            reject(e);
+          }
+        }
+      } catch (e) {
+        reject(e);
+      }
+    });
     this.pendingPublishPromises.set(track, publishPromise);
     try {
       const publication = await publishPromise;
@@ -1713,23 +1732,62 @@ export default class LocalParticipant extends Participant {
       onProgress?: (progress: number) => void;
     },
   ) {
-    const totalLength = file.size;
-    const header = new DataStream_Header({
-      totalLength: numberToBigInt(totalLength),
-      mimeType: options?.mimeType ?? file.type,
+    const writer = await this.streamBytes({
       streamId,
+      totalSize: file.size,
+      name: file.name,
+      mimeType: options?.mimeType ?? file.type,
       topic: options?.topic,
-      encryptionType: options?.encryptionType,
+      destinationIdentities: options?.destinationIdentities,
+    });
+    const reader = file.stream().getReader();
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) {
+        break;
+      }
+      await writer.write(value);
+    }
+    await writer.close();
+    return writer.info;
+  }
+
+  async streamBytes(options?: {
+    name?: string;
+    topic?: string;
+    attributes?: Record<string, string>;
+    destinationIdentities?: Array<string>;
+    streamId?: string;
+    mimeType?: string;
+    totalSize?: number;
+  }) {
+    const streamId = options?.streamId ?? crypto.randomUUID();
+    const destinationIdentities = options?.destinationIdentities;
+
+    const info: ByteStreamInfo = {
+      id: streamId,
+      mimeType: options?.mimeType ?? 'application/octet-stream',
+      topic: options?.topic ?? '',
+      timestamp: Date.now(),
+      attributes: options?.attributes,
+      size: options?.totalSize,
+      name: options?.name ?? 'unknown',
+    };
+
+    const header = new DataStream_Header({
+      totalLength: numberToBigInt(info.size ?? 0),
+      mimeType: info.mimeType,
+      streamId,
+      topic: info.topic,
       timestamp: numberToBigInt(Date.now()),
       contentHeader: {
         case: 'byteHeader',
         value: new DataStream_ByteHeader({
-          name: file.name,
+          name: info.name,
         }),
       },
     });
 
-    const destinationIdentities = options?.destinationIdentities;
     const packet = new DataPacket({
       destinationIdentities,
       value: {
@@ -1739,47 +1797,61 @@ export default class LocalParticipant extends Participant {
     });
 
     await this.engine.sendDataPacket(packet, DataPacket_Kind.RELIABLE);
-    function read(b: Blob): Promise<Uint8Array> {
-      return new Promise((resolve) => {
-        const fr = new FileReader();
-        fr.onload = () => {
-          resolve(new Uint8Array(fr.result as ArrayBuffer));
-        };
-        fr.readAsArrayBuffer(b);
-      });
-    }
-    const totalChunks = Math.ceil(totalLength / STREAM_CHUNK_SIZE);
-    for (let i = 0; i < totalChunks; i++) {
-      const chunkData = await read(
-        file.slice(i * STREAM_CHUNK_SIZE, Math.min((i + 1) * STREAM_CHUNK_SIZE, totalLength)),
-      );
-      await this.engine.waitForBufferStatusLow(DataPacket_Kind.RELIABLE);
-      const chunk = new DataStream_Chunk({
-        content: chunkData,
-        streamId,
-        chunkIndex: numberToBigInt(i),
-      });
-      const chunkPacket = new DataPacket({
-        destinationIdentities,
-        value: {
-          case: 'streamChunk',
-          value: chunk,
-        },
-      });
-      await this.engine.sendDataPacket(chunkPacket, DataPacket_Kind.RELIABLE);
-      options?.onProgress?.((i + 1) / totalChunks);
-    }
-    const trailer = new DataStream_Trailer({
-      streamId,
-    });
-    const trailerPacket = new DataPacket({
-      destinationIdentities,
-      value: {
-        case: 'streamTrailer',
-        value: trailer,
+
+    let chunkId = 0;
+    const writeMutex = new Mutex();
+    const engine = this.engine;
+    const log = this.log;
+
+    const writableStream = new WritableStream<Uint8Array>({
+      async write(chunk) {
+        const unlock = await writeMutex.lock();
+
+        let byteOffset = 0;
+        try {
+          while (byteOffset < chunk.byteLength) {
+            const subChunk = chunk.slice(byteOffset, byteOffset + STREAM_CHUNK_SIZE);
+            await engine.waitForBufferStatusLow(DataPacket_Kind.RELIABLE);
+            const chunkPacket = new DataPacket({
+              destinationIdentities,
+              value: {
+                case: 'streamChunk',
+                value: new DataStream_Chunk({
+                  content: subChunk,
+                  streamId,
+                  chunkIndex: numberToBigInt(chunkId),
+                }),
+              },
+            });
+            await engine.sendDataPacket(chunkPacket, DataPacket_Kind.RELIABLE);
+            chunkId += 1;
+            byteOffset += subChunk.byteLength;
+          }
+        } finally {
+          unlock();
+        }
+      },
+      async close() {
+        const trailer = new DataStream_Trailer({
+          streamId,
+        });
+        const trailerPacket = new DataPacket({
+          destinationIdentities,
+          value: {
+            case: 'streamTrailer',
+            value: trailer,
+          },
+        });
+        await engine.sendDataPacket(trailerPacket, DataPacket_Kind.RELIABLE);
+      },
+      abort(err) {
+        log.error('Sink error:', err);
       },
     });
-    await this.engine.sendDataPacket(trailerPacket, DataPacket_Kind.RELIABLE);
+
+    const byteWriter = new ByteStreamWriter(writableStream, info);
+
+    return byteWriter;
   }
 
   /**

package/src/room/track/create.ts

@@ -2,6 +2,7 @@ import DeviceManager from '../DeviceManager';
 import { audioDefaults, videoDefaults } from '../defaults';
 import { DeviceUnsupportedError, TrackInvalidError } from '../errors';
 import { mediaTrackToLocalTrack } from '../participant/publishUtils';
+import type { LoggerOptions } from '../types';
 import { isAudioTrack, isSafari17, isVideoTrack, unwrapConstraint } from '../utils';
 import LocalAudioTrack from './LocalAudioTrack';
 import type LocalTrack from './LocalTrack';
@@ -29,72 +30,126 @@ import {
  */
 export async function createLocalTracks(
   options?: CreateLocalTracksOptions,
+  loggerOptions?: LoggerOptions,
 ): Promise<Array<LocalTrack>> {
   // set default options to true
-  options ??= {};
-  options.audio ??= { deviceId: 'default' };
-  options.video ??= { deviceId: 'default' };
-
-  const { audioProcessor, videoProcessor } = extractProcessorsFromOptions(options);
-  const opts = mergeDefaultOptions(options, audioDefaults, videoDefaults);
+  const internalOptions = { ...(options ?? {}) };
+  let attemptExactMatch = false;
+  let retryAudioOptions: AudioCaptureOptions | undefined | boolean = options?.audio;
+  let retryVideoOptions: VideoCaptureOptions | undefined | boolean = options?.video;
+  // if the user passes a device id as a string, we default to exact match
+  if (
+    internalOptions.audio &&
+    typeof internalOptions.audio === 'object' &&
+    typeof internalOptions.audio.deviceId === 'string'
+  ) {
+    const deviceId: string = internalOptions.audio.deviceId;
+    internalOptions.audio.deviceId = { exact: deviceId };
+    attemptExactMatch = true;
+    retryAudioOptions = {
+      ...internalOptions.audio,
+      deviceId: { ideal: deviceId },
+    };
+  }
+  if (
+    internalOptions.video &&
+    typeof internalOptions.video === 'object' &&
+    typeof internalOptions.video.deviceId === 'string'
+  ) {
+    const deviceId: string = internalOptions.video.deviceId;
+    internalOptions.video.deviceId = { exact: deviceId };
+    attemptExactMatch = true;
+    retryVideoOptions = {
+      ...internalOptions.video,
+      deviceId: { ideal: deviceId },
+    };
+  }
+  // TODO if internal options don't have device Id specified, set it to 'default'
+  if (
+    internalOptions.audio === true ||
+    (typeof internalOptions.audio === 'object' && !internalOptions.audio.deviceId)
+  ) {
+    internalOptions.audio = { deviceId: 'default' };
+  }
+  if (
+    internalOptions.video === true ||
+    (typeof internalOptions.video === 'object' && !internalOptions.video.deviceId)
+  ) {
+    internalOptions.video = { deviceId: 'default' };
+  }
+  const { audioProcessor, videoProcessor } = extractProcessorsFromOptions(internalOptions);
+  const opts = mergeDefaultOptions(internalOptions, audioDefaults, videoDefaults);
   const constraints = constraintsForOptions(opts);
 
   // Keep a reference to the promise on DeviceManager and await it in getLocalDevices()
   // works around iOS Safari Bug https://bugs.webkit.org/show_bug.cgi?id=179363
   const mediaPromise = navigator.mediaDevices.getUserMedia(constraints);
 
-  if (options.audio) {
+  if (internalOptions.audio) {
     DeviceManager.userMediaPromiseMap.set('audioinput', mediaPromise);
     mediaPromise.catch(() => DeviceManager.userMediaPromiseMap.delete('audioinput'));
   }
-  if (options.video) {
+  if (internalOptions.video) {
     DeviceManager.userMediaPromiseMap.set('videoinput', mediaPromise);
     mediaPromise.catch(() => DeviceManager.userMediaPromiseMap.delete('videoinput'));
   }
+  try {
+    const stream = await mediaPromise;
+    return await Promise.all(
+      stream.getTracks().map(async (mediaStreamTrack) => {
+        const isAudio = mediaStreamTrack.kind === 'audio';
+        let trackOptions = isAudio ? opts!.audio : opts!.video;
+        if (typeof trackOptions === 'boolean' || !trackOptions) {
+          trackOptions = {};
+        }
+        let trackConstraints: MediaTrackConstraints | undefined;
+        const conOrBool = isAudio ? constraints.audio : constraints.video;
+        if (typeof conOrBool !== 'boolean') {
+          trackConstraints = conOrBool;
+        }
 
-  const stream = await mediaPromise;
-  return Promise.all(
-    stream.getTracks().map(async (mediaStreamTrack) => {
-      const isAudio = mediaStreamTrack.kind === 'audio';
-      let trackOptions = isAudio ? opts!.audio : opts!.video;
-      if (typeof trackOptions === 'boolean' || !trackOptions) {
-        trackOptions = {};
-      }
-      let trackConstraints: MediaTrackConstraints | undefined;
-      const conOrBool = isAudio ? constraints.audio : constraints.video;
-      if (typeof conOrBool !== 'boolean') {
-        trackConstraints = conOrBool;
-      }
-
-      // update the constraints with the device id the user gave permissions to in the permission prompt
-      // otherwise each track restart (e.g. mute - unmute) will try to initialize the device again -> causing additional permission prompts
-      const newDeviceId = mediaStreamTrack.getSettings().deviceId;
-      if (
-        trackConstraints?.deviceId &&
-        unwrapConstraint(trackConstraints.deviceId) !== newDeviceId
-      ) {
-        trackConstraints.deviceId = newDeviceId;
-      } else if (!trackConstraints) {
-        trackConstraints = { deviceId: newDeviceId };
-      }
+        // update the constraints with the device id the user gave permissions to in the permission prompt
+        // otherwise each track restart (e.g. mute - unmute) will try to initialize the device again -> causing additional permission prompts
+        const newDeviceId = mediaStreamTrack.getSettings().deviceId;
+        if (
+          trackConstraints?.deviceId &&
+          unwrapConstraint(trackConstraints.deviceId) !== newDeviceId
+        ) {
+          trackConstraints.deviceId = newDeviceId;
+        } else if (!trackConstraints) {
+          trackConstraints = { deviceId: newDeviceId };
+        }
 
-      const track = mediaTrackToLocalTrack(mediaStreamTrack, trackConstraints);
-      if (track.kind === Track.Kind.Video) {
-        track.source = Track.Source.Camera;
-      } else if (track.kind === Track.Kind.Audio) {
-        track.source = Track.Source.Microphone;
-      }
-      track.mediaStream = stream;
+        const track = mediaTrackToLocalTrack(mediaStreamTrack, trackConstraints, loggerOptions);
+        if (track.kind === Track.Kind.Video) {
+          track.source = Track.Source.Camera;
+        } else if (track.kind === Track.Kind.Audio) {
+          track.source = Track.Source.Microphone;
+        }
+        track.mediaStream = stream;
 
-      if (isAudioTrack(track) && audioProcessor) {
-        await track.setProcessor(audioProcessor);
-      } else if (isVideoTrack(track) && videoProcessor) {
-        await track.setProcessor(videoProcessor);
-      }
+        if (isAudioTrack(track) && audioProcessor) {
+          await track.setProcessor(audioProcessor);
+        } else if (isVideoTrack(track) && videoProcessor) {
+          await track.setProcessor(videoProcessor);
+        }
 
-      return track;
-    }),
-  );
+        return track;
+      }),
+    );
+  } catch (e) {
+    if (!attemptExactMatch) {
+      throw e;
+    }
+    return createLocalTracks(
+      {
+        ...options,
+        audio: retryAudioOptions,
+        video: retryVideoOptions,
+      },
+      loggerOptions,
+    );
+  }
 }
 
 /**

package/src/room/track/utils.ts

@@ -33,7 +33,7 @@ export function mergeDefaultOptions(
       clonedOptions.audio as Record<string, unknown>,
       audioDefaults as Record<string, unknown>,
     );
-    clonedOptions.audio.deviceId ??= 'default';
+    clonedOptions.audio.deviceId ??= { ideal: 'default' };
     if (audioProcessor || defaultAudioProcessor) {
       clonedOptions.audio.processor = audioProcessor ?? defaultAudioProcessor;
     }
@@ -43,7 +43,7 @@ export function mergeDefaultOptions(
       clonedOptions.video as Record<string, unknown>,
       videoDefaults as Record<string, unknown>,
     );
-    clonedOptions.video.deviceId ??= 'default';
+    clonedOptions.video.deviceId ??= { ideal: 'default' };
     if (videoProcessor || defaultVideoProcessor) {
       clonedOptions.video.processor = videoProcessor ?? defaultVideoProcessor;
     }
@@ -81,9 +81,9 @@ export function constraintsForOptions(options: CreateLocalTracksOptions): MediaS
         }
       });
       constraints.video = videoOptions;
-      constraints.video.deviceId ??= 'default';
+      constraints.video.deviceId ??= { ideal: 'default' };
     } else {
-      constraints.video = options.video ? { deviceId: 'default' } : false;
+      constraints.video = options.video ? { deviceId: { ideal: 'default' } } : false;
     }
   } else {
     constraints.video = false;
@@ -92,9 +92,9 @@ export function constraintsForOptions(options: CreateLocalTracksOptions): MediaS
   if (options.audio) {
     if (typeof options.audio === 'object') {
       constraints.audio = options.audio;
-      constraints.audio.deviceId ??= 'default';
+      constraints.audio.deviceId ??= { ideal: 'default' };
     } else {
-      constraints.audio = { deviceId: 'default' };
+      constraints.audio = { deviceId: { ideal: 'default' } };
     }
   } else {
     constraints.audio = false;