livekit-client 2.1.5 → 2.3.0

package/src/room/participant/LocalParticipant.ts

@@ -30,6 +30,7 @@ import type {
   VideoCaptureOptions,
 } from '../track/options';
 import { ScreenSharePresets, VideoPresets, isBackupCodec } from '../track/options';
+import type { TrackProcessor } from '../track/processor/types';
 import {
   constraintsForOptions,
   getLogContextFromTrack,
@@ -394,13 +395,13 @@ export default class LocalParticipant extends Participant {
    * @returns
    */
   async createTracks(options?: CreateLocalTracksOptions): Promise<LocalTrack[]> {
-    const opts = mergeDefaultOptions(
+    const mergedOptions = mergeDefaultOptions(
       options,
       this.roomOptions?.audioCaptureDefaults,
       this.roomOptions?.videoCaptureDefaults,
     );
 
-    const constraints = constraintsForOptions(opts);
+    const constraints = constraintsForOptions(mergedOptions);
     let stream: MediaStream | undefined;
     try {
       stream = await navigator.mediaDevices.getUserMedia(constraints);
@@ -425,29 +426,39 @@ export default class LocalParticipant extends Participant {
       this.cameraError = undefined;
     }
 
-    return stream.getTracks().map((mediaStreamTrack) => {
-      const isAudio = mediaStreamTrack.kind === 'audio';
-      let trackOptions = isAudio ? options!.audio : options!.video;
-      if (typeof trackOptions === 'boolean' || !trackOptions) {
-        trackOptions = {};
-      }
-      let trackConstraints: MediaTrackConstraints | undefined;
-      const conOrBool = isAudio ? constraints.audio : constraints.video;
-      if (typeof conOrBool !== 'boolean') {
-        trackConstraints = conOrBool;
-      }
-      const track = mediaTrackToLocalTrack(mediaStreamTrack, trackConstraints, {
-        loggerName: this.roomOptions.loggerName,
-        loggerContextCb: () => this.logContext,
-      });
-      if (track.kind === Track.Kind.Video) {
-        track.source = Track.Source.Camera;
-      } else if (track.kind === Track.Kind.Audio) {
-        track.source = Track.Source.Microphone;
-      }
-      track.mediaStream = stream;
-      return track;
-    });
+    return Promise.all(
+      stream.getTracks().map(async (mediaStreamTrack) => {
+        const isAudio = mediaStreamTrack.kind === 'audio';
+        let trackOptions = isAudio ? mergedOptions!.audio : mergedOptions!.video;
+        if (typeof trackOptions === 'boolean' || !trackOptions) {
+          trackOptions = {};
+        }
+        let trackConstraints: MediaTrackConstraints | undefined;
+        const conOrBool = isAudio ? constraints.audio : constraints.video;
+        if (typeof conOrBool !== 'boolean') {
+          trackConstraints = conOrBool;
+        }
+        const track = mediaTrackToLocalTrack(mediaStreamTrack, trackConstraints, {
+          loggerName: this.roomOptions.loggerName,
+          loggerContextCb: () => this.logContext,
+        });
+        if (track.kind === Track.Kind.Video) {
+          track.source = Track.Source.Camera;
+        } else if (track.kind === Track.Kind.Audio) {
+          track.source = Track.Source.Microphone;
+          track.setAudioContext(this.audioContext);
+        }
+        track.mediaStream = stream;
+        if (trackOptions.processor) {
+          if (track instanceof LocalAudioTrack) {
+            await track.setProcessor(trackOptions.processor as TrackProcessor<Track.Kind.Audio>);
+          } else {
+            await track.setProcessor(trackOptions.processor as TrackProcessor<Track.Kind.Video>);
+          }
+        }
+        return track;
+      }),
+    );
   }
 
   /**

package/src/room/participant/Participant.ts

@@ -1,8 +1,10 @@
 import {
   DataPacket_Kind,
   ParticipantInfo,
+  ParticipantInfo_Kind as ParticipantKind,
   ParticipantPermission,
   ConnectionQuality as ProtoQuality,
+  type SipDTMF,
   SubscriptionError,
 } from '@livekit/protocol';
 import { EventEmitter } from 'events';
@@ -45,6 +47,8 @@ function qualityFromProto(q: ProtoQuality): ConnectionQuality {
   }
 }
 
+export { ParticipantKind };
+
 export default class Participant extends (EventEmitter as new () => TypedEmitter<ParticipantEventCallbacks>) {
   protected participantInfo?: ParticipantInfo;
 
@@ -77,6 +81,8 @@ export default class Participant extends (EventEmitter as new () => TypedEmitter
 
   permissions?: ParticipantPermission;
 
+  protected _kind: ParticipantKind;
+
   private _connectionQuality: ConnectionQuality = ConnectionQuality.Unknown;
 
   protected audioContext?: AudioContext;
@@ -99,7 +105,11 @@ export default class Participant extends (EventEmitter as new () => TypedEmitter
   }
 
   get isAgent() {
-    return this.permissions?.agent ?? false;
+    return this.permissions?.agent || this.kind === ParticipantKind.AGENT;
+  }
+
+  get kind() {
+    return this._kind;
   }
 
   /** @internal */
@@ -109,6 +119,7 @@ export default class Participant extends (EventEmitter as new () => TypedEmitter
     name?: string,
     metadata?: string,
     loggerOptions?: LoggerOptions,
+    kind: ParticipantKind = ParticipantKind.STANDARD,
   ) {
     super();
 
@@ -123,6 +134,7 @@ export default class Participant extends (EventEmitter as new () => TypedEmitter
     this.audioTrackPublications = new Map();
     this.videoTrackPublications = new Map();
     this.trackPublications = new Map();
+    this._kind = kind;
   }
 
   getTrackPublications(): TrackPublication[] {
@@ -329,6 +341,7 @@ export type ParticipantEventCallbacks = {
   participantMetadataChanged: (prevMetadata: string | undefined, participant?: any) => void;
   participantNameChanged: (name: string) => void;
   dataReceived: (payload: Uint8Array, kind: DataPacket_Kind) => void;
+  sipDTMFReceived: (dtmf: SipDTMF) => void;
   transcriptionReceived: (
     transcription: TranscriptionSegment[],
     publication?: TrackPublication,

package/src/room/participant/RemoteParticipant.ts

@@ -16,7 +16,7 @@ import type { AudioOutputOptions } from '../track/options';
 import type { AdaptiveStreamSettings } from '../track/types';
 import { getLogContextFromTrack } from '../track/utils';
 import type { LoggerOptions } from '../types';
-import Participant from './Participant';
+import Participant, { ParticipantKind } from './Participant';
 import type { ParticipantEventCallbacks } from './Participant';
 
 export default class RemoteParticipant extends Participant {
@@ -33,8 +33,20 @@ export default class RemoteParticipant extends Participant {
   private audioOutput?: AudioOutputOptions;
 
   /** @internal */
-  static fromParticipantInfo(signalClient: SignalClient, pi: ParticipantInfo): RemoteParticipant {
-    return new RemoteParticipant(signalClient, pi.sid, pi.identity, pi.name, pi.metadata);
+  static fromParticipantInfo(
+    signalClient: SignalClient,
+    pi: ParticipantInfo,
+    loggerOptions: LoggerOptions,
+  ): RemoteParticipant {
+    return new RemoteParticipant(
+      signalClient,
+      pi.sid,
+      pi.identity,
+      pi.name,
+      pi.metadata,
+      loggerOptions,
+      pi.kind,
+    );
   }
 
   protected get logContext() {
@@ -53,8 +65,9 @@ export default class RemoteParticipant extends Participant {
     name?: string,
     metadata?: string,
     loggerOptions?: LoggerOptions,
+    kind: ParticipantKind = ParticipantKind.STANDARD,
   ) {
-    super(sid, identity || '', name, metadata, loggerOptions);
+    super(sid, identity || '', name, metadata, loggerOptions, kind);
     this.signalClient = signalClient;
     this.trackPublications = new Map();
     this.audioTrackPublications = new Map();

package/src/room/participant/publishUtils.ts

@@ -149,6 +149,10 @@ export function computeVideoEncodings(
     const browser = getBrowser();
     if (
       isSafari() ||
+      // Even tho RN runs M114, it does not produce SVC layers when a single encoding
+      // is provided. So we'll use the legacy SVC specification for now.
+      // TODO: when we upstream libwebrtc, this will need additional verification
+      isReactNative() ||
       (browser?.name === 'Chrome' && compareVersions(browser?.version, '113') < 0)
     ) {
       const bitratesRatio = sm.suffix == 'h' ? 2 : 3;

package/src/room/track/LocalTrack.ts

@@ -433,7 +433,7 @@ export default abstract class LocalTrack<
       this.emit(TrackEvent.UpstreamResumed, this);
 
       // this operation is noop if mediastreamtrack is already being sent
-      await this.sender.replaceTrack(this._mediaStreamTrack);
+      await this.sender.replaceTrack(this.mediaStreamTrack);
     } finally {
       unlock();
     }
@@ -468,16 +468,17 @@ export default abstract class LocalTrack<
     try {
       this.log.debug('setting up processor', this.logContext);
 
-      this.processorElement =
-        this.processorElement ?? (document.createElement(this.kind) as HTMLMediaElement);
+      const processorElement = document.createElement(this.kind) as HTMLMediaElement;
 
       const processorOptions = {
         kind: this.kind,
         track: this._mediaStreamTrack,
-        element: this.processorElement,
+        element: processorElement,
         audioContext: this.audioContext,
       };
       await processor.init(processorOptions);
+      this.log.debug('processor initialized', this.logContext);
+
       if (this.processor) {
         await this.stopProcessor();
       }
@@ -485,16 +486,17 @@ export default abstract class LocalTrack<
         throw TypeError('cannot set processor on track of unknown kind');
       }
 
-      attachToElement(this._mediaStreamTrack, this.processorElement);
-      this.processorElement.muted = true;
+      attachToElement(this._mediaStreamTrack, processorElement);
+      processorElement.muted = true;
 
-      this.processorElement
+      processorElement
         .play()
         .catch((error) =>
           this.log.error('failed to play processor element', { ...this.logContext, error }),
         );
 
       this.processor = processor;
+      this.processorElement = processorElement;
       if (this.processor.processedTrack) {
         for (const el of this.attachedElements) {
           if (el !== this.processorElement && showProcessedStreamLocally) {
@@ -521,15 +523,17 @@ export default abstract class LocalTrack<
    * @experimental
    * @returns
    */
-  async stopProcessor() {
+  async stopProcessor(keepElement = true) {
     if (!this.processor) return;
 
     this.log.debug('stopping processor', this.logContext);
     this.processor.processedTrack?.stop();
     await this.processor.destroy();
     this.processor = undefined;
-    this.processorElement?.remove();
-    this.processorElement = undefined;
+    if (!keepElement) {
+      this.processorElement?.remove();
+      this.processorElement = undefined;
+    }
     // apply original track constraints in case the processor changed them
     await this._mediaStreamTrack.applyConstraints(this._constraints);
     // force re-setting of the mediaStreamTrack on the sender

package/src/room/track/LocalVideoTrack.ts

@@ -276,7 +276,10 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
     }
   }
 
-  async setProcessor(processor: TrackProcessor<Track.Kind>, showProcessedStreamLocally = true) {
+  async setProcessor(
+    processor: TrackProcessor<Track.Kind.Video>,
+    showProcessedStreamLocally = true,
+  ) {
     await super.setProcessor(processor, showProcessedStreamLocally);
 
     if (this.processor?.processedTrack) {

package/src/room/track/create.ts

@@ -14,6 +14,7 @@ import type {
   VideoCaptureOptions,
 } from './options';
 import { ScreenSharePresets } from './options';
+import type { TrackProcessor } from './processor/types';
 import {
   constraintsForOptions,
   mergeDefaultOptions,
@@ -51,35 +52,44 @@ export async function createLocalTracks(
   }
 
   const stream = await mediaPromise;
-  return stream.getTracks().map((mediaStreamTrack) => {
-    const isAudio = mediaStreamTrack.kind === 'audio';
-    let trackOptions = isAudio ? options!.audio : options!.video;
-    if (typeof trackOptions === 'boolean' || !trackOptions) {
-      trackOptions = {};
-    }
-    let trackConstraints: MediaTrackConstraints | undefined;
-    const conOrBool = isAudio ? constraints.audio : constraints.video;
-    if (typeof conOrBool !== 'boolean') {
-      trackConstraints = conOrBool;
-    }
+  return Promise.all(
+    stream.getTracks().map(async (mediaStreamTrack) => {
+      const isAudio = mediaStreamTrack.kind === 'audio';
+      let trackOptions = isAudio ? options!.audio : options!.video;
+      if (typeof trackOptions === 'boolean' || !trackOptions) {
+        trackOptions = {};
+      }
+      let trackConstraints: MediaTrackConstraints | undefined;
+      const conOrBool = isAudio ? constraints.audio : constraints.video;
+      if (typeof conOrBool !== 'boolean') {
+        trackConstraints = conOrBool;
+      }
 
-    // update the constraints with the device id the user gave permissions to in the permission prompt
-    // otherwise each track restart (e.g. mute - unmute) will try to initialize the device again -> causing additional permission prompts
-    if (trackConstraints) {
-      trackConstraints.deviceId = mediaStreamTrack.getSettings().deviceId;
-    } else {
-      trackConstraints = { deviceId: mediaStreamTrack.getSettings().deviceId };
-    }
+      // update the constraints with the device id the user gave permissions to in the permission prompt
+      // otherwise each track restart (e.g. mute - unmute) will try to initialize the device again -> causing additional permission prompts
+      if (trackConstraints) {
+        trackConstraints.deviceId = mediaStreamTrack.getSettings().deviceId;
+      } else {
+        trackConstraints = { deviceId: mediaStreamTrack.getSettings().deviceId };
+      }
 
-    const track = mediaTrackToLocalTrack(mediaStreamTrack, trackConstraints);
-    if (track.kind === Track.Kind.Video) {
-      track.source = Track.Source.Camera;
-    } else if (track.kind === Track.Kind.Audio) {
-      track.source = Track.Source.Microphone;
-    }
-    track.mediaStream = stream;
-    return track;
-  });
+      const track = mediaTrackToLocalTrack(mediaStreamTrack, trackConstraints);
+      if (track.kind === Track.Kind.Video) {
+        track.source = Track.Source.Camera;
+      } else if (track.kind === Track.Kind.Audio) {
+        track.source = Track.Source.Microphone;
+      }
+      track.mediaStream = stream;
+      if (trackOptions.processor) {
+        if (track instanceof LocalAudioTrack) {
+          await track.setProcessor(trackOptions.processor as TrackProcessor<Track.Kind.Audio>);
+        } else if (track instanceof LocalVideoTrack) {
+          await track.setProcessor(trackOptions.processor as TrackProcessor<Track.Kind.Video>);
+        }
+      }
+      return track;
+    }),
+  );
 }
 
 /**

package/src/room/track/options.ts

@@ -1,4 +1,9 @@
 import type { Track } from './Track';
+import type {
+  AudioProcessorOptions,
+  TrackProcessor,
+  VideoProcessorOptions,
+} from './processor/types';
 
 export interface TrackPublishDefaults {
   /**
@@ -152,6 +157,11 @@ export interface VideoCaptureOptions {
   facingMode?: 'user' | 'environment' | 'left' | 'right';
 
   resolution?: VideoResolution;
+
+  /**
+   * initialize the track with a given processor
+   */
+  processor?: TrackProcessor<Track.Kind.Video, VideoProcessorOptions>;
 }
 
 export interface ScreenShareCaptureOptions {
@@ -245,6 +255,11 @@ export interface AudioCaptureOptions {
    * sample size or range of sample sizes which are acceptable and/or required.
    */
   sampleSize?: ConstrainULong;
+
+  /**
+   * initialize the track with a given processor
+   */
+  processor?: TrackProcessor<Track.Kind.Audio, AudioProcessorOptions>;
 }
 
 export interface AudioOutputOptions {