livekit-client 2.0.1 → 2.0.3

package/src/room/track/LocalTrack.ts

@@ -11,7 +11,9 @@ import type { TrackProcessor } from './processor/types';
 
 const defaultDimensionsTimeout = 1000;
 
-export default abstract class LocalTrack extends Track {
+export default abstract class LocalTrack<
+  TrackKind extends Track.Kind = Track.Kind,
+> extends Track<TrackKind> {
   /** @internal */
   sender?: RTCRtpSender;
 
@@ -34,10 +36,14 @@ export default abstract class LocalTrack extends Track {
 
   protected processorElement?: HTMLMediaElement;
 
-  protected processor?: TrackProcessor<this['kind']>;
+  protected processor?: TrackProcessor<TrackKind, any>;
 
   protected processorLock: Mutex;
 
+  protected audioContext?: AudioContext;
+
+  private restartLock: Mutex;
+
   /**
    *
    * @param mediaTrack
@@ -47,7 +53,7 @@ export default abstract class LocalTrack extends Track {
    */
   protected constructor(
     mediaTrack: MediaStreamTrack,
-    kind: Track.Kind,
+    kind: TrackKind,
     constraints?: MediaTrackConstraints,
     userProvidedTrack = false,
     loggerOptions?: LoggerOptions,
@@ -58,6 +64,7 @@ export default abstract class LocalTrack extends Track {
     this.muteLock = new Mutex();
     this.pauseUpstreamLock = new Mutex();
     this.processorLock = new Mutex();
+    this.restartLock = new Mutex();
     this.setMediaStreamTrack(mediaTrack, true);
 
     // added to satisfy TS compiler, constraints are synced with MediaStreamTrack
@@ -128,21 +135,28 @@ export default abstract class LocalTrack extends Track {
       this._constraints = newTrack.getConstraints();
     }
     let processedTrack: MediaStreamTrack | undefined;
-    if (this.processor && newTrack && this.processorElement) {
-      this.log.debug('restarting processor', this.logContext);
-      if (this.kind === 'unknown') {
-        throw TypeError('cannot set processor on track of unknown kind');
-      }
+    if (this.processor && newTrack) {
+      const unlock = await this.processorLock.lock();
+      try {
+        this.log.debug('restarting processor', this.logContext);
+        if (this.kind === 'unknown') {
+          throw TypeError('cannot set processor on track of unknown kind');
+        }
 
-      attachToElement(newTrack, this.processorElement);
-      // ensure the processorElement itself stays muted
-      this.processorElement.muted = true;
-      await this.processor.restart({
-        track: newTrack,
-        kind: this.kind,
-        element: this.processorElement,
-      });
-      processedTrack = this.processor.processedTrack;
+        if (this.processorElement) {
+          attachToElement(newTrack, this.processorElement);
+          // ensure the processorElement itself stays muted
+          this.processorElement.muted = true;
+        }
+        await this.processor.restart({
+          track: newTrack,
+          kind: this.kind,
+          element: this.processorElement,
+        });
+        processedTrack = this.processor.processedTrack;
+      } finally {
+        unlock();
+      }
     }
     if (this.sender) {
       await this.sender.replaceTrack(processedTrack ?? newTrack);
@@ -200,17 +214,17 @@ export default abstract class LocalTrack extends Track {
     return DeviceManager.getInstance().normalizeDeviceId(kind, deviceId, groupId);
   }
 
-  async mute(): Promise<LocalTrack> {
+  async mute() {
     this.setTrackMuted(true);
     return this;
   }
 
-  async unmute(): Promise<LocalTrack> {
+  async unmute() {
     this.setTrackMuted(false);
     return this;
   }
 
-  async replaceTrack(track: MediaStreamTrack, userProvidedTrack = true): Promise<LocalTrack> {
+  async replaceTrack(track: MediaStreamTrack, userProvidedTrack = true) {
     if (!this.sender) {
       throw new TrackInvalidError('unable to replace an unpublished track');
     }
@@ -227,45 +241,50 @@ export default abstract class LocalTrack extends Track {
     return this;
   }
 
-  protected async restart(constraints?: MediaTrackConstraints): Promise<LocalTrack> {
-    if (!constraints) {
-      constraints = this._constraints;
-    }
-    this.log.debug('restarting track with constraints', { ...this.logContext, constraints });
+  protected async restart(constraints?: MediaTrackConstraints) {
+    const unlock = await this.restartLock.lock();
+    try {
+      if (!constraints) {
+        constraints = this._constraints;
+      }
+      this.log.debug('restarting track with constraints', { ...this.logContext, constraints });
 
-    const streamConstraints: MediaStreamConstraints = {
-      audio: false,
-      video: false,
-    };
+      const streamConstraints: MediaStreamConstraints = {
+        audio: false,
+        video: false,
+      };
 
-    if (this.kind === Track.Kind.Video) {
-      streamConstraints.video = constraints;
-    } else {
-      streamConstraints.audio = constraints;
-    }
+      if (this.kind === Track.Kind.Video) {
+        streamConstraints.video = constraints;
+      } else {
+        streamConstraints.audio = constraints;
+      }
 
-    // these steps are duplicated from setMediaStreamTrack because we must stop
-    // the previous tracks before new tracks can be acquired
-    this.attachedElements.forEach((el) => {
-      detachTrack(this.mediaStreamTrack, el);
-    });
-    this._mediaStreamTrack.removeEventListener('ended', this.handleEnded);
-    // on Safari, the old audio track must be stopped before attempting to acquire
-    // the new track, otherwise the new track will stop with
-    // 'A MediaStreamTrack ended due to a capture failure`
-    this._mediaStreamTrack.stop();
+      // these steps are duplicated from setMediaStreamTrack because we must stop
+      // the previous tracks before new tracks can be acquired
+      this.attachedElements.forEach((el) => {
+        detachTrack(this.mediaStreamTrack, el);
+      });
+      this._mediaStreamTrack.removeEventListener('ended', this.handleEnded);
+      // on Safari, the old audio track must be stopped before attempting to acquire
+      // the new track, otherwise the new track will stop with
+      // 'A MediaStreamTrack ended due to a capture failure`
+      this._mediaStreamTrack.stop();
 
-    // create new track and attach
-    const mediaStream = await navigator.mediaDevices.getUserMedia(streamConstraints);
-    const newTrack = mediaStream.getTracks()[0];
-    newTrack.addEventListener('ended', this.handleEnded);
-    this.log.debug('re-acquired MediaStreamTrack', this.logContext);
+      // create new track and attach
+      const mediaStream = await navigator.mediaDevices.getUserMedia(streamConstraints);
+      const newTrack = mediaStream.getTracks()[0];
+      newTrack.addEventListener('ended', this.handleEnded);
+      this.log.debug('re-acquired MediaStreamTrack', this.logContext);
 
-    await this.setMediaStreamTrack(newTrack);
-    this._constraints = constraints;
+      await this.setMediaStreamTrack(newTrack);
+      this._constraints = constraints;
 
-    this.emit(TrackEvent.Restarted, this);
-    return this;
+      this.emit(TrackEvent.Restarted, this);
+      return this;
+    } finally {
+      unlock();
+    }
   }
 
   protected setTrackMuted(muted: boolean) {
@@ -408,7 +427,7 @@ export default abstract class LocalTrack extends Track {
    * @param showProcessedStreamLocally
    * @returns
    */
-  async setProcessor(processor: TrackProcessor<this['kind']>, showProcessedStreamLocally = true) {
+  async setProcessor(processor: TrackProcessor<TrackKind>, showProcessedStreamLocally = true) {
     const unlock = await this.processorLock.lock();
     try {
       this.log.debug('setting up processor', this.logContext);
@@ -418,7 +437,8 @@ export default abstract class LocalTrack extends Track {
       if (this.kind === 'unknown') {
         throw TypeError('cannot set processor on track of unknown kind');
       }
-      this.processorElement = this.processorElement ?? document.createElement(this.kind);
+      this.processorElement =
+        this.processorElement ?? (document.createElement(this.kind) as HTMLMediaElement);
 
       attachToElement(this._mediaStreamTrack, this.processorElement);
       this.processorElement.muted = true;
@@ -433,6 +453,7 @@ export default abstract class LocalTrack extends Track {
         kind: this.kind,
         track: this._mediaStreamTrack,
         element: this.processorElement,
+        audioContext: this.audioContext,
       };
 
       await processor.init(processorOptions);
@@ -446,6 +467,7 @@ export default abstract class LocalTrack extends Track {
         }
         await this.sender?.replaceTrack(this.processor.processedTrack);
       }
+      this.emit(TrackEvent.TrackProcessorUpdate, this.processor);
     } finally {
       unlock();
     }
@@ -471,8 +493,8 @@ export default abstract class LocalTrack extends Track {
     this.processor = undefined;
     this.processorElement?.remove();
     this.processorElement = undefined;
-
     await this.restart();
+    this.emit(TrackEvent.TrackProcessorUpdate);
   }
 
   protected abstract monitorSender(): void;

package/src/room/track/LocalVideoTrack.ts

@@ -30,7 +30,7 @@ export class SimulcastTrackInfo {
 
 const refreshSubscribedCodecAfterNewCodec = 5000;
 
-export default class LocalVideoTrack extends LocalTrack {
+export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
   /* @internal */
   signalClient?: SignalClient;
 
@@ -115,7 +115,7 @@ export default class LocalVideoTrack extends LocalTrack {
     }
   }
 
-  async mute(): Promise<LocalVideoTrack> {
+  async mute(): Promise<typeof this> {
     const unlock = await this.muteLock.lock();
     try {
       if (this.source === Track.Source.Camera && !this.isUserProvided) {
@@ -130,7 +130,7 @@ export default class LocalVideoTrack extends LocalTrack {
     }
   }
 
-  async unmute(): Promise<LocalVideoTrack> {
+  async unmute(): Promise<typeof this> {
     const unlock = await this.muteLock.lock();
     try {
       if (this.source === Track.Source.Camera && !this.isUserProvided) {

package/src/room/track/RemoteAudioTrack.ts

@@ -7,7 +7,7 @@ import RemoteTrack from './RemoteTrack';
 import { Track } from './Track';
 import type { AudioOutputOptions } from './options';
 
-export default class RemoteAudioTrack extends RemoteTrack {
+export default class RemoteAudioTrack extends RemoteTrack<Track.Kind.Audio> {
   private prevStats?: AudioReceiverStats;
 
   private elementVolume: number | undefined;

package/src/room/track/RemoteTrack.ts

@@ -3,14 +3,16 @@ import { monitorFrequency } from '../stats';
 import type { LoggerOptions } from '../types';
 import { Track } from './Track';
 
-export default abstract class RemoteTrack extends Track {
+export default abstract class RemoteTrack<
+  TrackKind extends Track.Kind = Track.Kind,
+> extends Track<TrackKind> {
   /** @internal */
   receiver?: RTCRtpReceiver;
 
   constructor(
     mediaTrack: MediaStreamTrack,
     sid: string,
-    kind: Track.Kind,
+    kind: TrackKind,
     receiver?: RTCRtpReceiver,
     loggerOptions?: LoggerOptions,
   ) {

package/src/room/track/RemoteVideoTrack.ts

@@ -12,7 +12,7 @@ import type { AdaptiveStreamSettings } from './types';
 
 const REACTION_DELAY = 100;
 
-export default class RemoteVideoTrack extends RemoteTrack {
+export default class RemoteVideoTrack extends RemoteTrack<Track.Kind.Video> {
   private prevStats?: VideoReceiverStats;
 
   private elementInfos: ElementInfo[] = [];

package/src/room/track/Track.ts

@@ -11,6 +11,7 @@ import { StreamState as ProtoStreamState } from '../../proto/livekit_rtc_pb';
 import { TrackEvent } from '../events';
 import type { LoggerOptions } from '../types';
 import { isFireFox, isSafari, isWeb } from '../utils';
+import type { TrackProcessor } from './processor/types';
 import { getLogContextFromTrack } from './utils';
 
 const BACKGROUND_REACTION_DELAY = 5000;
@@ -24,8 +25,10 @@ export enum VideoQuality {
   MEDIUM = ProtoQuality.MEDIUM,
   HIGH = ProtoQuality.HIGH,
 }
-export abstract class Track extends (EventEmitter as new () => TypedEventEmitter<TrackEventCallbacks>) {
-  kind: Track.Kind;
+export abstract class Track<
+  TrackKind extends Track.Kind = Track.Kind,
+> extends (EventEmitter as new () => TypedEventEmitter<TrackEventCallbacks>) {
+  readonly kind: TrackKind;
 
   attachedElements: HTMLMediaElement[] = [];
 
@@ -67,7 +70,7 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
 
   protected constructor(
     mediaTrack: MediaStreamTrack,
-    kind: Track.Kind,
+    kind: TrackKind,
     loggerOptions: LoggerOptions = {},
   ) {
     super();
@@ -500,4 +503,5 @@ export type TrackEventCallbacks = {
   elementDetached: (element: HTMLMediaElement) => void;
   upstreamPaused: (track: any) => void;
   upstreamResumed: (track: any) => void;
+  trackProcessorUpdate: (processor?: TrackProcessor<Track.Kind, any>) => void;
 };

package/src/room/track/processor/types.ts

@@ -1,3 +1,4 @@
+import type Room from '../../Room';
 import type { Track } from '../Track';
 
 /**
@@ -7,6 +8,7 @@ export type ProcessorOptions<T extends Track.Kind> = {
   kind: T;
   track: MediaStreamTrack;
   element?: HTMLMediaElement;
+  audioContext?: AudioContext;
 };
 
 /**
@@ -33,4 +35,6 @@ export interface TrackProcessor<
   restart: (opts: U) => Promise<void>;
   destroy: () => Promise<void>;
   processedTrack?: MediaStreamTrack;
+  onPublish?: (room: Room) => Promise<void>;
+  onUnpublish?: () => Promise<void>;
 }