livekit-client 1.4.4 → 1.6.0

package/src/room/track/LocalVideoTrack.ts

@@ -113,7 +113,7 @@ export default class LocalVideoTrack extends LocalTrack {
   }
 
   async getSenderStats(): Promise<VideoSenderStats[]> {
-    if (!this.sender) {
+    if (!this.sender?.getStats) {
       return [];
     }
 

package/src/room/track/RemoteTrackPublication.ts

@@ -22,6 +22,8 @@ export default class RemoteTrackPublication extends TrackPublication {
 
   protected videoDimensions?: Track.Dimensions;
 
+  protected fps?: number;
+
   constructor(kind: Track.Kind, id: string, name: string, autoSubscribe: boolean | undefined) {
     super(kind, id, name);
     this.subscribed = autoSubscribe;
@@ -143,6 +145,23 @@ export default class RemoteTrackPublication extends TrackPublication {
     this.emitTrackUpdate();
   }
 
+  setVideoFPS(fps: number) {
+    if (!this.isManualOperationAllowed()) {
+      return;
+    }
+
+    if (!(this.track instanceof RemoteVideoTrack)) {
+      return;
+    }
+
+    if (this.fps === fps) {
+      return;
+    }
+
+    this.fps = fps;
+    this.emitTrackUpdate();
+  }
+
   get videoQuality(): VideoQuality | undefined {
     return this.currentVideoQuality;
   }
@@ -256,6 +275,7 @@ export default class RemoteTrackPublication extends TrackPublication {
     const settings: UpdateTrackSettings = UpdateTrackSettings.fromPartial({
       trackSids: [this.trackSid],
       disabled: this.disabled,
+      fps: this.fps,
     });
     if (this.videoDimensions) {
       settings.width = this.videoDimensions.width;

package/src/room/track/RemoteVideoTrack.ts

@@ -118,6 +118,10 @@ export default class RemoteVideoTrack extends RemoteTrack {
    * @internal
    */
   stopObservingElementInfo(elementInfo: ElementInfo) {
+    if (!this.isAdaptiveStream) {
+      log.warn('stopObservingElementInfo ignored');
+      return;
+    }
     const stopElementInfos = this.elementInfos.filter((info) => info === elementInfo);
     for (const info of stopElementInfos) {
       info.stopObserving();

package/src/room/track/Track.ts

@@ -398,6 +398,7 @@ export type TrackEventCallbacks = {
   message: () => void;
   muted: (track?: any) => void;
   unmuted: (track?: any) => void;
+  restarted: (track?: any) => void;
   ended: (track?: any) => void;
   updateSettings: () => void;
   updateSubscription: () => void;

package/src/room/types.ts

@@ -0,0 +1,12 @@
+export type SimulationOptions = {
+  publish?: {
+    audio?: boolean;
+    video?: boolean;
+  };
+  participants?: {
+    count?: number;
+    aspectRatios?: Array<number>;
+    audio?: boolean;
+    video?: boolean;
+  };
+};

package/src/room/utils.ts

@@ -1,6 +1,9 @@
 import UAParser from 'ua-parser-js';
 import { ClientInfo, ClientInfo_SDK } from '../proto/livekit_models';
 import { protocolVersion, version } from '../version';
+import type LocalAudioTrack from './track/LocalAudioTrack';
+import type RemoteAudioTrack from './track/RemoteAudioTrack';
+import { getNewAudioContext } from './track/utils';
 
 const separator = '|';
 
@@ -178,22 +181,41 @@ let emptyVideoStreamTrack: MediaStreamTrack | undefined;
 
 export function getEmptyVideoStreamTrack() {
   if (!emptyVideoStreamTrack) {
-    const canvas = document.createElement('canvas');
-    // the canvas size is set to 16, because electron apps seem to fail with smaller values
-    canvas.width = 16;
-    canvas.height = 16;
-    canvas.getContext('2d')?.fillRect(0, 0, canvas.width, canvas.height);
-    // @ts-ignore
-    const emptyStream = canvas.captureStream();
-    [emptyVideoStreamTrack] = emptyStream.getTracks();
-    if (!emptyVideoStreamTrack) {
-      throw Error('Could not get empty media stream video track');
-    }
-    emptyVideoStreamTrack.enabled = false;
+    emptyVideoStreamTrack = createDummyVideoStreamTrack();
   }
   return emptyVideoStreamTrack;
 }
 
+export function createDummyVideoStreamTrack(
+  width: number = 16,
+  height: number = 16,
+  enabled: boolean = false,
+  paintContent: boolean = false,
+) {
+  const canvas = document.createElement('canvas');
+  // the canvas size is set to 16 by default, because electron apps seem to fail with smaller values
+  canvas.width = width;
+  canvas.height = height;
+  const ctx = canvas.getContext('2d');
+  ctx?.fillRect(0, 0, canvas.width, canvas.height);
+  if (paintContent && ctx) {
+    ctx.beginPath();
+    ctx.arc(width / 2, height / 2, 50, 0, Math.PI * 2, true);
+    ctx.closePath();
+    ctx.fillStyle = 'grey';
+    ctx.fill();
+  }
+  // @ts-ignore
+  const dummyStream = canvas.captureStream();
+  const [dummyTrack] = dummyStream.getTracks();
+  if (!dummyTrack) {
+    throw Error('Could not get empty media stream video track');
+  }
+  dummyTrack.enabled = enabled;
+
+  return dummyTrack;
+}
+
 let emptyAudioStreamTrack: MediaStreamTrack | undefined;
 
 export function getEmptyAudioStreamTrack() {
@@ -236,3 +258,119 @@ export class Future<T> {
     }).finally(() => this.onFinally?.());
   }
 }
+
+export type AudioAnalyserOptions = {
+  /**
+   * If set to true, the analyser will use a cloned version of the underlying mediastreamtrack, which won't be impacted by muting the track.
+   * Useful for local tracks when implementing things like "seems like you're muted, but trying to speak".
+   * Defaults to false
+   */
+  cloneTrack?: boolean;
+  /**
+   * see https://developer.mozilla.org/en-US/docs/Web/API/AnalyserNode/fftSize
+   */
+  fftSize?: number;
+  /**
+   * see https://developer.mozilla.org/en-US/docs/Web/API/AnalyserNode/smoothingTimeConstant
+   */
+  smoothingTimeConstant?: number;
+  /**
+   * see https://developer.mozilla.org/en-US/docs/Web/API/AnalyserNode/minDecibels
+   */
+  minDecibels?: number;
+  /**
+   * see https://developer.mozilla.org/en-US/docs/Web/API/AnalyserNode/maxDecibels
+   */
+  maxDecibels?: number;
+};
+
+/**
+ * Creates and returns an analyser web audio node that is attached to the provided track.
+ * Additionally returns a convenience method `calculateVolume` to perform instant volume readings on that track.
+ * Call the returned `cleanup` function to close the audioContext that has been created for the instance of this helper
+ */
+export function createAudioAnalyser(
+  track: LocalAudioTrack | RemoteAudioTrack,
+  options?: AudioAnalyserOptions,
+) {
+  const opts = {
+    cloneTrack: false,
+    fftSize: 2048,
+    smoothingTimeConstant: 0.8,
+    minDecibels: -100,
+    maxDecibels: -80,
+    ...options,
+  };
+  const audioContext = getNewAudioContext();
+
+  if (!audioContext) {
+    throw new Error('Audio Context not supported on this browser');
+  }
+  const streamTrack = opts.cloneTrack ? track.mediaStreamTrack.clone() : track.mediaStreamTrack;
+  const mediaStreamSource = audioContext.createMediaStreamSource(new MediaStream([streamTrack]));
+  const analyser = audioContext.createAnalyser();
+  analyser.minDecibels = opts.minDecibels;
+  analyser.maxDecibels = opts.maxDecibels;
+  analyser.fftSize = opts.fftSize;
+  analyser.smoothingTimeConstant = opts.smoothingTimeConstant;
+
+  mediaStreamSource.connect(analyser);
+  const dataArray = new Uint8Array(analyser.frequencyBinCount);
+
+  /**
+   * Calculates the current volume of the track in the range from 0 to 1
+   */
+  const calculateVolume = () => {
+    analyser.getByteFrequencyData(dataArray);
+    let sum = 0;
+    for (const amplitude of dataArray) {
+      sum += Math.pow(amplitude / 255, 2);
+    }
+    const volume = Math.sqrt(sum / dataArray.length);
+    return volume;
+  };
+
+  const cleanup = () => {
+    audioContext.close();
+    if (opts.cloneTrack) {
+      streamTrack.stop();
+    }
+  };
+
+  return { calculateVolume, analyser, cleanup };
+}
+
+export class Mutex {
+  private _locking: Promise<void>;
+
+  private _locks: number;
+
+  constructor() {
+    this._locking = Promise.resolve();
+    this._locks = 0;
+  }
+
+  isLocked() {
+    return this._locks > 0;
+  }
+
+  lock() {
+    this._locks += 1;
+
+    let unlockNext: () => void;
+
+    const willLock = new Promise<void>(
+      (resolve) =>
+        (unlockNext = () => {
+          this._locks -= 1;
+          resolve();
+        }),
+    );
+
+    const willUnlock = this._locking.then(() => unlockNext);
+
+    this._locking = this._locking.then(() => willLock);
+
+    return willUnlock;
+  }
+}