livekit-client 1.5.0 → 1.6.1

package/src/room/track/LocalVideoTrack.ts

@@ -113,7 +113,7 @@ export default class LocalVideoTrack extends LocalTrack {
   }
 
   async getSenderStats(): Promise<VideoSenderStats[]> {
-    if (!this.sender) {
+    if (!this.sender?.getStats) {
       return [];
     }
 

package/src/room/track/RemoteVideoTrack.ts

@@ -118,6 +118,10 @@ export default class RemoteVideoTrack extends RemoteTrack {
    * @internal
    */
   stopObservingElementInfo(elementInfo: ElementInfo) {
+    if (!this.isAdaptiveStream) {
+      log.warn('stopObservingElementInfo ignored');
+      return;
+    }
     const stopElementInfos = this.elementInfos.filter((info) => info === elementInfo);
     for (const info of stopElementInfos) {
       info.stopObserving();

package/src/room/track/Track.ts

@@ -121,7 +121,9 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
     // we'll want to re-attach it in that case
     attachToElement(this._mediaStreamTrack, element);
 
-    if (element instanceof HTMLAudioElement) {
+    // handle auto playback failures
+    const allMediaStreamTracks = (element.srcObject as MediaStream).getTracks();
+    if (allMediaStreamTracks.some((tr) => tr.kind === 'audio')) {
       // manually play audio to detect audio playback status
       element
         .play()
@@ -130,6 +132,17 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
         })
         .catch((e) => {
           this.emit(TrackEvent.AudioPlaybackFailed, e);
+          // If audio playback isn't allowed make sure we still play back the video
+          if (
+            element &&
+            allMediaStreamTracks.some((tr) => tr.kind === 'video') &&
+            e.name === 'NotAllowedError'
+          ) {
+            element.muted = true;
+            element.play().catch(() => {
+              // catch for Safari, exceeded options at this point to automatically play the media element
+            });
+          }
         });
     }
 
@@ -259,6 +272,13 @@ export function attachToElement(track: MediaStreamTrack, element: HTMLMediaEleme
     mediaStream.addTrack(track);
   }
 
+  element.autoplay = true;
+  // In case there are no audio tracks present on the mediastream, we set the element as muted to ensure autoplay works
+  element.muted = mediaStream.getAudioTracks().length === 0;
+  if (element instanceof HTMLVideoElement) {
+    element.playsInline = true;
+  }
+
   // avoid flicker
   if (element.srcObject !== mediaStream) {
     element.srcObject = mediaStream;
@@ -280,10 +300,6 @@ export function attachToElement(track: MediaStreamTrack, element: HTMLMediaEleme
       }, 0);
     }
   }
-  element.autoplay = true;
-  if (element instanceof HTMLVideoElement) {
-    element.playsInline = true;
-  }
 }
 
 /** @internal */
@@ -398,6 +414,7 @@ export type TrackEventCallbacks = {
   message: () => void;
   muted: (track?: any) => void;
   unmuted: (track?: any) => void;
+  restarted: (track?: any) => void;
   ended: (track?: any) => void;
   updateSettings: () => void;
   updateSubscription: () => void;

package/src/room/types.ts

@@ -0,0 +1,12 @@
+export type SimulationOptions = {
+  publish?: {
+    audio?: boolean;
+    video?: boolean;
+  };
+  participants?: {
+    count?: number;
+    aspectRatios?: Array<number>;
+    audio?: boolean;
+    video?: boolean;
+  };
+};

package/src/room/utils.ts

@@ -1,6 +1,9 @@
 import UAParser from 'ua-parser-js';
 import { ClientInfo, ClientInfo_SDK } from '../proto/livekit_models';
 import { protocolVersion, version } from '../version';
+import type LocalAudioTrack from './track/LocalAudioTrack';
+import type RemoteAudioTrack from './track/RemoteAudioTrack';
+import { getNewAudioContext } from './track/utils';
 
 const separator = '|';
 
@@ -178,22 +181,41 @@ let emptyVideoStreamTrack: MediaStreamTrack | undefined;
 
 export function getEmptyVideoStreamTrack() {
   if (!emptyVideoStreamTrack) {
-    const canvas = document.createElement('canvas');
-    // the canvas size is set to 16, because electron apps seem to fail with smaller values
-    canvas.width = 16;
-    canvas.height = 16;
-    canvas.getContext('2d')?.fillRect(0, 0, canvas.width, canvas.height);
-    // @ts-ignore
-    const emptyStream = canvas.captureStream();
-    [emptyVideoStreamTrack] = emptyStream.getTracks();
-    if (!emptyVideoStreamTrack) {
-      throw Error('Could not get empty media stream video track');
-    }
-    emptyVideoStreamTrack.enabled = false;
+    emptyVideoStreamTrack = createDummyVideoStreamTrack();
   }
   return emptyVideoStreamTrack;
 }
 
+export function createDummyVideoStreamTrack(
+  width: number = 16,
+  height: number = 16,
+  enabled: boolean = false,
+  paintContent: boolean = false,
+) {
+  const canvas = document.createElement('canvas');
+  // the canvas size is set to 16 by default, because electron apps seem to fail with smaller values
+  canvas.width = width;
+  canvas.height = height;
+  const ctx = canvas.getContext('2d');
+  ctx?.fillRect(0, 0, canvas.width, canvas.height);
+  if (paintContent && ctx) {
+    ctx.beginPath();
+    ctx.arc(width / 2, height / 2, 50, 0, Math.PI * 2, true);
+    ctx.closePath();
+    ctx.fillStyle = 'grey';
+    ctx.fill();
+  }
+  // @ts-ignore
+  const dummyStream = canvas.captureStream();
+  const [dummyTrack] = dummyStream.getTracks();
+  if (!dummyTrack) {
+    throw Error('Could not get empty media stream video track');
+  }
+  dummyTrack.enabled = enabled;
+
+  return dummyTrack;
+}
+
 let emptyAudioStreamTrack: MediaStreamTrack | undefined;
 
 export function getEmptyAudioStreamTrack() {
@@ -236,3 +258,119 @@ export class Future<T> {
     }).finally(() => this.onFinally?.());
   }
 }
+
+export type AudioAnalyserOptions = {
+  /**
+   * If set to true, the analyser will use a cloned version of the underlying mediastreamtrack, which won't be impacted by muting the track.
+   * Useful for local tracks when implementing things like "seems like you're muted, but trying to speak".
+   * Defaults to false
+   */
+  cloneTrack?: boolean;
+  /**
+   * see https://developer.mozilla.org/en-US/docs/Web/API/AnalyserNode/fftSize
+   */
+  fftSize?: number;
+  /**
+   * see https://developer.mozilla.org/en-US/docs/Web/API/AnalyserNode/smoothingTimeConstant
+   */
+  smoothingTimeConstant?: number;
+  /**
+   * see https://developer.mozilla.org/en-US/docs/Web/API/AnalyserNode/minDecibels
+   */
+  minDecibels?: number;
+  /**
+   * see https://developer.mozilla.org/en-US/docs/Web/API/AnalyserNode/maxDecibels
+   */
+  maxDecibels?: number;
+};
+
+/**
+ * Creates and returns an analyser web audio node that is attached to the provided track.
+ * Additionally returns a convenience method `calculateVolume` to perform instant volume readings on that track.
+ * Call the returned `cleanup` function to close the audioContext that has been created for the instance of this helper
+ */
+export function createAudioAnalyser(
+  track: LocalAudioTrack | RemoteAudioTrack,
+  options?: AudioAnalyserOptions,
+) {
+  const opts = {
+    cloneTrack: false,
+    fftSize: 2048,
+    smoothingTimeConstant: 0.8,
+    minDecibels: -100,
+    maxDecibels: -80,
+    ...options,
+  };
+  const audioContext = getNewAudioContext();
+
+  if (!audioContext) {
+    throw new Error('Audio Context not supported on this browser');
+  }
+  const streamTrack = opts.cloneTrack ? track.mediaStreamTrack.clone() : track.mediaStreamTrack;
+  const mediaStreamSource = audioContext.createMediaStreamSource(new MediaStream([streamTrack]));
+  const analyser = audioContext.createAnalyser();
+  analyser.minDecibels = opts.minDecibels;
+  analyser.maxDecibels = opts.maxDecibels;
+  analyser.fftSize = opts.fftSize;
+  analyser.smoothingTimeConstant = opts.smoothingTimeConstant;
+
+  mediaStreamSource.connect(analyser);
+  const dataArray = new Uint8Array(analyser.frequencyBinCount);
+
+  /**
+   * Calculates the current volume of the track in the range from 0 to 1
+   */
+  const calculateVolume = () => {
+    analyser.getByteFrequencyData(dataArray);
+    let sum = 0;
+    for (const amplitude of dataArray) {
+      sum += Math.pow(amplitude / 255, 2);
+    }
+    const volume = Math.sqrt(sum / dataArray.length);
+    return volume;
+  };
+
+  const cleanup = () => {
+    audioContext.close();
+    if (opts.cloneTrack) {
+      streamTrack.stop();
+    }
+  };
+
+  return { calculateVolume, analyser, cleanup };
+}
+
+export class Mutex {
+  private _locking: Promise<void>;
+
+  private _locks: number;
+
+  constructor() {
+    this._locking = Promise.resolve();
+    this._locks = 0;
+  }
+
+  isLocked() {
+    return this._locks > 0;
+  }
+
+  lock() {
+    this._locks += 1;
+
+    let unlockNext: () => void;
+
+    const willLock = new Promise<void>(
+      (resolve) =>
+        (unlockNext = () => {
+          this._locks -= 1;
+          resolve();
+        }),
+    );
+
+    const willUnlock = this._locking.then(() => unlockNext);
+
+    this._locking = this._locking.then(() => willLock);
+
+    return willUnlock;
+  }
+}