@100mslive/hms-video-store 0.8.5-alpha.0 → 0.8.5-alpha.2

package/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.8.5-alpha.0",
+  "version": "0.8.5-alpha.2",
   "license": "MIT",
   "main": "dist/index.cjs.js",
   "module": "dist/index.js",
@@ -41,7 +41,7 @@
   "author": "100ms",
   "sideEffects": false,
   "dependencies": {
-    "@100mslive/hms-video": "0.7.5-alpha.0",
+    "@100mslive/hms-video": "0.7.5-alpha.2",
     "eventemitter2": "^6.4.7",
     "immer": "^9.0.6",
     "reselect": "4.0.0",
@@ -66,5 +66,5 @@
     "url": "https://github.com/100mslive/hms-video-store/issues"
   },
   "homepage": "https://github.com/100mslive/hms-video-store#readme",
-  "gitHead": "8ea7a607a55e81770857c4ab8928e54a5c26a928"
+  "gitHead": "2c9b127a10783f0a5d2f73f4fdd8bed866800c7e"
 }

package/src/controller/beam/BeamSpeakerLabelsLogger.ts

@@ -0,0 +1,146 @@
+import { HMSLogger } from '../../common/ui-logger';
+import { HMSPeer, IHMSStore, selectIsConnectedToRoom, selectPeers } from '../../core';
+import { IHMSActions } from '../../core/IHMSActions';
+
+/**
+ * Log data of audio level and speaker speaking periodically to beam for transcript
+ * diarization.
+ */
+export class BeamSpeakerLabelsLogger {
+  private audioContext?: AudioContext;
+  private readonly intervalMs: number;
+  private shouldMonitor: boolean;
+  private hasStarted: boolean;
+  private unsubs: any[];
+  private readonly analysers: Record<string, AnalyserNode>;
+  private readonly store: IHMSStore;
+  private actions: IHMSActions;
+  constructor(store: IHMSStore, actions: IHMSActions) {
+    this.intervalMs = 100;
+    this.shouldMonitor = false;
+    this.hasStarted = false;
+    this.unsubs = [];
+    this.analysers = {};
+    this.store = store;
+    this.actions = actions;
+  }
+
+  async start() {
+    if (this.hasStarted) {
+      return;
+    }
+    this.hasStarted = true;
+    HMSLogger.d('starting audio level monitor for remote peers', this.store);
+    const isConnected = this.store.getState(selectIsConnectedToRoom);
+    HMSLogger.d('starting audio levels is connected to room', isConnected);
+    if (isConnected) {
+      await this.monitorAudioLevels();
+    }
+    const unsub = this.store.subscribe(this.monitorAudioLevels.bind(this), selectIsConnectedToRoom);
+    this.unsubs.push(unsub);
+  }
+
+  async stop() {
+    if (!this.hasStarted) {
+      return;
+    }
+    this.hasStarted = false;
+    this.shouldMonitor = false;
+    this.unsubs.forEach(unsub => unsub());
+    HMSLogger.d('stopped audio level monitor for remote peers');
+  }
+
+  async monitorAudioLevels() {
+    const isConnected = this.store.getState(selectIsConnectedToRoom);
+    if (!isConnected) {
+      if (this.shouldMonitor) {
+        HMSLogger.i('room no longer connected, stopping audio level monitoring for remote');
+        this.shouldMonitor = false;
+      }
+      return;
+    }
+    if (this.shouldMonitor) {
+      return;
+    }
+    HMSLogger.i('monitoring audio levels');
+    this.shouldMonitor = true;
+    const loop = () => {
+      if (this.shouldMonitor) {
+        this.logAllPeersAudioLevels();
+        setTimeout(loop, this.intervalMs);
+      } else {
+        HMSLogger.i('stopped monitoring audio levels');
+      }
+    };
+    setTimeout(loop, 1000);
+  }
+
+  // eslint-disable-next-line complexity
+  async logAllPeersAudioLevels() {
+    if (!window.__triggerBeamEvent__) {
+      return;
+    }
+    // optimise this to selectTracks instead of selecting peers
+    const allPeers = this.store.getState(selectPeers);
+    const peers = allPeers.filter(peer => !!peer.audioTrack);
+    const peerAudioLevels = [];
+    for (const peer of peers) {
+      // @ts-ignore
+      const sdkTrack = this.actions.hmsSDKTracks[peer.audioTrack];
+      const nativeStream: MediaStream = sdkTrack?.stream?.nativeStream;
+      if (!peer.joinedAt) {
+        continue;
+      }
+      if (nativeStream) {
+        const peerLevel = await this.getAudioLevel(peer, nativeStream);
+        if (peerLevel.level > 0) {
+          peerAudioLevels.push(peerLevel);
+        }
+      }
+    }
+    if (peerAudioLevels.length > 0) {
+      const payload = {
+        event: 'app-audio-level',
+        data: peerAudioLevels,
+      };
+      // HMSLogger.d('logging audio levels', peerAudioLevels);
+      window.__triggerBeamEvent__(JSON.stringify(payload));
+    }
+  }
+
+  async getAudioLevel(peer: HMSPeer, stream: MediaStream) {
+    if (!this.analysers[stream.id]) {
+      this.analysers[stream.id] = this.createAnalyserNode(stream);
+    }
+    const analyserNode = this.analysers[stream.id];
+    const level = this.calculateAudioLevel(analyserNode);
+    return {
+      peerId: peer.id,
+      peerName: peer.name,
+      level,
+    };
+  }
+
+  createAnalyserNode(stream: MediaStream) {
+    if (!this.audioContext) {
+      this.audioContext = new AudioContext();
+    }
+    const analyser = this.audioContext.createAnalyser();
+    const source = this.audioContext.createMediaStreamSource(stream);
+    source.connect(analyser);
+    return analyser;
+  }
+
+  calculateAudioLevel(analyserNode: AnalyserNode) {
+    const data = new Uint8Array(analyserNode.fftSize);
+    analyserNode.getByteTimeDomainData(data);
+    const lowest = 0.009;
+    let max = lowest;
+    for (const frequency of data) {
+      max = Math.max(max, (frequency - 128) / 128);
+    }
+    const normalized = (Math.log(lowest) - Math.log(max)) / Math.log(lowest);
+    const percent = Math.ceil(Math.min(Math.max(normalized * 100, 0), 100));
+    return percent;
+  }
+}

package/src/core/IHMSActions.ts

@@ -10,6 +10,9 @@ import {
   HMSScreenShareConfig,
   HMSVideoPlugin,
   HMSVideoTrackSettings,
+  TokenRequest,
+  TokenRequestOptions,
+  TokenResult,
 } from '@100mslive/hms-video';
 import { HLSConfig, RTMPRecordingConfig } from './hmsSDKStore/sdkTypes';
 import {
@@ -472,4 +475,12 @@ export interface IHMSActions {
    **/
   setAppData(key: string, value: Record<string | number, any>, merge?: boolean): void;
   setAppData(key: string, value: any): void;
+
+  getAuthTokenByRoomCode(tokenRequest: TokenRequest, tokenRequestOptions?: TokenRequestOptions): Promise<TokenResult>;
+
+  /**
+   * enable sending audio speaker data to beam
+   * @alpha
+   */
+  enableBeamSpeakerLabelsLogging(): Promise<void>;
 }

package/src/core/hmsSDKStore/HMSReactiveStore.ts

@@ -25,6 +25,7 @@ declare global {
   interface Window {
     __hms: HMSReactiveStore;
     __beam: BeamControllerStore;
+    __triggerBeamEvent__: (args: any) => void;
   }
 }
 

package/src/core/hmsSDKStore/HMSSDKActions.ts

@@ -30,6 +30,7 @@ import { HMSPlaylist } from './HMSPlaylist';
 import { NamedSetState } from './internalTypes';
 import * as sdkTypes from './sdkTypes';
 import { HMSLogger } from '../../common/ui-logger';
+import { BeamSpeakerLabelsLogger } from '../../controller/beam/BeamSpeakerLabelsLogger';
 import { IHMSActions } from '../IHMSActions';
 import { IHMSStore } from '../IHMSStore';
 import {
@@ -104,6 +105,7 @@ export class HMSSDKActions implements IHMSActions {
   // private actionBatcher: ActionBatcher;
   audioPlaylist!: IHMSPlaylistActions;
   videoPlaylist!: IHMSPlaylistActions;
+  private beamSpeakerLabelsLogger?: BeamSpeakerLabelsLogger;
 
   constructor(store: IHMSStore, sdk: HMSSdk, notificationManager: HMSNotifications) {
     this.store = store;
@@ -144,12 +146,12 @@ export class HMSSDKActions implements IHMSActions {
       if (track instanceof SDKHMSRemoteVideoTrack) {
         //@ts-ignore
         if (layer === HMSSimulcastLayer.NONE) {
-          HMSLogger.w(`layer ${HMSSimulcastLayer.NONE} will be ignored`);
+          HMSLogger.d(`layer ${HMSSimulcastLayer.NONE} will be ignored`);
           return;
         }
         const alreadyInSameState = this.store.getState(selectVideoTrackByID(trackId))?.preferredLayer === layer;
         if (alreadyInSameState) {
-          HMSLogger.w(`preferred layer is already ${layer}`);
+          HMSLogger.d(`preferred layer is already ${layer}`);
           return;
         }
         this.setState(draftStore => {
@@ -160,13 +162,20 @@ export class HMSSDKActions implements IHMSActions {
         }, 'setPreferredLayer');
         await track.setPreferredLayer(layer);
       } else {
-        HMSLogger.w(`track ${trackId} is not a remote video track`);
+        HMSLogger.d(`track ${trackId} is not a remote video track`);
       }
     } else {
       this.logPossibleInconsistency(`track ${trackId} not present, unable to set preffer layer`);
     }
   }
 
+  getAuthTokenByRoomCode(
+    tokenRequest: sdkTypes.TokenRequest,
+    tokenRequestOptions?: sdkTypes.TokenRequestOptions,
+  ): Promise<sdkTypes.TokenResult> {
+    return this.sdk.getAuthTokenByRoomCode(tokenRequest, tokenRequestOptions);
+  }
+
   async preview(config: sdkTypes.HMSPreviewConfig) {
     if (this.isRoomJoinCalled) {
       this.logPossibleInconsistency('attempting to call preview after join was called');
@@ -222,6 +231,9 @@ export class HMSSDKActions implements IHMSActions {
       .leave(notifyServer)
       .then(() => {
         this.resetState('leave');
+        if (this.beamSpeakerLabelsLogger) {
+          this.beamSpeakerLabelsLogger.stop().catch(HMSLogger.e);
+        }
         HMSLogger.i('left room');
       })
       .catch(err => {
@@ -393,12 +405,12 @@ export class HMSSDKActions implements IHMSActions {
   async detachVideo(trackID: string, videoElement: HTMLVideoElement) {
     const sdkTrack = this.hmsSDKTracks[trackID];
     if (sdkTrack?.type === 'video') {
-      (sdkTrack as SDKHMSVideoTrack).removeSink(videoElement);
+      await this.sdk.detachVideo(sdkTrack as SDKHMSVideoTrack, videoElement);
     } else {
       if (videoElement) {
         videoElement.srcObject = null; // so chrome can clean up
       }
-      this.logPossibleInconsistency('no video track found to remove sink');
+      HMSLogger.d('possible inconsistency detected - no video track found to remove sink');
     }
   }
 
@@ -664,6 +676,14 @@ export class HMSSDKActions implements IHMSActions {
     }
   }
 
+  async enableBeamSpeakerLabelsLogging() {
+    if (!this.beamSpeakerLabelsLogger) {
+      HMSLogger.i('enabling beam speaker labels logging');
+      this.beamSpeakerLabelsLogger = new BeamSpeakerLabelsLogger(this.store, this);
+      await this.beamSpeakerLabelsLogger.start();
+    }
+  }
+
   private resetState(reason = 'resetState') {
     this.isRoomJoinCalled = false;
     this.hmsSDKTracks = {};
@@ -789,7 +809,7 @@ export class HMSSDKActions implements IHMSActions {
   private async attachVideoInternal(trackID: string, videoElement: HTMLVideoElement) {
     const sdkTrack = this.hmsSDKTracks[trackID];
     if (sdkTrack && sdkTrack.type === 'video') {
-      await (sdkTrack as SDKHMSVideoTrack).addSink(videoElement);
+      await this.sdk.attachVideo(sdkTrack as SDKHMSVideoTrack, videoElement);
     } else {
       this.logPossibleInconsistency('no video track found to add sink');
     }

package/src/core/hmsSDKStore/sdkTypes.ts

@@ -50,6 +50,9 @@ import {
   RTMPRecordingConfig,
   ScreenCaptureHandle,
   simulcastMapping,
+  TokenRequest,
+  TokenRequestOptions,
+  TokenResult,
 } from '@100mslive/hms-video';
 
 export {
@@ -107,4 +110,7 @@ export type {
   RID,
   ScreenCaptureHandle,
   HMSPreferredSimulcastLayer,
+  TokenRequest,
+  TokenRequestOptions,
+  TokenResult,
 };

package/src/core/index.ts

@@ -39,6 +39,9 @@ export type {
   HMSScreenShareConfig,
   ScreenCaptureHandle,
   HMSPreferredSimulcastLayer,
+  TokenRequest,
+  TokenRequestOptions,
+  TokenResult,
 } from './hmsSDKStore/sdkTypes';
 
 export * from '../controller/beam/BeamController';