@livedigital/client 2.43.0 → 2.44.0-audio-observer.1

package/dist/types/channelAudioObserver.d.ts

@@ -0,0 +1,28 @@
+import { TrackLabel } from './common';
+export declare enum ChannelAudioObserverEvents {
+    DominantSpeaker = "dominant-speaker",
+    PeersVolumes = "peers-volumes",
+    Silence = "silence"
+}
+export declare type DominantSpeakerEvent = {
+    event: ChannelAudioObserverEvents.DominantSpeaker;
+    data: {
+        peerId: string;
+        producerId: string;
+        trackLabel: TrackLabel.Microphone;
+    };
+};
+export declare type SilenceEvent = {
+    event: ChannelAudioObserverEvents.Silence;
+};
+export declare type PeerVolume = {
+    peerId: string;
+    producerId: string;
+    value: number;
+    trackLabel: TrackLabel.Microphone;
+};
+export declare type PeersVolumesEvent = {
+    event: ChannelAudioObserverEvents.PeersVolumes;
+    volumes: PeerVolume[];
+};
+export declare type AudioObserverEvents = DominantSpeakerEvent | SilenceEvent | PeersVolumesEvent;

package/dist/types/common.d.ts

@@ -1,6 +1,6 @@
 /// <reference types="node" />
 import { MediaKind, RtpEncodingParameters, RtpParameters } from 'mediasoup-client/lib/types';
-import { ConsumerOptions } from 'mediasoup-client/lib/Consumer';
+import { Consumer, ConsumerOptions } from 'mediasoup-client/lib/Consumer';
 import { RtpCapabilities } from 'mediasoup-client/src/RtpParameters';
 import { ProducerCodecOptions } from 'mediasoup-client/lib/Producer';
 import { ConnectionState } from 'mediasoup-client/src/Transport';
@@ -232,6 +232,15 @@ export declare type CreateConsumerPayload = {
     channelId?: string;
     producerPeerId: string;
 };
+export declare type CreateDataConsumerPayload = {
+    producerId: string;
+    appId?: string;
+    channelId?: string;
+};
+export declare type CreateConsumerResponse = {
+    consumer: Consumer;
+    isProducerPaused: boolean;
+};
 export declare enum DeviceErrors {
     NotFoundError = "NotFoundError",
     NoDevices = "NoDevices",

package/dist/types/engine.d.ts

@@ -8,6 +8,7 @@ import ChannelEventHandler from '../engine/handlers/ChannelEventHandler';
 import MediaSoupEventHandler from '../engine/handlers/MediaSoupEventHandler';
 import { LoadBalancerApiClientParams } from '../engine/network/LoadBalancerClient';
 import { Logger, LogLevel, LogMessageHandler } from './common';
+import ChannelAudioObserverEventHandler from '../engine/handlers/ChannelAudioObserverEventHandler';
 export declare type IssuesHandler = (issues: IssueDetectorResult) => void;
 export declare type NetworkScoresUpdatedHandler = (networks: NetworkScores) => void;
 export interface CreateIssueDetectorParams {
@@ -44,11 +45,16 @@ export interface CreateChannelEventHandlerParams {
     engine: Engine;
     onLogMessage?: LogMessageHandler;
 }
+export interface CreateAudioObserverEventHandlerParams {
+    engine: Engine;
+    onLogMessage?: LogMessageHandler;
+}
 export interface EngineDependenciesFactory {
     createSystem: (params: CreateSystemParams) => System;
     createMedia: (params: CreateMediaParams) => Media;
     createNetwork: (params: CreateNetworkParams) => Network;
     createChannelEventHandler: (params: CreateChannelEventHandlerParams) => ChannelEventHandler;
     createMediaSoupEventHandler: (params: CreateMediaSoupEventHandlerParams) => MediaSoupEventHandler;
+    createAudioObserverEventHandler: (params: CreateAudioObserverEventHandlerParams) => ChannelAudioObserverEventHandler;
     createIssueDetector: (params: CreateIssueDetectorParams) => WebRTCIssueDetector;
 }

package/package.json

@@ -2,7 +2,7 @@
   "name": "@livedigital/client",
   "author": "vlprojects",
   "license": "MIT",
-  "version": "2.43.0",
+  "version": "2.44.0-audio-observer.1",
   "private": false,
   "bugs": {
     "url": "https://github.com/vlprojects/livedigital-sdk/issues"

package/src/constants/events.ts

@@ -8,6 +8,7 @@ export const CHANNEL_EVENTS = {
   activityConfirmationAcquired: 'channel.activityConfirmationAcquired',
   activityConfirmationRequired: 'channel.activityConfirmationRequired',
   activityConfirmationExpired: 'channel.activityConfirmationExpired',
+  getAudioObserverProducer: 'channel.getAudioObserverProducer',
 };
 
 export const CLIENT_EVENTS = {
@@ -21,6 +22,11 @@ export const CLIENT_EVENTS = {
   devicesListUpdated: 'devices-list-updated',
   transportConnectionTimeout: 'transport-connection-timeout',
   trackPublishingFailed: 'track-publishing-failed',
+  activeSpeakerChanged: 'active-speaker-changed',
+};
+
+export const TRACK_EVENTS = {
+  volumeChanged: 'volume-changed',
 };
 
 export const INTERNAL_CLIENT_EVENTS = {
@@ -66,6 +72,7 @@ export const MEDIASOUP_EVENTS = {
   producerSetMaxSpatialLayer: 'producer.setMaxSpatialLayer',
   producerRequestMaxSpatialLayer: 'producer.requestMaxSpatialLayer',
   createConsumer: 'consumer.create',
+  createDataConsumer: 'dataConsumer.create',
   closeConsumer: 'consumer.close',
   pauseConsumer: 'consumer.pause',
   resumeConsumer: 'consumer.resume',

package/src/engine/DefaultEngineDependenciesFactory.ts

@@ -28,6 +28,7 @@ import {
 } from '../types/engine';
 import SocketIO from './network/Socket';
 import LoadBalancerApiClient from './network/LoadBalancerClient';
+import ChannelAudioObserverEventHandler from './handlers/ChannelAudioObserverEventHandler';
 
 /* eslint-disable class-methods-use-this */
 class DefaultEngineDependenciesFactory implements EngineDependenciesFactory {
@@ -63,6 +64,10 @@ class DefaultEngineDependenciesFactory implements EngineDependenciesFactory {
     return new MediaSoupEventHandler(params);
   }
 
+  createAudioObserverEventHandler(params: CreateMediaSoupEventHandlerParams): ChannelAudioObserverEventHandler {
+    return new ChannelAudioObserverEventHandler(params);
+  }
+
   createIssueDetector(params: CreateIssueDetectorParams): WebRTCIssueDetector {
     const logger = params.logger ?? console;
     const compositeStatsParser = new CompositeRTCStatsParser({

package/src/engine/handlers/ChannelAudioObserverEventHandler.ts

@@ -0,0 +1,129 @@
+import { serializeError } from 'serialize-error';
+import { LogMessageHandler, TrackLabel } from '../../types/common';
+import Engine from '../index';
+import Logger from '../Logger';
+import InvalidPayloadError from '../../errors/InvalidPayloadError';
+import {
+  AudioObserverEvents,
+  ChannelAudioObserverEvents,
+  DominantSpeakerEvent, PeersVolumesEvent,
+} from '../../types/channelAudioObserver';
+
+class ChannelAudioObserverEventHandler {
+  private readonly engine: Engine;
+
+  private readonly logger: Logger;
+
+  constructor(params: { engine: Engine, onLogMessage?: LogMessageHandler }) {
+    const { engine, onLogMessage } = params;
+    this.engine = engine;
+    this.logger = new Logger({
+      logLevel: engine.logLevel,
+      namespace: 'ChannelAudioObserverEventHandler',
+      onLogMessage,
+    });
+  }
+
+  public handle(data: string): void {
+    try {
+      const payload = JSON.parse(data);
+      ChannelAudioObserverEventHandler.validateEventPayload(payload);
+      switch (payload.event) {
+        case ChannelAudioObserverEvents.DominantSpeaker: {
+          this.handleDominantSpeaker(payload);
+          break;
+        }
+        case ChannelAudioObserverEvents.PeersVolumes: {
+          this.handlePeerVolumes(payload);
+          break;
+        }
+        case ChannelAudioObserverEvents.Silence: {
+          this.handleSilence();
+          break;
+        }
+        default:
+          break;
+      }
+    } catch (error) {
+      this.logger.warn('Failed to handle audio observer event', {
+        error: serializeError(error),
+        eventData: data,
+      });
+    }
+  }
+
+  private handleDominantSpeaker(payload: DominantSpeakerEvent): void {
+    const { peerId } = payload.data;
+    const peer = this.engine.getPeerById(peerId);
+    this.engine.setActiveSpeakerPeer(peer);
+  }
+
+  private handlePeerVolumes(payload: PeersVolumesEvent): void {
+    payload.volumes.forEach((item) => {
+      const { peerId, trackLabel, value } = item;
+      const track = this.engine.getPeerById(peerId)?.tracks.get(trackLabel);
+      if (!track) {
+        return;
+      }
+
+      track.setVolume(value);
+    });
+  }
+
+  private handleSilence(): void {
+    const tracks = this.engine.peers.flatMap((peer) => Array.from(peer.tracks.values()));
+    tracks.forEach((track) => track.setVolume(0));
+    this.engine.setActiveSpeakerPeer(undefined);
+  }
+
+  private static validateEventPayload(payload: AudioObserverEvents): void {
+    switch (payload.event) {
+      case ChannelAudioObserverEvents.Silence: {
+        break;
+      }
+
+      case ChannelAudioObserverEvents.DominantSpeaker: {
+        if (!payload.data) {
+          ChannelAudioObserverEventHandler.throwInvalidPayload();
+        }
+
+        const { peerId, producerId, trackLabel } = payload.data;
+        if (typeof peerId !== 'string' || typeof producerId !== 'string' || trackLabel !== TrackLabel.Microphone) {
+          ChannelAudioObserverEventHandler.throwInvalidPayload();
+        }
+        break;
+      }
+
+      case ChannelAudioObserverEvents.PeersVolumes: {
+        if (!Array.isArray(payload.volumes)) {
+          ChannelAudioObserverEventHandler.throwInvalidPayload();
+        }
+
+        payload.volumes.forEach((item) => {
+          const {
+            peerId, producerId, value, trackLabel,
+          } = item;
+          if (
+            typeof peerId !== 'string'
+            || typeof producerId !== 'string'
+            || typeof value !== 'number'
+            || trackLabel !== TrackLabel.Microphone
+          ) {
+            ChannelAudioObserverEventHandler.throwInvalidPayload();
+          }
+        });
+        break;
+      }
+
+      default: {
+        ChannelAudioObserverEventHandler.throwInvalidPayload();
+      }
+    }
+  }
+
+  static throwInvalidPayload(): void {
+    throw new InvalidPayloadError('Invalid channel audio observer event handler payload');
+  }
+}
+
+export default ChannelAudioObserverEventHandler;

package/src/engine/index.ts

@@ -1,6 +1,7 @@
 import { UnsupportedError as MediasoupUnsupportedError } from 'mediasoup-client/lib/errors';
 import { RtpCapabilities } from 'mediasoup-client/lib/RtpParameters';
 import WebRTCIssueDetector from 'webrtc-issue-detector';
+import { DataConsumer } from 'mediasoup-client/lib/DataConsumer';
 import {
   CreateCameraVideoTrackOptions,
   CreateMicrophoneAudioTrackOptions,
@@ -35,6 +36,8 @@ import { LogLevels } from '../constants/common';
 import { LoadBalancerApiClientParams } from './network/LoadBalancerClient';
 import validateAppData from '../helpers/appDataValidator';
 import AudioTrack from './media/tracks/AudioTrack';
+import NeedJoinFirstError from '../errors/NeedJoinFirstError';
+import ChannelAudioObserverEventHandler from './handlers/ChannelAudioObserverEventHandler';
 
 type EngineParams = {
   clientEventEmitter: EnhancedEventEmitter,
@@ -66,6 +69,8 @@ class Engine {
 
   private mediaSoupEventsHandler: MediaSoupEventHandler;
 
+  private channelAudioObserverEventHandler: ChannelAudioObserverEventHandler;
+
   private initialized = false;
 
   private isJoined = false;
@@ -80,6 +85,10 @@ class Engine {
 
   private webRtcIssueDetector?: WebRTCIssueDetector;
 
+  private audioObserver?: DataConsumer;
+
+  private activeSpeakerPeer?: Peer;
+
   constructor(params: EngineParams) {
     const {
       clientEventEmitter,
@@ -114,6 +123,10 @@ class Engine {
       engine: this,
       onLogMessage: params.onLogMessage,
     });
+    this.channelAudioObserverEventHandler = dependenciesFactory.createAudioObserverEventHandler({
+      engine: this,
+      onLogMessage: params.onLogMessage,
+    });
 
     this.logger = new Logger({
       logLevel: this.logLevel,
@@ -176,6 +189,7 @@ class Engine {
       await this.network.closeSendTransport();
       await this.network.closeReceiveTransport();
       await this.media.clearTracks();
+      this.audioObserver?.close();
 
       this.logger.debug('release()', {
         socketId: this.mySocketId,
@@ -197,6 +211,10 @@ class Engine {
     });
   }
 
+  public get activeSpeaker(): Peer | undefined {
+    return this.activeSpeakerPeer;
+  }
+
   public get peers(): Peer[] {
     return Array.from(this.peersRepository.values());
   }
@@ -213,6 +231,10 @@ class Engine {
     return this.network.socket.id;
   }
 
+  public getPeerById(id: string): Peer | undefined {
+    return this.peersRepository.get(id);
+  }
+
   public async confirmActivity(): Promise<void> {
     if (!this.channel || this.isChannelJoining) {
       throw new Error('Connect to the channel first');
@@ -411,6 +433,15 @@ class Engine {
     return this.media.isNoiseSuppressorLoaded;
   }
 
+  setActiveSpeakerPeer(peer?: Peer): void {
+    if (this.activeSpeaker === peer) {
+      return;
+    }
+
+    this.activeSpeakerPeer = peer;
+    this.clientEventEmitter.safeEmit(CLIENT_EVENTS.activeSpeakerChanged, { peer });
+  }
+
   private async connectToSocketServerWithRetry(params: { channelId: string, role: Role }): Promise<void> {
     const connectToSocketServerAction = async () => this.connectToSocketServer(params);
     return retryAsync(connectToSocketServerAction, {
@@ -496,6 +527,7 @@ class Engine {
       this.mediaSoupEventsHandler.subscribeToEvents();
       this.isJoined = true;
       this.logger.debug('Successfully joined channel', { channelId: params.channelId });
+      await this.createAudioObserver();
     } catch (error) {
       this.logger.error('performJoin()', { error });
       throw error;
@@ -576,6 +608,36 @@ class Engine {
 
     myPeer.tracks.set(peerTrack.label, peerTrack);
   }
+
+  private async createAudioObserver(): Promise<void> {
+    if (!this.isJoined) {
+      throw new NeedJoinFirstError('Need to join first');
+    }
+
+    const logCtx = {
+      channelId: this.channelId,
+      appId: this.appId,
+      peerId: this.myPeer?.id,
+    };
+
+    try {
+      const { producerId } = await this.network.getAudioObserverProducer() as { producerId: string };
+      const audioObserver = await this.network.createDataConsumer({
+        producerId,
+        appId: this.appId,
+        channelId: this.channelId,
+      });
+
+      audioObserver.on('message', (data) => {
+        this.channelAudioObserverEventHandler.handle(data);
+      });
+
+      this.audioObserver = audioObserver;
+      this.logger.debug('Successfully create audio observer', logCtx);
+    } catch (error) {
+      this.logger.error('Failed to create audio observer', logCtx);
+    }
+  }
 }
 
 export default Engine;

package/src/engine/media/tracks/PeerTrack.ts

@@ -10,7 +10,7 @@ import {
   LogMessageHandler,
 } from '../../../types/common';
 import Logger from '../../Logger';
-import { MEDIASOUP_EVENTS, PEER_EVENTS } from '../../../constants/events';
+import { MEDIASOUP_EVENTS, PEER_EVENTS, TRACK_EVENTS } from '../../../constants/events';
 import PeerConsumer from '../../PeerConsumer';
 import Engine from '../../index';
 import EnhancedEventEmitter from '../../../EnhancedEventEmitter';
@@ -52,6 +52,8 @@ class PeerTrack {
 
   #muted = false;
 
+  #volumeLevel: number = 0;
+
   readonly observer = new EnhancedEventEmitter();
 
   constructor(payload: PeerTrackConstructor) {
@@ -70,6 +72,10 @@ class PeerTrack {
     this.#peerEventEmitter.safeEmit(PEER_EVENTS.trackStart, this);
   }
 
+  get volume(): number {
+    return this.#volumeLevel;
+  }
+
   get isRemote(): boolean {
     return !!this.consumer;
   }
@@ -408,6 +414,15 @@ class PeerTrack {
     }
   }
 
+  setVolume(value: number): void {
+    if (this.#volumeLevel === value) {
+      return;
+    }
+
+    this.#volumeLevel = value;
+    this.observer.safeEmit(TRACK_EVENTS.volumeChanged, { value });
+  }
+
   private async closeConsumer(): Promise<void> {
     try {
       if (!this.consumer) {