@livedigital/client 2.42.0 → 2.44.0-audio-observer.1

package/src/engine/network/index.ts

@@ -5,10 +5,11 @@ import {
   TransportOptions,
 } from 'mediasoup-client/lib/Transport';
 import { Device } from 'mediasoup-client';
-import { Consumer } from 'mediasoup-client/lib/Consumer';
+import { DataConsumer, DataConsumerOptions } from 'mediasoup-client/lib/DataConsumer';
 import SocketIO from './Socket';
 import {
   CreateConsumerPayload,
+  CreateConsumerResponse,
   LogLevel,
   LogMessageHandler,
   ProduceParams,
@@ -17,6 +18,7 @@ import {
   TransportsStateInfo,
   TransportStateInfo,
   UpdatePeerAppDataPayload,
+  CreateDataConsumerPayload,
 } from '../../types/common';
 import LoadBalancerApiClient from './LoadBalancerClient';
 import { CHANNEL_EVENTS, MEDIASOUP_EVENTS, MEDIASOUP_TRANSPORT_EVENTS } from '../../constants/events';
@@ -108,25 +110,24 @@ class Network {
       localDirection: 'send',
     }) as TransportOptions;
 
-    this.sendTransport = mediasoupDevice.createSendTransport(sendTransportOptions);
+    const transport = mediasoupDevice.createSendTransport(sendTransportOptions);
     this.logger.debug('createSendTransport()', { transport: this.sendTransport });
 
-    // eslint-disable-next-line @typescript-eslint/no-misused-promises
-    this.sendTransport.on(MEDIASOUP_TRANSPORT_EVENTS.connect, async (
+    transport.on(MEDIASOUP_TRANSPORT_EVENTS.connect, async (
       { dtlsParameters }: { dtlsParameters: DtlsParameters },
       callback: () => void,
       errback: (error: Error) => void,
     ) => {
       try {
         await this.socket.request(MEDIASOUP_EVENTS.transportConnect, {
-          transportId: this.sendTransport?.id,
+          transportId: transport.id,
           dtlsParameters,
         });
 
         callback();
 
         this.logger.debug('sendTransport.connect()', {
-          transportId: this.sendTransport?.id,
+          transportId: transport.id,
           dtlsRole: dtlsParameters.role,
         });
       } catch (error) {
@@ -135,14 +136,14 @@ class Network {
       }
     });
 
-    this.sendTransport.on(MEDIASOUP_TRANSPORT_EVENTS.produce, async (
+    transport.on(MEDIASOUP_TRANSPORT_EVENTS.produce, async (
       parameters: ProduceParams,
       callback: (a: unknown) => void,
       errback: (error: Error) => void,
     ) => {
       try {
         const { id } = await this.socket.request(MEDIASOUP_EVENTS.transportProduce, {
-          transportId: this.sendTransport?.id,
+          transportId: transport.id,
           kind: parameters.kind,
           rtpParameters: parameters.rtpParameters,
           appData: parameters.appData,
@@ -152,68 +153,67 @@ class Network {
 
         this.logger.debug('sendTransport.produce()', {
           producerId: id,
-          transportId: this.sendTransport?.id,
+          transportId: transport.id,
           kind: parameters.kind,
         });
       } catch (error) {
         this.logger.error('sendTransport.produce()', {
           error,
-          transportId: this.sendTransport?.id,
+          transportId: transport.id,
           kind: parameters.kind,
         });
         errback(error);
       }
     });
 
-    this.sendTransport.on(MEDIASOUP_EVENTS.transportStateChange, (state: RTCPeerConnectionState) => {
+    transport.on(MEDIASOUP_EVENTS.transportStateChange, (state: RTCPeerConnectionState) => {
       this.logger.debug('sendTransport.state', { state });
       if (state === 'failed') {
-        if (this.sendTransport) {
-          this.restartIce(this.sendTransport);
-          this.logger.warn('sendTransport.restartIce()', {
-            state,
-            sendTransportId: this.sendTransport?.id,
-          });
-        }
+        this.restartIce(transport);
+        this.logger.warn('sendTransport.restartIce()', {
+          state,
+          sendTransportId: transport.id,
+        });
       }
 
       setTimeout(() => {
-        if (this.sendTransport?.connectionState === 'disconnected' && this.socket.connection?.connected) {
+        if (transport.connectionState === 'disconnected' && this.socket.connection?.connected) {
           this.logger.warn('sendTransport is closed before websocket is closed', {
-            sendTransportId: this.sendTransport.id,
+            sendTransportId: transport.id,
           });
         }
       }, 3000);
     });
+
+    this.sendTransport = transport;
   }
 
   async createRecvTransport(mediasoupDevice: Device): Promise<void> {
     if (this.receiveTransport) {
-      this.logger.warn('createRecvTransport()', { message: 'Receive transport already exists' });
+      this.logger.warn('createReceiveTransport()', { message: 'Receive transport already exists' });
       return;
     }
 
     const recvTransportOptions = await this.socket.request(MEDIASOUP_EVENTS.transportCreate, {
       localDirection: 'receive',
     }) as TransportOptions;
-    this.receiveTransport = mediasoupDevice.createRecvTransport(recvTransportOptions);
-    this.logger.debug('createRecvTransport()', { transport: this.receiveTransport });
+    const transport = mediasoupDevice.createRecvTransport(recvTransportOptions);
+    this.logger.debug('createReceiveTransport()', { transport: this.receiveTransport });
 
-    // eslint-disable-next-line @typescript-eslint/no-misused-promises
-    this.receiveTransport.on(MEDIASOUP_TRANSPORT_EVENTS.connect, async (
+    transport.on(MEDIASOUP_TRANSPORT_EVENTS.connect, async (
       { dtlsParameters }: { dtlsParameters: DtlsParameters },
       callback: () => void,
       errback: (error: Error) => void,
     ) => {
       try {
         await this.socket.request(MEDIASOUP_EVENTS.transportConnect, {
-          transportId: this.receiveTransport?.id,
+          transportId: transport.id,
           dtlsParameters,
         });
         callback();
 
         this.logger.debug('receiveTransport.connect()', {
-          transportId: this.receiveTransport?.id,
+          transportId: transport.id,
           dtlsRole: dtlsParameters.role,
         });
       } catch (error) {
@@ -222,23 +222,23 @@ class Network {
       }
     });
 
-    this.receiveTransport.on(MEDIASOUP_EVENTS.transportStateChange, (state: RTCPeerConnectionState) => {
+    transport.on(MEDIASOUP_EVENTS.transportStateChange, (state: RTCPeerConnectionState) => {
       this.logger.debug('receiveTransport.state', { state });
       if (state === 'failed') {
-        if (this.receiveTransport) {
-          this.restartIce(this.receiveTransport);
-          this.logger.debug('receiveTransport.restartIce()', { recvTransportId: this.receiveTransport.id });
-        }
+        this.restartIce(transport);
+        this.logger.debug('receiveTransport.restartIce()', { recvTransportId: transport.id });
       }
 
       setTimeout(() => {
-        if (this.receiveTransport?.connectionState === 'disconnected' && this.socket.connection?.connected) {
+        if (transport.connectionState === 'disconnected' && this.socket.connection?.connected) {
           this.logger.warn('receiveTransport is closed before websocket is closed', {
-            recvTransportId: this.receiveTransport.id,
+            recvTransportId: transport.id,
           });
         }
       }, 3000);
     });
+
+    this.receiveTransport = transport;
   }
 
   async getIceParameters(transport: Transport): Promise<IceParameters> {
@@ -271,11 +271,11 @@ class Network {
     appId,
     channelId,
     producerPeerId,
-  }: CreateConsumerPayload): Promise<{ consumer: Consumer, isProducerPaused: boolean }> {
+  }: CreateConsumerPayload): Promise<CreateConsumerResponse> {
     const transport = this.receiveTransport;
     if (!transport) {
-      this.logger.error('createConsumer()', { reason: 'missing receiveTransport' });
-      throw new Error('Client not initialized');
+      this.logger.error('createConsumer()', { reason: 'missing receive transport' });
+      throw new Error('Failed to create consumer');
     }
 
     this.logger.debug('createConsumer()', { producerId });
@@ -301,6 +301,34 @@ class Network {
     };
   }
 
+  async createDataConsumer({
+    producerId,
+    appId,
+    channelId,
+  }: CreateDataConsumerPayload): Promise<DataConsumer> {
+    const transport = this.receiveTransport;
+    if (!transport) {
+      this.logger.error('createDataConsumer()', { reason: 'missing receive transport' });
+      throw new Error('Failed to create consumer');
+    }
+
+    this.logger.debug('createDataConsumer()', { producerId });
+    const options = await this.socket.request(MEDIASOUP_EVENTS.createDataConsumer, {
+      dataProducerId: producerId,
+      transportId: transport.id,
+      appData: {
+        peerId: this.socket.id,
+        producerId,
+        appId,
+        channelId,
+      },
+    }) as DataConsumerOptions;
+
+    const consumer = await transport.consumeData(options);
+    this.logger.debug('Data consumer created', { consumer });
+    return consumer;
+  }
+
   async closeRemoteConsumer(consumerId: string): Promise<SocketResponse> {
     return this.socket.request(MEDIASOUP_EVENTS.closeConsumer, { peerId: this.socket.id, consumerId });
   }
@@ -317,6 +345,10 @@ class Network {
     return this.socket.request(CHANNEL_EVENTS.updatePeerAppData, { peerId, appData });
   }
 
+  async getAudioObserverProducer(): Promise<SocketResponse> {
+    return this.socket.request(CHANNEL_EVENTS.getAudioObserverProducer);
+  }
+
   async getTransportsStateInfo(): Promise<TransportsStateInfo> {
     const [receive, send] = await Promise.all([
       this.getParsedTransportStats(this.receiveTransport),

package/src/errors/InvalidPayloadError.ts

@@ -0,0 +1,9 @@
+import LivedigitalSDKError from './LivedigitalSDKError';
+
+class InvalidPayloadError extends LivedigitalSDKError {
+  constructor(msg?: string, errorCode?: string) {
+    super(msg, errorCode ?? 'invalid_payload');
+  }
+}
+
+export default InvalidPayloadError;

package/src/errors/NeedJoinFirstError.ts

@@ -0,0 +1,9 @@
+import LivedigitalSDKError from './LivedigitalSDKError';
+
+class NeedJoinFirstError extends LivedigitalSDKError {
+  constructor(msg?: string, errorCode?: string) {
+    super(msg, errorCode ?? 'need_join_first');
+  }
+}
+
+export default NeedJoinFirstError;

package/src/types/channelAudioObserver.ts

@@ -0,0 +1,34 @@
+import { TrackLabel } from './common';
+
+export enum ChannelAudioObserverEvents {
+  DominantSpeaker = 'dominant-speaker',
+  PeersVolumes = 'peers-volumes',
+  Silence = 'silence',
+}
+
+export type DominantSpeakerEvent = {
+  event: ChannelAudioObserverEvents.DominantSpeaker;
+  data: {
+    peerId: string;
+    producerId: string;
+    trackLabel: TrackLabel.Microphone;
+  }
+};
+
+export type SilenceEvent = {
+  event: ChannelAudioObserverEvents.Silence;
+};
+
+export type PeerVolume = {
+  peerId: string;
+  producerId: string;
+  value: number;
+  trackLabel: TrackLabel.Microphone;
+};
+
+export type PeersVolumesEvent = {
+  event: ChannelAudioObserverEvents.PeersVolumes;
+  volumes: PeerVolume[]
+};
+
+export type AudioObserverEvents = DominantSpeakerEvent | SilenceEvent | PeersVolumesEvent;

package/src/types/common.ts

@@ -3,7 +3,7 @@ import {
   RtpEncodingParameters,
   RtpParameters,
 } from 'mediasoup-client/lib/types';
-import { ConsumerOptions } from 'mediasoup-client/lib/Consumer';
+import { Consumer, ConsumerOptions } from 'mediasoup-client/lib/Consumer';
 import { RtpCapabilities } from 'mediasoup-client/src/RtpParameters';
 import { ProducerCodecOptions } from 'mediasoup-client/lib/Producer';
 import { ConnectionState } from 'mediasoup-client/src/Transport';
@@ -287,6 +287,17 @@ export type CreateConsumerPayload = {
   producerPeerId: string,
 };
 
+export type CreateDataConsumerPayload = {
+  producerId: string,
+  appId?: string,
+  channelId?: string,
+};
+
+export type CreateConsumerResponse = {
+  consumer: Consumer,
+  isProducerPaused: boolean,
+};
+
 export enum DeviceErrors {
   NotFoundError = 'NotFoundError',
   NoDevices = 'NoDevices',
@@ -376,3 +387,8 @@ export interface TrackWithEncodings {
   getCodecOptions(): ProducerCodecOptions;
   getEncodings(): RtpEncodingParameters[];
 }
+
+export type ActivityConfirmationRequiredPayload = {
+  channelId: string;
+  time: number;
+};

package/src/types/engine.ts

@@ -8,6 +8,7 @@ import ChannelEventHandler from '../engine/handlers/ChannelEventHandler';
 import MediaSoupEventHandler from '../engine/handlers/MediaSoupEventHandler';
 import { LoadBalancerApiClientParams } from '../engine/network/LoadBalancerClient';
 import { Logger, LogLevel, LogMessageHandler } from './common';
+import ChannelAudioObserverEventHandler from '../engine/handlers/ChannelAudioObserverEventHandler';
 
 export type IssuesHandler = (issues: IssueDetectorResult) => void;
 
@@ -54,11 +55,17 @@ export interface CreateChannelEventHandlerParams {
   onLogMessage?: LogMessageHandler;
 }
 
+export interface CreateAudioObserverEventHandlerParams {
+  engine: Engine;
+  onLogMessage?: LogMessageHandler;
+}
+
 export interface EngineDependenciesFactory {
   createSystem: (params: CreateSystemParams) => System;
   createMedia: (params: CreateMediaParams) => Media;
   createNetwork: (params: CreateNetworkParams) => Network;
   createChannelEventHandler: (params: CreateChannelEventHandlerParams) => ChannelEventHandler;
   createMediaSoupEventHandler: (params: CreateMediaSoupEventHandlerParams) => MediaSoupEventHandler;
+  createAudioObserverEventHandler: (params: CreateAudioObserverEventHandlerParams) => ChannelAudioObserverEventHandler;
   createIssueDetector: (params: CreateIssueDetectorParams) => WebRTCIssueDetector;
 }