@livedigital/client 2.15.0 → 2.17.0

package/dist/types/common.d.ts

@@ -113,12 +113,14 @@ export declare enum ConnectionQuality {
 }
 export declare type EncoderConfig = {};
 export declare type VideoCodec = 'h264' | 'vp8';
+export declare type AudioCodec = 'opus';
 export declare type VideoEncoderConfig = EncoderConfig & {
     preferredCodec?: VideoCodec;
     encodings?: RtpEncodingParameters[];
     videoGoogleStartBitrate?: number;
 };
 export declare type AudioEncoderConfig = EncoderConfig & {
+    preferredCodec?: AudioCodec;
     enableFec?: boolean;
 };
 export declare type CreateTrackOptions = {
@@ -204,3 +206,9 @@ export declare type RemoteConsumerOptions = ConsumerOptions & {
 };
 export declare type LogMessageHandler = (msg: any, ...meta: any) => void;
 export declare type LogLevel = 3 | 4 | 6 | 7;
+export declare type TransportConnectionTimeoutPayload = {
+    reason: 'ice' | 'dtls';
+    transportId: string;
+    direction: 'receive' | 'send';
+    timeout: number;
+};

package/package.json

@@ -2,7 +2,7 @@
   "name": "@livedigital/client",
   "author": "vlprojects",
   "license": "MIT",
-  "version": "2.15.0",
+  "version": "2.17.0",
   "private": false,
   "bugs": {
     "url": "https://github.com/vlprojects/livedigital-sdk/issues"

package/src/constants/events.ts

@@ -11,6 +11,7 @@ export const CLIENT_EVENTS = {
   peerLeft: 'peer-left',
   channelRejoinRequired: 'channel-rejoin-required',
   devicesListUpdated: 'devices-list-updated',
+  transportConnectionTimeout: 'transport-connection-timeout',
 };
 
 export const PEER_EVENTS = {
@@ -51,6 +52,7 @@ export const MEDIASOUP_EVENTS = {
   transportClose: 'transport.close',
   transportConnect: 'transport.connect',
   transportProduce: 'transport.produce',
+  transportConnectionTimeout: 'transport.connectionTimeout',
   transportStateChange: 'connectionstatechange',
   transportGetIceParameters: 'getIceParameters',
 };

package/src/engine/handlers/MediaSoupEventHandler.ts

@@ -4,10 +4,12 @@ import {
   ConsumerScoreChangedPayload,
   ProducerData,
   ProducerRequestMaxSpatialLayer,
-  ProducerScoreChangedPayload, ProducerSetMaxSpatialLayer,
+  ProducerScoreChangedPayload,
+  ProducerSetMaxSpatialLayer,
+  TransportConnectionTimeoutPayload,
 } from '../../types/common';
 import Engine from '../index';
-import { MEDIASOUP_EVENTS } from '../../constants/events';
+import { CLIENT_EVENTS, MEDIASOUP_EVENTS } from '../../constants/events';
 import Logger from '../Logger';
 import VideoTrack from '../media/tracks/VideoTrack';
 
@@ -138,6 +140,30 @@ class MediaSoupEventHandler {
 
       peer.observer.safeEmit(MEDIASOUP_EVENTS.producerSetMaxSpatialLayer, { peerId, producerId, spatialLayer });
     });
+
+    connection.on(MEDIASOUP_EVENTS.transportConnectionTimeout, async ({
+      reason,
+      transportId,
+      direction,
+      timeout,
+    }: TransportConnectionTimeoutPayload) => {
+      const transport = [
+        this.engine.network.receiveTransport,
+        this.engine.network.sendTransport,
+      ].find((item) => item?.id === transportId);
+
+      if (!transport || transport.connectionState === 'connected') {
+        return;
+      }
+
+      this.logger.warn(MEDIASOUP_EVENTS.transportConnectionTimeout, {
+        reason,
+        transportId,
+        direction,
+        timeout,
+      });
+      this.engine.clientEventEmitter.emit(CLIENT_EVENTS.transportConnectionTimeout, { direction, reason });
+    });
   }
 
   private async handleProducerSetMaxSpatialLayer({

package/src/engine/index.ts

@@ -69,7 +69,7 @@ class Engine {
 
   private channel?: string;
 
-  private isRoomJoining = false;
+  private isChannelJoining = false;
 
   private readonly logger: Logger;
 
@@ -187,14 +187,14 @@ class Engine {
   public async join(params: JoinChannelParams): Promise<void> {
     try {
       this.logger.debug('join()', { params });
-      this.isRoomJoining = true;
+      this.isChannelJoining = true;
       await this.connectToSocketServerWithRetry(params);
       await this.performJoin(params);
     } catch (error) {
       this.logger.error('join()', { error });
       throw error;
     } finally {
-      this.isRoomJoining = false;
+      this.isChannelJoining = false;
     }
   }
 
@@ -512,13 +512,13 @@ class Engine {
         const stopListening = () => this.network.socket.observer.removeListener('state', onSocketStateChange);
         const isStateNotExpected = [SocketIOEvents.Disconnected, SocketIOEvents.Reconnecting].includes(state);
 
-        if (error || (this.isRoomJoining && isStateNotExpected)) {
+        if (error || (this.isChannelJoining && isStateNotExpected)) {
           stopListening();
           reject(error || 'Not expected socket state for new connection');
           return;
         }
 
-        if (this.isRoomJoining && [SocketIOEvents.Connected, SocketIOEvents.Reconnected].includes(state)) {
+        if (this.isChannelJoining && [SocketIOEvents.Connected, SocketIOEvents.Reconnected].includes(state)) {
           stopListening();
           resolve();
         }

package/src/engine/media/index.ts

@@ -45,18 +45,13 @@ class Media {
   }
 
   getTrackCodec(track: Track): RtpCodecCapability | undefined {
-    if (!(track instanceof VideoTrack)) {
+    const { rtpCapabilities, rtpCapabilities: { codecs } } = this.mediasoupDevice;
+    if (!codecs) {
+      this.#logger.error('getTrackCodec()', { track, rtpCapabilities });
       return undefined;
     }
 
-    if (!this.mediasoupDevice.rtpCapabilities?.codecs) {
-      return undefined;
-    }
-
-    return this.mediasoupDevice
-      .rtpCapabilities
-      .codecs
-      .find((c) => c.mimeType.toLowerCase() === `video/${track.getPreferredCodec()}`);
+    return codecs.find((c) => c.mimeType.toLowerCase() === `${track.kind}/${track.getPreferredCodec()}`);
   }
 
   private createTracks(stream: MediaStream): Track[] {

package/src/engine/media/tracks/AudioTrack.ts

@@ -1,5 +1,5 @@
 import { ProducerCodecOptions } from 'mediasoup-client/lib/Producer';
-import { AudioEncoderConfig } from '../../../types/common';
+import { AudioCodec, AudioEncoderConfig } from '../../../types/common';
 import BaseTrack from './BaseTrack';
 import TrackWithCodecOptions from './TrackWithCodecOptions';
 
@@ -13,6 +13,15 @@ class AudioTrack extends BaseTrack implements TrackWithCodecOptions {
       opusFec: this.getEncoderConfig().enableFec || true,
     };
   }
+
+  getPreferredCodec(): AudioCodec {
+    const { preferredCodec } = this.getEncoderConfig();
+    if (preferredCodec) {
+      return preferredCodec;
+    }
+
+    return 'opus';
+  }
 }
 
 export default AudioTrack;

package/src/engine/media/tracks/VideoTrack.ts

@@ -1,7 +1,7 @@
 import { ProducerCodecOptions } from 'mediasoup-client/lib/Producer';
 import { RtpEncodingParameters } from 'mediasoup-client/lib/RtpParameters';
 import { WEBCAM_SIMULCAST_ENCODINGS } from '../../../constants/simulcastEncodings';
-import { VideoCodec, VideoEncoderConfig } from '../../../types/common';
+import { TrackLabel, VideoCodec, VideoEncoderConfig } from '../../../types/common';
 import BaseTrack from './BaseTrack';
 import TrackWithCodecOptions from './TrackWithCodecOptions';
 import TrackWithEncodings from './TrackWithEncodings';
@@ -20,7 +20,16 @@ class VideoTrack extends BaseTrack implements TrackWithCodecOptions, TrackWithEn
   }
 
   getPreferredCodec(): VideoCodec {
-    return this.getEncoderConfig().preferredCodec || 'h264';
+    const { preferredCodec } = this.getEncoderConfig();
+    if (preferredCodec) {
+      return preferredCodec;
+    }
+
+    if (this.getLabel() === TrackLabel.ScreenVideo) {
+      return 'vp8';
+    }
+
+    return 'h264';
   }
 
   getEncodings(): RtpEncodingParameters[] {

package/src/engine/network/Socket.ts

@@ -49,7 +49,11 @@ class SocketIO {
   }
 
   connect(serverUrl: string): void {
-    const connection = io(serverUrl);
+    const connection = io(serverUrl, {
+      transports: ['polling', 'websocket'],
+      upgrade: true,
+      rememberUpgrade: false,
+    });
 
     this.connection = connection;
     this.serverUrl = serverUrl;

package/src/engine/network/index.ts

@@ -92,7 +92,7 @@ class Network {
 
     const sendTransportOptions = await this.socket.request(MEDIASOUP_EVENTS.transportCreate, {
       sctpCapabilities: mediasoupDevice.sctpCapabilities,
-      localDirection: 'out',
+      localDirection: 'send',
     }) as TransportOptions;
 
     this.sendTransport = mediasoupDevice.createSendTransport(sendTransportOptions);
@@ -118,7 +118,6 @@ class Network {
       }
     });
 
-    // eslint-disable-next-line @typescript-eslint/no-misused-promises
     this.sendTransport.on(MEDIASOUP_TRANSPORT_EVENTS.produce, async (
       parameters: ProduceParams,
       callback: (a: unknown) => void,
@@ -158,8 +157,7 @@ class Network {
     }
 
     const recvTransportOptions = await this.socket.request(MEDIASOUP_EVENTS.transportCreate, {
-      sctpCapabilities: mediasoupDevice.sctpCapabilities,
-      localDirection: 'in',
+      localDirection: 'receive',
     }) as TransportOptions;
     this.receiveTransport = mediasoupDevice.createRecvTransport(recvTransportOptions);
     this.logger.debug('createRecvTransport()', { transport: this.receiveTransport });

package/src/engine/wid/WebRTCIssueDetector.ts

@@ -10,9 +10,11 @@ import {
 import QualityLimitationsIssueDetector from './detectors/QualityLimitationsIssueDetector';
 import FramesDroppedIssueDetector from './detectors/FramesDroppedIssueDetector';
 import FramesEncodedSentIssueDetector from './detectors/FramesEncodedSentIssueDetector';
-import NetworkIssueDetector from './detectors/NetworkIssueDetector';
+import InboundNetworkIssueDetector from './detectors/InboundNetworkIssueDetector';
+import OutboundNetworkIssueDetector from './detectors/OutboundNetworkIssueDetector';
 import NetworkMediaSyncIssueDetector from './detectors/NetworkMediaSyncIssueDetector';
 import AvailableOutgoingBitrateIssueDetector from './detectors/AvailableOutgoingBitrateIssueDetector';
+import VideoCodecMismatchDetector from './detectors/VideoCodecMismatchDetector';
 
 class WebRTCIssueDetector {
   private readonly webrtcStats: WebRTCStats;
@@ -35,6 +37,7 @@ class WebRTCIssueDetector {
     // Move instantiation from the constructor
     this.webrtcStats = new WebRTCStats({
       getStatsInterval: this.getStatsInterval,
+      remote: true,
     });
 
     (window as unknown as WIDWindow).wid = this;
@@ -44,9 +47,11 @@ class WebRTCIssueDetector {
       new QualityLimitationsIssueDetector(),
       new FramesDroppedIssueDetector(),
       new FramesEncodedSentIssueDetector(),
-      new NetworkIssueDetector(),
+      new InboundNetworkIssueDetector(),
+      new OutboundNetworkIssueDetector(),
       new NetworkMediaSyncIssueDetector(),
       new AvailableOutgoingBitrateIssueDetector(),
+      new VideoCodecMismatchDetector(),
     ];
 
     this.webrtcStats.on('stats', (event) => {

package/src/engine/wid/detectors/{NetworkIssueDetector.ts → InboundNetworkIssueDetector.ts}

@@ -6,7 +6,7 @@ import {
   IssueType,
 } from '../types';
 
-class NetworkIssueDetector implements IssueDetector {
+class InboundNetworkIssueDetector implements IssueDetector {
   #lastProcessedStats: { [connectionId: string]: WebRTCStatsEventData } = {};
 
   detect(data: WebRTCStatsEventData): IssueDetectorResult {
@@ -89,16 +89,17 @@ class NetworkIssueDetector implements IssueDetector {
     if (isPoorConnectionQuality) {
       issues.push({
         type: IssueType.Network,
-        reason: IssueReason.LowMOS,
+        reason: IssueReason.LowInboundMOS,
         iceCandidate: data.connection.local.id,
         debug,
+        data: mos,
       });
     }
 
     if (isNetworkIssue) {
       issues.push({
         type: IssueType.Network,
-        reason: IssueReason.NetworkQuality,
+        reason: IssueReason.InboundNetworkQuality,
         iceCandidate: data.connection.local.id,
         debug,
       });
@@ -116,7 +117,7 @@ class NetworkIssueDetector implements IssueDetector {
     if (isNetworkMediaLatencyIssue) {
       issues.push({
         type: IssueType.Network,
-        reason: IssueReason.NetworkMediaLatency,
+        reason: IssueReason.InboundNetworkMediaLatency,
         iceCandidate: data.connection.local.id,
         debug,
       });
@@ -135,4 +136,4 @@ class NetworkIssueDetector implements IssueDetector {
   }
 }
 
-export default NetworkIssueDetector;
+export default InboundNetworkIssueDetector;

package/src/engine/wid/detectors/OutboundNetworkIssueDetector.ts

@@ -0,0 +1,115 @@
+import { WebRTCStatsEventData } from '@peermetrics/webrtc-stats';
+import {
+  IssueDetector,
+  IssueDetectorResult,
+  IssueReason,
+  IssueType,
+} from '../types';
+
+class OutboundNetworkIssueDetector implements IssueDetector {
+  #lastProcessedStats: { [connectionId: string]: WebRTCStatsEventData } = {};
+
+  detect(data: WebRTCStatsEventData): IssueDetectorResult {
+    const issues = this.processData(data);
+    this.#lastProcessedStats[data.connection.id] = data;
+    return issues;
+  }
+
+  private processData(data: WebRTCStatsEventData): IssueDetectorResult {
+    const issues: IssueDetectorResult = [];
+    const remoteInboundRTPStreamsStats = [
+      ...data.remote?.audio.inbound || [],
+      ...data.remote?.video.inbound || [],
+    ];
+
+    if (!remoteInboundRTPStreamsStats.length) {
+      return issues;
+    }
+
+    const previousStats = this.#lastProcessedStats[data.connection.id];
+    if (!previousStats) {
+      return issues;
+    }
+
+    const previousRemoteInboundRTPStreamsStats = [
+      ...previousStats.remote?.audio.inbound || [],
+      ...previousStats.remote?.video.inbound || [],
+    ];
+
+    const { packetsSent } = data.connection;
+    const lastPacketsSent = previousStats.connection.packetsSent;
+
+    const rtpNetworkStats = remoteInboundRTPStreamsStats.reduce((stats, currentStreamStats) => {
+      const previousStreamStats = previousRemoteInboundRTPStreamsStats
+        .find((stream) => stream.ssrc === currentStreamStats.ssrc);
+
+      return {
+        sumJitter: stats.sumJitter + currentStreamStats.jitter,
+        packetsLost: stats.packetsLost + currentStreamStats.packetsLost,
+        lastPacketsLost: stats.lastPacketsLost + (previousStreamStats?.packetsLost || 0),
+      };
+    }, {
+      sumJitter: 0,
+      packetsLost: 0,
+      lastPacketsLost: 0,
+    });
+
+    const rtt = (1e3 * data.connection.currentRoundTripTime) || 0;
+    const { sumJitter } = rtpNetworkStats;
+    const avgJitter = sumJitter / remoteInboundRTPStreamsStats.length;
+
+    const deltaPacketSent = packetsSent - lastPacketsSent;
+    const deltaPacketLost = rtpNetworkStats.packetsLost - rtpNetworkStats.lastPacketsLost;
+
+    const packetsLoss = deltaPacketSent && deltaPacketLost
+      ? Math.round((deltaPacketLost * 100) / (deltaPacketSent + deltaPacketLost))
+      : 0;
+
+    const effectiveLatency = rtt + (avgJitter * 2) + 10;
+    let rFactor = effectiveLatency < 160
+      ? 93.2 - (effectiveLatency / 40)
+      : 93.2 - (effectiveLatency / 120) - 10;
+    rFactor -= (packetsLoss * 2.5);
+    const mos = 1 + (0.035) * rFactor + (0.000007) * rFactor * (rFactor - 60) * (100 - rFactor);
+
+    const isHighPacketsLoss = packetsLoss > 5;
+    const isHighJitter = avgJitter >= 200;
+    const isPoorConnectionQuality = mos < 3.5;
+    const isNetworkMediaLatencyIssue = isHighPacketsLoss && isHighJitter;
+    const isNetworkIssue = (!isHighPacketsLoss && isHighJitter) || isHighJitter || isHighPacketsLoss;
+    const debug = `packetLoss: ${packetsLoss}%, jitter: ${avgJitter}, rtt: ${rtt},`
+      + ` MOS: ${mos}`;
+
+    if (isNetworkMediaLatencyIssue) {
+      issues.push({
+        type: IssueType.Network,
+        reason: IssueReason.OutboundNetworkMediaLatency,
+        iceCandidate: data.connection.local.id,
+        debug,
+      });
+    }
+
+    if (isNetworkIssue) {
+      issues.push({
+        type: IssueType.Network,
+        reason: IssueReason.OutboundNetworkQuality,
+        iceCandidate: data.connection.local.id,
+        debug,
+      });
+    }
+
+    if (isPoorConnectionQuality) {
+      issues.push({
+        type: IssueType.Network,
+        reason: IssueReason.LowOutboundMOS,
+        iceCandidate: data.connection.local.id,
+        debug,
+        data: mos,
+      });
+    }
+
+    return issues;
+  }
+}
+
+export default OutboundNetworkIssueDetector;

package/src/engine/wid/detectors/VideoCodecMismatchDetector.ts

@@ -0,0 +1,39 @@
+import { WebRTCStatsEventData } from '@peermetrics/webrtc-stats';
+import {
+  IssueDetector,
+  IssueDetectorResult,
+  IssueReason,
+  IssueType,
+} from '../types';
+
+class VideoCodecMismatchDetector implements IssueDetector {
+  #lastProcessedStats: { [connectionId: string]: WebRTCStatsEventData } = {};
+
+  detect(data: WebRTCStatsEventData): IssueDetectorResult {
+    const issues = this.processData(data);
+    this.#lastProcessedStats[data.connection.id] = data;
+    return issues;
+  }
+
+  private processData(data: WebRTCStatsEventData): IssueDetectorResult {
+    const issues: IssueDetectorResult = [];
+    const previousInboundRTPVideoStreamsStats = this.#lastProcessedStats[data.connection.id]?.video.inbound;
+
+    data.video.inbound.forEach((streamStats) => {
+      const previousStreamStats = previousInboundRTPVideoStreamsStats?.find((item) => item.ssrc === streamStats.ssrc);
+      if (streamStats.decoderImplementation === 'unknown' && previousStreamStats?.decoderImplementation !== 'unknown') {
+        issues.push({
+          type: IssueType.Stream,
+          reason: IssueReason.VideoCodecMismatchIssue,
+          ssrc: streamStats.ssrc,
+          trackIdentifier: streamStats.track.trackIdentifier,
+          debug: `mimeType: ${streamStats.mimeType}, decoderImplementation: ${streamStats.decoderImplementation}`,
+        });
+      }
+    });
+
+    return issues;
+  }
+}
+
+export default VideoCodecMismatchDetector;

package/src/engine/wid/types.ts

@@ -26,18 +26,23 @@ export enum IssueType {
   Network = 'network',
   CPU = 'cpu',
   Server = 'server',
+  Stream = 'stream',
 }
 
 export enum IssueReason {
-  NetworkQuality = 'network-quality',
-  NetworkMediaLatency = 'network-media-latency',
+  OutboundNetworkQuality = 'outbound-network-quality',
+  InboundNetworkQuality = 'inbound-network-quality',
+  OutboundNetworkMediaLatency = 'outbound-network-media-latency',
+  InboundNetworkMediaLatency = 'inbound-network-media-latency',
   NetworkMediaSyncFailure = 'network-media-sync-failure',
   OutboundNetworkThroughput = 'outbound-network-throughput',
   InboundNetworkThroughput = 'inbound-network-throughput',
   EncoderCPUThrottling = 'encoder-cpu-throttling',
   DecoderCPUThrottling = 'decoder-cpu-throttling',
   ServerIssue = 'server-issue',
-  LowMOS = 'low-mean-opinion-score',
+  VideoCodecMismatchIssue = 'codec-mismatch',
+  LowInboundMOS = 'low-inbound-mean-opinion-score',
+  LowOutboundMOS = 'low-outbound-mean-opinion-score',
 }
 
 export type IssuePayload = {
@@ -45,5 +50,7 @@ export type IssuePayload = {
   reason: IssueReason,
   ssrc?: number,
   iceCandidate?: string,
+  data?: number;
   debug?: string,
+  trackIdentifier?: string,
 };

package/src/types/common.ts

@@ -135,6 +135,8 @@ export type EncoderConfig = {};
 
 export type VideoCodec = 'h264' | 'vp8';
 
+export type AudioCodec = 'opus';
+
 export type VideoEncoderConfig = EncoderConfig & {
   preferredCodec?: VideoCodec,
   encodings?: RtpEncodingParameters[],
@@ -142,6 +144,7 @@ export type VideoEncoderConfig = EncoderConfig & {
 };
 
 export type AudioEncoderConfig = EncoderConfig & {
+  preferredCodec?: AudioCodec,
   enableFec?: boolean;
 };
 
@@ -235,3 +238,10 @@ export type RemoteConsumerOptions = ConsumerOptions & {
 export type LogMessageHandler = (msg: any, ...meta: any) => void;
 
 export type LogLevel = 3 | 4 | 6 | 7;
+
+export type TransportConnectionTimeoutPayload = {
+  reason: 'ice' | 'dtls',
+  transportId: string,
+  direction: 'receive' | 'send',
+  timeout: number,
+};