@observertc/observer-js 0.41.7 → 0.42.0

package/src/ObservedPeerConnection.ts

@@ -1,22 +1,27 @@
 import { EventEmitter } from 'events';
 import { PeerConnectionTransport } from '@observertc/sample-schemas-js';
 import { ObservedClient } from './ObservedClient';
-import { ObservedInboundTrack, ObservedInboundTrackModel } from './ObservedInboundTrack';
-import { ObservedOutboundTrack, ObservedOutboundTrackModel } from './ObservedOutboundTrack';
 import { CallEventReport, PeerConnectionTransportReport } from '@observertc/report-schemas-js';
 import { ObservedICE } from './ObservedICE';
 import { ObservedDataChannel } from './ObservedDataChannel';
 import { PartialBy } from './common/utils';
+import { ObservedInboundAudioTrack, ObservedInboundAudioTrackModel } from './ObservedInboundAudioTrack';
+import { ObservedInboundVideoTrack, ObservedInboundVideoTrackModel } from './ObservedInboundVideoTrack';
+import { ObservedOutboundAudioTrack, ObservedOutboundAudioTrackModel } from './ObservedOutboundAudioTrack';
+import { ObservedOutboundVideoTrack, ObservedOutboundVideoTrackModel } from './ObservedOutboundVideoTrack';
+import { CalculatedScore, calculateLatencyMOS } from './common/CalculatedScore';
+import { ClientIssue } from './monitors/CallSummary';
 
 export type ObservedPeerConnectionEvents = {
 	update: [{
 		elapsedTimeInMs: number;
 	}],
 	close: [],
-	newinboudaudiotrack: [ObservedInboundTrack<'audio'>],
-	newinboudvideotrack: [ObservedInboundTrack<'video'>],
-	newoutboundaudiotrack: [ObservedOutboundTrack<'audio'>],
-	newoutboundvideotrack: [ObservedOutboundTrack<'video'>],
+	score: [CalculatedScore],
+	newinboudaudiotrack: [ObservedInboundAudioTrack],
+	newinboudvideotrack: [ObservedInboundVideoTrack],
+	newoutboundaudiotrack: [ObservedOutboundAudioTrack],
+	newoutboundvideotrack: [ObservedOutboundVideoTrack],
 	newdatachannel: [ObservedDataChannel],
 };
 
@@ -39,7 +44,11 @@ export class ObservedPeerConnection extends EventEmitter {
 	public visited = true;
 	
 	private _elapsedTimeSinceLastUpdate?: number;
+	private _statsTimestamp?: number;
+	
+	public ωpendingIssuesForScores: ClientIssue[] = [];
 
+	public score?: CalculatedScore;
 	public totalInboundPacketsLost = 0;
 	public totalInboundPacketsReceived = 0;
 	public totalOutboundPacketsSent = 0;
@@ -77,6 +86,7 @@ export class ObservedPeerConnection extends EventEmitter {
 	public receivingVideoBitrate = 0;
     
 	public avgRttInMs?: number;
+	public avgJitter?: number;
 
 	private _closed = false;
 	private _updated = Date.now();
@@ -84,10 +94,10 @@ export class ObservedPeerConnection extends EventEmitter {
 	private _marker?: string;
 
 	public readonly ICE = ObservedICE.create(this);
-	private readonly _inboundAudioTracks = new Map<string, ObservedInboundTrack<'audio'>>();
-	private readonly _inboundVideoTracks = new Map<string, ObservedInboundTrack<'video'>>();
-	private readonly _outboundAudioTracks = new Map<string, ObservedOutboundTrack<'audio'>>();
-	private readonly _outboundVideoTracks = new Map<string, ObservedOutboundTrack<'video'>>();
+	private readonly _inboundAudioTracks = new Map<string, ObservedInboundAudioTrack>();
+	private readonly _inboundVideoTracks = new Map<string, ObservedInboundVideoTrack>();
+	private readonly _outboundAudioTracks = new Map<string, ObservedOutboundAudioTrack>();
+	private readonly _outboundVideoTracks = new Map<string, ObservedOutboundVideoTrack>();
 	private readonly _dataChannels = new Map<number, ObservedDataChannel>();
 	
 	public constructor(
@@ -155,19 +165,19 @@ export class ObservedPeerConnection extends EventEmitter {
 		return this._updated;
 	}
 
-	public get inboundAudioTracks(): ReadonlyMap<string, ObservedInboundTrack<'audio'>> {
+	public get inboundAudioTracks(): ReadonlyMap<string, ObservedInboundAudioTrack> {
 		return this._inboundAudioTracks;
 	}
 
-	public get inboundVideoTracks(): ReadonlyMap<string, ObservedInboundTrack<'video'>> {
+	public get inboundVideoTracks(): ReadonlyMap<string, ObservedInboundVideoTrack> {
 		return this._inboundVideoTracks;
 	}
 
-	public get outboundAudioTracks(): ReadonlyMap<string, ObservedOutboundTrack<'audio'>> {
+	public get outboundAudioTracks(): ReadonlyMap<string, ObservedOutboundAudioTrack> {
 		return this._outboundAudioTracks;
 	}
 
-	public get outboundVideoTracks(): ReadonlyMap<string, ObservedOutboundTrack<'video'>> {
+	public get outboundVideoTracks(): ReadonlyMap<string, ObservedOutboundVideoTrack> {
 		return this._outboundVideoTracks;
 	}
 
@@ -218,13 +228,19 @@ export class ObservedPeerConnection extends EventEmitter {
 
 		Array.from(this._inboundAudioTracks.values()).forEach((track) => track.close());
 		Array.from(this._inboundVideoTracks.values()).forEach((track) => track.close());
-		Array.from(this._outboundAudioTracks.values()).forEach((track) => track.close());
-		Array.from(this._outboundVideoTracks.values()).forEach((track) => track.close());
+		Array.from(this._outboundAudioTracks.values()).forEach((track) => {
+			this.client.call.sfuStreamIdToOutboundAudioTrack.delete(track.sfuStreamId ?? '');
+			track.close();
+		});
+		Array.from(this._outboundVideoTracks.values()).forEach((track) => {
+			this.client.call.sfuStreamIdToOutboundVideoTrack.delete(track.sfuStreamId ?? '');
+			track.close();
+		});
 
 		this.emit('close');
 	}
 
-	public getTrack(trackId: string): ObservedInboundTrack<'audio'> | ObservedInboundTrack<'video'> | ObservedOutboundTrack<'audio'> | ObservedOutboundTrack<'video'> | undefined {
+	public getTrack(trackId: string): ObservedInboundAudioTrack | ObservedInboundVideoTrack | ObservedOutboundAudioTrack | ObservedOutboundVideoTrack | undefined {
 		return this._inboundAudioTracks.get(trackId) ?? this._inboundVideoTracks.get(trackId) ?? this._outboundAudioTracks.get(trackId) ?? this._outboundVideoTracks.get(trackId);
 	}
 
@@ -251,8 +267,119 @@ export class ObservedPeerConnection extends EventEmitter {
 		this.ICE.resetMetrics();
 	}
 
+	public update(sample: PeerConnectionTransport, timestamp: number) {
+		if (this._closed) return;
+		if (sample.peerConnectionId !== this._model.peerConnectionId) throw new Error(`TransportId mismatch. PeerConnectionId: ${ this._model.peerConnectionId } TransportId: ${ sample.transportId}`);
+
+		this._sample = sample;
+		if (this._model.label !== sample.label) {
+			this._model.label = sample.label;
+		}
+		const now = Date.now();
+		const report: PeerConnectionTransportReport = {
+			serviceId: this.client.call.serviceId,
+			roomId: this.client.call.roomId,
+			callId: this.client.call.callId,
+			clientId: this.client.clientId,
+			userId: this.client.userId,
+			mediaUnitId: this.client.mediaUnitId,
+			...sample,
+			timestamp,
+			sampleSeq: -1, // deprecated
+			marker: this.marker,
+		};
+
+		this.reports.addPeerConnectionTransportReports(report);
+		this._elapsedTimeSinceLastUpdate = now - this._updated;		
+		this.visited = true;
+		this._updated = now;
+		this._statsTimestamp = timestamp;
+		this.emit('update', {
+			elapsedTimeInMs: this._elapsedTimeSinceLastUpdate,
+		});
+	}
+
+	public createInboundAudioTrack(config: ObservedInboundAudioTrackModel): ObservedInboundAudioTrack {
+		if (this._closed) throw new Error(`PeerConnection ${this.peerConnectionId} is closed`);
+		
+		const result = new ObservedInboundAudioTrack(config, this);
+
+		result.on('close', () => {
+			this._inboundAudioTracks.delete(result.trackId);
+		});
+		this._inboundAudioTracks.set(result.trackId, result);
+
+		this.emit('newinboudaudiotrack', result);
+
+		return result;
+	}
+
+	public createInboundVideoTrack(config: ObservedInboundVideoTrackModel): ObservedInboundVideoTrack {
+		if (this._closed) throw new Error(`PeerConnection ${this.peerConnectionId} is closed`);
+		
+		const result = new ObservedInboundVideoTrack(config, this);
+
+		result.on('close', () => {
+			this._inboundVideoTracks.delete(result.trackId);
+		});
+		this._inboundVideoTracks.set(result.trackId, result);
+
+		this.emit('newinboudvideotrack', result);
+
+		return result;
+	}
+
+	public createOutboundAudioTrack(config: ObservedOutboundAudioTrackModel): ObservedOutboundAudioTrack {
+		if (this._closed) throw new Error(`PeerConnection ${this.peerConnectionId} is closed`);
+		
+		const result = new ObservedOutboundAudioTrack(config, this);
+
+		result.on('close', () => {
+			this._outboundAudioTracks.delete(result.trackId);
+		});
+		this._outboundAudioTracks.set(result.trackId, result);
+
+		this.emit('newoutboundaudiotrack', result);
+
+		return result;
+	}
+
+	public createOutboundVideoTrack(config: ObservedOutboundVideoTrackModel): ObservedOutboundVideoTrack {
+		if (this._closed) throw new Error(`PeerConnection ${this.peerConnectionId} is closed`);
+		
+		const result = new ObservedOutboundVideoTrack(config, this);
+
+		result.on('close', () => {
+			this._outboundVideoTracks.delete(result.trackId);
+		});
+		this._outboundVideoTracks.set(result.trackId, result);
+
+		this.emit('newoutboundvideotrack', result);
+
+		return result;
+	}
+
+	public createDataChannel(channelId: number) {
+		if (this._closed) throw new Error(`PeerConnection ${this.peerConnectionId} is closed`);
+		
+		const result = new ObservedDataChannel({
+			channelId,
+		}, this);
+
+		result.on('close', () => {
+			this._dataChannels.delete(result.channelId);
+		});
+		this._dataChannels.set(result.channelId, result);
+
+		this.emit('newdatachannel', result);
+
+		return result;
+	}
+
 	public updateMetrics() {
 		let sumRttInMs = 0;
+		let sumJitter = 0;
+		const trackScores: CalculatedScore[] = [];
 
 		this._inboundAudioTracks.forEach((track) => {
 			track.updateMetrics();
@@ -266,7 +393,10 @@ export class ObservedPeerConnection extends EventEmitter {
 			
 			this.receivingAudioBitrate += track.bitrate;
 
-			sumRttInMs = (track.rttInMs ?? 0);
+			sumRttInMs += (track.rttInMs ?? 0);
+			sumJitter += (track.jitter ?? 0);
+
+			track.score && trackScores.push(track.score);
 		});
 
 		this._inboundVideoTracks.forEach((track) => {
@@ -281,7 +411,10 @@ export class ObservedPeerConnection extends EventEmitter {
 			
 			this.receivingVideoBitrate += track.bitrate;
 
-			sumRttInMs = (track.rttInMs ?? 0);
+			sumRttInMs += (track.rttInMs ?? 0);
+			sumJitter += (track.jitter ?? 0);
+
+			track.score && trackScores.push(track.score);
 		});
 
 		this._outboundAudioTracks.forEach((track) => {
@@ -295,7 +428,10 @@ export class ObservedPeerConnection extends EventEmitter {
 			
 			this.sendingAudioBitrate += track.bitrate;
 
-			sumRttInMs = (track.rttInMs ?? 0);
+			sumRttInMs += (track.rttInMs ?? 0);
+			sumJitter += (track.jitter ?? 0);
+
+			track.score && trackScores.push(track.score);
 		});
 
 		this._outboundVideoTracks.forEach((track) => {
@@ -309,7 +445,10 @@ export class ObservedPeerConnection extends EventEmitter {
 			
 			this.sendingVideoBitrate += track.bitrate;
 
-			sumRttInMs = (track.rttInMs ?? 0);
+			sumRttInMs += (track.rttInMs ?? 0);
+			sumJitter += (track.jitter ?? 0);
+
+			track.score && trackScores.push(track.score);
 		});
 
 		this._dataChannels.forEach((channel) => {
@@ -320,6 +459,8 @@ export class ObservedPeerConnection extends EventEmitter {
 		const iceRttInMs = this.ICE.stats?.currentRoundTripTime;
 		let nrOfBelongings = this._inboundAudioTracks.size + this._inboundVideoTracks.size + this._outboundAudioTracks.size + this._outboundVideoTracks.size;
 
+		this.avgJitter = 0 < nrOfBelongings ? sumJitter / nrOfBelongings : undefined;
+
 		if (iceRttInMs) {
 			sumRttInMs += iceRttInMs;
 			nrOfBelongings += 1;
@@ -347,125 +488,58 @@ export class ObservedPeerConnection extends EventEmitter {
 			this.sendingPacketsPerSecond = 0;
 			this.receivingPacketsPerSecond = 0;
 		}
-	}
-
-	public update(sample: PeerConnectionTransport, timestamp: number) {
-		if (this._closed) return;
-		if (sample.peerConnectionId !== this._model.peerConnectionId) throw new Error(`TransportId mismatch. PeerConnectionId: ${ this._model.peerConnectionId } TransportId: ${ sample.transportId}`);
-
-		this._sample = sample;
-		if (this._model.label !== sample.label) {
-			this._model.label = sample.label;
-		}
-		const now = Date.now();
-		const report: PeerConnectionTransportReport = {
-			serviceId: this.client.call.serviceId,
-			roomId: this.client.call.roomId,
-			callId: this.client.call.callId,
-			clientId: this.client.clientId,
-			userId: this.client.userId,
-			mediaUnitId: this.client.mediaUnitId,
-			...sample,
-			timestamp,
-			sampleSeq: -1, // deprecated
-			marker: this.marker,
-		};
-
-		this.reports.addPeerConnectionTransportReports(report);
-		this._elapsedTimeSinceLastUpdate = now - this._updated;		
-		this.visited = true;
-		this._updated = now;
-		this.emit('update', {
-			elapsedTimeInMs: this._elapsedTimeSinceLastUpdate,
-		});
-	}
-
-	public createInboundAudioTrack(config: Omit<ObservedInboundTrackModel<'audio'>, 'kind'>): ObservedInboundTrack<'audio'> {
-		if (this._closed) throw new Error(`PeerConnection ${this.peerConnectionId} is closed`);
-		
-		const result = new ObservedInboundTrack<'audio'>({
-			kind: 'audio',
-			...config,
-		}, this);
-
-		result.on('close', () => {
-			this._inboundAudioTracks.delete(result.trackId);
-		});
-		this._inboundAudioTracks.set(result.trackId, result);
-
-		this.emit('newinboudaudiotrack', result);
-
-		return result;
-	}
 
-	public createInboundVideoTrack(config: Omit<ObservedInboundTrackModel<'video'>, 'kind'>): ObservedInboundTrack<'video'> {
-		if (this._closed) throw new Error(`PeerConnection ${this.peerConnectionId} is closed`);
-		
-		const result = new ObservedInboundTrack<'video'>({
-			kind: 'video',
-			...config,
-		}, this);
-
-		result.on('close', () => {
-			this._inboundVideoTracks.delete(result.trackId);
-		});
-		this._inboundVideoTracks.set(result.trackId, result);
-
-		this.emit('newinboudvideotrack', result);
-
-		return result;
+		// calculate quality score
+		this._updateQualityScore(trackScores);
 	}
 
-	public createOutboundAudioTrack(config: Omit<ObservedOutboundTrackModel<'audio'>, 'kind'>): ObservedOutboundTrack<'audio'> {
-		if (this._closed) throw new Error(`PeerConnection ${this.peerConnectionId} is closed`);
-		
-		const result = new ObservedOutboundTrack<'audio'>({
-			kind: 'audio',
-			...config,
-		}, this);
-
-		result.on('close', () => {
-			this._outboundAudioTracks.delete(result.trackId);
+	private _updateQualityScore(trackScores: CalculatedScore[]) {
+		const mosScore = calculateLatencyMOS({
+			avgJitter: this.avgJitter ?? 0,
+			rttInMs: this.avgRttInMs ?? 0,
+			packetsLoss: this.deltaInboundPacketsLost,
 		});
-		this._outboundAudioTracks.set(result.trackId, result);
-
-		this.emit('newoutboundaudiotrack', result);
-
-		return result;
-	}
-
-	public createOutboundVideoTrack(config: Omit<ObservedOutboundTrackModel<'video'>, 'kind'>): ObservedOutboundTrack<'video'> {
-		if (this._closed) throw new Error(`PeerConnection ${this.peerConnectionId} is closed`);
+		const score: CalculatedScore = {
+			remarks: [ {
+				severity: 'none',
+				text: `MOS score: ${mosScore}`,
+			} ],
+			score: mosScore / 10.0, // normalize between 0 and 2
+			timestamp: this._statsTimestamp ?? Date.now(),
+		};
 		
-		const result = new ObservedOutboundTrack<'video'>({
-			kind: 'video',
-			...config,
-		}, this);
-
-		result.on('close', () => {
-			this._outboundVideoTracks.delete(result.trackId);
-		});
-		this._outboundVideoTracks.set(result.trackId, result);
+		let sumTrackScores = 0;
 
-		this.emit('newoutboundvideotrack', result);
-
-		return result;
-	}
-
-	public createDataChannel(channelId: number) {
-		if (this._closed) throw new Error(`PeerConnection ${this.peerConnectionId} is closed`);
-		
-		const result = new ObservedDataChannel({
-			channelId,
-		}, this);
+		for (const trackScore of trackScores) {
+			sumTrackScores += trackScore.score;
+		}
 
-		result.on('close', () => {
-			this._dataChannels.delete(result.channelId);
+		score.score += sumTrackScores / trackScores.length;
+		score.remarks.push({
+			severity: 'none',
+			text: `Track scores: ${sumTrackScores / trackScores.length}`,
 		});
-		this._dataChannels.set(result.channelId, result);
 
-		this.emit('newdatachannel', result);
+		for (const pendingScore of this.ωpendingIssuesForScores) {
+			switch (pendingScore.severity) {
+				case 'critical':
+					score.score = 0.0;
+					break;
+				case 'major':
+					score.score *= 0.5;
+					break;
+				case 'minor':
+					score.score *= 0.8;
+					break;
+			}
+			score.remarks.push({
+				severity: pendingScore.severity,
+				text: pendingScore.description ?? 'Issue occurred',
+			});
+		}
+		this.ωpendingIssuesForScores = [];
+		this.score = score;
 
-		return result;
+		this.emit('score', score);
 	}
 }

package/src/RemoteTrackAssigner.ts

@@ -0,0 +1,68 @@
+import { ObservedInboundAudioTrack } from './ObservedInboundAudioTrack';
+import { ObservedInboundVideoTrack } from './ObservedInboundVideoTrack';
+import { ObservedOutboundAudioTrack } from './ObservedOutboundAudioTrack';
+import { ObservedOutboundVideoTrack } from './ObservedOutboundVideoTrack';
+
+export class RemoteTrackAssigner {
+	public readonly sfuStreamIdToOutboundVideoTracks = new Map<string, ObservedOutboundVideoTrack>();
+	public readonly sfuStreamIdToOutboundAudioTracks = new Map<string, ObservedOutboundAudioTrack>();
+	public readonly pendingInboundVideoTracks = new Map<string, ObservedInboundVideoTrack[]>();
+	public readonly pendingInboundAudioTracks = new Map<string, ObservedInboundAudioTrack[]>();
+
+	public addOutboundAudioTrack(track: ObservedOutboundAudioTrack) {
+		const untilSfuStreamIdListener = () => {
+			if (!track.sfuStreamId) return;
+
+			this.sfuStreamIdToOutboundAudioTracks.set(track.sfuStreamId, track);
+			track.off('update', untilSfuStreamIdListener);
+
+			// we need to assign to the pending inbound tracks
+		};
+
+		track.on('update', untilSfuStreamIdListener);
+		untilSfuStreamIdListener();
+	}
+
+	public addOutboundVideoTrack(track: ObservedOutboundVideoTrack) {
+		const untilSfuStreamIdListener = () => {
+			if (!track.sfuStreamId) return;
+
+			this.sfuStreamIdToOutboundVideoTracks.set(track.sfuStreamId, track);
+			track.off('update', untilSfuStreamIdListener);
+
+			// we need to assign to the pending inbound tracks
+		};
+
+		track.on('update', untilSfuStreamIdListener);
+		untilSfuStreamIdListener();
+	}
+
+	public addInboundAudioTrack(track: ObservedInboundAudioTrack) {
+		const untilSfuStreamIdListener = () => {
+			if (!track.sfuStreamId) return;
+			const outboundTrack = this.sfuStreamIdToOutboundAudioTracks.get(track.sfuStreamId);
+	
+			if (!outboundTrack) {
+				this.pendingInboundAudioTracks.set(
+					track.sfuStreamId, 
+					(this.pendingInboundAudioTracks.get(track.sfuStreamId) || []).concat(track)
+				);
+
+				return;
+			}
+			
+			if (outboundTrack) {
+				track.remoteOutboundTrack = outboundTrack;
+			} else {
+				this.pendingInboundAudioTracks.set(
+					track.sfuStreamId, 
+					(this.pendingInboundAudioTracks.get(track.sfuStreamId) || []).concat(track)
+				);
+			}
+	
+		};
+
+		track.on('update', untilSfuStreamIdListener);
+	}
+	
+}