@observertc/observer-js 0.30.0-e12fcb4.0 → 0.30.2-beta

package/src/ObservedClient.ts

@@ -1,4 +1,4 @@
-import { ClientSample, IceLocalCandidate } from '@observertc/sample-schemas-js';
+import { ClientSample } from '@observertc/sample-schemas-js';
 import { ObservedCall } from './ObservedCall';
 import * as Models from './models/Models';
 import { StorageProvider } from './storages/StorageProvider';
@@ -9,8 +9,9 @@ import { createLogger } from './common/logger';
 import { CallMetaType, createCallMetaReport } from './common/callMetaReports';
 // eslint-disable-next-line camelcase
 import { Samples_ClientSample_Browser, Samples_ClientSample_Engine, Samples_ClientSample_OperationSystem, Samples_ClientSample_Platform } from './models/samples_pb';
-import { IceCandidatePairReport } from '@observertc/report-schemas-js';
 import { isValidUuid } from './common/utils';
+import { createClientJoinedEventReport, createClientLeftEventReport } from './common/callEventReports';
+import { CallEventType } from './common/CallEventType';
 
 const logger = createLogger('ObservedClient');
 
@@ -21,13 +22,17 @@ export type ObservedClientConfig<AppData extends Record<string, unknown> = Recor
 	appData: AppData;
 	overflowingProcessingThreshold?: number;
 	joined?: number,
+
+	/**
+	 * before client-monitor-js 3.4.0 the client joined event was generated by the observer, this flag is to keep the compatibility with the old behavior
+	 */
+	addJoinAndDetachClientEvent?: boolean;
 };
 
 export type ObservedClientEvents = {
 	update: [],
 	close: [],
 	'processing-sample-overflow': [],
-	'candidate-pair-change': [],
 	newpeerconnection: [ObservedPeerConnection],
 };
 
@@ -65,6 +70,21 @@ export class ObservedClient<AppData extends Record<string, unknown> = Record<str
 
 		result._overflowingProcessingThreshold = config.overflowingProcessingThreshold ?? 0;
 
+		if (config.addJoinAndDetachClientEvent) {
+			call.reports.addCallEventReport(createClientJoinedEventReport(
+				call.serviceId,
+				result.mediaUnitId,
+				call.roomId,
+				call.callId,
+				result.clientId,
+				result.joined,
+				result.userId,
+				result.marker,
+			));
+		}
+
+		result.once('close', () => result._addClientLeftReport());
+
 		return result;
 	}
 
@@ -76,9 +96,39 @@ export class ObservedClient<AppData extends Record<string, unknown> = Record<str
 	private _updated = Date.now();
 	private _acceptedSample = 0;
 	private _processingSample = 0;
-	private _iceLocalCandidate?: IceLocalCandidate;
-	private _iceRemoteCandidate?: IceLocalCandidate;
-	
+	private _timeZoneOffsetInHours?: number;
+	private _leaveReported = false;
+
+	public totalInboundPacketsLost = 0;
+	public totalInboundPacketsReceived = 0;
+	public totalOutboundPacketsLost = 0;
+	public totalOutboundPacketsReceived = 0;
+	public totalOutboundPacketsSent = 0;
+	public totalDataChannelBytesSent = 0;
+	public totalDataChannelBytesReceived = 0;
+	public totalSentAudioBytes = 0;
+	public totalSentVideoBytes = 0;
+	public totalReceivedAudioBytes = 0;
+	public totalReceivedVideoBytes = 0;
+
+	public deltaInboundPacketsLost?: number;
+	public deltaInboundPacketsReceived?: number;
+	public deltaOutboundPacketsLost?: number;
+	public deltaOutboundPacketsReceived?: number;
+	public deltaOutboundPacketsSent?: number;
+	public deltaDataChannelBytesSent?: number;
+	public deltaDataChannelBytesReceived?: number;
+	public deltaReceivedAudioBytes?: number;
+	public deltaReceivedVideoBytes?: number;
+	public deltaSentAudioBytes?: number;
+	public deltaSentVideoBytes?: number;
+
+	public avgRttInS?: number;
+	public sendingAudioBitrate?: number;
+	public sendingVideoBitrate?: number;
+	public receivingAudioBitrate?: number;
+	public receivingVideoBitrate?: number;
+
 	public readonly mediaDevices: string[] = [];
 	public readonly codecs: string[] = [];
 	public readonly userMediaErrors: string[] = [];
@@ -130,12 +180,8 @@ export class ObservedClient<AppData extends Record<string, unknown> = Record<str
 		return this._model.userId;
 	}
 
-	public get iceLocalCandidate() {
-		return this._iceLocalCandidate;
-	}
-
-	public get iceRemoteCandidate() {
-		return this._iceRemoteCandidate;
+	public get timeZoneOffsetInHours() {
+		return this._timeZoneOffsetInHours;
 	}
 
 	public get marker() {
@@ -209,6 +255,11 @@ export class ObservedClient<AppData extends Record<string, unknown> = Record<str
 				executeSave = true;
 			}
 
+			if (this._timeZoneOffsetInHours !== sample.timeZoneOffsetInHours) {
+				this._timeZoneOffsetInHours = sample.timeZoneOffsetInHours;
+				executeSave = true;
+			}
+
 			if (sample.os && (
 				this._model.operationSystem?.name !== sample.os.name || 
 				this._model.operationSystem?.version !== sample.os.version ||
@@ -348,6 +399,23 @@ export class ObservedClient<AppData extends Record<string, unknown> = Record<str
 			}
 
 			for (const { timestamp = Date.now(), ...callEvent } of sample.customCallEvents ?? []) {
+				const peerConnection = this._peerConnections.get(callEvent.peerConnectionId ?? '');
+				
+				switch (callEvent.name) {
+					case CallEventType.CLIENT_LEFT:
+						this._leaveReported = true;
+						break;
+					case CallEventType.ICE_CONNECTION_STATE_CHANGED: 
+						peerConnection && callEvent.value && peerConnection.emit('iceconnectionstatechange', callEvent.value);
+						break;
+					case CallEventType.ICE_GATHERING_STATE_CHANGED:
+						peerConnection && callEvent.value && peerConnection.emit('icegatheringstatechange', callEvent.value);
+						break;
+					case CallEventType.PEER_CONNECTION_STATE_CHANGED:
+						peerConnection && callEvent.value && peerConnection.emit('connectionstatechange', callEvent.value);
+						break;
+				}
+
 				this.reports.addCallEventReport({
 					serviceId: this.serviceId,
 					mediaUnitId: this.mediaUnitId,
@@ -446,75 +514,6 @@ export class ObservedClient<AppData extends Record<string, unknown> = Record<str
 
 				this.reports.addCallMetaReport(callMetaReport);
 			}
-			let selectedLocalCandidateId: string | undefined;
-			let selectedRemoteCandidateId: string | undefined;
-
-			for (const candidatePair of sample.iceCandidatePairs ?? []) {
-				if (candidatePair.nominated) {
-					if (!selectedLocalCandidateId) selectedLocalCandidateId = candidatePair.localCandidateId;
-					if (!selectedRemoteCandidateId) selectedRemoteCandidateId = candidatePair.remoteCandidateId;
-				}
-
-				const report: IceCandidatePairReport = {
-					serviceId: this.serviceId,
-					mediaUnitId: this.mediaUnitId,
-					roomId: this.roomId,
-					callId: this.callId,
-					clientId: this.clientId,
-					userId: this.userId,
-					timestamp: sample.timestamp,
-					...candidatePair,
-					sampleSeq: -1, // deprecated
-				};
-		
-				this.reports.addIceCandidatePairReport(report);
-			}
-	
-			let changeIceCandidatePair = false;
-
-			for (const iceLocalCandidate of sample.iceLocalCandidates ?? []) {
-				if (
-					iceLocalCandidate.id && selectedLocalCandidateId &&
-					iceLocalCandidate.id === selectedLocalCandidateId && 
-					this._iceLocalCandidate?.id !== iceLocalCandidate.id
-				) {
-					this._iceLocalCandidate = iceLocalCandidate;
-					changeIceCandidatePair = true;
-				}
-				const callMetaReport = createCallMetaReport(
-					this.serviceId, 
-					this.mediaUnitId, 
-					this.roomId, 
-					this.callId, 
-					this.clientId, {
-						type: CallMetaType.ICE_LOCAL_CANDIDATE,
-						payload: iceLocalCandidate,
-					}, this.userId);
-
-				this.reports.addCallMetaReport(callMetaReport);
-			}
-	
-			for (const iceRemoteCandidate of sample.iceRemoteCandidates ?? []) {
-				if (
-					iceRemoteCandidate.id && selectedRemoteCandidateId &&
-					iceRemoteCandidate.id === selectedRemoteCandidateId && 
-					this._iceRemoteCandidate?.id !== iceRemoteCandidate.id
-				) {
-					this._iceRemoteCandidate = iceRemoteCandidate;
-					changeIceCandidatePair = true;
-				}
-				const callMetaReport = createCallMetaReport(
-					this.serviceId, 
-					this.mediaUnitId, 
-					this.roomId, 
-					this.callId, 
-					this.clientId, {
-						type: CallMetaType.ICE_REMOTE_CANDIDATE,
-						payload: iceRemoteCandidate,
-					}, this.userId);
-
-				this.reports.addCallMetaReport(callMetaReport);
-			}
 
 			for (const transport of sample.pcTransports ?? []) {
 				try {
@@ -640,9 +639,78 @@ export class ObservedClient<AppData extends Record<string, unknown> = Record<str
 					logger.error(`Error creating outbound video track: ${(err as Error)?.message}`);
 				}
 			}
-			
-			if (changeIceCandidatePair) {
-				this.emit('candidate-pair-change');
+
+			for (const iceLocalCandidate of sample.iceLocalCandidates ?? []) {
+				if (!iceLocalCandidate.peerConnectionId) {
+					logger.warn(`Local ice candidate without peerConnectionId: ${JSON.stringify(iceLocalCandidate)}`);
+					continue;
+				}
+				const peerConnection = this._peerConnections.get(iceLocalCandidate.peerConnectionId);
+
+				if (!peerConnection) {
+					logger.debug(`Peer connection ${iceLocalCandidate.peerConnectionId} not found for ice local candidate ${iceLocalCandidate.id}`);
+					continue;
+				}
+
+				peerConnection.ICE.addLocalCandidate(iceLocalCandidate);
+			}
+	
+			for (const iceRemoteCandidate of sample.iceRemoteCandidates ?? []) {
+				if (!iceRemoteCandidate.peerConnectionId) {
+					logger.warn(`Remote ice candidate without peerConnectionId: ${JSON.stringify(iceRemoteCandidate)}`);
+					continue;
+				}
+				const peerConnection = this._peerConnections.get(iceRemoteCandidate.peerConnectionId);
+
+				if (!peerConnection) {
+					logger.debug(`Peer connection ${iceRemoteCandidate.peerConnectionId} not found for ice remote candidate ${iceRemoteCandidate.id}`);
+					continue;
+				}
+
+				peerConnection.ICE.addRemoteCandidate(iceRemoteCandidate);
+			}
+
+			for (const candidatePair of sample.iceCandidatePairs ?? []) {
+				const peerConnection = this._peerConnections.get(candidatePair.peerConnectionId);
+
+				if (!peerConnection) {
+					logger.debug(`Peer connection ${candidatePair.peerConnectionId} not found for ice candidate pair ${candidatePair.localCandidateId}, ${candidatePair.remoteCandidateId}`);
+
+					continue;
+				}
+
+				peerConnection.ICE.update(candidatePair, sample.timestamp);
+			}
+
+			// close entries that are idle for too long
+			if (this.call.observer.config.maxEntryIdleTimeInMs && this._updated < sample.timestamp) {
+
+				for (const peerConnection of this._peerConnections.values()) {
+					if (this.call.observer.config.maxEntryIdleTimeInMs < sample.timestamp - peerConnection.updated) {
+						logger.debug(`Closing peer connection ${peerConnection.peerConnectionId} due to inactivity`);
+						peerConnection.close();
+
+						continue;
+					}
+					for (const track of peerConnection.inboundAudioTracks.values()) {
+						if (this.call.observer.config.maxEntryIdleTimeInMs < sample.timestamp - track.updated) {
+							logger.debug(`Closing inbound audio track ${track.trackId} due to inactivity`);
+							track.close();
+						}
+					}
+					for (const track of peerConnection.inboundVideoTracks.values()) {
+						if (this.call.observer.config.maxEntryIdleTimeInMs < sample.timestamp - track.updated) {
+							logger.debug(`Closing inbound video track ${track.trackId} due to inactivity`);
+							track.close();
+						}
+					}
+					for (const track of peerConnection.outboundAudioTracks.values()) {
+						if (this.call.observer.config.maxEntryIdleTimeInMs < sample.timestamp - track.updated) {
+							logger.debug(`Closing outbound audio track ${track.trackId} due to inactivity`);
+							track.close();
+						}
+					}
+				}
 			}
 			
 			if (executeSave) await this._save();
@@ -678,4 +746,20 @@ export class ObservedClient<AppData extends Record<string, unknown> = Record<str
 		
 		return this._storageProvider.clientStorage.set(this.clientId, this._model);
 	}
+
+	private _addClientLeftReport() {
+		if (this._leaveReported) return;
+		this._leaveReported = true;
+
+		this.reports.addCallEventReport(createClientLeftEventReport(
+			this.serviceId,
+			this.mediaUnitId,
+			this.roomId,
+			this.callId,
+			this.clientId,
+			this.joined,
+			this.userId,
+			this.marker,
+		));
+	}
 }

package/src/ObservedICE.ts

@@ -0,0 +1,176 @@
+import { IceCandidatePair, IceLocalCandidate, IceRemoteCandidate } from '@observertc/sample-schemas-js';
+import { EventEmitter } from 'events';
+import { ObservedPeerConnection } from './ObservedPeerConnection';
+import { IceCandidatePairReport } from '@observertc/report-schemas-js';
+import { CallMetaType, createCallMetaReport } from './common/callMetaReports';
+
+export type ObservedICEEvents = {
+	update: [],
+	'new-local-candidate': [IceLocalCandidate],
+	'new-remote-candidate': [IceRemoteCandidate],
+	'new-selected-candidate-pair': [{
+		localCandidate: IceLocalCandidate,
+		remoteCandidate: IceRemoteCandidate,
+	}],
+	close: [],
+};
+
+export declare interface ObservedICE {
+	on<U extends keyof ObservedICEEvents>(event: U, listener: (...args: ObservedICEEvents[U]) => void): this;
+	off<U extends keyof ObservedICEEvents>(event: U, listener: (...args: ObservedICEEvents[U]) => void): this;
+	once<U extends keyof ObservedICEEvents>(event: U, listener: (...args: ObservedICEEvents[U]) => void): this;
+	emit<U extends keyof ObservedICEEvents>(event: U, ...args: ObservedICEEvents[U]): boolean;
+}
+
+export class ObservedICE extends EventEmitter {
+	public static create(peerConnection: ObservedPeerConnection,) {
+		return new ObservedICE(peerConnection);
+	}
+
+	private readonly _localCandidates = new Map<string, IceLocalCandidate>();
+	private readonly _remoteCandidates = new Map<string, IceLocalCandidate>();
+	
+	private _selectedLocalCandidateId?: string;
+	private _selectedRemoteCandidateId?: string;
+	private _updated = Date.now();
+	private _stats?: IceCandidatePair;
+	private _closed = false;
+    
+	private constructor(
+		public readonly peerConnection: ObservedPeerConnection,
+	) {
+		super();
+	}
+
+	public get reports() {
+		return this.peerConnection.reports;
+	}
+
+	public get localCandidates(): ReadonlyMap<string, IceLocalCandidate> {
+		return this._localCandidates;
+	}
+
+	public get remoteCandidates(): ReadonlyMap<string, IceRemoteCandidate> {
+		return this._remoteCandidates;
+	}
+
+	public get selectedLocalCandidate() {
+		return this._localCandidates.get(this._selectedLocalCandidateId ?? '');
+	}
+
+	public get selectedRemoteCandidate() {
+		return this._remoteCandidates.get(this._selectedRemoteCandidateId ?? '');
+	}
+
+	public get stats() {
+		return this._stats;
+	}
+
+	public get updated() {
+		return this._updated;
+	}
+
+	public get closed() {
+		return this._closed;
+	}
+
+	public close() {
+		if (this._closed) return;
+		this._closed = true;
+		this.emit('close');
+	}
+
+	public addLocalCandidate(candidate: IceLocalCandidate) {
+		if (!candidate.id) return;
+
+		const newCandidate = !this._localCandidates.has(candidate.id);
+
+		this._localCandidates.set(candidate.id, candidate);
+
+		if (newCandidate) {
+			const callMetaReport = createCallMetaReport(
+				this.peerConnection.client.serviceId, 
+				this.peerConnection.client.mediaUnitId, 
+				this.peerConnection.client.roomId, 
+				this.peerConnection.client.callId, 
+				this.peerConnection.client.clientId, {
+					type: CallMetaType.ICE_LOCAL_CANDIDATE,
+					payload: candidate,
+				}, this.peerConnection.client.userId);
+
+			this.reports.addCallMetaReport(callMetaReport);
+
+			this.emit('new-local-candidate', candidate);
+		}
+	}
+
+	public addRemoteCandidate(candidate: IceRemoteCandidate) {
+		if (!candidate.id) return;
+
+		const newCandidate = !this._remoteCandidates.has(candidate.id);
+
+		this._remoteCandidates.set(candidate.id, candidate);
+
+		if (newCandidate) {
+			const callMetaReport = createCallMetaReport(
+				this.peerConnection.client.serviceId, 
+				this.peerConnection.client.mediaUnitId, 
+				this.peerConnection.client.roomId, 
+				this.peerConnection.client.callId, 
+				this.peerConnection.client.clientId, {
+					type: CallMetaType.ICE_REMOTE_CANDIDATE,
+					payload: candidate,
+				}, this.peerConnection.client.userId);
+
+			this.reports.addCallMetaReport(callMetaReport);
+            
+			this.emit('new-remote-candidate', candidate);
+		}
+	}
+
+	public update(candidatePair: IceCandidatePair, timestamp: number) {
+
+		const report: IceCandidatePairReport = {
+			serviceId: this.peerConnection.client.serviceId,
+			mediaUnitId: this.peerConnection.client.mediaUnitId,
+			roomId: this.peerConnection.client.roomId,
+			callId: this.peerConnection.client.callId,
+			clientId: this.peerConnection.client.clientId,
+			userId: this.peerConnection.client.userId,
+			timestamp,
+			...candidatePair,
+			sampleSeq: -1, // deprecated
+		};
+
+		this.reports.addIceCandidatePairReport(report);
+
+		if (!candidatePair.nominated) return;
+
+		this._stats = candidatePair;
+
+		if (
+			candidatePair.localCandidateId && 
+            candidatePair.localCandidateId !== this._selectedLocalCandidateId && 
+            this._localCandidates.has(candidatePair.localCandidateId) &&
+            candidatePair.remoteCandidateId &&
+            candidatePair.remoteCandidateId !== this._selectedRemoteCandidateId &&
+            this._remoteCandidates.has(candidatePair.remoteCandidateId)
+		) {
+			this._selectedLocalCandidateId = candidatePair.localCandidateId;
+			this._selectedRemoteCandidateId = candidatePair.remoteCandidateId;
+
+			const localCandidate = this._localCandidates.get(candidatePair.localCandidateId);
+			const remoteCandidate = this._remoteCandidates.get(candidatePair.remoteCandidateId);
+
+			if (localCandidate && remoteCandidate) {
+				this.emit('new-selected-candidate-pair', {
+					localCandidate,
+					remoteCandidate,
+				});
+			}
+		}
+
+		this._updated = timestamp;
+		this.emit('update');
+	}
+}

package/src/ObservedInboundTrack.ts

@@ -28,6 +28,14 @@ export type ObservedInboundTrackStats<K extends MediaKind> = ObservedInboundTrac
 	bitrate: number;
 	fractionLost: number;
 	rttInMs?: number;
+	lostPackets?: number;
+	receivedPackets?: number;
+	receivedFrames?: number;
+	decodedFrames?: number;
+	droppedFrames?: number;
+	receivedSamples?: number;
+	silentConcealedSamples?: number;
+	fractionLoss?: number;
 };
 
 // {
@@ -76,6 +84,14 @@ export class ObservedInboundTrack<Kind extends MediaKind> extends EventEmitter	{
 	private _remoteOutboundTrack?: ObservedOutboundTrack<Kind>;
 	public bitrate = -1;
 	public rttInMs = -1;
+	public lostPackets = 0;
+	public receivedPackets = 0;
+	public receivedFrames = 0;
+	public decodedFrames = 0;
+	public droppedFrames = 0;
+	public receivedSamples = 0;
+	public silentConcealedSamples = 0;
+	public fractionLoss = 0;
 	
 	private constructor(
 		private readonly _model: Models.InboundTrack,
@@ -211,19 +227,52 @@ export class ObservedInboundTrack<Kind extends MediaKind> extends EventEmitter	{
 		const lostPackets = (sample.packetsLost ?? 0) - (lastStat?.packetsLost ?? 0);
 		const receivedPackets = (sample.packetsReceived ?? 0) - (lastStat?.packetsReceived ?? 0);
 		const fractionLost = 0 < receivedPackets && 0 < lostPackets ? (lostPackets / (receivedPackets + lostPackets)) : 0;
+		let decodedFrames: number | undefined;
+		let droppedFrames: number | undefined;
+		let receivedFrames: number | undefined;
+		let silentConcealedSamples: number | undefined;
+
+		if (this.kind === 'video') {
+			const videoSample = sample as InboundVideoTrack;
+			const lastVideoStat = lastStat as InboundVideoTrack | undefined;
+
+			decodedFrames = (videoSample.framesDecoded ?? 0) - (lastVideoStat?.framesDecoded ?? 0);
+			droppedFrames = (videoSample.framesDropped ?? 0) - (lastVideoStat?.framesDropped ?? 0);
+			receivedFrames = (videoSample.framesReceived ?? 0) - (lastVideoStat?.framesReceived ?? 0);
+		} else if (this.kind === 'audio') {
+			const audioSample = sample as InboundAudioTrack;
+			const lastAudioStat = lastStat as InboundAudioTrack | undefined;
+
+			silentConcealedSamples = (audioSample.silentConcealedSamples ?? 0) - (lastAudioStat?.silentConcealedSamples ?? 0);
+		}
+
 		const stats: ObservedInboundTrackStats<Kind> = {
 			...sample,
 			fractionLost,
 			rttInMs,
 			bitrate,
 			ssrc: sample.ssrc,
+			lostPackets,
+			receivedPackets,
+			receivedFrames,
+			decodedFrames,
+			droppedFrames,
+			silentConcealedSamples,
 		};
 
 		this._stats.set(sample.ssrc, stats);
 
 		this.bitrate = [ ...this._stats.values() ].reduce((acc, stat) => acc + stat.bitrate, 0);
 		this.rttInMs = [ ...this._stats.values() ].reduce((acc, stat) => acc + (stat.rttInMs ?? 0), 0) / (this._stats.size || 1);
-		
+		this.lostPackets = [ ...this._stats.values() ].reduce((acc, stat) => acc + (stat.lostPackets ?? 0), 0);
+		this.receivedPackets = [ ...this._stats.values() ].reduce((acc, stat) => acc + (stat.receivedPackets ?? 0), 0);
+		this.receivedFrames = [ ...this._stats.values() ].reduce((acc, stat) => acc + (stat.receivedFrames ?? 0), 0);
+		this.decodedFrames = [ ...this._stats.values() ].reduce((acc, stat) => acc + (stat.decodedFrames ?? 0), 0);
+		this.droppedFrames = [ ...this._stats.values() ].reduce((acc, stat) => acc + (stat.droppedFrames ?? 0), 0);
+		this.receivedSamples = [ ...this._stats.values() ].reduce((acc, stat) => acc + (stat.receivedSamples ?? 0), 0);
+		this.silentConcealedSamples = [ ...this._stats.values() ].reduce((acc, stat) => acc + (stat.silentConcealedSamples ?? 0), 0);
+		this.fractionLoss = 0 < this.receivedPackets && 0 < this.lostPackets ? (this.lostPackets / (this.receivedPackets + this.lostPackets)) : 0;
+
 		if (executeSave) await this._save();
 
 		this._updated = timestamp;