@observertc/observer-js 0.9.11-beta → 0.9.12-beta

package/src/middlewares/VisitObservedCallsMiddleware.ts

@@ -8,31 +8,29 @@ import { visitInboundAudioTrack } from './visitors/visitInboundAudioTrack';
 import { visitOutboundAudioTrack } from './visitors/visitOutboundAudioTrack';
 import { visitOutboundVideoTrack } from './visitors/visitOutboundVideoTarck';
 import { visitInboundVideoTrack } from './visitors/visitInboundVideoTrack';
+import { InboundTrack } from '../models/Models';
+import { StorageProvider } from '../storages/StorageProvider';
+import { findRemoteMatches as findRemoteMatchFunc } from '../processes/findRemoteMatches';
 
 export const logger = createLogger('VisitObservedCallsMiddleware');
 
+const emptyMap = new Map();
+
 export function createVisitObservedCallsMiddleware(
+	storages: StorageProvider,
 	reports: ReportsCollector,
-	fetchSamples: boolean
+	fetchSamples: boolean,
+	findRemoteMatches?: boolean,
 ): Middleware<TransactionContext> {
 	const process = async (transaction: TransactionContext) => {
 		const {
 			observedCalls,
 			clients,
 			updatedPeerConnections,
-			deletedPeerConnections,
-
 			updatedInboundAudioTracks,
-			deletedInboundAudioTracks,
-
 			updatedInboundVideoTracks,
-			deletedInboundVideoTracks,
-
 			updatedOutboundAudioTracks,
-			deletedOutboundAudioTracks,
-
 			updatedOutboundVideoTracks,
-			deletedOutboundVideoTracks,
 		} = transaction;
 
 		const visitedPeerConnectionIds = new Set<string>();
@@ -40,6 +38,23 @@ export function createVisitObservedCallsMiddleware(
 		const visitedInboundVideoTrackIds = new Set<string>();
 		const visitedOutboundAudioTrackIds = new Set<string>();
 		const visitedOutboundVideoTrackIds = new Set<string>();
+
+		const {
+			remoteTrackIds,
+			remotePeerConnectionIds,
+			remoteClientIds
+		} = !findRemoteMatches ? {
+			remoteTrackIds: emptyMap,
+			remotePeerConnectionIds: emptyMap,
+			remoteClientIds: emptyMap,
+		} : await findRemoteMatchFunc(
+			storages,
+			clients,
+			new Map<string, InboundTrack>([...updatedInboundAudioTracks, ...updatedInboundVideoTracks])
+		)
+		
+		const now = BigInt(Date.now());
+		const fakeTouched = { touched: BigInt(0) };
 		
 		for (const observedCall of observedCalls.observedCalls()) {
 			const { callId, serviceId, roomId } = observedCall;
@@ -48,7 +63,6 @@ export function createVisitObservedCallsMiddleware(
 				const { clientId } = observedClient;
 				const storedClient = clients.get(clientId);
 
-				// console.warn("dskfjhdkfhsdkf", observedClient, storedClient);
 				if (!storedClient) {
 					// should not happen as client joined must have run before this
 					logger.warn('Client has not been registered');
@@ -61,7 +75,7 @@ export function createVisitObservedCallsMiddleware(
 					reports, 
 					fetchSamples
 				);
-
+				
 				for (const observedPeerConnection of observedClient.observedPeerConnections()) {
 					const { peerConnectionId } = observedPeerConnection;
 
@@ -75,7 +89,8 @@ export function createVisitObservedCallsMiddleware(
 						);
 						continue;
 					}
-
+					
+					storedPeerConnection.touched = now;
 					visitedPeerConnectionIds.add(peerConnectionId);
 
 					for (const observedInboundAudioTrack of observedPeerConnection.inboundAudioTracks()) {
@@ -85,10 +100,14 @@ export function createVisitObservedCallsMiddleware(
 							storedPeerConnection,
 							updatedInboundAudioTracks,
 							reports,
-							fetchSamples
+							fetchSamples,
+							remoteTrackIds.get(trackId),
+							remotePeerConnectionIds.get(trackId),
+							remoteClientIds.get(trackId),
 						);
 
 						visitedInboundAudioTrackIds.add(trackId);
+						(updatedInboundAudioTracks.get(trackId) ?? fakeTouched).touched = now;
 					}
 
 					for (const observedInboundVideoTrack of observedPeerConnection.inboundVideoTracks()) {
@@ -99,10 +118,14 @@ export function createVisitObservedCallsMiddleware(
 							storedPeerConnection,
 							updatedInboundVideoTracks,
 							reports,
-							fetchSamples
+							fetchSamples,
+							remoteTrackIds.get(trackId),
+							remotePeerConnectionIds.get(trackId),
+							remoteClientIds.get(trackId),
 						);
 
 						visitedInboundVideoTrackIds.add(trackId);
+						(updatedInboundVideoTracks.get(trackId) ?? fakeTouched).touched = now;
 					}
 
 					for (const observedOutboundAudioTrack of observedPeerConnection.outboundAudioTracks()) {
@@ -117,6 +140,7 @@ export function createVisitObservedCallsMiddleware(
 						);
 
 						visitedOutboundAudioTrackIds.add(trackId);
+						(updatedOutboundAudioTracks.get(trackId) ?? fakeTouched).touched = now;
 					}
 
 					for (const observedOutboundVideoTrack of observedPeerConnection.outboundVideoTracks()) {
@@ -131,120 +155,11 @@ export function createVisitObservedCallsMiddleware(
 						);
 
 						visitedOutboundVideoTrackIds.add(trackId);
+						(updatedOutboundVideoTracks.get(trackId) ?? fakeTouched).touched = now;
 					}
 				}
 			}
 		}
-
-		for (const [peerConnectionId, peerConnection] of Array.from(updatedPeerConnections.entries())) {
-			if (visitedPeerConnectionIds.has(peerConnectionId)) {
-				continue;
-			}
-			// delete PeerConnection
-			const { serviceId, callId, clientId } = peerConnection;
-
-			if (!serviceId || !callId || !clientId) {
-				continue;
-			}
-
-			const storedClient = clients.get(clientId);
-
-			updatedPeerConnections.delete(peerConnectionId);
-			deletedPeerConnections.add(peerConnectionId);
-
-			// also update the client model
-			if (storedClient) {
-				storedClient.peerConnectionIds = storedClient.peerConnectionIds.filter((pcId) => pcId !== peerConnectionId);
-			}
-		}
-
-		for (const [trackId, inboundAudioTrack] of Array.from(updatedInboundAudioTracks.entries())) {
-			if (visitedInboundAudioTrackIds.has(trackId)) {
-				continue;
-			}
-			// delete PeerConnection
-			const { serviceId, callId, clientId, peerConnectionId } = inboundAudioTrack;
-
-			if (!serviceId || !callId || !clientId || !peerConnectionId) {
-				continue;
-			}
-
-			const storedPeerConnection = updatedPeerConnections.get(peerConnectionId);
-
-			updatedInboundAudioTracks.delete(peerConnectionId);
-			deletedInboundAudioTracks.add(peerConnectionId);
-
-			// also update the peer connection model
-			if (storedPeerConnection) {
-				storedPeerConnection.inboundTrackIds = storedPeerConnection.inboundTrackIds.filter((tId) => tId !== trackId);
-			}
-		}
-
-		for (const [trackId, inboundVideoTrack] of Array.from(updatedInboundVideoTracks.entries())) {
-			if (visitedInboundVideoTrackIds.has(trackId)) {
-				continue;
-			}
-			// delete PeerConnection
-			const { serviceId, callId, clientId, peerConnectionId } = inboundVideoTrack;
-
-			if (!serviceId || !callId || !clientId || !peerConnectionId) {
-				continue;
-			}
-
-			const storedPeerConnection = updatedPeerConnections.get(peerConnectionId);
-
-			updatedInboundVideoTracks.delete(peerConnectionId);
-			deletedInboundVideoTracks.add(peerConnectionId);
-
-			// also update the peer connection model
-			if (storedPeerConnection) {
-				storedPeerConnection.inboundTrackIds = storedPeerConnection.inboundTrackIds.filter((tId) => tId !== trackId);
-			}
-		}
-
-		for (const [trackId, OutboundAudioTrack] of Array.from(updatedOutboundAudioTracks.entries())) {
-			if (visitedOutboundAudioTrackIds.has(trackId)) {
-				continue;
-			}
-			// delete PeerConnection
-			const { serviceId, callId, clientId, peerConnectionId } = OutboundAudioTrack;
-
-			if (!serviceId || !callId || !clientId || !peerConnectionId) {
-				continue;
-			}
-
-			const storedPeerConnection = updatedPeerConnections.get(peerConnectionId);
-
-			updatedOutboundAudioTracks.delete(peerConnectionId);
-			deletedOutboundAudioTracks.add(peerConnectionId);
-
-			// also update the peer connection model
-			if (storedPeerConnection) {
-				storedPeerConnection.outboundTrackIds = storedPeerConnection.outboundTrackIds.filter((tId) => tId !== trackId);
-			}
-		}
-
-		for (const [trackId, OutboundVideoTrack] of Array.from(updatedOutboundVideoTracks.entries())) {
-			if (visitedOutboundVideoTrackIds.has(trackId)) {
-				continue;
-			}
-			// delete PeerConnection
-			const { serviceId, callId, clientId, peerConnectionId } = OutboundVideoTrack;
-
-			if (!serviceId || !callId || !clientId || !peerConnectionId) {
-				continue;
-			}
-
-			const storedPeerConnection = updatedPeerConnections.get(peerConnectionId);
-
-			updatedOutboundVideoTracks.delete(peerConnectionId);
-			deletedOutboundVideoTracks.add(peerConnectionId);
-
-			// also update the peer connection model
-			if (storedPeerConnection) {
-				storedPeerConnection.outboundTrackIds = storedPeerConnection.outboundTrackIds.filter((tId) => tId !== trackId);
-			}
-		}
 	};
 	const result = async (context: TransactionContext, next?: Middleware<TransactionContext>) => {
 		await process(context);

package/src/middlewares/visitors/visitClient.ts

@@ -15,7 +15,7 @@ export function visitClient(
 	reports: ReportsCollector,
 	fetchSamples: boolean
 ) {
-	const { mediaUnitId, clientId } = observedClient;
+	const { mediaUnitId, clientId, userId } = observedClient;
 
 	const { callId, serviceId, roomId } = observedClient.call;
 
@@ -34,7 +34,7 @@ export function visitClient(
 			const callMetaReport = createCallMetaReport(serviceId, mediaUnitId, roomId, callId, clientId, {
 				type: CallMetaType.OPERATION_SYSTEM,
 				payload: clientSample.os,
-			});
+			}, userId);
 			reports.addCallMetaReport(callMetaReport);
 		}
 
@@ -51,7 +51,7 @@ export function visitClient(
 			const callMetaReport = createCallMetaReport(serviceId, mediaUnitId, roomId, callId, clientId, {
 				type: CallMetaType.ENGINE,
 				payload: clientSample.engine,
-			});
+			}, userId);
 			reports.addCallMetaReport(callMetaReport);
 		}
 
@@ -69,7 +69,7 @@ export function visitClient(
 			const callMetaReport = createCallMetaReport(serviceId, mediaUnitId, roomId, callId, clientId, {
 				type: CallMetaType.PLATFORM,
 				payload: clientSample.platform,
-			});
+			}, userId);
 			reports.addCallMetaReport(callMetaReport);
 		}
 
@@ -86,7 +86,7 @@ export function visitClient(
 			const callMetaReport = createCallMetaReport(serviceId, mediaUnitId, roomId, callId, clientId, {
 				type: CallMetaType.BROWSER,
 				payload: clientSample.browser,
-			});
+			}, userId);
 			reports.addCallMetaReport(callMetaReport);
 		}
 
@@ -95,7 +95,7 @@ export function visitClient(
 				const callMetaReport = createCallMetaReport(serviceId, mediaUnitId, roomId, callId, clientId, {
 					type: CallMetaType.MEDIA_CONSTRAINT,
 					payload: mediaConstraint,
-				});
+				}, userId);
 				reports.addCallMetaReport(callMetaReport);
 			}
 		}
@@ -106,7 +106,7 @@ export function visitClient(
 				const callMetaReport = createCallMetaReport(serviceId, mediaUnitId, roomId, callId, clientId, {
 					type: CallMetaType.LOCAL_SDP,
 					payload: localSDP,
-				});
+				}, userId);
 				reports.addCallMetaReport(callMetaReport);
 			}
 		}
@@ -119,6 +119,7 @@ export function visitClient(
 					roomId,
 					callId,
 					clientId,
+					userId,
 					timestamp: Date.now(),
 					payload: extensionStats.payload,
 					extensionType: extensionStats.type,
@@ -134,6 +135,7 @@ export function visitClient(
 					roomId,
 					callId,
 					clientId,
+					userId,
 					timestamp: Date.now(),
 					...callEvent,
 				});
@@ -145,7 +147,7 @@ export function visitClient(
 				const callMetaReport = createCallMetaReport(serviceId, mediaUnitId, roomId, callId, clientId, {
 					type: CallMetaType.USER_MEDIA_ERROR,
 					payload: userMediaError,
-				});
+				}, userId);
 				reports.addCallMetaReport(callMetaReport);
 			}
 		}
@@ -155,7 +157,7 @@ export function visitClient(
 				const callMetaReport = createCallMetaReport(serviceId, mediaUnitId, roomId, callId, clientId, {
 					type: CallMetaType.CERTIFICATE,
 					payload: certificate,
-				});
+				}, userId);
 				reports.addCallMetaReport(callMetaReport);
 			}
 		}
@@ -165,7 +167,7 @@ export function visitClient(
 				const callMetaReport = createCallMetaReport(serviceId, mediaUnitId, roomId, callId, clientId, {
 					type: CallMetaType.CODEC,
 					payload: codec,
-				});
+				}, userId);
 				reports.addCallMetaReport(callMetaReport);
 			}
 		}
@@ -175,7 +177,7 @@ export function visitClient(
 				const callMetaReport = createCallMetaReport(serviceId, mediaUnitId, roomId, callId, clientId, {
 					type: CallMetaType.ICE_SERVER,
 					payload: iceServer,
-				});
+				}, userId);
 				reports.addCallMetaReport(callMetaReport);
 			}
 		}
@@ -185,7 +187,7 @@ export function visitClient(
 				const callMetaReport = createCallMetaReport(serviceId, mediaUnitId, roomId, callId, clientId, {
 					type: CallMetaType.MEDIA_DEVICE,
 					payload: mediaDevice,
-				});
+				}, userId);
 				reports.addCallMetaReport(callMetaReport);
 			}
 		}
@@ -195,7 +197,7 @@ export function visitClient(
 				const callMetaReport = createCallMetaReport(serviceId, mediaUnitId, roomId, callId, clientId, {
 					type: CallMetaType.MEDIA_SOURCE,
 					payload: mediaSource,
-				});
+				}, userId);
 				reports.addCallMetaReport(callMetaReport);
 			}
 		}

package/src/middlewares/visitors/visitInboundAudioTrack.ts

@@ -10,9 +10,12 @@ export function visitInboundAudioTrack(
 	storedPeerConnection: Models.PeerConnection,
 	storedInboundTracks: Map<string, Models.InboundTrack>,
 	reports: ReportsCollector,
-	fetchSamples: boolean
+	fetchSamples: boolean,
+	remoteTrackId?: string,
+	remotePeerConnectionId?: string,
+	remoteClientId?: string,
 ) {
-	const { trackId } = observedInboundAudioTrack;
+	const { trackId, sfuStreamId, sfuSinkId } = observedInboundAudioTrack;
 
 	const { peerConnectionId } = observedInboundAudioTrack.peerConnection;
 
@@ -25,7 +28,7 @@ export function visitInboundAudioTrack(
 	} = observedInboundAudioTrack.peerConnection.client;
 
 	const { serviceId, roomId, callId } = observedInboundAudioTrack.peerConnection.client.call;
-
+	
 	let storedInboundAudioTrack = storedInboundTracks.get(trackId);
 	if (!storedInboundAudioTrack) {
 		storedInboundAudioTrack = new Models.InboundTrack({
@@ -33,10 +36,13 @@ export function visitInboundAudioTrack(
 			roomId,
 			callId,
 			clientId,
+			kind: 'audio',
 			peerConnectionId,
 			mediaUnitId,
 			trackId,
-
+			sfuSinkId,
+			sfuStreamId,
+			
 			userId,
 			marker,
 		});
@@ -58,11 +64,15 @@ export function visitInboundAudioTrack(
 			roomId,
 			callId,
 			clientId,
+			userId,
 			mediaUnitId,
 			peerConnectionId,
 			...inboundAudioSample,
 			timestamp,
 			sampleSeq: -1,
+			remoteClientId,
+			remotePeerConnectionId,
+			remoteTrackId,
 		};
 		reports.addInboundAudioTrackReport(report);
 
@@ -93,6 +103,6 @@ export function visitInboundAudioTrack(
 			statsMap.set(ssrc, videoStats);
 		}
 	}
-
+	storedInboundAudioTrack.ssrc = [...statsMap.keys()];
 	storedInboundAudioTrack.videoStats = Array.from(statsMap.values());
 }

package/src/middlewares/visitors/visitInboundVideoTrack.ts

@@ -12,9 +12,12 @@ export function visitInboundVideoTrack(
 	storedPeerConnection: Models.PeerConnection,
 	storedInboundTracks: Map<string, Models.InboundTrack>,
 	reports: ReportsCollector,
-	fetchSamples: boolean
+	fetchSamples: boolean,
+	remoteTrackId?: string,
+	remotePeerConnectionId?: string,
+	remoteClientId?: string,
 ) {
-	const { trackId } = observedInboundVideoTrack;
+	const { trackId, sfuStreamId, sfuSinkId } = observedInboundVideoTrack;
 
 	const { peerConnectionId } = observedInboundVideoTrack.peerConnection;
 
@@ -35,9 +38,13 @@ export function visitInboundVideoTrack(
 			roomId,
 			callId,
 			clientId,
+			kind: 'video',
 			peerConnectionId,
 			mediaUnitId,
 			trackId,
+			
+			sfuStreamId, 
+			sfuSinkId,
 
 			userId,
 			marker,
@@ -59,11 +66,15 @@ export function visitInboundVideoTrack(
 			roomId,
 			callId,
 			clientId,
+			userId,
 			mediaUnitId,
 			peerConnectionId,
 			...inboundVideoSample,
 			timestamp,
 			sampleSeq: -1,
+			remoteClientId,
+			remotePeerConnectionId,
+			remoteTrackId,
 		};
 		reports.addInboundVideoTrackReport(report);
 
@@ -96,5 +107,6 @@ export function visitInboundVideoTrack(
 			statsMap.set(ssrc, videoStats);
 		}
 	}
+	storedInboundVideoTrack.ssrc = [...statsMap.keys()];
 	storedInboundVideoTrack.videoStats = Array.from(statsMap.values());
 }

package/src/middlewares/visitors/visitOutboundAudioTrack.ts

@@ -12,7 +12,7 @@ export function visitOutboundAudioTrack(
 	reports: ReportsCollector,
 	fetchSamples: boolean
 ) {
-	const { trackId } = observedOutboundAudioTrack;
+	const { trackId, sfuStreamId } = observedOutboundAudioTrack;
 
 	const { peerConnectionId } = observedOutboundAudioTrack.peerConnection;
 
@@ -33,10 +33,12 @@ export function visitOutboundAudioTrack(
 			roomId,
 			callId,
 			clientId,
+			kind: 'audio',
 			peerConnectionId,
 			mediaUnitId,
 			trackId,
-
+			
+			sfuStreamId,
 			userId,
 			marker,
 		});
@@ -57,6 +59,7 @@ export function visitOutboundAudioTrack(
 			roomId,
 			callId,
 			clientId,
+			userId,
 			mediaUnitId,
 			peerConnectionId,
 			...outboundAudioSample,
@@ -88,5 +91,6 @@ export function visitOutboundAudioTrack(
 			statsMap.set(ssrc, audioStats);
 		}
 	}
+	storedOutboundAudioTrack.ssrc = [...statsMap.keys()];
 	storedOutboundAudioTrack.audioStats = Array.from(statsMap.values());
 }

package/src/middlewares/visitors/visitOutboundVideoTarck.ts

@@ -12,7 +12,7 @@ export function visitOutboundVideoTrack(
 	reports: ReportsCollector,
 	fetchSamples: boolean
 ) {
-	const { trackId } = observedOutboundVideoTrack;
+	const { trackId, sfuStreamId } = observedOutboundVideoTrack;
 
 	const { peerConnectionId } = observedOutboundVideoTrack.peerConnection;
 
@@ -33,10 +33,12 @@ export function visitOutboundVideoTrack(
 			roomId,
 			callId,
 			clientId,
+			kind: 'video',
 			peerConnectionId,
 			mediaUnitId,
 			trackId,
 
+			sfuStreamId,
 			userId,
 			marker,
 		});
@@ -57,6 +59,7 @@ export function visitOutboundVideoTrack(
 			roomId,
 			callId,
 			clientId,
+			userId,
 			mediaUnitId,
 			peerConnectionId,
 			...outboundVideoSample,
@@ -90,5 +93,6 @@ export function visitOutboundVideoTrack(
 			statsMap.set(ssrc, videoStats);
 		}
 	}
+	storedOutboundVideoTrack.ssrc = [...statsMap.keys()];
 	storedOutboundVideoTrack.videoStats = Array.from(statsMap.values());
 }

package/src/middlewares/visitors/visitPeerConnection.ts

@@ -2,7 +2,7 @@ import { ObservedPeerConnection } from '../../samples/ObservedPeerConnection';
 import * as Models from '../../models/Models';
 import { ReportsCollector } from '../../common/ReportsCollector';
 import { logger } from '../VisitObservedCallsMiddleware';
-import { PeerConnectionTransportReport } from '@observertc/report-schemas-js';
+import { IceCandidatePairReport, PeerConnectionTransportReport } from '@observertc/report-schemas-js';
 import { Samples_ClientSample_IceCandidatePair, Samples_ClientSample_IceLocalCandidate, Samples_ClientSample_IceRemoteCandidate, Samples_ClientSample_PeerConnectionTransport } from '../../models/samples_pb';
 
 export function visitPeerConnection(
@@ -47,6 +47,7 @@ export function visitPeerConnection(
 			roomId,
 			callId,
 			clientId,
+			userId,
 			mediaUnitId,
 			...pcTransport,
 			timestamp,
@@ -55,6 +56,21 @@ export function visitPeerConnection(
 		reports.addPeerConnectionTransportReports(report);
 	}
 
+	for (const iceCandidatePair of observedPeerConnection.iceCandidatePairs()) {
+		const report: IceCandidatePairReport = {
+			serviceId,
+			roomId,
+			callId,
+			userId,
+			clientId,
+			mediaUnitId,
+			...iceCandidatePair,
+			timestamp,
+			sampleSeq: -1, // deprecated
+		};
+		reports.addIceCandidatePairReport(report);
+	}
+
 	if (fetchSamples) {
 		storedPeerConnection.icelocalCandidates = [];
 		for (const iceLocalCandidate of observedPeerConnection.iceLocalCandidates()) {