@observertc/observer-js 0.9.11-beta → 0.9.12-beta

package/lib/samples/ObservedSfus.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ObservedSfusBuilder = void 0;
+const utils_1 = require("../common/utils");
 const ObservedSfu_1 = require("./ObservedSfu");
 class ObservedSfusBuilder {
     constructor() {
@@ -18,19 +19,21 @@ class ObservedSfusBuilder {
     }
     build() {
         const observedSfus = new Map();
-        const sfuIds = function* () {
+        const sfuIdsGenerator = function* () {
             for (const observedSfu of observedSfus.values()) {
                 yield observedSfu.sfuId;
             }
         };
-        const sfuTransportIds = function* () {
+        const sfuIds = () => (0, utils_1.iteratorConverter)(sfuIdsGenerator());
+        const sfuTransportIdsGenerator = function* () {
             for (const observedSfu of observedSfus.values()) {
                 for (const observedTransport of observedSfu.observedSfuTransports()) {
                     yield observedTransport.transportId;
                 }
             }
         };
-        const sfuInboundRtpPadIds = function* () {
+        const sfuTransportIds = () => (0, utils_1.iteratorConverter)(sfuTransportIdsGenerator());
+        const sfuInboundRtpPadIdsGenerator = function* () {
             for (const observedSfu of observedSfus.values()) {
                 for (const observedTransport of observedSfu.observedSfuTransports()) {
                     for (const sfuInboundRtpPad of observedTransport.inboundRtpPads()) {
@@ -39,7 +42,8 @@ class ObservedSfusBuilder {
                 }
             }
         };
-        const sfuOutboundRtpPadIds = function* () {
+        const sfuInboundRtpPadIds = () => (0, utils_1.iteratorConverter)(sfuInboundRtpPadIdsGenerator());
+        const sfuOutboundRtpPadIdsGenerator = function* () {
             for (const observedSfu of observedSfus.values()) {
                 for (const observedTransport of observedSfu.observedSfuTransports()) {
                     for (const sfuOutboundRtpPad of observedTransport.outboundRtpPads()) {
@@ -48,6 +52,7 @@ class ObservedSfusBuilder {
                 }
             }
         };
+        const sfuOutboundRtpPadIds = () => (0, utils_1.iteratorConverter)(sfuOutboundRtpPadIdsGenerator());
         const result = {
             sfuIds,
             sfuTransportIds,
@@ -60,6 +65,7 @@ class ObservedSfusBuilder {
             const observedSfu = builder.build();
             observedSfus.set(observedSfu.sfuId, observedSfu);
         }
+        this._builders.clear();
         return result;
     }
 }

package/lib/storages/ObserverStorage.d.ts

@@ -79,5 +79,9 @@ export interface ObserverStorage<K, V> {
      * Returns an async iterable iterator for the key-value pairs in the storage.
      */
     [Symbol.asyncIterator](): AsyncIterableIterator<[K, V]>;
+    /**
+     * Returns an iterable iterator for the key-value pairs stored in this local instance of the storage.
+     */
+    localEntries(): IterableIterator<[K, V]>;
 }
 //# sourceMappingURL=ObserverStorage.d.ts.map

package/lib/storages/ObserverStorage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ObserverStorage.d.ts","sourceRoot":"","sources":["../../src/storages/ObserverStorage.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,WAAW,eAAe,CAAC,CAAC,EAAE,CAAC;IACpC;;OAEG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAExB;;OAEG;IACH,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvB;;;;;OAKG;IACH,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;IAEpC;;;;;OAKG;IACH,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAEtD;;;;;;OAMG;IACH,GAAG,CAAC,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;IAE9C;;;;;OAKG;IACH,MAAM,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAE/D;;;;;;OAMG;IACH,MAAM,CAAC,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;IAEjD;;;;;;;;OAQG;IACH,SAAS,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;IAE9E;;;;;OAKG;IACH,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;IAEvC;;;;;OAKG;IACH,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAEzD;;OAEG;IACH,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,qBAAqB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;CACxD"}
+{"version":3,"file":"ObserverStorage.d.ts","sourceRoot":"","sources":["../../src/storages/ObserverStorage.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,MAAM,WAAW,eAAe,CAAC,CAAC,EAAE,CAAC;IACpC;;OAEG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAExB;;OAEG;IACH,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvB;;;;;OAKG;IACH,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;IAEpC;;;;;OAKG;IACH,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAEtD;;;;;;OAMG;IACH,GAAG,CAAC,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;IAE9C;;;;;OAKG;IACH,MAAM,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAE/D;;;;;;OAMG;IACH,MAAM,CAAC,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;IAEjD;;;;;;;;OAQG;IACH,SAAS,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;IAE9E;;;;;OAKG;IACH,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;IAEvC;;;;;OAKG;IACH,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAEzD;;OAEG;IACH,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,qBAAqB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAExD;;OAEG;IACH,YAAY,IAAI,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;CACzC"}

package/lib/storages/SimpleStorage.d.ts

@@ -14,5 +14,6 @@ export declare class SimpleStorage<K, V> implements ObserverStorage<K, V> {
     remove(key: K): Promise<V | undefined>;
     removeAll(keys: Iterable<K>): Promise<ReadonlyMap<K, V>>;
     [Symbol.asyncIterator](): AsyncIterableIterator<[K, V]>;
+    localEntries(): IterableIterator<[K, V]>;
 }
 //# sourceMappingURL=SimpleStorage.d.ts.map

package/lib/storages/SimpleStorage.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"SimpleStorage.d.ts","sourceRoot":"","sources":["../../src/storages/SimpleStorage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAEpD,qBAAa,aAAa,CAAC,CAAC,EAAE,CAAC,CAAE,YAAW,eAAe,CAAC,CAAC,EAAE,CAAC,CAAC;aAG7B,EAAE,EAAE,MAAM;IAF7C,OAAO,CAAC,IAAI,CAAmB;gBAEI,EAAE,EAAE,MAAM;IAEhC,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC;IAIvB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAItB,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC;IAInC,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAWrD,GAAG,CAAC,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC;IAM7C,MAAM,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAY9D,MAAM,CAAC,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC;IAQhD,SAAS,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAajE,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC;IAMtC,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAYvD,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,qBAAqB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;CAKrE"}
+{"version":3,"file":"SimpleStorage.d.ts","sourceRoot":"","sources":["../../src/storages/SimpleStorage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAEpD,qBAAa,aAAa,CAAC,CAAC,EAAE,CAAC,CAAE,YAAW,eAAe,CAAC,CAAC,EAAE,CAAC,CAAC;aAG7B,EAAE,EAAE,MAAM;IAF7C,OAAO,CAAC,IAAI,CAAmB;gBAEI,EAAE,EAAE,MAAM;IAEhC,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC;IAIvB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAItB,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC;IAInC,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAWrD,GAAG,CAAC,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC;IAM7C,MAAM,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAY9D,MAAM,CAAC,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC;IAQhD,SAAS,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAajE,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC;IAMtC,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAYvD,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,qBAAqB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAM9D,YAAY,IAAI,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;CAG/C"}

package/lib/storages/SimpleStorage.js

@@ -82,5 +82,8 @@ class SimpleStorage {
             yield entry;
         }
     }
+    localEntries() {
+        return this._map.entries();
+    }
 }
 exports.SimpleStorage = SimpleStorage;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@observertc/observer-js",
-  "version": "0.9.11-beta",
+  "version": "0.9.12-beta",
   "description": "Server Side NodeJS Library for processing ObserveRTC Samples",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",

package/src/Evaluator.ts

@@ -21,6 +21,8 @@ export type EvaluatorEvents = {
 
 export type EvaluatorConfig = {
 	fetchSamples: boolean;
+	maxIdleTimeInMs?: number,
+	findRemoteMatches?: boolean,
 };
 
 export type EvaluatorProcess = (evaluatorContext: EvaluatorContext) => Promise<void>;
@@ -50,7 +52,9 @@ export class Evaluator {
 		this._transactionProcessor = createTransactionProcessor(
 			_storageProvider,
 			_reportsCollector,
-			this.config.fetchSamples
+			this.config.fetchSamples,
+			BigInt(this.config.maxIdleTimeInMs ?? 10000),
+			this.config.findRemoteMatches,
 		);
 
 		this._customProcessor = createProcessor();
@@ -106,7 +110,7 @@ export class Evaluator {
 	public addObservedSamples(samples: SourcesEvents['observed-samples']): void {
 		// this triggers the evaluation
 		const { observedCalls, observedSfus } = samples;
-
+		
 		const evaluatorContext: EvaluatorContext = {
 			observedCalls,
 			observedSfus,
@@ -142,7 +146,7 @@ export class Evaluator {
 			sfuEvents: [],
 			sfuExtensionStats: [],
 		};
-
+		
 		const callOperations = createCallOperationContext(this._clientOperations, evaluatorContext);
 		this._clientOperations.clear();
 

package/src/Observer.ts

@@ -53,6 +53,7 @@ export class Observer {
 				},
 				evaluator: {
 					fetchSamples: true,
+					maxIdleTimeInMs: 300 * 1000,
 				},
 				sink: {},
 				logLevel: 'info',

package/src/common/callMetaReports.ts

@@ -82,6 +82,7 @@ export function createCallMetaReport(
 	callId: string,
 	clientId: string,
 	reportType: CallMetaReportType,
+	userId?: string,
 	timestamp?: number
 ) {
 	const report: CallMetaReport = {
@@ -91,6 +92,7 @@ export function createCallMetaReport(
 		roomId,
 		callId,
 		clientId,
+		userId,
 		payload: JSON.stringify(reportType.payload),
 		timestamp: timestamp ?? Date.now(),
 	};

package/src/common/utils.ts

@@ -1,3 +1,14 @@
 export type PartialBy<T, K extends keyof T> = Omit<T, K> & Partial<Pick<T, K>>;
 
 export type Writable<T> = T extends object ? { -readonly [K in keyof T]: Writable<T[K]> } : T;
+
+export function iteratorConverter<T>(generator: Generator<T, void, undefined>): IterableIterator<T> {
+	return {
+		[Symbol.iterator](): IterableIterator<T> {
+			return this;
+		  },
+		  next(): IteratorResult<T> {
+			return generator.next();
+		  },
+	}
+}

package/src/middlewares/DeleteOutdatedCallEntries.ts

@@ -0,0 +1,151 @@
+import { Middleware } from './Middleware';
+import { TransactionContext } from './TransactionContext';
+import { createLogger } from '../common/logger';
+import { StorageProvider } from '..';
+
+export const logger = createLogger('DeleteOutdatedCallEntries');
+
+export function createDeleteOutdatedCallEntries(
+	storages: StorageProvider,
+	maxIdleTimeInMs: BigInt,
+): Middleware<TransactionContext> {
+	const process = async (transaction: TransactionContext) => {
+		const {
+			clients,
+			
+			evaluatorContext,
+			
+			updatedPeerConnections,
+			deletedPeerConnections,
+
+			updatedInboundAudioTracks,
+			deletedInboundAudioTracks,
+
+			updatedInboundVideoTracks,
+			deletedInboundVideoTracks,
+
+			updatedOutboundAudioTracks,
+			deletedOutboundAudioTracks,
+
+			updatedOutboundVideoTracks,
+			deletedOutboundVideoTracks,
+		} = transaction;
+
+		const now = BigInt(Date.now());
+		const deletedCallIds = new Set(evaluatorContext.endedCalls.map(c => c.callId).filter(id => !!id))
+		const deletedClientIds = new Set(evaluatorContext.detachedClients.map(c => c.clientId).filter(id => !!id));
+		for (const [peerConnectionId, peerConnection] of storages.peerConnectionStorage.localEntries()) {
+			if (updatedPeerConnections.has(peerConnectionId)) {
+				continue;
+			}
+			
+			const { touched, clientId, callId } = peerConnection;
+			if (touched && now - touched < maxIdleTimeInMs) {
+				if (
+					!deletedCallIds.has(callId ?? "not exists") &&
+					!deletedClientIds.has(clientId ?? "not exists")
+				){
+					continue;
+				}
+			}
+
+			deletedPeerConnections.add(peerConnectionId);
+
+			if (clientId) {
+				let storedClient = clients.get(clientId);
+				if (storedClient) {
+					storedClient.peerConnectionIds = storedClient.peerConnectionIds.filter((pcId) => pcId !== peerConnectionId);
+				} else {
+					storedClient = await storages.clientStorage.get(clientId);
+					if (storedClient) {
+						storedClient.peerConnectionIds = storedClient.peerConnectionIds.filter((pcId) => pcId !== peerConnectionId);
+						await storages.clientStorage.set(clientId, storedClient);
+					}
+				}
+			}
+		}
+
+		for (const [trackId, inboundTrack] of storages.inboundTrackStorage.localEntries()) {
+			if (updatedInboundAudioTracks.has(trackId) || updatedInboundVideoTracks.has(trackId)) {
+				continue;
+			}
+
+			const { touched, clientId, callId, peerConnectionId } = inboundTrack;
+			if (touched && now - touched < maxIdleTimeInMs) {
+				if (
+					!deletedCallIds.has(callId ?? "not exists") &&
+					!deletedClientIds.has(clientId ?? "not exists") && 
+					!deletedPeerConnections.has(peerConnectionId ?? "not exists")
+				){
+					continue;
+				}
+			}
+
+			switch (inboundTrack.kind) {
+				case 'audio':
+					deletedInboundAudioTracks.add(trackId);
+					break;
+				case 'video':
+					deletedInboundVideoTracks.add(trackId);
+			}
+
+			if (peerConnectionId) {
+				let storedPeerConnection = updatedPeerConnections.get(peerConnectionId);
+				if (!storedPeerConnection) {
+					storedPeerConnection = await storages.peerConnectionStorage.get(peerConnectionId);
+					if (storedPeerConnection) {
+						updatedPeerConnections.set(peerConnectionId, storedPeerConnection);
+					}
+				}
+				if (storedPeerConnection) {
+					storedPeerConnection.inboundTrackIds = storedPeerConnection.inboundTrackIds.filter((tId) => tId !== trackId);
+				}
+			}
+		}
+
+		// console.warn("auodiaussdoi", updatedOutboundAudioTracks, updatedOutboundVideoTracks);
+		for (const [trackId, outboundTrack] of storages.outboundTrackStorage.localEntries()) {
+			if (updatedOutboundAudioTracks.has(trackId) || updatedOutboundVideoTracks.has(trackId)) {
+				continue;
+			}
+
+			const { touched, clientId, callId, peerConnectionId } = outboundTrack;
+			// console.warn("deprecated outb tracks", outboundTrack, now, touched, maxIdleTimeInMs, touched && now - touched < maxIdleTimeInMs, callId)
+			if (touched && now - touched < maxIdleTimeInMs) {
+				if (
+					!deletedCallIds.has(callId ?? "not exists") &&
+					!deletedClientIds.has(clientId ?? "not exists") && 
+					!deletedPeerConnections.has(peerConnectionId ?? "not exists")
+				){
+					continue;
+				}
+			}
+
+			switch (outboundTrack.kind) {
+				case 'audio':
+					deletedOutboundAudioTracks.add(trackId);
+					break;
+				case 'video':
+					deletedOutboundVideoTracks.add(trackId);
+			}
+			
+			if (peerConnectionId) {
+				let storedPeerConnection = updatedPeerConnections.get(peerConnectionId);
+				if (!storedPeerConnection) {
+					storedPeerConnection = await storages.peerConnectionStorage.get(peerConnectionId);
+					if (storedPeerConnection) {
+						updatedPeerConnections.set(peerConnectionId, storedPeerConnection);
+					}
+				}
+				if (storedPeerConnection) {
+					storedPeerConnection.outboundTrackIds = storedPeerConnection.outboundTrackIds.filter((tId) => tId !== trackId);
+				}
+			}
+		}
+	};
+	const result = async (context: TransactionContext, next?: Middleware<TransactionContext>) => {
+		await process(context);
+		if (next) await next(context);
+	};
+	return result;
+}

package/src/middlewares/DeleteOutdatedSfuEntries.ts

@@ -0,0 +1,159 @@
+import { Middleware } from './Middleware';
+import { TransactionContext } from './TransactionContext';
+import { createLogger } from '../common/logger';
+import { StorageProvider } from '..';
+
+export const logger = createLogger('DeleteOutdatedSfuEntries');
+
+export function createDeleteOutdatedSfuEntries(
+	storages: StorageProvider,
+	maxIdleTimeInMs: BigInt,
+): Middleware<TransactionContext> {
+	const process = async (transaction: TransactionContext) => {
+		const {
+			updatedSfus,
+			deletedSfus,
+			
+			updatedSfuTransports,
+			deletedSfuTransports,
+
+			updatedSfuInboundRtpPads,
+			deletedSfuInboundRtpPads,
+
+			updatedSfuOutboundRtpPads,
+			deletedSfuOutboundRtpPads,
+
+			updatedSfuSctpChannels,
+			deletedSfuSctpChannels
+
+		} = transaction;
+
+		const now = BigInt(Date.now());
+		for (const [sfuTransportId, sfuTransport] of storages.sfuTransportStorage.localEntries()) {
+			if (updatedSfuTransports.has(sfuTransportId)) {
+				continue;
+			}
+			
+			const { touched, sfuId } = sfuTransport;
+			if (touched && now - touched < maxIdleTimeInMs) {
+				if (!deletedSfus.has(sfuId ?? "not exists")){
+					continue;
+				}
+			}
+
+			deletedSfuTransports.add(sfuTransportId);
+
+			if (sfuId) {
+				let storedSfu = updatedSfus.get(sfuId);
+				if (storedSfu) {
+					storedSfu.sfuTransportIds = storedSfu.sfuTransportIds.filter((transportId) => transportId !== sfuTransportId);
+				} else {
+					storedSfu = await storages.sfuStorage.get(sfuId);
+					if (storedSfu) {
+						storedSfu.sfuTransportIds = storedSfu.sfuTransportIds.filter((transportId) => transportId !== sfuTransportId);
+						await storages.sfuStorage.set(sfuId, storedSfu);
+					}
+				}
+			}
+		}
+
+		for (const [sfuInboundPadId, sfuInboundPad] of storages.sfuInboundRtpPadStorage.localEntries()) {
+			if (updatedSfuInboundRtpPads.has(sfuInboundPadId)) {
+				continue;
+			}
+
+			const { touched, sfuTransportId, sfuId } = sfuInboundPad;
+			if (touched && now - touched < maxIdleTimeInMs) {
+				if (
+					!deletedSfuTransports.has(sfuTransportId ?? "not exists") && 
+					!deletedSfus.has(sfuId ?? "not exists")
+				){
+					continue;
+				}
+			}
+
+			deletedSfuInboundRtpPads.add(sfuInboundPadId);
+
+			if (sfuTransportId) {
+				let storedSfuTransport = updatedSfuTransports.get(sfuTransportId);
+				if (!storedSfuTransport) {
+					storedSfuTransport = await storages.sfuTransportStorage.get(sfuTransportId);
+					if (storedSfuTransport) {
+						updatedSfuTransports.set(sfuTransportId, storedSfuTransport);
+					}
+				}
+				if (storedSfuTransport) {
+					storedSfuTransport.inboundRtpPadIds = storedSfuTransport.inboundRtpPadIds.filter((tId) => tId !== sfuInboundPadId);
+				}
+			}
+		}
+
+
+		for (const [sfuOutboundPadId, sfuOutboundPad] of storages.sfuOutboundRtpPadStorage.localEntries()) {
+			if (updatedSfuOutboundRtpPads.has(sfuOutboundPadId)) {
+				continue;
+			}
+
+			const { touched, sfuTransportId, sfuId } = sfuOutboundPad;
+			if (touched && now - touched < maxIdleTimeInMs) {
+				if (
+					!deletedSfuTransports.has(sfuTransportId ?? "not exists") && 
+					!deletedSfus.has(sfuId ?? "not exists")
+				){
+					continue;
+				}
+			}
+
+			deletedSfuOutboundRtpPads.add(sfuOutboundPadId);
+
+			if (sfuTransportId) {
+				let storedSfuTransport = updatedSfuTransports.get(sfuTransportId);
+				if (!storedSfuTransport) {
+					storedSfuTransport = await storages.sfuTransportStorage.get(sfuTransportId);
+					if (storedSfuTransport) {
+						updatedSfuTransports.set(sfuTransportId, storedSfuTransport);
+					}
+				}
+				if (storedSfuTransport) {
+					storedSfuTransport.outboundRtpPadIds = storedSfuTransport.outboundRtpPadIds.filter((tId) => tId !== sfuOutboundPadId);
+				}
+			}
+		}
+
+		for (const [sctpChannelId, sctpChannel] of storages.sfuSctpChannelStorage.localEntries()) {
+			if (updatedSfuSctpChannels.has(sctpChannelId)) {
+				continue;
+			}
+
+			const { touched, sfuTransportId, sfuId } = sctpChannel;
+			if (touched && now - touched < maxIdleTimeInMs) {
+				if (
+					!deletedSfuTransports.has(sfuTransportId ?? "not exists") && 
+					!deletedSfus.has(sfuId ?? "not exists")
+				){
+					continue;
+				}
+			}
+
+			deletedSfuSctpChannels.add(sctpChannelId);
+
+			if (sfuTransportId) {
+				let storedSfuTransport = updatedSfuTransports.get(sfuTransportId);
+				if (!storedSfuTransport) {
+					storedSfuTransport = await storages.sfuTransportStorage.get(sfuTransportId);
+					if (storedSfuTransport) {
+						updatedSfuTransports.set(sfuTransportId, storedSfuTransport);
+					}
+				}
+				if (storedSfuTransport) {
+					storedSfuTransport.sctpChannelIds = storedSfuTransport.sctpChannelIds.filter((tId) => tId !== sctpChannelId);
+				}
+			}
+		}
+	};
+	const result = async (context: TransactionContext, next?: Middleware<TransactionContext>) => {
+		await process(context);
+		if (next) await next(context);
+	};
+	return result;
+}

package/src/middlewares/TransactionContext.ts

@@ -67,41 +67,73 @@ export async function createTransactionContext(
 		sfuSctpChannelStorage,
 	} = storageProvider;
 
-	const [clients, sfus] = await Promise.all([clientStorage.getAll(observedCalls.clientIds()), sfuStorage.getAll([])]);
+	const [
+		[
+			clients, 
+			peerConnections,
+			inboundAudioTrackEntries,
+			inboundVideoTrackEntries,
+			outboundAudioTrackEntries,
+			outboundVideoTrackEntries,
+		],
+		[
+			sfus,
+			sfuTransports,
+			sfuInboundRtpPads,
+			sfuOutboundRtpPads,
+			sfuSctpChannels
+		]
+	] = await Promise.all([
+		Promise.all([
+			clientStorage.getAll(observedCalls.clientIds()), 
+			peerConnectionStorage.getAll(observedCalls.peerConnectionIds()),
+			inboundTrackStorage.getAll(observedCalls.inboundAudioTrackIds()),
+			inboundTrackStorage.getAll(observedCalls.inboundVideoTrackIds()),
+			outboundTrackStorage.getAll(observedCalls.outboundAudioTrackIds()),
+			outboundTrackStorage.getAll(observedCalls.outboundVideoTrackIds()),
+		]),
+		Promise.all([
+			sfuStorage.getAll(observedSfus.sfuIds()),
+			sfuTransportStorage.getAll(observedSfus.sfuTransportIds()),
+			sfuInboundRtpPadStorage.getAll(observedCalls.peerConnectionIds()),
+			sfuOutboundRtpPadStorage.getAll(observedCalls.peerConnectionIds()),
+			sfuSctpChannelStorage.getAll(observedCalls.peerConnectionIds()),
+		])
+	]);
 
-	const peerConnectionIds = Array.from(clients.values()).flatMap((client) => client.peerConnectionIds);
+	// const peerConnectionIds = Array.from(clients.values()).flatMap((client) => client.peerConnectionIds);
 
-	const sfuTransportIds = Array.from(sfus.values()).flatMap((sfu) => sfu.sfuTransportIds);
+	// const sfuTransportIds = Array.from(sfus.values()).flatMap((sfu) => sfu.sfuTransportIds);
 
-	const [peerConnections, sfuTransports] = await Promise.all([
-		peerConnectionStorage.getAll(peerConnectionIds),
-		sfuTransportStorage.getAll(sfuTransportIds),
-	]);
+	// const [peerConnections, sfuTransports] = await Promise.all([
+	// 	peerConnectionStorage.getAll(peerConnectionIds),
+	// 	sfuTransportStorage.getAll(sfuTransportIds),
+	// ]);
 
-	const peerConnectionValues = Array.from(peerConnections.values());
-	const inboundTrackIds = peerConnectionValues.flatMap((pc) => pc.inboundTrackIds);
-	const outboundTrackIds = peerConnectionValues.flatMap((pc) => pc.outboundTrackIds);
-
-	const sfuTransportValues = Array.from(sfuTransports.values());
-	const sfuInboundRtpPadIds = sfuTransportValues.flatMap((sfuTransport) => sfuTransport.inboundRtpPadIds);
-	const sfuOutboundRtpPadIds = sfuTransportValues.flatMap((sfuTransport) => sfuTransport.outboundRtpPadIds);
-	const sfuSctpChannelIds = sfuTransportValues.flatMap((sfuTransport) => sfuTransport.sctpChannelIds);
-
-	const [inboundTracks, outboundTracks, sfuInboundRtpPads, sfuOutboundRtpPads, sfuSctpChannels] = await Promise.all([
-		inboundTrackStorage.getAll(inboundTrackIds),
-		outboundTrackStorage.getAll(outboundTrackIds),
-		sfuInboundRtpPadStorage.getAll(sfuInboundRtpPadIds),
-		sfuOutboundRtpPadStorage.getAll(sfuOutboundRtpPadIds),
-		sfuSctpChannelStorage.getAll(sfuSctpChannelIds),
-	]);
+	// const peerConnectionValues = Array.from(peerConnections.values());
+	// const inboundTrackIds = peerConnectionValues.flatMap((pc) => pc.inboundTrackIds);
+	// const outboundTrackIds = peerConnectionValues.flatMap((pc) => pc.outboundTrackIds);
+
+	// const sfuTransportValues = Array.from(sfuTransports.values());
+	// const sfuInboundRtpPadIds = sfuTransportValues.flatMap((sfuTransport) => sfuTransport.inboundRtpPadIds);
+	// const sfuOutboundRtpPadIds = sfuTransportValues.flatMap((sfuTransport) => sfuTransport.outboundRtpPadIds);
+	// const sfuSctpChannelIds = sfuTransportValues.flatMap((sfuTransport) => sfuTransport.sctpChannelIds);
+
+	// const [inboundTracks, outboundTracks, sfuInboundRtpPads, sfuOutboundRtpPads, sfuSctpChannels] = await Promise.all([
+	// 	inboundTrackStorage.getAll(inboundTrackIds),
+	// 	outboundTrackStorage.getAll(outboundTrackIds),
+	// 	sfuInboundRtpPadStorage.getAll(sfuInboundRtpPadIds),
+	// 	sfuOutboundRtpPadStorage.getAll(sfuOutboundRtpPadIds),
+	// 	sfuSctpChannelStorage.getAll(sfuSctpChannelIds),
+	// ]);
 
-	const inboundTrackEntries = Array.from(inboundTracks);
-	const outboundTrackEntries = Array.from(outboundTracks);
+	// const inboundTrackEntries = Array.from(inboundTracks);
+	// const outboundTrackEntries = Array.from(outboundTracks);
 
-	const inboundAudioTrackEntries = inboundTrackEntries.filter((e) => e[1].kind === 'audio');
-	const inboundVideoTrackEntries = inboundTrackEntries.filter((e) => e[1].kind === 'video');
-	const outboundAudioTrackEntries = outboundTrackEntries.filter((e) => e[1].kind === 'audio');
-	const outboundVideoTrackEntries = outboundTrackEntries.filter((e) => e[1].kind === 'video');
+	// const inboundAudioTrackEntries = inboundTrackEntries.filter((e) => e[1].kind === 'audio');
+	// const inboundVideoTrackEntries = inboundTrackEntries.filter((e) => e[1].kind === 'video');
+	// const outboundAudioTrackEntries = outboundTrackEntries.filter((e) => e[1].kind === 'audio');
+	// const outboundVideoTrackEntries = outboundTrackEntries.filter((e) => e[1].kind === 'video');
 
 	const transactionContext: TransactionContext = {
 		id: uuid(),

package/src/middlewares/TransactoinProcessor.ts

@@ -1,6 +1,8 @@
 import { ReportsCollector } from '../common/ReportsCollector';
 import { StorageProvider } from '../storages/StorageProvider';
 import { createCommitTransactionMiddleware } from './CommitTransactionMiddleware';
+import { createDeleteOutdatedCallEntries } from './DeleteOutdatedCallEntries';
+import { createDeleteOutdatedSfuEntries } from './DeleteOutdatedSfuEntries';
 import { Processor, createProcessor } from './Processor';
 import { TransactionContext } from './TransactionContext';
 import { createVisitObservedCallsMiddleware } from './VisitObservedCallsMiddleware';
@@ -9,12 +11,16 @@ import { createVisitObservedSfusMiddleware } from './VisitObservedSfusMiddleware
 export function createTransactionProcessor(
 	storageProvider: StorageProvider,
 	reportsCollector: ReportsCollector,
-	fetchSamples: boolean
+	fetchSamples: boolean,
+	maxIdleTimeInMs: BigInt,
+	findRemoteMatches?: boolean,
 ): Processor<TransactionContext> {
 	return createProcessor(
 		// all of these middlewares must execute sequentially!
-		createVisitObservedCallsMiddleware(reportsCollector, fetchSamples),
+		createVisitObservedCallsMiddleware(storageProvider, reportsCollector, fetchSamples, findRemoteMatches),
 		createVisitObservedSfusMiddleware(reportsCollector, fetchSamples),
+		createDeleteOutdatedCallEntries(storageProvider, maxIdleTimeInMs),
+		createDeleteOutdatedSfuEntries(storageProvider, maxIdleTimeInMs),
 
 		// must be the last one!
 		createCommitTransactionMiddleware(storageProvider, reportsCollector)