@observertc/observer-js 0.40.32 → 0.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/ObservedPeerConnection.d.ts +3 -0
- package/lib/ObservedPeerConnection.d.ts.map +1 -1
- package/lib/ObservedPeerConnection.js +12 -2
- package/lib/Observer.d.ts +2 -0
- package/lib/Observer.d.ts.map +1 -1
- package/lib/Observer.js +41 -4
- package/lib/monitors/SfuServerMonitor.d.ts +79 -0
- package/lib/monitors/SfuServerMonitor.d.ts.map +1 -0
- package/lib/monitors/SfuServerMonitor.js +198 -0
- package/package.json +1 -1
- package/src/ObservedPeerConnection.ts +13 -3
- package/src/Observer.ts +50 -4
- package/src/monitors/SfuServerMonitor.ts +285 -0
|
@@ -37,6 +37,7 @@ export declare class ObservedPeerConnection extends EventEmitter {
|
|
|
37
37
|
readonly client: ObservedClient;
|
|
38
38
|
readonly created: number;
|
|
39
39
|
visited: boolean;
|
|
40
|
+
private _elapsedTimeSinceLastUpdate?;
|
|
40
41
|
totalInboundPacketsLost: number;
|
|
41
42
|
totalInboundPacketsReceived: number;
|
|
42
43
|
totalOutboundPacketsSent: number;
|
|
@@ -63,6 +64,8 @@ export declare class ObservedPeerConnection extends EventEmitter {
|
|
|
63
64
|
deltaReceivedVideoPackets: number;
|
|
64
65
|
deltaSentAudioBytes: number;
|
|
65
66
|
deltaSentVideoBytes: number;
|
|
67
|
+
receivingPacketsPerSecond: number;
|
|
68
|
+
sendingPacketsPerSecond: number;
|
|
66
69
|
sendingAudioBitrate: number;
|
|
67
70
|
sendingVideoBitrate: number;
|
|
68
71
|
receivingAudioBitrate: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ObservedPeerConnection.d.ts","sourceRoot":"","sources":["../src/ObservedPeerConnection.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AACxE,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,oBAAoB,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC;AACzF,OAAO,EAAE,qBAAqB,EAAE,0BAA0B,EAAE,MAAM,yBAAyB,CAAC;AAC5F,OAAO,EAAE,eAAe,EAAiC,MAAM,+BAA+B,CAAC;AAC/F,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAE3C,MAAM,MAAM,4BAA4B,GAAG;IAC1C,MAAM,EAAE;QAAC;YACR,eAAe,EAAE,MAAM,CAAC;SACxB;KAAC,CAAC;IACH,KAAK,EAAE,EAAE,CAAC;IACV,mBAAmB,EAAE,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC;IACrD,mBAAmB,EAAE,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC;IACrD,qBAAqB,EAAE,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC,CAAC;IACxD,qBAAqB,EAAE,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC,CAAC;IACxD,cAAc,EAAE,CAAC,mBAAmB,CAAC,CAAC;CACtC,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG;IACzC,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG,IAAI,CAAC,uBAAuB,EAAE,aAAa,GAAG,OAAO,CAAC,CAAC;AAEjG,MAAM,CAAC,OAAO,WAAW,sBAAsB;IAC9C,EAAE,CAAC,CAAC,SAAS,MAAM,4BAA4B,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,4BAA4B,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,IAAI,CAAC;IAC/H,GAAG,CAAC,CAAC,SAAS,MAAM,4BAA4B,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,4BAA4B,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,IAAI,CAAC;IAChI,IAAI,CAAC,CAAC,SAAS,MAAM,4BAA4B,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,4BAA4B,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,IAAI,CAAC;IACjI,IAAI,CAAC,CAAC,SAAS,MAAM,4BAA4B,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,IAAI,EAAE,4BAA4B,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC;CAChH;AAED,qBAAa,sBAAuB,SAAQ,YAAY;
|
|
1
|
+
{"version":3,"file":"ObservedPeerConnection.d.ts","sourceRoot":"","sources":["../src/ObservedPeerConnection.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AACxE,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,oBAAoB,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC;AACzF,OAAO,EAAE,qBAAqB,EAAE,0BAA0B,EAAE,MAAM,yBAAyB,CAAC;AAC5F,OAAO,EAAE,eAAe,EAAiC,MAAM,+BAA+B,CAAC;AAC/F,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAE3C,MAAM,MAAM,4BAA4B,GAAG;IAC1C,MAAM,EAAE;QAAC;YACR,eAAe,EAAE,MAAM,CAAC;SACxB;KAAC,CAAC;IACH,KAAK,EAAE,EAAE,CAAC;IACV,mBAAmB,EAAE,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC;IACrD,mBAAmB,EAAE,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC;IACrD,qBAAqB,EAAE,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC,CAAC;IACxD,qBAAqB,EAAE,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC,CAAC;IACxD,cAAc,EAAE,CAAC,mBAAmB,CAAC,CAAC;CACtC,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG;IACzC,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,2BAA2B,GAAG,IAAI,CAAC,uBAAuB,EAAE,aAAa,GAAG,OAAO,CAAC,CAAC;AAEjG,MAAM,CAAC,OAAO,WAAW,sBAAsB;IAC9C,EAAE,CAAC,CAAC,SAAS,MAAM,4BAA4B,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,4BAA4B,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,IAAI,CAAC;IAC/H,GAAG,CAAC,CAAC,SAAS,MAAM,4BAA4B,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,4BAA4B,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,IAAI,CAAC;IAChI,IAAI,CAAC,CAAC,SAAS,MAAM,4BAA4B,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,4BAA4B,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,IAAI,CAAC;IACjI,IAAI,CAAC,CAAC,SAAS,MAAM,4BAA4B,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,IAAI,EAAE,4BAA4B,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC;CAChH;AAED,qBAAa,sBAAuB,SAAQ,YAAY;IAyDtD,OAAO,CAAC,QAAQ,CAAC,MAAM;aACP,MAAM,EAAE,cAAc;IAzDvC,SAAgB,OAAO,SAAc;IAC9B,OAAO,UAAQ;IAEtB,OAAO,CAAC,2BAA2B,CAAC,CAAS;IAEtC,uBAAuB,SAAK;IAC5B,2BAA2B,SAAK;IAChC,wBAAwB,SAAK;IAC7B,yBAAyB,SAAK;IAC9B,6BAA6B,SAAK;IAElC,mBAAmB,SAAK;IACxB,mBAAmB,SAAK;IACxB,qBAAqB,SAAK;IAC1B,qBAAqB,SAAK;IAC1B,yBAAyB,SAAK;IAC9B,yBAAyB,SAAK;IAC9B,uBAAuB,SAAK;IAC5B,uBAAuB,SAAK;IAE5B,uBAAuB,SAAK;IAC5B,2BAA2B,SAAK;IAChC,wBAAwB,SAAK;IAC7B,yBAAyB,SAAK;IAC9B,6BAA6B,SAAK;IAClC,yBAAyB,SAAK;IAC9B,sBAAsB,SAAK;IAE3B,uBAAuB,SAAK;IAC5B,uBAAuB,SAAK;IAC5B,yBAAyB,SAAK;IAC9B,yBAAyB,SAAK;IAC9B,mBAAmB,SAAK;IACxB,mBAAmB,SAAK;IACxB,yBAAyB,SAAK;IAC9B,uBAAuB,SAAK;IAC5B,mBAAmB,SAAK;IACxB,mBAAmB,SAAK;IACxB,qBAAqB,SAAK;IAC1B,qBAAqB,SAAK;IAE1B,UAAU,CAAC,EAAE,MAAM,CAAC;IAE3B,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,QAAQ,CAAc;IAC9B,OAAO,CAAC,OAAO,CAAC,CAA8B;IAC9C,OAAO,CAAC,OAAO,CAAC,CAAS;IAEzB,SAAgB,GAAG,cAA4B;IAC/C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAoD;IACxF,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAoD;IACxF,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAqD;IAC1F,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAqD;IAC1F,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA0C;gBAGtD,MAAM,EAAE,2BAA2B,EACpC,MAAM,EAAE,cAAc;IAMvC,IAAW,SAAS,WAEnB;IAED,IAAW,MAAM,WAEhB;IAED,IAAW,MAAM,WAEhB;IAED,IAAW,QAAQ,WAElB;IAED,IAAW,WAAW,WAErB;IAED,IAAW,KAAK,IAIQ,MAAM,GAAG,SAAS,CAFzC;IAED,IAAW,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAEzC;IAED,IAAW,SAAS,YAEnB;IAED,IAAW,wBAAwB,uBAElC;IAED,IAAW,wBAAwB,uBAElC;IAED,IAAW,gBAAgB,IAAI,MAAM,CAGpC;IAED,IAAW,OAAO,kDAEjB;IAED,IAAW,KAAK,IAAI,2BAA2B,GAAG,SAAS,CAE1D;IAED,IAAW,OAAO,IAAI,MAAM,CAE3B;IAED,IAAW,kBAAkB,IAAI,WAAW,CAAC,MAAM,EAAE,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAElF;IAED,IAAW,kBAAkB,IAAI,WAAW,CAAC,MAAM,EAAE,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAElF;IAED,IAAW,mBAAmB,IAAI,WAAW,CAAC,MAAM,EAAE,qBAAqB,CAAC,OAAO,CAAC,CAAC,CAEpF;IAED,IAAW,mBAAmB,IAAI,WAAW,CAAC,MAAM,EAAE,qBAAqB,CAAC,OAAO,CAAC,CAAC,CAEpF;IAED,IAAW,YAAY,IAAI,WAAW,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAElE;IAED,IAAW,UAAU,WAEpB;IAED,IAAW,MAAM,IAIQ,MAAM,GAAG,SAAS,CAF1C;IAED,IAAW,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,EAQ1C;IAEM,cAAc,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,CAAC,eAAe,EAAE,WAAW,GAAG,QAAQ,GAAG,QAAQ,GAAG,UAAU,GAAG,QAAQ,GAAG,kBAAkB,GAAG,QAAQ,CAAC,EAAE,WAAW,CAAC;IAetK,IAAW,MAAM,YAEhB;IAEM,KAAK;IAYL,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,oBAAoB,CAAC,OAAO,CAAC,GAAG,oBAAoB,CAAC,OAAO,CAAC,GAAG,qBAAqB,CAAC,OAAO,CAAC,GAAG,qBAAqB,CAAC,OAAO,CAAC,GAAG,SAAS;IAItK,YAAY;IAuBZ,aAAa;IAkGb,MAAM,CAAC,MAAM,EAAE,uBAAuB,EAAE,SAAS,EAAE,MAAM;IA+BzD,uBAAuB,CAAC,MAAM,EAAE,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,GAAG,oBAAoB,CAAC,OAAO,CAAC;IAkBhH,uBAAuB,CAAC,MAAM,EAAE,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,GAAG,oBAAoB,CAAC,OAAO,CAAC;IAkBhH,wBAAwB,CAAC,MAAM,EAAE,IAAI,CAAC,0BAA0B,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,GAAG,qBAAqB,CAAC,OAAO,CAAC;IAkBnH,wBAAwB,CAAC,MAAM,EAAE,IAAI,CAAC,0BAA0B,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC,GAAG,qBAAqB,CAAC,OAAO,CAAC;IAkBnH,iBAAiB,CAAC,SAAS,EAAE,MAAM;CAgB1C"}
|
|
@@ -39,6 +39,8 @@ class ObservedPeerConnection extends events_1.EventEmitter {
|
|
|
39
39
|
this.deltaReceivedVideoPackets = 0;
|
|
40
40
|
this.deltaSentAudioBytes = 0;
|
|
41
41
|
this.deltaSentVideoBytes = 0;
|
|
42
|
+
this.receivingPacketsPerSecond = 0;
|
|
43
|
+
this.sendingPacketsPerSecond = 0;
|
|
42
44
|
this.sendingAudioBitrate = 0;
|
|
43
45
|
this.sendingVideoBitrate = 0;
|
|
44
46
|
this.receivingAudioBitrate = 0;
|
|
@@ -238,6 +240,14 @@ class ObservedPeerConnection extends events_1.EventEmitter {
|
|
|
238
240
|
this.totalReceivedVideoPackets += this.deltaReceivedVideoPackets;
|
|
239
241
|
this.totalSentAudioPackets += this.deltaSentAudioBytes;
|
|
240
242
|
this.totalSentVideoPackets += this.deltaSentVideoBytes;
|
|
243
|
+
if (this._elapsedTimeSinceLastUpdate && 0 < this._elapsedTimeSinceLastUpdate) {
|
|
244
|
+
this.sendingPacketsPerSecond = this.deltaOutboundPacketsSent / (this._elapsedTimeSinceLastUpdate / 1000);
|
|
245
|
+
this.receivingPacketsPerSecond = this.deltaInboundPacketsReceived / (this._elapsedTimeSinceLastUpdate / 1000);
|
|
246
|
+
}
|
|
247
|
+
else {
|
|
248
|
+
this.sendingPacketsPerSecond = 0;
|
|
249
|
+
this.receivingPacketsPerSecond = 0;
|
|
250
|
+
}
|
|
241
251
|
}
|
|
242
252
|
update(sample, timestamp) {
|
|
243
253
|
if (this._closed)
|
|
@@ -249,13 +259,13 @@ class ObservedPeerConnection extends events_1.EventEmitter {
|
|
|
249
259
|
this._model.label = sample.label;
|
|
250
260
|
}
|
|
251
261
|
const now = Date.now();
|
|
252
|
-
const elapsedTimeInMs = now - this._updated;
|
|
253
262
|
const report = Object.assign(Object.assign({ serviceId: this.client.call.serviceId, roomId: this.client.call.roomId, callId: this.client.call.callId, clientId: this.client.clientId, userId: this.client.userId, mediaUnitId: this.client.mediaUnitId }, sample), { timestamp, sampleSeq: -1, marker: this.marker });
|
|
254
263
|
this.reports.addPeerConnectionTransportReports(report);
|
|
264
|
+
this._elapsedTimeSinceLastUpdate = now - this._updated;
|
|
255
265
|
this.visited = true;
|
|
256
266
|
this._updated = now;
|
|
257
267
|
this.emit('update', {
|
|
258
|
-
elapsedTimeInMs,
|
|
268
|
+
elapsedTimeInMs: this._elapsedTimeSinceLastUpdate,
|
|
259
269
|
});
|
|
260
270
|
}
|
|
261
271
|
createInboundAudioTrack(config) {
|
package/lib/Observer.d.ts
CHANGED
|
@@ -8,6 +8,7 @@ import { ObservedSfu, ObservedSfuModel } from './ObservedSfu';
|
|
|
8
8
|
import { CallSummaryMonitor, CallSummaryMonitorConfig } from './monitors/CallSummaryMonitor';
|
|
9
9
|
import { TurnUsageMonitor } from './monitors/TurnUsageMonitor';
|
|
10
10
|
import { ClientIssueMonitor } from './monitors/ClientIssueMonitor';
|
|
11
|
+
import { SfuServerMonitor } from './monitors/SfuServerMonitor';
|
|
11
12
|
export type ObserverEvents = {
|
|
12
13
|
'newcall': [ObservedCall];
|
|
13
14
|
'newsfu': [ObservedSfu];
|
|
@@ -59,6 +60,7 @@ export declare class Observer extends EventEmitter {
|
|
|
59
60
|
timeoutAfterCallClose?: number;
|
|
60
61
|
}): CallSummaryMonitor;
|
|
61
62
|
createTurnUsageMonitor(): TurnUsageMonitor;
|
|
63
|
+
createSfuServerMonitor(): SfuServerMonitor;
|
|
62
64
|
createClientIssueMonitor(): ClientIssueMonitor;
|
|
63
65
|
get observedCalls(): ReadonlyMap<string, ObservedCall>;
|
|
64
66
|
get observedSfus(): ReadonlyMap<string, ObservedSfu>;
|
package/lib/Observer.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Observer.d.ts","sourceRoot":"","sources":["../src/Observer.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACjE,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAE3C,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAC7F,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAG/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;
|
|
1
|
+
{"version":3,"file":"Observer.d.ts","sourceRoot":"","sources":["../src/Observer.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACjE,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAE3C,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,MAAM,+BAA+B,CAAC;AAC7F,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAG/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAI/D,MAAM,MAAM,cAAc,GAAG;IAC5B,SAAS,EAAE,CAAC,YAAY,CAAC,CAAC;IAC1B,QAAQ,EAAE,CAAC,WAAW,CAAC,CAAC;IACxB,SAAS,EAAE,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,EAAE,EAAE,CAAC;CACZ,CAAA;AAED,MAAM,MAAM,cAAc,GAAG;IAE5B;;;;;;OAMG;IACH,gBAAgB,EAAE,MAAM,CAAC;IAEzB;;;;OAIG;IACH,kBAAkB,EAAE,MAAM,CAAC;IAE3B,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,qBAAqB,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC3C,CAAC;AAEF,MAAM,CAAC,OAAO,WAAW,QAAQ;IAChC,EAAE,CAAC,CAAC,SAAS,MAAM,cAAc,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,IAAI,CAAC;IACnG,GAAG,CAAC,CAAC,SAAS,MAAM,cAAc,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,IAAI,CAAC;IACpG,IAAI,CAAC,CAAC,SAAS,MAAM,cAAc,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,IAAI,CAAC;IACrG,IAAI,CAAC,CAAC,SAAS,MAAM,cAAc,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC;CACpF;AAED,qBAAa,QAAS,SAAQ,YAAY;aA2BxB,MAAM,EAAE,cAAc;WA1BzB,MAAM,CAAC,cAAc,EAAE,OAAO,CAAC,cAAc,CAAC,GAAG,QAAQ;IAkBvE,SAAgB,OAAO,mBAA0B;IACjD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAmC;IAClE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAkC;IAChE,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAsF;IAEhH,OAAO,CAAC,YAAY,CAAC,CAAgC;IACrD,OAAO,CAAC,OAAO,CAAS;gBAEP,MAAM,EAAE,cAAc;IAyBhC,kBAAkB,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACpF,MAAM,EAAE,SAAS,CAAC,iBAAiB,EAAE,WAAW,CAAC,GAAG;QAAE,OAAO,EAAE,CAAC,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAAC,eAAe,CAAC,EAAE,OAAO,CAAA;KAAE,GAC1I,YAAY,CAAC,CAAC,CAAC;IA2CX,iBAAiB,CAAC,OAAO,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACzF,KAAK,EAAE,gBAAgB,EACvB,OAAO,EAAE,OAAO,GACd,WAAW,CAAC,OAAO,CAAC;IAoBhB,wBAAwB,CAAC,OAAO,CAAC,EAAE,wBAAwB,GAAG;QAAE,qBAAqB,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,kBAAkB;IAkCrH,sBAAsB;IA+CtB,sBAAsB;IA6CtB,wBAAwB;IA0B/B,IAAW,aAAa,IAAI,WAAW,CAAC,MAAM,EAAE,YAAY,CAAC,CAE5D;IAED,IAAW,YAAY,IAAI,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,CAE1D;IAED,IAAW,MAAM,YAEhB;IAEM,KAAK;IAWZ,OAAO,CAAC,YAAY;CAepB"}
|
package/lib/Observer.js
CHANGED
|
@@ -21,6 +21,7 @@ const ObservedSfu_1 = require("./ObservedSfu");
|
|
|
21
21
|
const CallSummaryMonitor_1 = require("./monitors/CallSummaryMonitor");
|
|
22
22
|
const TurnUsageMonitor_1 = require("./monitors/TurnUsageMonitor");
|
|
23
23
|
const ClientIssueMonitor_1 = require("./monitors/ClientIssueMonitor");
|
|
24
|
+
const SfuServerMonitor_1 = require("./monitors/SfuServerMonitor");
|
|
24
25
|
const logger = (0, logger_1.createLogger)('Observer');
|
|
25
26
|
class Observer extends events_1.EventEmitter {
|
|
26
27
|
static create(providedConfig) {
|
|
@@ -154,10 +155,46 @@ class Observer extends events_1.EventEmitter {
|
|
|
154
155
|
call.on('newclient', onNewClient);
|
|
155
156
|
};
|
|
156
157
|
monitor.once('close', () => {
|
|
157
|
-
this._monitors.delete(
|
|
158
|
+
this._monitors.delete(TurnUsageMonitor_1.TurnUsageMonitor.name);
|
|
158
159
|
this.off('newcall', onNewCall);
|
|
159
160
|
});
|
|
160
|
-
this._monitors.set(
|
|
161
|
+
this._monitors.set(TurnUsageMonitor_1.TurnUsageMonitor.name, monitor);
|
|
162
|
+
this.on('newcall', onNewCall);
|
|
163
|
+
this.once('close', () => {
|
|
164
|
+
monitor.close();
|
|
165
|
+
});
|
|
166
|
+
return monitor;
|
|
167
|
+
}
|
|
168
|
+
createSfuServerMonitor() {
|
|
169
|
+
if (this._closed)
|
|
170
|
+
throw new Error('Cannot create a turn usage monitor on a closed observer');
|
|
171
|
+
const existingMonitor = this._monitors.get(SfuServerMonitor_1.SfuServerMonitor.name);
|
|
172
|
+
if (existingMonitor)
|
|
173
|
+
return existingMonitor;
|
|
174
|
+
const monitor = new SfuServerMonitor_1.SfuServerMonitor({
|
|
175
|
+
tooHighRttAlertSettings: {
|
|
176
|
+
minNumberOfPeerConnections: 10,
|
|
177
|
+
percentageOfPeerConnectionsHighWatermark: 0.5,
|
|
178
|
+
percentageOfPeerConnectionsLowWatermark: 0.2,
|
|
179
|
+
threshold: 'rtt-gt-300',
|
|
180
|
+
}
|
|
181
|
+
});
|
|
182
|
+
const onNewCall = (call) => {
|
|
183
|
+
const onNewClient = (client) => {
|
|
184
|
+
const onNewPeerConnection = (pc) => {
|
|
185
|
+
monitor.addPeerConnection(pc);
|
|
186
|
+
};
|
|
187
|
+
client.once('close', () => client.off('newpeerconnection', onNewPeerConnection));
|
|
188
|
+
client.on('newpeerconnection', onNewPeerConnection);
|
|
189
|
+
};
|
|
190
|
+
call.once('close', () => call.off('newclient', onNewClient));
|
|
191
|
+
call.on('newclient', onNewClient);
|
|
192
|
+
};
|
|
193
|
+
monitor.once('close', () => {
|
|
194
|
+
this._monitors.delete(SfuServerMonitor_1.SfuServerMonitor.name);
|
|
195
|
+
this.off('newcall', onNewCall);
|
|
196
|
+
});
|
|
197
|
+
this._monitors.set(SfuServerMonitor_1.SfuServerMonitor.name, monitor);
|
|
161
198
|
this.on('newcall', onNewCall);
|
|
162
199
|
this.once('close', () => {
|
|
163
200
|
monitor.close();
|
|
@@ -173,10 +210,10 @@ class Observer extends events_1.EventEmitter {
|
|
|
173
210
|
const monitor = new ClientIssueMonitor_1.ClientIssueMonitor();
|
|
174
211
|
const onNewCall = (call) => monitor.addCall(call);
|
|
175
212
|
monitor.once('close', () => {
|
|
176
|
-
this._monitors.delete(
|
|
213
|
+
this._monitors.delete(ClientIssueMonitor_1.ClientIssueMonitor.name);
|
|
177
214
|
this.off('newcall', onNewCall);
|
|
178
215
|
});
|
|
179
|
-
this._monitors.set(
|
|
216
|
+
this._monitors.set(ClientIssueMonitor_1.ClientIssueMonitor.name, monitor);
|
|
180
217
|
this.on('newcall', onNewCall);
|
|
181
218
|
this.once('close', () => {
|
|
182
219
|
monitor.close();
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import { EventEmitter } from 'events';
|
|
3
|
+
import { ObservedPeerConnection } from '../ObservedPeerConnection';
|
|
4
|
+
export type SfuServerMonitorMetricsRecord = {
|
|
5
|
+
sfuId: string;
|
|
6
|
+
receivedPacketsPerSecond: number;
|
|
7
|
+
sentPacketsPerSecond: number;
|
|
8
|
+
receivingAudioBitrate: number;
|
|
9
|
+
receivingVideoBitrate: number;
|
|
10
|
+
sendingAudioBitrate: number;
|
|
11
|
+
sendingVideoBitrate: number;
|
|
12
|
+
numberOfPeerConnectionsRttLt50: number;
|
|
13
|
+
numberOfPeerConnectionsRttLt150: number;
|
|
14
|
+
numberOfPeerConnectionsRttLt300: number;
|
|
15
|
+
numberOfPeerConnectionsRttOver300: number;
|
|
16
|
+
numberOfPeerConnections: number;
|
|
17
|
+
};
|
|
18
|
+
export type SfuServerMonitorConfig = {
|
|
19
|
+
tooHighRttAlertSettings: {
|
|
20
|
+
/**
|
|
21
|
+
* The threshold for the RTT that will trigger the alert
|
|
22
|
+
*/
|
|
23
|
+
threshold: 'rtt-gt-300' | 'rtt-gt-150' | 'rtt-gt-50';
|
|
24
|
+
/**
|
|
25
|
+
* The percentage of peer connections that need to be above the threshold to trigger the alert
|
|
26
|
+
*/
|
|
27
|
+
percentageOfPeerConnectionsHighWatermark: number;
|
|
28
|
+
/**
|
|
29
|
+
* The percentage of peer connections that need to be above the threshold to clear the alert
|
|
30
|
+
*/
|
|
31
|
+
percentageOfPeerConnectionsLowWatermark: number;
|
|
32
|
+
/**
|
|
33
|
+
* The minimum number of clients that need to be connected to trigger the alert
|
|
34
|
+
*/
|
|
35
|
+
minNumberOfPeerConnections: number;
|
|
36
|
+
};
|
|
37
|
+
};
|
|
38
|
+
export type SfuServerMonitorEvents = {
|
|
39
|
+
'metrics': [SfuServerMonitorMetricsRecord];
|
|
40
|
+
'high-rtt-started': [
|
|
41
|
+
{
|
|
42
|
+
sfuId: string;
|
|
43
|
+
threshold: SfuServerMonitorConfig['tooHighRttAlertSettings']['threshold'];
|
|
44
|
+
percentageOfPeerConnections: number;
|
|
45
|
+
}
|
|
46
|
+
];
|
|
47
|
+
'high-rtt-cleared': [
|
|
48
|
+
{
|
|
49
|
+
sfuId: string;
|
|
50
|
+
}
|
|
51
|
+
];
|
|
52
|
+
'too-many-clients': [{
|
|
53
|
+
ip: string;
|
|
54
|
+
numberOfClients: number;
|
|
55
|
+
}];
|
|
56
|
+
'close': [];
|
|
57
|
+
};
|
|
58
|
+
export declare interface SfuServerMonitor {
|
|
59
|
+
on<U extends keyof SfuServerMonitorEvents>(event: U, listener: (...args: SfuServerMonitorEvents[U]) => void): this;
|
|
60
|
+
off<U extends keyof SfuServerMonitorEvents>(event: U, listener: (...args: SfuServerMonitorEvents[U]) => void): this;
|
|
61
|
+
once<U extends keyof SfuServerMonitorEvents>(event: U, listener: (...args: SfuServerMonitorEvents[U]) => void): this;
|
|
62
|
+
emit<U extends keyof SfuServerMonitorEvents>(event: U, ...args: SfuServerMonitorEvents[U]): boolean;
|
|
63
|
+
}
|
|
64
|
+
export declare class SfuServerMonitor extends EventEmitter {
|
|
65
|
+
readonly config: SfuServerMonitorConfig;
|
|
66
|
+
private _closed;
|
|
67
|
+
private readonly _peerConnections;
|
|
68
|
+
private readonly _metrics;
|
|
69
|
+
private readonly _rttAlertsOn;
|
|
70
|
+
constructor(config: SfuServerMonitorConfig);
|
|
71
|
+
get stats(): SfuServerMonitorMetricsRecord[];
|
|
72
|
+
get closed(): boolean;
|
|
73
|
+
close(): void;
|
|
74
|
+
addPeerConnection(peerConnection: ObservedPeerConnection): void;
|
|
75
|
+
private _checkHighRttForClients;
|
|
76
|
+
private _increase;
|
|
77
|
+
private _subtract;
|
|
78
|
+
}
|
|
79
|
+
//# sourceMappingURL=SfuServerMonitor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"SfuServerMonitor.d.ts","sourceRoot":"","sources":["../../src/monitors/SfuServerMonitor.ts"],"names":[],"mappings":";AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAC;AAEnE,MAAM,MAAM,6BAA6B,GAAG;IAC3C,KAAK,EAAE,MAAM,CAAC;IACd,wBAAwB,EAAE,MAAM,CAAC;IACjC,oBAAoB,EAAE,MAAM,CAAC;IAC7B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,8BAA8B,EAAE,MAAM,CAAC;IACvC,+BAA+B,EAAE,MAAM,CAAC;IACxC,+BAA+B,EAAE,MAAM,CAAC;IACxC,iCAAiC,EAAE,MAAM,CAAC;IAC1C,uBAAuB,EAAE,MAAM,CAAC;CAChC,CAAA;AAED,MAAM,MAAM,sBAAsB,GAAG;IACpC,uBAAuB,EAAE;QAExB;;WAES;QACT,SAAS,EAAE,YAAY,GAAG,YAAY,GAAG,WAAW,CAAC;QAErD;;WAEG;QACH,wCAAwC,EAAE,MAAM,CAAC;QAEjD;;WAEG;QACH,uCAAuC,EAAE,MAAM,CAAC;QAEhD;;WAEG;QACH,0BAA0B,EAAE,MAAM,CAAC;KACnC,CAAA;CACD,CAAA;AAED,MAAM,MAAM,sBAAsB,GAAG;IACpC,SAAS,EAAE,CAAC,6BAA6B,CAAC,CAAC;IAC3C,kBAAkB,EAAE;QAAC;YACpB,KAAK,EAAE,MAAM,CAAC;YACd,SAAS,EAAE,sBAAsB,CAAC,yBAAyB,CAAC,CAAC,WAAW,CAAC,CAAC;YAC1E,2BAA2B,EAAE,MAAM,CAAC;SACpC;KAAC,CAAC;IACH,kBAAkB,EAAE;QAAC;YACpB,KAAK,EAAE,MAAM,CAAC;SACd;KAAC,CAAC;IACH,kBAAkB,EAAE,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,eAAe,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC9D,OAAO,EAAE,EAAE,CAAC;CACZ,CAAA;AAYD,MAAM,CAAC,OAAO,WAAW,gBAAgB;IACxC,EAAE,CAAC,CAAC,SAAS,MAAM,sBAAsB,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,sBAAsB,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,IAAI,CAAC;IACnH,GAAG,CAAC,CAAC,SAAS,MAAM,sBAAsB,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,sBAAsB,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,IAAI,CAAC;IACpH,IAAI,CAAC,CAAC,SAAS,MAAM,sBAAsB,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,sBAAsB,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,IAAI,CAAC;IACrH,IAAI,CAAC,CAAC,SAAS,MAAM,sBAAsB,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,IAAI,EAAE,sBAAsB,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC;CACpG;AAED,qBAAa,gBAAiB,SAAQ,YAAY;aAOhC,MAAM,EAAE,sBAAsB;IAN/C,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAA2C;IAC5E,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAoD;IAC7E,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAqB;gBAGjC,MAAM,EAAE,sBAAsB;IAK/C,IAAW,KAAK,oCAEf;IAED,IAAW,MAAM,YAEhB;IAEM,KAAK;IAUL,iBAAiB,CAAC,cAAc,EAAE,sBAAsB;IA4E/D,OAAO,CAAC,uBAAuB;IA8D/B,OAAO,CAAC,SAAS;IAqBjB,OAAO,CAAC,SAAS;CAqBjB"}
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.SfuServerMonitor = void 0;
|
|
4
|
+
const events_1 = require("events");
|
|
5
|
+
class SfuServerMonitor extends events_1.EventEmitter {
|
|
6
|
+
constructor(config) {
|
|
7
|
+
super();
|
|
8
|
+
this.config = config;
|
|
9
|
+
this._closed = false;
|
|
10
|
+
this._peerConnections = new Map();
|
|
11
|
+
this._metrics = new Map();
|
|
12
|
+
this._rttAlertsOn = new Set();
|
|
13
|
+
}
|
|
14
|
+
get stats() {
|
|
15
|
+
return Array.from(this._metrics.values());
|
|
16
|
+
}
|
|
17
|
+
get closed() {
|
|
18
|
+
return this._closed;
|
|
19
|
+
}
|
|
20
|
+
close() {
|
|
21
|
+
if (this.closed)
|
|
22
|
+
return;
|
|
23
|
+
this._closed = true;
|
|
24
|
+
this._peerConnections.clear();
|
|
25
|
+
this._metrics.clear();
|
|
26
|
+
this._rttAlertsOn.clear();
|
|
27
|
+
this.emit('close');
|
|
28
|
+
}
|
|
29
|
+
addPeerConnection(peerConnection) {
|
|
30
|
+
if (this.closed)
|
|
31
|
+
return;
|
|
32
|
+
let registered = false;
|
|
33
|
+
const onUpdate = () => {
|
|
34
|
+
const sfuId = peerConnection.client.sfuId;
|
|
35
|
+
const peerConnectionRecord = this._peerConnections.get(peerConnection.peerConnectionId);
|
|
36
|
+
if (!sfuId)
|
|
37
|
+
return;
|
|
38
|
+
let metrics = this._metrics.get(sfuId);
|
|
39
|
+
if (!metrics) {
|
|
40
|
+
metrics = {
|
|
41
|
+
sfuId,
|
|
42
|
+
receivedPacketsPerSecond: 0,
|
|
43
|
+
sentPacketsPerSecond: 0,
|
|
44
|
+
receivingAudioBitrate: 0,
|
|
45
|
+
receivingVideoBitrate: 0,
|
|
46
|
+
sendingAudioBitrate: 0,
|
|
47
|
+
sendingVideoBitrate: 0,
|
|
48
|
+
numberOfPeerConnectionsRttLt150: 0,
|
|
49
|
+
numberOfPeerConnectionsRttLt300: 0,
|
|
50
|
+
numberOfPeerConnectionsRttLt50: 0,
|
|
51
|
+
numberOfPeerConnectionsRttOver300: 0,
|
|
52
|
+
numberOfPeerConnections: 0,
|
|
53
|
+
};
|
|
54
|
+
this._metrics.set(sfuId, metrics);
|
|
55
|
+
}
|
|
56
|
+
if (!registered) {
|
|
57
|
+
registered = true;
|
|
58
|
+
++metrics.numberOfPeerConnections;
|
|
59
|
+
}
|
|
60
|
+
if (peerConnectionRecord) {
|
|
61
|
+
this._subtract(metrics, peerConnectionRecord);
|
|
62
|
+
}
|
|
63
|
+
this._increase(metrics, peerConnection);
|
|
64
|
+
this._peerConnections.set(peerConnection.peerConnectionId, {
|
|
65
|
+
lastSendingAudioBitrate: peerConnection.sendingAudioBitrate,
|
|
66
|
+
lastSendingVideoBitrate: peerConnection.sendingVideoBitrate,
|
|
67
|
+
lastReceivingAudioBitrate: peerConnection.receivingAudioBitrate,
|
|
68
|
+
lastReceivingVideoBitrate: peerConnection.receivingVideoBitrate,
|
|
69
|
+
lastAvgRttInMs: peerConnection.avgRttInMs,
|
|
70
|
+
lastReceivingPacketsPerSecond: peerConnection.receivingPacketsPerSecond,
|
|
71
|
+
lastSendingPacketsPerSecond: peerConnection.sendingPacketsPerSecond,
|
|
72
|
+
});
|
|
73
|
+
this._checkHighRttForClients(metrics);
|
|
74
|
+
};
|
|
75
|
+
peerConnection.once('close', () => {
|
|
76
|
+
var _a, _b;
|
|
77
|
+
const address = (_b = (_a = peerConnection.ICE.selectedRemoteCandidate) === null || _a === void 0 ? void 0 : _a.address) !== null && _b !== void 0 ? _b : '';
|
|
78
|
+
const peerConnectionRecord = this._peerConnections.get(peerConnection.peerConnectionId);
|
|
79
|
+
const metrics = this._metrics.get(address);
|
|
80
|
+
if (metrics) {
|
|
81
|
+
if (peerConnectionRecord) {
|
|
82
|
+
this._subtract(metrics, peerConnectionRecord);
|
|
83
|
+
}
|
|
84
|
+
if (registered) {
|
|
85
|
+
if (--metrics.numberOfPeerConnections === 0) {
|
|
86
|
+
this._metrics.delete(address);
|
|
87
|
+
this._rttAlertsOn.delete(address);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
this._peerConnections.delete(peerConnection.peerConnectionId);
|
|
92
|
+
peerConnection.off('update', onUpdate);
|
|
93
|
+
});
|
|
94
|
+
peerConnection.on('update', onUpdate);
|
|
95
|
+
}
|
|
96
|
+
_checkHighRttForClients(metrics) {
|
|
97
|
+
const totalClients = metrics.numberOfPeerConnections;
|
|
98
|
+
const { percentageOfPeerConnectionsHighWatermark, percentageOfPeerConnectionsLowWatermark, minNumberOfPeerConnections, threshold } = this.config.tooHighRttAlertSettings;
|
|
99
|
+
if (metrics.numberOfPeerConnections < minNumberOfPeerConnections)
|
|
100
|
+
return;
|
|
101
|
+
const rttLt150Percentage = (metrics.numberOfPeerConnectionsRttLt150 + metrics.numberOfPeerConnectionsRttLt300 + metrics.numberOfPeerConnectionsRttOver300) / totalClients;
|
|
102
|
+
const rttLt300Percentage = (metrics.numberOfPeerConnectionsRttLt150 + metrics.numberOfPeerConnectionsRttLt300) / totalClients;
|
|
103
|
+
const rttOver300Percentage = metrics.numberOfPeerConnectionsRttOver300 / totalClients;
|
|
104
|
+
const alertOn = this._rttAlertsOn.has(metrics.sfuId);
|
|
105
|
+
switch (threshold) {
|
|
106
|
+
case 'rtt-gt-50': {
|
|
107
|
+
if (!alertOn && percentageOfPeerConnectionsHighWatermark < rttLt150Percentage) {
|
|
108
|
+
this.emit('high-rtt-started', {
|
|
109
|
+
sfuId: metrics.sfuId,
|
|
110
|
+
threshold: 'rtt-gt-50',
|
|
111
|
+
percentageOfPeerConnections: rttLt150Percentage
|
|
112
|
+
});
|
|
113
|
+
this._rttAlertsOn.add(metrics.sfuId);
|
|
114
|
+
}
|
|
115
|
+
else if (alertOn && rttLt150Percentage < percentageOfPeerConnectionsLowWatermark) {
|
|
116
|
+
this.emit('high-rtt-cleared', { sfuId: metrics.sfuId });
|
|
117
|
+
this._rttAlertsOn.delete(metrics.sfuId);
|
|
118
|
+
}
|
|
119
|
+
break;
|
|
120
|
+
}
|
|
121
|
+
case 'rtt-gt-150': {
|
|
122
|
+
if (!alertOn && percentageOfPeerConnectionsHighWatermark < rttLt300Percentage) {
|
|
123
|
+
this.emit('high-rtt-started', {
|
|
124
|
+
sfuId: metrics.sfuId,
|
|
125
|
+
threshold: 'rtt-gt-150',
|
|
126
|
+
percentageOfPeerConnections: rttLt300Percentage
|
|
127
|
+
});
|
|
128
|
+
this._rttAlertsOn.add(metrics.sfuId);
|
|
129
|
+
}
|
|
130
|
+
else if (alertOn && rttLt300Percentage < percentageOfPeerConnectionsLowWatermark) {
|
|
131
|
+
this.emit('high-rtt-cleared', { sfuId: metrics.sfuId });
|
|
132
|
+
this._rttAlertsOn.delete(metrics.sfuId);
|
|
133
|
+
}
|
|
134
|
+
break;
|
|
135
|
+
}
|
|
136
|
+
case 'rtt-gt-300': {
|
|
137
|
+
if (!alertOn && percentageOfPeerConnectionsHighWatermark < rttOver300Percentage) {
|
|
138
|
+
this.emit('high-rtt-started', {
|
|
139
|
+
sfuId: metrics.sfuId,
|
|
140
|
+
threshold: 'rtt-gt-300',
|
|
141
|
+
percentageOfPeerConnections: rttOver300Percentage
|
|
142
|
+
});
|
|
143
|
+
this._rttAlertsOn.add(metrics.sfuId);
|
|
144
|
+
}
|
|
145
|
+
else if (alertOn && rttOver300Percentage < percentageOfPeerConnectionsLowWatermark) {
|
|
146
|
+
this.emit('high-rtt-cleared', { sfuId: metrics.sfuId });
|
|
147
|
+
this._rttAlertsOn.delete(metrics.sfuId);
|
|
148
|
+
}
|
|
149
|
+
break;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
_increase(metrics, peerConnection) {
|
|
154
|
+
metrics.sendingAudioBitrate += peerConnection.receivingAudioBitrate;
|
|
155
|
+
metrics.sendingVideoBitrate += peerConnection.receivingVideoBitrate;
|
|
156
|
+
metrics.receivingAudioBitrate += peerConnection.sendingAudioBitrate;
|
|
157
|
+
metrics.receivingVideoBitrate += peerConnection.sendingVideoBitrate;
|
|
158
|
+
metrics.receivedPacketsPerSecond += peerConnection.sendingPacketsPerSecond;
|
|
159
|
+
metrics.sentPacketsPerSecond += peerConnection.receivingPacketsPerSecond;
|
|
160
|
+
if (peerConnection.avgRttInMs) {
|
|
161
|
+
if (peerConnection.avgRttInMs < 50) {
|
|
162
|
+
++metrics.numberOfPeerConnectionsRttLt50;
|
|
163
|
+
}
|
|
164
|
+
else if (peerConnection.avgRttInMs < 150) {
|
|
165
|
+
++metrics.numberOfPeerConnectionsRttLt150;
|
|
166
|
+
}
|
|
167
|
+
else if (peerConnection.avgRttInMs < 300) {
|
|
168
|
+
++metrics.numberOfPeerConnectionsRttLt300;
|
|
169
|
+
}
|
|
170
|
+
else {
|
|
171
|
+
++metrics.numberOfPeerConnectionsRttOver300;
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
_subtract(metrics, peerConnectionRecord) {
|
|
176
|
+
metrics.sendingAudioBitrate -= peerConnectionRecord.lastReceivingAudioBitrate;
|
|
177
|
+
metrics.sendingVideoBitrate -= peerConnectionRecord.lastReceivingVideoBitrate;
|
|
178
|
+
metrics.receivingAudioBitrate -= peerConnectionRecord.lastSendingAudioBitrate;
|
|
179
|
+
metrics.receivingVideoBitrate -= peerConnectionRecord.lastSendingVideoBitrate;
|
|
180
|
+
metrics.receivedPacketsPerSecond -= peerConnectionRecord.lastSendingPacketsPerSecond;
|
|
181
|
+
metrics.sentPacketsPerSecond -= peerConnectionRecord.lastReceivingPacketsPerSecond;
|
|
182
|
+
if (peerConnectionRecord.lastAvgRttInMs) {
|
|
183
|
+
if (peerConnectionRecord.lastAvgRttInMs < 50) {
|
|
184
|
+
--metrics.numberOfPeerConnectionsRttLt50;
|
|
185
|
+
}
|
|
186
|
+
else if (peerConnectionRecord.lastAvgRttInMs < 150) {
|
|
187
|
+
--metrics.numberOfPeerConnectionsRttLt150;
|
|
188
|
+
}
|
|
189
|
+
else if (peerConnectionRecord.lastAvgRttInMs < 300) {
|
|
190
|
+
--metrics.numberOfPeerConnectionsRttLt300;
|
|
191
|
+
}
|
|
192
|
+
else {
|
|
193
|
+
--metrics.numberOfPeerConnectionsRttOver300;
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
exports.SfuServerMonitor = SfuServerMonitor;
|
package/package.json
CHANGED
|
@@ -37,6 +37,8 @@ export declare interface ObservedPeerConnection {
|
|
|
37
37
|
export class ObservedPeerConnection extends EventEmitter {
|
|
38
38
|
public readonly created = Date.now();
|
|
39
39
|
public visited = true;
|
|
40
|
+
|
|
41
|
+
private _elapsedTimeSinceLastUpdate?: number;
|
|
40
42
|
|
|
41
43
|
public totalInboundPacketsLost = 0;
|
|
42
44
|
public totalInboundPacketsReceived = 0;
|
|
@@ -67,6 +69,8 @@ export class ObservedPeerConnection extends EventEmitter {
|
|
|
67
69
|
public deltaReceivedVideoPackets = 0;
|
|
68
70
|
public deltaSentAudioBytes = 0;
|
|
69
71
|
public deltaSentVideoBytes = 0;
|
|
72
|
+
public receivingPacketsPerSecond = 0;
|
|
73
|
+
public sendingPacketsPerSecond = 0;
|
|
70
74
|
public sendingAudioBitrate = 0;
|
|
71
75
|
public sendingVideoBitrate = 0;
|
|
72
76
|
public receivingAudioBitrate = 0;
|
|
@@ -336,6 +340,13 @@ export class ObservedPeerConnection extends EventEmitter {
|
|
|
336
340
|
this.totalSentAudioPackets += this.deltaSentAudioBytes;
|
|
337
341
|
this.totalSentVideoPackets += this.deltaSentVideoBytes;
|
|
338
342
|
|
|
343
|
+
if (this._elapsedTimeSinceLastUpdate && 0 < this._elapsedTimeSinceLastUpdate) {
|
|
344
|
+
this.sendingPacketsPerSecond = this.deltaOutboundPacketsSent / (this._elapsedTimeSinceLastUpdate / 1000);
|
|
345
|
+
this.receivingPacketsPerSecond = this.deltaInboundPacketsReceived / (this._elapsedTimeSinceLastUpdate / 1000);
|
|
346
|
+
} else {
|
|
347
|
+
this.sendingPacketsPerSecond = 0;
|
|
348
|
+
this.receivingPacketsPerSecond = 0;
|
|
349
|
+
}
|
|
339
350
|
}
|
|
340
351
|
|
|
341
352
|
public update(sample: PeerConnectionTransport, timestamp: number) {
|
|
@@ -347,7 +358,6 @@ export class ObservedPeerConnection extends EventEmitter {
|
|
|
347
358
|
this._model.label = sample.label;
|
|
348
359
|
}
|
|
349
360
|
const now = Date.now();
|
|
350
|
-
const elapsedTimeInMs = now - this._updated;
|
|
351
361
|
const report: PeerConnectionTransportReport = {
|
|
352
362
|
serviceId: this.client.call.serviceId,
|
|
353
363
|
roomId: this.client.call.roomId,
|
|
@@ -362,11 +372,11 @@ export class ObservedPeerConnection extends EventEmitter {
|
|
|
362
372
|
};
|
|
363
373
|
|
|
364
374
|
this.reports.addPeerConnectionTransportReports(report);
|
|
365
|
-
|
|
375
|
+
this._elapsedTimeSinceLastUpdate = now - this._updated;
|
|
366
376
|
this.visited = true;
|
|
367
377
|
this._updated = now;
|
|
368
378
|
this.emit('update', {
|
|
369
|
-
elapsedTimeInMs,
|
|
379
|
+
elapsedTimeInMs: this._elapsedTimeSinceLastUpdate,
|
|
370
380
|
});
|
|
371
381
|
}
|
|
372
382
|
|
package/src/Observer.ts
CHANGED
|
@@ -11,6 +11,7 @@ import { TurnUsageMonitor } from './monitors/TurnUsageMonitor';
|
|
|
11
11
|
import { ObservedClient } from './ObservedClient';
|
|
12
12
|
import { ObservedPeerConnection } from './ObservedPeerConnection';
|
|
13
13
|
import { ClientIssueMonitor } from './monitors/ClientIssueMonitor';
|
|
14
|
+
import { SfuServerMonitor } from './monitors/SfuServerMonitor';
|
|
14
15
|
|
|
15
16
|
const logger = createLogger('Observer');
|
|
16
17
|
|
|
@@ -237,11 +238,56 @@ export class Observer extends EventEmitter {
|
|
|
237
238
|
};
|
|
238
239
|
|
|
239
240
|
monitor.once('close', () => {
|
|
240
|
-
this._monitors.delete(
|
|
241
|
+
this._monitors.delete(TurnUsageMonitor.name);
|
|
241
242
|
this.off('newcall', onNewCall);
|
|
242
243
|
});
|
|
243
244
|
|
|
244
|
-
this._monitors.set(
|
|
245
|
+
this._monitors.set(TurnUsageMonitor.name, monitor);
|
|
246
|
+
this.on('newcall', onNewCall);
|
|
247
|
+
|
|
248
|
+
this.once('close', () => {
|
|
249
|
+
monitor.close();
|
|
250
|
+
});
|
|
251
|
+
|
|
252
|
+
return monitor;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
public createSfuServerMonitor() {
|
|
256
|
+
if (this._closed) throw new Error('Cannot create a turn usage monitor on a closed observer');
|
|
257
|
+
|
|
258
|
+
const existingMonitor = this._monitors.get(SfuServerMonitor.name);
|
|
259
|
+
|
|
260
|
+
if (existingMonitor) return existingMonitor as SfuServerMonitor;
|
|
261
|
+
|
|
262
|
+
const monitor = new SfuServerMonitor({
|
|
263
|
+
tooHighRttAlertSettings: {
|
|
264
|
+
minNumberOfPeerConnections: 10,
|
|
265
|
+
percentageOfPeerConnectionsHighWatermark: 0.5,
|
|
266
|
+
percentageOfPeerConnectionsLowWatermark: 0.2,
|
|
267
|
+
threshold: 'rtt-gt-300',
|
|
268
|
+
}
|
|
269
|
+
});
|
|
270
|
+
|
|
271
|
+
const onNewCall = (call: ObservedCall) => {
|
|
272
|
+
const onNewClient = (client: ObservedClient) => {
|
|
273
|
+
const onNewPeerConnection = (pc: ObservedPeerConnection) => {
|
|
274
|
+
monitor.addPeerConnection(pc);
|
|
275
|
+
};
|
|
276
|
+
|
|
277
|
+
client.once('close', () => client.off('newpeerconnection', onNewPeerConnection));
|
|
278
|
+
client.on('newpeerconnection', onNewPeerConnection);
|
|
279
|
+
};
|
|
280
|
+
|
|
281
|
+
call.once('close', () => call.off('newclient', onNewClient));
|
|
282
|
+
call.on('newclient', onNewClient);
|
|
283
|
+
};
|
|
284
|
+
|
|
285
|
+
monitor.once('close', () => {
|
|
286
|
+
this._monitors.delete(SfuServerMonitor.name);
|
|
287
|
+
this.off('newcall', onNewCall);
|
|
288
|
+
});
|
|
289
|
+
|
|
290
|
+
this._monitors.set(SfuServerMonitor.name, monitor);
|
|
245
291
|
this.on('newcall', onNewCall);
|
|
246
292
|
|
|
247
293
|
this.once('close', () => {
|
|
@@ -263,11 +309,11 @@ export class Observer extends EventEmitter {
|
|
|
263
309
|
const onNewCall = (call: ObservedCall) => monitor.addCall(call);
|
|
264
310
|
|
|
265
311
|
monitor.once('close', () => {
|
|
266
|
-
this._monitors.delete(
|
|
312
|
+
this._monitors.delete(ClientIssueMonitor.name);
|
|
267
313
|
this.off('newcall', onNewCall);
|
|
268
314
|
});
|
|
269
315
|
|
|
270
|
-
this._monitors.set(
|
|
316
|
+
this._monitors.set(ClientIssueMonitor.name, monitor);
|
|
271
317
|
this.on('newcall', onNewCall);
|
|
272
318
|
|
|
273
319
|
this.once('close', () => {
|
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
import { EventEmitter } from 'events';
|
|
2
|
+
import { ObservedPeerConnection } from '../ObservedPeerConnection';
|
|
3
|
+
|
|
4
|
+
export type SfuServerMonitorMetricsRecord = {
|
|
5
|
+
sfuId: string,
|
|
6
|
+
receivedPacketsPerSecond: number,
|
|
7
|
+
sentPacketsPerSecond: number,
|
|
8
|
+
receivingAudioBitrate: number,
|
|
9
|
+
receivingVideoBitrate: number,
|
|
10
|
+
sendingAudioBitrate: number,
|
|
11
|
+
sendingVideoBitrate: number,
|
|
12
|
+
numberOfPeerConnectionsRttLt50: number,
|
|
13
|
+
numberOfPeerConnectionsRttLt150: number,
|
|
14
|
+
numberOfPeerConnectionsRttLt300: number,
|
|
15
|
+
numberOfPeerConnectionsRttOver300: number,
|
|
16
|
+
numberOfPeerConnections: number,
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export type SfuServerMonitorConfig = {
|
|
20
|
+
tooHighRttAlertSettings: {
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* The threshold for the RTT that will trigger the alert
|
|
24
|
+
*/
|
|
25
|
+
threshold: 'rtt-gt-300' | 'rtt-gt-150' | 'rtt-gt-50',
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* The percentage of peer connections that need to be above the threshold to trigger the alert
|
|
29
|
+
*/
|
|
30
|
+
percentageOfPeerConnectionsHighWatermark: number,
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* The percentage of peer connections that need to be above the threshold to clear the alert
|
|
34
|
+
*/
|
|
35
|
+
percentageOfPeerConnectionsLowWatermark: number,
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* The minimum number of clients that need to be connected to trigger the alert
|
|
39
|
+
*/
|
|
40
|
+
minNumberOfPeerConnections: number,
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export type SfuServerMonitorEvents = {
|
|
45
|
+
'metrics': [SfuServerMonitorMetricsRecord],
|
|
46
|
+
'high-rtt-started': [{
|
|
47
|
+
sfuId: string,
|
|
48
|
+
threshold: SfuServerMonitorConfig['tooHighRttAlertSettings']['threshold'],
|
|
49
|
+
percentageOfPeerConnections: number,
|
|
50
|
+
}],
|
|
51
|
+
'high-rtt-cleared': [{
|
|
52
|
+
sfuId: string,
|
|
53
|
+
}],
|
|
54
|
+
'too-many-clients': [{ ip: string, numberOfClients: number }],
|
|
55
|
+
'close': [],
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
type PeerConnectionRecord = {
|
|
59
|
+
lastSendingPacketsPerSecond: number,
|
|
60
|
+
lastReceivingPacketsPerSecond: number,
|
|
61
|
+
lastSendingAudioBitrate: number,
|
|
62
|
+
lastSendingVideoBitrate: number,
|
|
63
|
+
lastReceivingAudioBitrate: number,
|
|
64
|
+
lastReceivingVideoBitrate: number,
|
|
65
|
+
lastAvgRttInMs?: number,
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export declare interface SfuServerMonitor {
|
|
69
|
+
on<U extends keyof SfuServerMonitorEvents>(event: U, listener: (...args: SfuServerMonitorEvents[U]) => void): this;
|
|
70
|
+
off<U extends keyof SfuServerMonitorEvents>(event: U, listener: (...args: SfuServerMonitorEvents[U]) => void): this;
|
|
71
|
+
once<U extends keyof SfuServerMonitorEvents>(event: U, listener: (...args: SfuServerMonitorEvents[U]) => void): this;
|
|
72
|
+
emit<U extends keyof SfuServerMonitorEvents>(event: U, ...args: SfuServerMonitorEvents[U]): boolean;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export class SfuServerMonitor extends EventEmitter {
|
|
76
|
+
private _closed = false;
|
|
77
|
+
private readonly _peerConnections = new Map<string, PeerConnectionRecord>();
|
|
78
|
+
private readonly _metrics = new Map<string, SfuServerMonitorMetricsRecord>();
|
|
79
|
+
private readonly _rttAlertsOn = new Set<string>();
|
|
80
|
+
|
|
81
|
+
public constructor(
|
|
82
|
+
public readonly config: SfuServerMonitorConfig
|
|
83
|
+
) {
|
|
84
|
+
super();
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
public get stats() {
|
|
88
|
+
return Array.from(this._metrics.values());
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
public get closed() {
|
|
92
|
+
return this._closed;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
public close() {
|
|
96
|
+
if (this.closed) return;
|
|
97
|
+
this._closed = true;
|
|
98
|
+
this._peerConnections.clear();
|
|
99
|
+
this._metrics.clear();
|
|
100
|
+
this._rttAlertsOn.clear();
|
|
101
|
+
|
|
102
|
+
this.emit('close');
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
public addPeerConnection(peerConnection: ObservedPeerConnection) {
|
|
106
|
+
if (this.closed) return;
|
|
107
|
+
let registered = false;
|
|
108
|
+
|
|
109
|
+
const onUpdate = () => {
|
|
110
|
+
const sfuId = peerConnection.client.sfuId;
|
|
111
|
+
const peerConnectionRecord = this._peerConnections.get(peerConnection.peerConnectionId);
|
|
112
|
+
|
|
113
|
+
if (!sfuId) return;
|
|
114
|
+
|
|
115
|
+
let metrics = this._metrics.get(sfuId);
|
|
116
|
+
|
|
117
|
+
if (!metrics) {
|
|
118
|
+
metrics = {
|
|
119
|
+
sfuId,
|
|
120
|
+
receivedPacketsPerSecond: 0,
|
|
121
|
+
sentPacketsPerSecond: 0,
|
|
122
|
+
receivingAudioBitrate: 0,
|
|
123
|
+
receivingVideoBitrate: 0,
|
|
124
|
+
sendingAudioBitrate: 0,
|
|
125
|
+
sendingVideoBitrate: 0,
|
|
126
|
+
numberOfPeerConnectionsRttLt150: 0,
|
|
127
|
+
numberOfPeerConnectionsRttLt300: 0,
|
|
128
|
+
numberOfPeerConnectionsRttLt50: 0,
|
|
129
|
+
numberOfPeerConnectionsRttOver300: 0,
|
|
130
|
+
numberOfPeerConnections: 0,
|
|
131
|
+
};
|
|
132
|
+
this._metrics.set(sfuId, metrics);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
if (!registered) {
|
|
136
|
+
registered = true;
|
|
137
|
+
++metrics.numberOfPeerConnections;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
if (peerConnectionRecord) {
|
|
141
|
+
this._subtract(metrics, peerConnectionRecord);
|
|
142
|
+
}
|
|
143
|
+
this._increase(metrics, peerConnection);
|
|
144
|
+
|
|
145
|
+
this._peerConnections.set(peerConnection.peerConnectionId, {
|
|
146
|
+
lastSendingAudioBitrate: peerConnection.sendingAudioBitrate,
|
|
147
|
+
lastSendingVideoBitrate: peerConnection.sendingVideoBitrate,
|
|
148
|
+
lastReceivingAudioBitrate: peerConnection.receivingAudioBitrate,
|
|
149
|
+
lastReceivingVideoBitrate: peerConnection.receivingVideoBitrate,
|
|
150
|
+
lastAvgRttInMs: peerConnection.avgRttInMs,
|
|
151
|
+
lastReceivingPacketsPerSecond: peerConnection.receivingPacketsPerSecond,
|
|
152
|
+
lastSendingPacketsPerSecond: peerConnection.sendingPacketsPerSecond,
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
this._checkHighRttForClients(metrics);
|
|
156
|
+
};
|
|
157
|
+
|
|
158
|
+
peerConnection.once('close', () => {
|
|
159
|
+
const address = peerConnection.ICE.selectedRemoteCandidate?.address ?? '';
|
|
160
|
+
const peerConnectionRecord = this._peerConnections.get(peerConnection.peerConnectionId);
|
|
161
|
+
const metrics = this._metrics.get(address);
|
|
162
|
+
|
|
163
|
+
if (metrics) {
|
|
164
|
+
if (peerConnectionRecord) {
|
|
165
|
+
this._subtract(metrics, peerConnectionRecord);
|
|
166
|
+
}
|
|
167
|
+
if (registered) {
|
|
168
|
+
if (--metrics.numberOfPeerConnections === 0) {
|
|
169
|
+
this._metrics.delete(address);
|
|
170
|
+
this._rttAlertsOn.delete(address);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
this._peerConnections.delete(peerConnection.peerConnectionId);
|
|
175
|
+
|
|
176
|
+
peerConnection.off('update', onUpdate);
|
|
177
|
+
});
|
|
178
|
+
peerConnection.on('update', onUpdate);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
private _checkHighRttForClients(metrics: SfuServerMonitorMetricsRecord) {
|
|
182
|
+
const totalClients = metrics.numberOfPeerConnections;
|
|
183
|
+
const {
|
|
184
|
+
percentageOfPeerConnectionsHighWatermark,
|
|
185
|
+
percentageOfPeerConnectionsLowWatermark,
|
|
186
|
+
minNumberOfPeerConnections,
|
|
187
|
+
threshold
|
|
188
|
+
} = this.config.tooHighRttAlertSettings;
|
|
189
|
+
|
|
190
|
+
if (metrics.numberOfPeerConnections < minNumberOfPeerConnections) return;
|
|
191
|
+
|
|
192
|
+
const rttLt150Percentage = (metrics.numberOfPeerConnectionsRttLt150 + metrics.numberOfPeerConnectionsRttLt300 + metrics.numberOfPeerConnectionsRttOver300) / totalClients;
|
|
193
|
+
const rttLt300Percentage = (metrics.numberOfPeerConnectionsRttLt150 + metrics.numberOfPeerConnectionsRttLt300) / totalClients;
|
|
194
|
+
const rttOver300Percentage = metrics.numberOfPeerConnectionsRttOver300 / totalClients;
|
|
195
|
+
const alertOn = this._rttAlertsOn.has(metrics.sfuId);
|
|
196
|
+
|
|
197
|
+
switch (threshold) {
|
|
198
|
+
case 'rtt-gt-50': {
|
|
199
|
+
if (!alertOn && percentageOfPeerConnectionsHighWatermark < rttLt150Percentage) {
|
|
200
|
+
this.emit('high-rtt-started', {
|
|
201
|
+
sfuId: metrics.sfuId,
|
|
202
|
+
threshold: 'rtt-gt-50',
|
|
203
|
+
percentageOfPeerConnections: rttLt150Percentage
|
|
204
|
+
});
|
|
205
|
+
this._rttAlertsOn.add(metrics.sfuId);
|
|
206
|
+
} else if (alertOn && rttLt150Percentage < percentageOfPeerConnectionsLowWatermark) {
|
|
207
|
+
this.emit('high-rtt-cleared', { sfuId: metrics.sfuId });
|
|
208
|
+
this._rttAlertsOn.delete(metrics.sfuId);
|
|
209
|
+
}
|
|
210
|
+
break;
|
|
211
|
+
}
|
|
212
|
+
case 'rtt-gt-150': {
|
|
213
|
+
if (!alertOn && percentageOfPeerConnectionsHighWatermark < rttLt300Percentage) {
|
|
214
|
+
this.emit('high-rtt-started', {
|
|
215
|
+
sfuId: metrics.sfuId,
|
|
216
|
+
threshold: 'rtt-gt-150',
|
|
217
|
+
percentageOfPeerConnections: rttLt300Percentage
|
|
218
|
+
});
|
|
219
|
+
this._rttAlertsOn.add(metrics.sfuId);
|
|
220
|
+
} else if (alertOn && rttLt300Percentage < percentageOfPeerConnectionsLowWatermark) {
|
|
221
|
+
this.emit('high-rtt-cleared', { sfuId: metrics.sfuId });
|
|
222
|
+
this._rttAlertsOn.delete(metrics.sfuId);
|
|
223
|
+
}
|
|
224
|
+
break;
|
|
225
|
+
}
|
|
226
|
+
case 'rtt-gt-300': {
|
|
227
|
+
if (!alertOn && percentageOfPeerConnectionsHighWatermark < rttOver300Percentage) {
|
|
228
|
+
this.emit('high-rtt-started', {
|
|
229
|
+
sfuId: metrics.sfuId,
|
|
230
|
+
threshold: 'rtt-gt-300',
|
|
231
|
+
percentageOfPeerConnections: rttOver300Percentage
|
|
232
|
+
});
|
|
233
|
+
this._rttAlertsOn.add(metrics.sfuId);
|
|
234
|
+
} else if (alertOn && rttOver300Percentage < percentageOfPeerConnectionsLowWatermark) {
|
|
235
|
+
this.emit('high-rtt-cleared', { sfuId: metrics.sfuId });
|
|
236
|
+
this._rttAlertsOn.delete(metrics.sfuId);
|
|
237
|
+
}
|
|
238
|
+
break;
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
private _increase(metrics: SfuServerMonitorMetricsRecord, peerConnection: ObservedPeerConnection) {
|
|
244
|
+
metrics.sendingAudioBitrate += peerConnection.receivingAudioBitrate;
|
|
245
|
+
metrics.sendingVideoBitrate += peerConnection.receivingVideoBitrate;
|
|
246
|
+
metrics.receivingAudioBitrate += peerConnection.sendingAudioBitrate;
|
|
247
|
+
metrics.receivingVideoBitrate += peerConnection.sendingVideoBitrate;
|
|
248
|
+
metrics.receivedPacketsPerSecond += peerConnection.sendingPacketsPerSecond;
|
|
249
|
+
metrics.sentPacketsPerSecond += peerConnection.receivingPacketsPerSecond;
|
|
250
|
+
|
|
251
|
+
if (peerConnection.avgRttInMs) {
|
|
252
|
+
if (peerConnection.avgRttInMs < 50) {
|
|
253
|
+
++metrics.numberOfPeerConnectionsRttLt50;
|
|
254
|
+
} else if (peerConnection.avgRttInMs < 150) {
|
|
255
|
+
++metrics.numberOfPeerConnectionsRttLt150;
|
|
256
|
+
} else if (peerConnection.avgRttInMs < 300) {
|
|
257
|
+
++metrics.numberOfPeerConnectionsRttLt300;
|
|
258
|
+
} else {
|
|
259
|
+
++metrics.numberOfPeerConnectionsRttOver300;
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
private _subtract(metrics: SfuServerMonitorMetricsRecord, peerConnectionRecord: PeerConnectionRecord) {
|
|
265
|
+
metrics.sendingAudioBitrate -= peerConnectionRecord.lastReceivingAudioBitrate;
|
|
266
|
+
metrics.sendingVideoBitrate -= peerConnectionRecord.lastReceivingVideoBitrate;
|
|
267
|
+
metrics.receivingAudioBitrate -= peerConnectionRecord.lastSendingAudioBitrate;
|
|
268
|
+
metrics.receivingVideoBitrate -= peerConnectionRecord.lastSendingVideoBitrate;
|
|
269
|
+
metrics.receivedPacketsPerSecond -= peerConnectionRecord.lastSendingPacketsPerSecond;
|
|
270
|
+
metrics.sentPacketsPerSecond -= peerConnectionRecord.lastReceivingPacketsPerSecond;
|
|
271
|
+
|
|
272
|
+
if (peerConnectionRecord.lastAvgRttInMs) {
|
|
273
|
+
if (peerConnectionRecord.lastAvgRttInMs < 50) {
|
|
274
|
+
--metrics.numberOfPeerConnectionsRttLt50;
|
|
275
|
+
} else if (peerConnectionRecord.lastAvgRttInMs < 150) {
|
|
276
|
+
--metrics.numberOfPeerConnectionsRttLt150;
|
|
277
|
+
} else if (peerConnectionRecord.lastAvgRttInMs < 300) {
|
|
278
|
+
--metrics.numberOfPeerConnectionsRttLt300;
|
|
279
|
+
} else {
|
|
280
|
+
--metrics.numberOfPeerConnectionsRttOver300;
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
}
|